openc3 7.0.1 → 7.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 36f12f777f56b7d64a19cfd89b8886352b6f0fb06bbda59d95d5643b8d4c71f6
-  data.tar.gz: 1ebffbb1399279e391a47dde42e6f8b9492be10b389c8e63d148aab402ee614c
+  metadata.gz: 719b55747a7004cc44da1976c3f15eec6c67a3fca702b56792862f59f02cb8b5
+  data.tar.gz: c80082e75c059dac71d15a56264cba9c968f82c494e564e33b598c75ac01a6c2
 SHA512:
-  metadata.gz: ee971acf5bfd378f5a238263b467385286997900bea7fe549b8458c707568fb3a2265b261ee8794601bd42db7d0c2e8a983cf3d81a52c0c5069b9be0b41c4966
-  data.tar.gz: 5ba0ea7fb684a6e3e029864289612975db9555d1754f30f65baf4cf3193208516ea74029bb889b4575014b7953c45990c7e515dcf8f43d292bf8fc844de15a6b
+  metadata.gz: 5b3c1ca82717f1debd25ab4a6ac0239aa9baee2b5f736f61b7f52103397b608ab3de8fd0d50e10f0a4481b507b2a519987a4c4c68f7d0f3770cd48f3e165b8eb
+  data.tar.gz: d2bc96742a2555be7c8149dd728155a76ba7c1a16474eb26a36388bdedaacc1fba9c447ab373beebd8fd58ac81248541531de1a163bc156be195c76e34b515f2

data/bin/openc3cli

@@ -45,7 +45,8 @@ require 'irb/completion'
 require 'digest'
 require 'argon2'
 
-$redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME']}:#{ENV['OPENC3_REDIS_PORT']}"
+$redis_shardnum = ENV['OPENC3_SHARDNUM'] || "0"
+$redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME'].to_s.gsub("SHARDNUM", $redis_shardnum)}:#{ENV['OPENC3_REDIS_PORT']}"
 
 ERROR_CODE = 1
 
@@ -443,6 +444,9 @@ def unload_plugin(plugin_name, scope:)
       plugin_model = OpenC3::PluginModel.get_model(name: plugin_name, scope: scope)
       plugin_model.destroy
       OpenC3::LocalMode.remove_local_plugin(plugin_name, scope: scope)
+      # Remove the backing gem now that no PluginModel references it,
+      # so it disappears from the admin Packages tab.
+      OpenC3::PluginModel.cleanup_gem(plugin_name, scope: scope)
       OpenC3::Logger.info("PluginModel destroyed: #{plugin_name}", scope: scope)
     rescue => e
       abort("Error uninstalling plugin: #{scope}: #{plugin_name}: #{e.formatted}")
@@ -1143,6 +1147,48 @@ if not ARGV[0].nil? # argument(s) given
     end
     cli_pkg_uninstall(ARGV[1], scope: ARGV[2])
 
+  when 'reingest'
+    # Internal command spawned by StorageController via ProcessManager so the
+    # reingest runs in its own process and System singleton resets cannot
+    # collide with the cmd-tlm-api Rails server.
+    if ARGV[1].nil? || ARGV[2].nil? || ARGV[1] == '--help' || ARGV[1] == '-h'
+      puts "Usage: cli reingest JOB_ID SCOPE"
+      exit(ARGV[1].nil? ? 1 : 0)
+    end
+    require 'openc3/utilities/reingest_job'
+    require 'openc3/models/reingest_job_model'
+    job_id = ARGV[1]
+    scope = ARGV[2]
+    job = OpenC3::ReingestJobModel.get_model(name: job_id, scope: scope)
+    if job.nil?
+      OpenC3::Logger.error("Reingest job #{job_id} not found in scope #{scope}")
+      exit(1)
+    end
+    begin
+      OpenC3::ReingestJob.new(
+        job_id: job_id,
+        files: job.files,
+        path: job.path,
+        bucket: job.bucket,
+        scope: scope,
+        target_version: job.target_version,
+      ).run
+    rescue Exception => e
+      # ReingestJob#run already marks Crashed for errors raised during the run.
+      # This catches failures from the constructor itself (or anything before
+      # run gets its rescue installed) so the model doesn't sit in Queued forever.
+      OpenC3::Logger.error("Reingest job #{job_id} crashed before run: #{e.formatted}")
+      begin
+        job.state = 'Crashed'
+        job.error = e.message
+        job.finished_at = Time.now.utc.iso8601
+        job.update
+      rescue => e2
+        OpenC3::Logger.error("Reingest job #{job_id} failed to mark Crashed: #{e2.message}")
+      end
+      exit(1)
+    end
+
   when 'generate'
     # To test against a local copy call this file from the root cosmos directory like this:
     # ruby -Iopenc3/lib openc3/bin/openc3cli generate ...
@@ -1390,8 +1436,9 @@ if not ARGV[0].nil? # argument(s) given
         end
       end
     end
-    # Unless explicitly disabled, ensure the tools bucket is public
-    unless ENV.fetch("OPENC3_NO_BUCKET_POLICY", false)
+    # Unless explicitly disabled, ensure the tools bucket is public.
+    # OPENC3_TOOLS_BUCKET_PRIVATE keeps the tools bucket private; the cmd-tlm-api proxies reads via ToolsController.
+    unless ENV.fetch("OPENC3_NO_BUCKET_POLICY", false) || ENV.fetch("OPENC3_TOOLS_BUCKET_PRIVATE", false)
       client.ensure_public(ENV['OPENC3_TOOLS_BUCKET'])
     end
     # Always ensure the scriptrunner policy is in place since it is required for script execution

data/data/config/interface_modifiers.yaml

@@ -212,7 +212,9 @@ WORK_DIR:
     WORK_DIR '/openc3/lib/openc3/microservices'
 PORT:
   summary: Open port for the microservice
-  description: Kubernetes needs a Service to be applied to open a port so this is required for Kubernetes support
+  description:
+    Kubernetes needs a Service to be applied to open a port so this is required for Kubernetes support
+    See [Exposing Microservices](/docs/guides/exposing-microservices) for more information.
   since: 5.7.0
   parameters:
     - name: Number

data/data/config/item_modifiers.yaml

@@ -193,4 +193,4 @@ HIDDEN:
   description: This item will not appear in PacketViewer or Item Choosers.
     It also hides this item from appearing in the Script Runner popup helper
     when writing scripts. The item will also not be included in decom data.
-  since: 6.10.1
+  since: 6.10.0

data/data/config/microservice.yaml

@@ -40,7 +40,9 @@ MICROSERVICE:
           WORK_DIR .
     PORT:
       summary: Open port for the microservice
-      description: Kubernetes needs a Service to be applied to open a port so this is required for Kubernetes support
+      description:
+        Kubernetes needs a Service to be applied to open a port so this is required for Kubernetes support.
+        See [Exposing Microservices](/docs/guides/exposing-microservices) for more information.
       since: 5.0.10
       parameters:
         - name: Number
@@ -168,11 +170,22 @@ MICROSERVICE:
       since: 6.0.0
       parameters:
         - name: Shard
-          required: false
+          required: true
           description: Shard number starting from 0
           values: \d+
       example: |
         SHARD 0
+    DB_SHARD:
+      summary: Shard for target database database if sharding Redis/TSDB
+      description: DB Shard. Only used if running multiple database shards typically in Kubernetes
+      since: 7.1.0
+      parameters:
+        - name: DB Shard
+          required: true
+          description: DB Shard number starting from 0
+          values: \d+
+      example: |
+        DB_SHARD 0
     STOPPED:
       summary: Initially creates the microservice in a stopped state (not enabled)
       since: 6.2.0

data/data/config/parameter_modifiers.yaml

@@ -82,14 +82,15 @@ WRITE_CONVERSION:
     [INST inst_cmds.txt](https://github.com/OpenC3/cosmos/blob/main/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST/cmd_tlm/inst_cmds.txt)
     or [INST2 inst_cmds.txt](https://github.com/OpenC3/cosmos/blob/main/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST2/cmd_tlm/inst_cmds.txt).
 
-    :::info Multiple write conversions on command parameters
+    :::info[Multiple write conversions on command parameters]
     When a command is built, each item gets written (and write conversions are run)
     to set the default value. Then items are written (again write conversions are run)
     with user provided values. Thus write conversions can be run twice. Also there are
     no guarantees which parameters have already been written. The packet itself has a
-    given_values() method which can be used to retrieve a hash of the user provided
+    `given_values` attribute which can be used to retrieve a hash of the user provided
     values to the command. That can be used to check parameter values passed in.
     :::
+
   parameters:
     - name: Class Filename
       required: true
@@ -105,9 +106,48 @@ WRITE_CONVERSION:
         to the class constructor.
       values: .*
   ruby_example: |
-    WRITE_CONVERSION ip_write_conversion.rb
+    # Example command with a WRITE_CONVERSION that sets a command parameter
+    # based on the given values of other parameters
+    COMMAND INST BLOCK BIG_ENDIAN "Send variable block of data"
+      APPEND_PARAMETER BYTE 8 UINT MIN MAX 0x55 "Byte to duplicate"
+        FORMAT_STRING "0x%0X"
+      APPEND_PARAMETER LENGTH 32 UINT MIN MAX 0 "Length of data"
+      APPEND_PARAMETER DATA 0 BLOCK "" "Variable block of data"
+        WRITE_CONVERSION block_conversion.rb
+        HIDDEN # Because we're filling it in with a conversion
+
+    # Implemented in INST/lib/block_conversion.rb:
+    require 'openc3/conversions/conversion'
+    module OpenC3
+      class BlockConversion < Conversion
+        def call(value, packet, buffer)
+          # Use the packet.given_values hash to access user provided values to the command
+          byte = packet.given_values['BYTE'] || 0x55
+          length = packet.given_values['LENGTH'] || 0
+          [byte].pack('C') * length
+        end
+      end
+    end
   python_example: |
-    WRITE_CONVERSION openc3/conversions/ip_write_conversion.py
+    # Example command with a WRITE_CONVERSION that sets a command parameter
+    # based on the given values of other parameters
+    COMMAND INST BLOCK BIG_ENDIAN "Send variable block of data"
+      APPEND_PARAMETER BYTE 8 UINT MIN MAX 0x55 "Byte to duplicate"
+        FORMAT_STRING "0x%0X"
+      APPEND_PARAMETER LENGTH 32 UINT MIN MAX 0 "Length of data"
+      APPEND_PARAMETER DATA 0 BLOCK "" "Variable block of data"
+        WRITE_CONVERSION block_conversion.py
+        HIDDEN # Because we're filling it in with a conversion
+
+    # Implemented in INST/lib/block_conversion.py:
+    from openc3.conversions.conversion import Conversion
+    class BlockConversion(Conversion):
+        def call(self, value, packet, buffer):
+            # Use the packet.given_values hash to access user provided values to the command
+            byte = packet.given_values.get('BYTE', 0x55)
+            length = packet.given_values.get('LENGTH', 0)
+            return bytes([byte]) * length
+
 POLY_WRITE_CONVERSION:
   summary: Adds a polynomial conversion factor to the current command parameter
   description: See [Polynomial Conversion](/docs/configuration/conversions#polynomial_conversion) for more information.
@@ -127,14 +167,15 @@ GENERIC_WRITE_CONVERSION_START:
     value. The GENERIC_WRITE_CONVERSION_END keyword specifies that all lines of
     code for the conversion have been given.
 
-    :::info Multiple write conversions on command parameters
+    :::info[Multiple write conversions on command parameters]
     When a command is built, each item gets written (and write conversions are run)
     to set the default value. Then items are written (again write conversions are run)
     with user provided values. Thus write conversions can be run twice. Also there are
     no guarantees which parameters have already been written. The packet itself has a
-    given_values() method which can be used to retrieve a hash of the user provided
+    `given_values` attribute which can be used to retrieve a hash of the user provided
     values to the command. That can be used to check parameter values passed in.
     :::
+
   warning: Generic conversions are not a good long term solution. Consider creating
     a conversion class and using WRITE_CONVERSION instead. WRITE_CONVERSION is easier
     to debug and higher performance.
@@ -171,4 +212,5 @@ HIDDEN:
   summary: Hides this parameter from all the OpenC3 tools
   description: This item will not appear in CmdSender.
     It also hides this item from appearing in the Script Runner popup helper
-    when writing scripts. The parameter should not be provided to commands.
+    when writing scripts. The parameter should not be provided to commands.
+  since: 6.10.0

data/data/config/plugins.yaml

@@ -38,6 +38,7 @@ VARIABLE_STATE:
     VARIABLE log_retain_time 172800
     VARIABLE_STATE "24 hours" 86400
     VARIABLE_STATE "48 hours" 172800
+    VARIABLE_STATE 60 # Both description and value are set to 60
   parameters:
     - name: State Description
       required: false

data/data/config/target.yaml

@@ -159,3 +159,14 @@ TARGET:
           values: \d+
       example: |
         SHARD 0
+    DB_SHARD:
+      summary: Shard for target database database if sharding Redis/TSDB
+      description: DB Shard. Only used if running multiple database shards typically in Kubernetes
+      since: 7.1.0
+      parameters:
+        - name: DB Shard
+          required: true
+          description: DB Shard number starting from 0
+          values: \d+
+      example: |
+        DB_SHARD 0

data/data/config/target_config.yaml

@@ -2,8 +2,12 @@
 LANGUAGE:
   summary: Programming language of the target interfaces and microservices
   description: The target language must be either Ruby or Python. The language
-    determines how the target's interfaces and microservices are run. Note that
-    both Ruby and Python still use ERB to perform templating.
+    determines how the target's interfaces and microservices are run. A target
+    must pick one language for its interfaces and microservices — you cannot
+    mix Ruby and Python interfaces/microservices within the same target. Scripts
+    executed in Script Runner are independent of this setting and may be written
+    in either Ruby or Python regardless of the target's LANGUAGE. Note that both
+    Ruby and Python still use ERB to perform templating.
   example: LANGUAGE python
   parameters:
     - language: Programming language

data/lib/openc3/api/api.rb

@@ -17,6 +17,7 @@
 
 require 'openc3/script/extract'
 require 'openc3/script/api_shared'
+require 'openc3/api/calendar_api'
 require 'openc3/api/cmd_api'
 require 'openc3/api/config_api'
 require 'openc3/api/interface_api'

data/lib/openc3/api/calendar_api.rb

@@ -0,0 +1,183 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'date'
+require 'openc3/models/timeline_model'
+require 'openc3/models/activity_model'
+require 'openc3/topics/timeline_topic'
+
+module OpenC3
+  module Api
+    # NOTE: These methods are intentionally NOT added to WHITELIST. Their signatures
+    # match openc3/lib/openc3/script/calendar.rb (no manual:/token: kwargs), so they
+    # cannot be dispatched via JSON-RPC (which auto-injects manual/token from headers).
+    # The script-side calendar methods reach the server through the timeline/activity
+    # HTTP controllers, which call these helpers after performing their own
+    # authorization.
+
+    # Returns an array of all timelines for the given scope.
+    def list_timelines(scope: $openc3_scope)
+      ret = []
+      TimelineModel.all.each do |timeline, value|
+        if scope == timeline.split('__')[0]
+          ret << value
+        end
+      end
+      ret
+    end
+
+    # Creates a new timeline and deploys its microservice.
+    # @return [Hash] the created timeline as a hash
+    def create_timeline(name, color: nil, scope: $openc3_scope)
+      model = TimelineModel.new(name: name, color: color, scope: scope)
+      model.create()
+      model.deploy()
+      model.as_json()
+    end
+
+    # @return [Hash, nil] the timeline as a hash, or nil if not found
+    def get_timeline(name, scope: $openc3_scope)
+      model = TimelineModel.get(name: name, scope: scope)
+      return nil if model.nil?
+      model.as_json()
+    end
+
+    # Updates the color of an existing timeline.
+    # @return [Hash, nil] the updated timeline as a hash, or nil if not found
+    def set_timeline_color(name, color, scope: $openc3_scope)
+      model = TimelineModel.get(name: name, scope: scope)
+      return nil if model.nil?
+      model.color = color
+      model.update()
+      model.notify(kind: 'updated')
+      model.as_json()
+    end
+
+    # Updates the execute flag of an existing timeline.
+    # @return [Hash, nil] the updated timeline as a hash, or nil if not found
+    def set_timeline_execute(name, enable, scope: $openc3_scope)
+      model = TimelineModel.get(name: name, scope: scope)
+      return nil if model.nil?
+      model.execute = enable
+      model.update()
+      model.notify(kind: 'updated')
+      model.as_json()
+    end
+
+    # Deletes a timeline (and optionally all of its activities when force is true).
+    # @return [Hash, nil] {'name' => name}, or nil if not found
+    def delete_timeline(name, force: false, scope: $openc3_scope)
+      model = TimelineModel.get(name: name, scope: scope)
+      return nil if model.nil?
+      TimelineModel.delete(name: name, scope: scope, force: force)
+      model.undeploy()
+      model.notify(kind: 'deleted')
+      { 'name' => name }
+    end
+
+    # Creates a new activity on the specified timeline.
+    # username is read from data['username'] if present and is used for the audit event.
+    # @return [Hash] the created activity as a hash
+    def create_timeline_activity(name, kind:, start:, stop:, data: {}, recurring: nil, scope: $openc3_scope)
+      data ||= {}
+      hash = {
+        kind: kind,
+        start: _cal_to_epoch(start),
+        stop: _cal_to_epoch(stop),
+        data: data,
+      }
+      if recurring
+        recurring = recurring.dup
+        if recurring['end']
+          recurring['end'] = _cal_to_epoch(recurring['end'])
+        end
+        hash[:recurring] = recurring
+      end
+      model = ActivityModel.from_json(hash, name: name, scope: scope)
+      model.create(username: data['username'])
+      model.as_json()
+    end
+
+    # Updates an existing activity on the specified timeline.
+    # @return [Hash, nil] the updated activity as a hash, or nil if not found
+    def update_timeline_activity(name, id:, kind:, start:, stop:, uuid:, data: {}, scope: $openc3_scope)
+      data ||= {}
+      model = ActivityModel.score(name: name, score: id.to_i, uuid: uuid, scope: scope)
+      return nil if model.nil?
+      model.update(
+        start: _cal_to_epoch(start),
+        stop: _cal_to_epoch(stop),
+        kind: kind,
+        data: data,
+        username: data['username'],
+      )
+      model.as_json()
+    end
+
+    # @return [Hash, nil] the activity as a hash, or nil if not found
+    def get_timeline_activity(name, start, uuid, scope: $openc3_scope)
+      model = ActivityModel.score(name: name, score: start.to_i, uuid: uuid, scope: scope)
+      return nil if model.nil?
+      model.as_json()
+    end
+
+    # Returns activities on the timeline in the given window.
+    # When start/stop are nil, defaults to a window of [now - 7 days, now + 7 days].
+    # When limit is nil, defaults to one event per minute over the window.
+    # @return [Array<Hash>] the matching activities
+    def get_timeline_activities(name, start: nil, stop: nil, limit: nil, scope: $openc3_scope)
+      now = DateTime.now.new_offset(0)
+      start_score = start.nil? ? (now - 7).strftime('%s').to_i : _cal_to_epoch(start)
+      stop_score = stop.nil? ? (now + 7).strftime('%s').to_i : _cal_to_epoch(stop)
+      limit ||= ((stop_score - start_score) / 60).to_i
+      ActivityModel.get(name: name, scope: scope, start: start_score, stop: stop_score, limit: limit)
+    end
+
+    # Removes an activity (or all members of its recurring group when recurring is truthy).
+    # @return [Integer] number of activities removed (0 indicates not found)
+    def delete_timeline_activity(name, start, uuid, recurring: nil, scope: $openc3_scope)
+      ActivityModel.destroy(name: name, scope: scope, score: start.to_i, uuid: uuid, recurring: recurring)
+    end
+
+    # @return [Integer] count of activities on the timeline
+    def count_timeline_activities(name, scope: $openc3_scope)
+      ActivityModel.count(name: name, scope: scope)
+    end
+
+    # Commits an event to an existing activity.
+    # @return [Hash, nil] the activity as a hash, or nil if not found
+    def commit_timeline_activity(name, start, uuid, status:, message: nil, scope: $openc3_scope)
+      model = ActivityModel.score(name: name, score: start.to_i, uuid: uuid, scope: scope)
+      return nil if model.nil?
+      model.commit(status: status, message: message)
+      model.as_json()
+    end
+
+    # Convert a value to an epoch integer. Accepts Integer/Numeric (treated as already-epoch),
+    # numeric strings, and ISO-style date/time strings or DateTime/Time objects.
+    def _cal_to_epoch(value)
+      case value
+      when Integer
+        value
+      when Numeric
+        value.to_i
+      when DateTime, Time, Date
+        value.to_datetime.strftime('%s').to_i
+      else
+        s = value.to_s
+        return s.to_i if s.match?(/\A-?\d+\z/)
+        DateTime.parse(s).strftime('%s').to_i
+      end
+    end
+  end
+end

data/lib/openc3/api/cmd_api.rb

@@ -183,7 +183,8 @@ module OpenC3
       authorize(permission: 'cmd_info', target_name: target_name, packet_name: command_name, manual: manual, scope: scope, token: token)
       TargetModel.packet(target_name, command_name, type: :CMD, scope: scope)
       topic = "#{scope}__COMMAND__{#{target_name}}__#{command_name}"
-      msg_id, msg_hash = Topic.get_newest_message(topic)
+      db_shard = Store.db_shard_for_target(target_name, scope: scope)
+      msg_id, msg_hash = Topic.get_newest_message(topic, db_shard: db_shard)
       if msg_id
         msg_hash['buffer'] = msg_hash['buffer'].b
         return msg_hash

data/lib/openc3/api/metrics_api.rb

@@ -28,9 +28,11 @@ module OpenC3
     DELAY_METRICS['log_topic_delta_seconds'] = 0.0
     DELAY_METRICS['router_topic_delta_seconds'] = 0.0
     DELAY_METRICS['text_log_topic_delta_seconds'] = 0.0
+    DELAY_METRICS['tsdb_ingest_topic_delta_seconds'] = 0.0
 
     DURATION_METRICS = {}
     DURATION_METRICS['decom_duration_seconds'] = 0.0
+    DURATION_METRICS['tsdb_ingest_duration_seconds'] = 0.0
 
     SUM_METRICS = {}
     SUM_METRICS['cleanup_total'] = 0
@@ -48,6 +50,8 @@ module OpenC3
     SUM_METRICS['router_directive_total'] = 0
     SUM_METRICS['text_log_total'] = 0
     SUM_METRICS['text_log_error_total'] = 0
+    SUM_METRICS['tsdb_ingest_total'] = 0
+    SUM_METRICS['tsdb_ingest_error_total'] = 0
 
     def get_metrics(manual: false, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'system', manual: manual, scope: scope, token: token)
@@ -79,7 +83,13 @@ module OpenC3
       result.merge!(duration_metrics)
       result.merge!(sum_metrics)
 
-      result.merge!(MetricModel.redis_metrics)
+      redis_metrics = MetricModel.redis_metrics
+      redis_metrics.each do |_db_shard, values|
+        values.each do |key, value|
+          existing = result[key]
+          result[key] = value if existing.nil? or value > existing
+        end
+      end
 
       return result
     end

data/lib/openc3/api/tlm_api.rb

@@ -117,7 +117,7 @@ module OpenC3
     # @param packet_name [String] Packet name of the packet
     # @param item_hash [Hash] Hash of item_name and value for each item you want to change from the current value table
     # @param type [Symbol] Telemetry type, :RAW, :CONVERTED (default), :FORMATTED
-    def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, manual: false, scope: $openc3_scope, token: $openc3_token)
+    def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, manual: false, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
       type = type.to_s.intern
       target_name = target_name.upcase
@@ -155,9 +155,9 @@ module OpenC3
 
       # Use an interface microservice if it exists, other use the decom microservice
       if interface_name
-        InterfaceTopic.inject_tlm(interface_name, target_name, packet_name, item_hash, type: type, scope: scope)
+        InterfaceTopic.inject_tlm(interface_name, target_name, packet_name, item_hash, type: type, stored: stored, scope: scope)
       else
-        DecomInterfaceTopic.inject_tlm(target_name, packet_name, item_hash, type: type, scope: scope)
+        DecomInterfaceTopic.inject_tlm(target_name, packet_name, item_hash, type: type, stored: stored, scope: scope)
       end
     end
 
@@ -221,7 +221,8 @@ module OpenC3
         return msg_hash
       else
         topic = "#{scope}__TELEMETRY__{#{target_name}}__#{packet_name}"
-        msg_id, msg_hash = Topic.get_newest_message(topic)
+        db_shard = Store.db_shard_for_target(target_name, scope: scope)
+        msg_id, msg_hash = Topic.get_newest_message(topic, db_shard: db_shard)
         if msg_id
           msg_hash['buffer'] = msg_hash['buffer'].b
           return msg_hash
@@ -446,7 +447,8 @@ module OpenC3
         packet_name = packet_name.upcase
         authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
         topic = "#{scope}__DECOM__{#{target_name}}__#{packet_name}"
-        id, _ = Topic.get_newest_message(topic)
+        db_shard = Store.db_shard_for_target(target_name, scope: scope)
+        id, = Topic.get_newest_message(topic, db_shard: db_shard)
         results[topic] = id ? id : '0-0'
       end
       results.to_a.join(SUBSCRIPTION_DELIMITER)
@@ -463,7 +465,20 @@ module OpenC3
       authorize(permission: 'tlm', manual: manual, scope: scope, token: token)
       # Split the list of topic, ID values and turn it into a hash for easy updates
       lookup = Hash[*id.split(SUBSCRIPTION_DELIMITER)]
-      xread = Topic.read_topics(lookup.keys, lookup.values, nil, count) # Always don't block
+      # Group topics by db_shard for multi-shard support
+      db_shard_groups = {}
+      lookup.each do |topic, offset|
+        target_name = topic.match(/__\{?([^}_]+)\}?__/)[1] rescue nil
+        db_shard = Store.db_shard_for_target(target_name, scope: scope)
+        db_shard_groups[db_shard] ||= { topics: [], offsets: [] }
+        db_shard_groups[db_shard][:topics] << topic
+        db_shard_groups[db_shard][:offsets] << offset
+      end
+      xread = {}
+      db_shard_groups.each do |db_shard, group|
+        result = Topic.read_topics(group[:topics], group[:offsets], nil, count, db_shard: db_shard) # Always don't block
+        xread.merge!(result) if result
+      end
       # Return the original ID and and empty array if we didn't get anything
       packets = []
       return [id, packets] if xread.empty?

data/lib/openc3/core_ext/faraday.rb

@@ -1,6 +1,6 @@
 # Remove warnings in CGI
 saved_verbose = $VERBOSE
-$VERBOSE = false
+$VERBOSE = nil
 require 'faraday'
 $VERBOSE = saved_verbose
 

data/lib/openc3/io/json_api.rb

@@ -65,7 +65,7 @@ module OpenC3
 
     def _request(*method_params, **kw_params)
       kw_params[:scope] = $openc3_scope unless kw_params[:scope]
-      kw_params[:json] = true unless kw_params[:json]
+      kw_params[:json] = true # This is JsonApi so should always be speaking json
       @json_api.request(*method_params, **kw_params)
     end
   end

data/lib/openc3/logs/log_writer.rb

@@ -237,7 +237,9 @@ module OpenC3
                   # Now that the file is in S3, trim the Redis stream up until the previous file.
                   # This keeps one minute of data in Redis
                   instance.cleanup_offsets[index].each do |redis_topic, cleanup_offset|
-                    Topic.trim_topic(redis_topic, cleanup_offset)
+                    target_match = redis_topic.match(/__\{?([^}_]+)\}?__/)
+                    db_shard = target_match ? Store.db_shard_for_target(target_match[1]) : 0
+                    Topic.trim_topic(redis_topic, cleanup_offset, db_shard: db_shard)
                   end
                   indexes_to_clear << index
                 end