openc3 7.0.0 → 7.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 0a06901f01bc9d8e7b6f0b123296da67475c08ea9cf50ef788a86b6da1308fcb
-  data.tar.gz: 34732b912cea1b4677f3e94ac0ba9f437e99217ccf54f9250816176ef5b63db1
+  metadata.gz: fa824943ce72cce586cdfe03c1a5a8395120801bd845174c25433a15bf986306
+  data.tar.gz: 620cc66576e14e7e696419e5be0b5704190452d29efc35a315a489099b20f82e
 SHA512:
-  metadata.gz: 5cda606ccb010606ad4b432685f53428cbf10b216f1cd06f2f55baac28fb723514dc3f8dfeadafc0958734237d3a7faf3f42439695481145914434e2a4bb1a58
-  data.tar.gz: 98532ea424c421554d45a6d155fadfc27e67ddf969a96be7653daebb30d5530dbb7601e13b5d3a3afa7dfd56718c161c060e81d67a5ee967a9248225554e1a03
+  metadata.gz: 1f392d51b2e5870128a995aa9e5717f9a4ed3bf0b8f55952b9141ec34898406734a2a834cf18d09b78d6e82f5675c2c0f3269e2ef62964ec58f2b96b8933ce80
+  data.tar.gz: a92a073ff672da55a44b85b700cb5b6a908208e9a10ddcad77828f02de0f877c3db2252d7d93857ab5b65d55b39594dfc109d72028a82cbfeaa6e468878eef94

data/bin/openc3cli

@@ -45,7 +45,8 @@ require 'irb/completion'
 require 'digest'
 require 'argon2'
 
-$redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME']}:#{ENV['OPENC3_REDIS_PORT']}"
+$redis_shardnum = ENV['OPENC3_SHARDNUM'] || "0"
+$redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME'].to_s.gsub("SHARDNUM", $redis_shardnum)}:#{ENV['OPENC3_REDIS_PORT']}"
 
 ERROR_CODE = 1
 
@@ -75,6 +76,7 @@ def print_usage
   puts "  cli pkguninstall PKGFILENAME SCOPE  # Uninstall loaded package (Ruby gem or python package)"
   puts "  cli xtce_converter                # Convert to and from the XTCE format. Run with --help for more info."
   puts "  cli cstol_converter               # Converts CSTOL files (.prc) to COSMOS. Run with --help for more info."
+  puts "  cli setpassword                   # Set the initial password from OPENC3_API_PASSWORD env var"
   puts ""
 end
 
@@ -588,35 +590,44 @@ def run_bridge(filename, params)
   end
 end
 
-def cli_script_monitor(script_id)
+def cli_script_monitor(script_id, format: 'text')
   ret_code = ERROR_CODE
   require 'openc3/script'
+  require 'openc3/utilities/ctrf'
   begin
     OpenC3::RunningScriptWebSocketApi.new(id: script_id) do |api|
       while (resp = api.read) do
         # see ScriptRunner.vue for types and states
         case resp['type']
         when 'file'
-          puts "Filename #{resp['filename']} scope #{resp['scope']}"
+          puts "Filename #{resp['filename']} scope #{resp['scope']}" unless format == 'ctrf'
         when 'line'
           fn = resp['filename'].nil? ? '<no file>' : resp['filename']
-          puts "At [#{fn}:#{resp['line_no']}] state [#{resp['state']}]"
+          puts "At [#{fn}:#{resp['line_no']}] state [#{resp['state']}]" unless format == 'ctrf'
           if resp['state'] == 'error' or resp['state'] == 'crashed'
             $script_interrupt_text = ''
-            puts 'script failed'
+            puts 'script failed' unless format == 'ctrf'
             break
           end
         when 'output'
-          puts resp['line']
+          puts resp['line'] unless format == 'ctrf'
         when 'complete'
           $script_interrupt_text = ''
-          puts 'script complete'
+          if resp['report']
+            if format == 'ctrf'
+              ctrf_data = OpenC3::Ctrf.convert_report(resp['report'])
+              puts JSON.pretty_generate(ctrf_data)
+            else
+              puts resp['report']
+            end
+          end
+          puts 'script complete' unless format == 'ctrf'
           ret_code = 0
           break
         # These conditions are all handled by the else
         # when 'running', 'breakpoint', 'waiting', 'time'
         else
-          puts resp.pretty_inspect
+          puts resp.pretty_inspect unless format == 'ctrf'
         end
       end
     end
@@ -692,10 +703,10 @@ def cli_script_run(args, options)
     puts id
     $script_interrupt_text = "  Script #{args[1]} still running remotely.\n" # for Ctrl-C
     if (options[:wait] < 1) then
-      ret_code = cli_script_monitor(id)
+      ret_code = cli_script_monitor(id, format: options[:format])
     else
       Timeout::timeout(options[:wait], nil, "--wait #{options[:wait]} exceeded") do
-        ret_code = cli_script_monitor(id)
+        ret_code = cli_script_monitor(id, format: options[:format])
       rescue Timeout::ExitException, Timeout::Error => e
         # Timeout exceptions are also raised by the Websocket API, so we check
         if e.message =~ /^--wait /
@@ -795,7 +806,7 @@ rescue => e
 end
 
 def cli_script(args=[])
-  options = {scope: 'DEFAULT', disconnect: false, wait: 0, verbose: false}
+  options = {scope: 'DEFAULT', disconnect: false, wait: 0, verbose: false, format: 'text'}
   option_parser = OptionParser.new do |opts|
     opts.banner = "Usage: script --scope SCOPE [init | list | spawn | run]\n" +
       "  init                               Initialize running scripts (Enterprise Only)\n" +
@@ -812,6 +823,9 @@ def cli_script(args=[])
     opts.on("--scope SCOPE", "Run with specified scope (default = DEFAULT)") do |arg|
       options[:scope] = arg
     end
+    opts.on("--format FORMAT", "Output format: text or ctrf (default = text)") do |arg|
+      options[:format] = arg
+    end
     opts.on("--suite SUITE", "Run with specified suite") do |arg|
       options[:suite] = arg
     end
@@ -892,6 +906,25 @@ def cli_script(args=[])
   exit(ret_code)
 end
 
+def set_password
+  password = ENV['OPENC3_API_PASSWORD']
+  argon2_profile = ENV["OPENC3_ARGON2_PROFILE"]&.to_sym || :rfc_9106_low_memory
+  if password.nil? or password.empty?
+    abort "OPENC3_API_PASSWORD environment variable is required"
+  end
+  if password.length < 8
+    abort "Password must be at least 8 characters"
+  end
+  redis = Redis.new(url: $redis_url, username: ENV['OPENC3_REDIS_USERNAME'], password: ENV['OPENC3_REDIS_PASSWORD'])
+  if redis.exists('OPENC3__TOKEN') == 1
+    abort "Password has already been set. Use the web interface to change the password."
+  end
+  pw_hash = Argon2::Password.create(password, profile: argon2_profile)
+  redis.set('OPENC3__TOKEN', pw_hash)
+  puts "Password set successfully."
+  exit 0
+end
+
 def migrate_password_hash
   password = ENV['OPENC3_API_PASSWORD']
   argon2_profile = ENV["OPENC3_ARGON2_PROFILE"]&.to_sym || :rfc_9106_low_memory
@@ -1111,6 +1144,48 @@ if not ARGV[0].nil? # argument(s) given
     end
     cli_pkg_uninstall(ARGV[1], scope: ARGV[2])
 
+  when 'reingest'
+    # Internal command spawned by StorageController via ProcessManager so the
+    # reingest runs in its own process and System singleton resets cannot
+    # collide with the cmd-tlm-api Rails server.
+    if ARGV[1].nil? || ARGV[2].nil? || ARGV[1] == '--help' || ARGV[1] == '-h'
+      puts "Usage: cli reingest JOB_ID SCOPE"
+      exit(ARGV[1].nil? ? 1 : 0)
+    end
+    require 'openc3/utilities/reingest_job'
+    require 'openc3/models/reingest_job_model'
+    job_id = ARGV[1]
+    scope = ARGV[2]
+    job = OpenC3::ReingestJobModel.get_model(name: job_id, scope: scope)
+    if job.nil?
+      OpenC3::Logger.error("Reingest job #{job_id} not found in scope #{scope}")
+      exit(1)
+    end
+    begin
+      OpenC3::ReingestJob.new(
+        job_id: job_id,
+        files: job.files,
+        path: job.path,
+        bucket: job.bucket,
+        scope: scope,
+        target_version: job.target_version,
+      ).run
+    rescue Exception => e
+      # ReingestJob#run already marks Crashed for errors raised during the run.
+      # This catches failures from the constructor itself (or anything before
+      # run gets its rescue installed) so the model doesn't sit in Queued forever.
+      OpenC3::Logger.error("Reingest job #{job_id} crashed before run: #{e.formatted}")
+      begin
+        job.state = 'Crashed'
+        job.error = e.message
+        job.finished_at = Time.now.utc.iso8601
+        job.update
+      rescue => e2
+        OpenC3::Logger.error("Reingest job #{job_id} failed to mark Crashed: #{e2.message}")
+      end
+      exit(1)
+    end
+
   when 'generate'
     # To test against a local copy call this file from the root cosmos directory like this:
     # ruby -Iopenc3/lib openc3/bin/openc3cli generate ...
@@ -1358,8 +1433,9 @@ if not ARGV[0].nil? # argument(s) given
         end
       end
     end
-    # Unless explicitly disabled, ensure the tools bucket is public
-    unless ENV.fetch("OPENC3_NO_BUCKET_POLICY", false)
+    # Unless explicitly disabled, ensure the tools bucket is public.
+    # OPENC3_TOOLS_BUCKET_PRIVATE keeps the tools bucket private; the cmd-tlm-api proxies reads via ToolsController.
+    unless ENV.fetch("OPENC3_NO_BUCKET_POLICY", false) || ENV.fetch("OPENC3_TOOLS_BUCKET_PRIVATE", false)
       client.ensure_public(ENV['OPENC3_TOOLS_BUCKET'])
     end
     # Always ensure the scriptrunner policy is in place since it is required for script execution
@@ -1377,6 +1453,22 @@ if not ARGV[0].nil? # argument(s) given
     end
     run_migrations(ARGV[1])
 
+  when 'setpassword'
+    if ARGV[1] == '--help' || ARGV[1] == '-h'
+      puts "Usage: cli setpassword"
+      puts ""
+      puts "Set the initial COSMOS password from the OPENC3_API_PASSWORD environment variable."
+      puts "This allows you to skip the password creation screen in the web interface."
+      puts ""
+      puts "The password must be at least 8 characters. This command will fail if a"
+      puts "password has already been set."
+      puts ""
+      puts "Options:"
+      puts "  -h, --help        Show this help message"
+      exit 0
+    end
+    set_password()
+
   when 'migratepassword'
     migrate_password_hash()
 

data/bin/pipinstall

@@ -1,13 +1,45 @@
 #!/bin/sh
-uv venv "$PYTHONUSERBASE"
+uv venv "$PYTHONUSERBASE" --allow-existing
 echo "uv pip install $@"
 uv pip install --python "$PYTHONUSERBASE" "$@"
 if [ $? -eq 0 ]; then
     echo "Command succeeded"
-else
-    echo "Command failed - retrying with --no-index"
-    uv pip install --python "$PYTHONUSERBASE" --no-index "$@"
-    if [ $? -ne 0 ]; then
-        echo "ERROR: uv pip install failed"
+    exit 0
+fi
+
+# Collect the last arg and all preceding args
+LAST_ARG=""
+OPTS=""
+PREV=""
+for ARG in "$@"; do
+    if [ -n "$PREV" ]; then
+        OPTS="${OPTS} ${PREV}"
+    fi
+    PREV="$ARG"
+done
+LAST_ARG="$PREV"
+
+# If last arg is a directory with pyproject.toml, the build may have failed
+# (e.g. Poetry package-mode = false). Try compiling and installing declared
+# dependencies only, without attempting to build the package itself.
+if [ -d "$LAST_ARG" ] && [ -f "$LAST_ARG/pyproject.toml" ]; then
+    echo "Warning: Failed to build Python package, attempting to install declared dependencies from pyproject.toml"
+    TMPFILE=$(mktemp)
+    uv pip compile ${OPTS} "${LAST_ARG}/pyproject.toml" > "$TMPFILE"
+    if [ $? -eq 0 ] && [ -s "$TMPFILE" ]; then
+        uv pip install --python "$PYTHONUSERBASE" ${OPTS} -r "$TMPFILE"
+        if [ $? -eq 0 ]; then
+            echo "Dependencies installed successfully"
+            rm -f "$TMPFILE"
+            exit 0
+        fi
     fi
+    rm -f "$TMPFILE"
+fi
+
+echo "Warning: Install failed - retrying with --no-index"
+uv pip install --python "$PYTHONUSERBASE" --no-index "$@"
+if [ $? -ne 0 ]; then
+    echo "ERROR: uv pip install failed"
+    exit 1
 fi

data/data/config/command_modifiers.yaml

@@ -199,6 +199,7 @@ HIDDEN:
   summary: Hides this command from all OpenC3 tools such as Command Sender and Handbook Creator
   description: Hidden commands do not appear in the Script Runner popup helper when writing scripts.
     The command still exists in the system and can be sent by scripts.
+  since: 6.10.1
 DISABLED:
   summary: Disables this command from being sent
   description: Hides the command and also disables it from being sent by scripts.

data/data/config/item_modifiers.yaml

@@ -192,4 +192,5 @@ HIDDEN:
   summary: Hides this item from all the OpenC3 tools
   description: This item will not appear in PacketViewer or Item Choosers.
     It also hides this item from appearing in the Script Runner popup helper
-    when writing scripts. The item will also not be included in decom data.
+    when writing scripts. The item will also not be included in decom data.
+  since: 6.10.0

data/data/config/microservice.yaml

@@ -168,11 +168,22 @@ MICROSERVICE:
       since: 6.0.0
       parameters:
         - name: Shard
-          required: false
+          required: true
           description: Shard number starting from 0
           values: \d+
       example: |
         SHARD 0
+    DB_SHARD:
+      summary: Shard for target database database if sharding Redis/TSDB
+      description: DB Shard. Only used if running multiple database shards typically in Kubernetes
+      since: 7.1.0
+      parameters:
+        - name: DB Shard
+          required: true
+          description: DB Shard number starting from 0
+          values: \d+
+      example: |
+        DB_SHARD 0
     STOPPED:
       summary: Initially creates the microservice in a stopped state (not enabled)
       since: 6.2.0

data/data/config/parameter_modifiers.yaml

@@ -82,14 +82,15 @@ WRITE_CONVERSION:
     [INST inst_cmds.txt](https://github.com/OpenC3/cosmos/blob/main/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST/cmd_tlm/inst_cmds.txt)
     or [INST2 inst_cmds.txt](https://github.com/OpenC3/cosmos/blob/main/openc3-cosmos-init/plugins/packages/openc3-cosmos-demo/targets/INST2/cmd_tlm/inst_cmds.txt).
 
-    :::info Multiple write conversions on command parameters
+    :::info[Multiple write conversions on command parameters]
     When a command is built, each item gets written (and write conversions are run)
     to set the default value. Then items are written (again write conversions are run)
     with user provided values. Thus write conversions can be run twice. Also there are
     no guarantees which parameters have already been written. The packet itself has a
-    given_values() method which can be used to retrieve a hash of the user provided
+    `given_values` attribute which can be used to retrieve a hash of the user provided
     values to the command. That can be used to check parameter values passed in.
     :::
+
   parameters:
     - name: Class Filename
       required: true
@@ -105,9 +106,48 @@ WRITE_CONVERSION:
         to the class constructor.
       values: .*
   ruby_example: |
-    WRITE_CONVERSION ip_write_conversion.rb
+    # Example command with a WRITE_CONVERSION that sets a command parameter
+    # based on the given values of other parameters
+    COMMAND INST BLOCK BIG_ENDIAN "Send variable block of data"
+      APPEND_PARAMETER BYTE 8 UINT MIN MAX 0x55 "Byte to duplicate"
+        FORMAT_STRING "0x%0X"
+      APPEND_PARAMETER LENGTH 32 UINT MIN MAX 0 "Length of data"
+      APPEND_PARAMETER DATA 0 BLOCK "" "Variable block of data"
+        WRITE_CONVERSION block_conversion.rb
+        HIDDEN # Because we're filling it in with a conversion
+
+    # Implemented in INST/lib/block_conversion.rb:
+    require 'openc3/conversions/conversion'
+    module OpenC3
+      class BlockConversion < Conversion
+        def call(value, packet, buffer)
+          # Use the packet.given_values hash to access user provided values to the command
+          byte = packet.given_values['BYTE'] || 0x55
+          length = packet.given_values['LENGTH'] || 0
+          [byte].pack('C') * length
+        end
+      end
+    end
   python_example: |
-    WRITE_CONVERSION openc3/conversions/ip_write_conversion.py
+    # Example command with a WRITE_CONVERSION that sets a command parameter
+    # based on the given values of other parameters
+    COMMAND INST BLOCK BIG_ENDIAN "Send variable block of data"
+      APPEND_PARAMETER BYTE 8 UINT MIN MAX 0x55 "Byte to duplicate"
+        FORMAT_STRING "0x%0X"
+      APPEND_PARAMETER LENGTH 32 UINT MIN MAX 0 "Length of data"
+      APPEND_PARAMETER DATA 0 BLOCK "" "Variable block of data"
+        WRITE_CONVERSION block_conversion.py
+        HIDDEN # Because we're filling it in with a conversion
+
+    # Implemented in INST/lib/block_conversion.py:
+    from openc3.conversions.conversion import Conversion
+    class BlockConversion(Conversion):
+        def call(self, value, packet, buffer):
+            # Use the packet.given_values hash to access user provided values to the command
+            byte = packet.given_values.get('BYTE', 0x55)
+            length = packet.given_values.get('LENGTH', 0)
+            return bytes([byte]) * length
+
 POLY_WRITE_CONVERSION:
   summary: Adds a polynomial conversion factor to the current command parameter
   description: See [Polynomial Conversion](/docs/configuration/conversions#polynomial_conversion) for more information.
@@ -127,14 +167,15 @@ GENERIC_WRITE_CONVERSION_START:
     value. The GENERIC_WRITE_CONVERSION_END keyword specifies that all lines of
     code for the conversion have been given.
 
-    :::info Multiple write conversions on command parameters
+    :::info[Multiple write conversions on command parameters]
     When a command is built, each item gets written (and write conversions are run)
     to set the default value. Then items are written (again write conversions are run)
     with user provided values. Thus write conversions can be run twice. Also there are
     no guarantees which parameters have already been written. The packet itself has a
-    given_values() method which can be used to retrieve a hash of the user provided
+    `given_values` attribute which can be used to retrieve a hash of the user provided
     values to the command. That can be used to check parameter values passed in.
     :::
+
   warning: Generic conversions are not a good long term solution. Consider creating
     a conversion class and using WRITE_CONVERSION instead. WRITE_CONVERSION is easier
     to debug and higher performance.
@@ -171,4 +212,5 @@ HIDDEN:
   summary: Hides this parameter from all the OpenC3 tools
   description: This item will not appear in CmdSender.
     It also hides this item from appearing in the Script Runner popup helper
-    when writing scripts. The parameter should not be provided to commands.
+    when writing scripts. The parameter should not be provided to commands.
+  since: 6.10.0

data/data/config/table_parameter_modifiers.yaml

@@ -3,7 +3,9 @@ HIDDEN:
   summary: Indicates that the parameter should not be shown to the user in the Table Manager GUI
   description: Hidden parameters still exist and will be saved to the resulting
     binary. This is useful for padding and other essential but non-user editable fields.
+  since: 6.10.1
 UNEDITABLE:
   summary: Indicates that the parameter should be shown to the user but not editable.
-  description: Uneditable parameters are useful for control fields which the user
+  description:
+    Uneditable parameters are useful for control fields which the user
     may be interested in but should not be able to edit.

data/data/config/target.yaml

@@ -159,3 +159,14 @@ TARGET:
           values: \d+
       example: |
         SHARD 0
+    DB_SHARD:
+      summary: Shard for target database database if sharding Redis/TSDB
+      description: DB Shard. Only used if running multiple database shards typically in Kubernetes
+      since: 7.1.0
+      parameters:
+        - name: DB Shard
+          required: true
+          description: DB Shard number starting from 0
+          values: \d+
+      example: |
+        DB_SHARD 0

data/data/config/target_config.yaml

@@ -2,8 +2,12 @@
 LANGUAGE:
   summary: Programming language of the target interfaces and microservices
   description: The target language must be either Ruby or Python. The language
-    determines how the target's interfaces and microservices are run. Note that
-    both Ruby and Python still use ERB to perform templating.
+    determines how the target's interfaces and microservices are run. A target
+    must pick one language for its interfaces and microservices — you cannot
+    mix Ruby and Python interfaces/microservices within the same target. Scripts
+    executed in Script Runner are independent of this setting and may be written
+    in either Ruby or Python regardless of the target's LANGUAGE. Note that both
+    Ruby and Python still use ERB to perform templating.
   example: LANGUAGE python
   parameters:
     - language: Programming language

data/lib/openc3/accessors/template_accessor.rb

@@ -53,6 +53,9 @@ module OpenC3
       return nil if item.data_type == :DERIVED
       configure()
 
+      # No template items to read (e.g. command with fixed template string)
+      return nil if @item_keys.empty?
+
       # Scan the response for all the variables in brackets <VARIABLE>
       values = buffer.scan(@read_regexp)[0]
       if !values || (values.length != @item_keys.length)
@@ -73,6 +76,12 @@ module OpenC3
       result = {}
       configure()
 
+      # No template items to read (e.g. command with fixed template string)
+      if @item_keys.empty?
+        items.each { |item| result[item.name] = nil }
+        return result
+      end
+
       # Scan the response for all the variables in brackets <VARIABLE>
       values = buffer.scan(@read_regexp)[0]
       if !values || (values.length != @item_keys.length)

data/lib/openc3/api/cmd_api.rb

@@ -183,7 +183,8 @@ module OpenC3
       authorize(permission: 'cmd_info', target_name: target_name, packet_name: command_name, manual: manual, scope: scope, token: token)
       TargetModel.packet(target_name, command_name, type: :CMD, scope: scope)
       topic = "#{scope}__COMMAND__{#{target_name}}__#{command_name}"
-      msg_id, msg_hash = Topic.get_newest_message(topic)
+      db_shard = Store.db_shard_for_target(target_name, scope: scope)
+      msg_id, msg_hash = Topic.get_newest_message(topic, db_shard: db_shard)
       if msg_id
         msg_hash['buffer'] = msg_hash['buffer'].b
         return msg_hash

data/lib/openc3/api/metrics_api.rb

@@ -28,9 +28,11 @@ module OpenC3
     DELAY_METRICS['log_topic_delta_seconds'] = 0.0
     DELAY_METRICS['router_topic_delta_seconds'] = 0.0
     DELAY_METRICS['text_log_topic_delta_seconds'] = 0.0
+    DELAY_METRICS['tsdb_ingest_topic_delta_seconds'] = 0.0
 
     DURATION_METRICS = {}
     DURATION_METRICS['decom_duration_seconds'] = 0.0
+    DURATION_METRICS['tsdb_ingest_duration_seconds'] = 0.0
 
     SUM_METRICS = {}
     SUM_METRICS['cleanup_total'] = 0
@@ -48,6 +50,8 @@ module OpenC3
     SUM_METRICS['router_directive_total'] = 0
     SUM_METRICS['text_log_total'] = 0
     SUM_METRICS['text_log_error_total'] = 0
+    SUM_METRICS['tsdb_ingest_total'] = 0
+    SUM_METRICS['tsdb_ingest_error_total'] = 0
 
     def get_metrics(manual: false, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'system', manual: manual, scope: scope, token: token)
@@ -79,7 +83,13 @@ module OpenC3
       result.merge!(duration_metrics)
       result.merge!(sum_metrics)
 
-      result.merge!(MetricModel.redis_metrics)
+      redis_metrics = MetricModel.redis_metrics
+      redis_metrics.each do |_db_shard, values|
+        values.each do |key, value|
+          existing = result[key]
+          result[key] = value if existing.nil? or value > existing
+        end
+      end
 
       return result
     end

data/lib/openc3/api/tlm_api.rb

@@ -117,7 +117,7 @@ module OpenC3
     # @param packet_name [String] Packet name of the packet
     # @param item_hash [Hash] Hash of item_name and value for each item you want to change from the current value table
     # @param type [Symbol] Telemetry type, :RAW, :CONVERTED (default), :FORMATTED
-    def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, manual: false, scope: $openc3_scope, token: $openc3_token)
+    def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, manual: false, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
       type = type.to_s.intern
       target_name = target_name.upcase
@@ -155,9 +155,9 @@ module OpenC3
 
       # Use an interface microservice if it exists, other use the decom microservice
       if interface_name
-        InterfaceTopic.inject_tlm(interface_name, target_name, packet_name, item_hash, type: type, scope: scope)
+        InterfaceTopic.inject_tlm(interface_name, target_name, packet_name, item_hash, type: type, stored: stored, scope: scope)
       else
-        DecomInterfaceTopic.inject_tlm(target_name, packet_name, item_hash, type: type, scope: scope)
+        DecomInterfaceTopic.inject_tlm(target_name, packet_name, item_hash, type: type, stored: stored, scope: scope)
       end
     end
 
@@ -221,7 +221,8 @@ module OpenC3
         return msg_hash
       else
         topic = "#{scope}__TELEMETRY__{#{target_name}}__#{packet_name}"
-        msg_id, msg_hash = Topic.get_newest_message(topic)
+        db_shard = Store.db_shard_for_target(target_name, scope: scope)
+        msg_id, msg_hash = Topic.get_newest_message(topic, db_shard: db_shard)
         if msg_id
           msg_hash['buffer'] = msg_hash['buffer'].b
           return msg_hash
@@ -446,7 +447,8 @@ module OpenC3
         packet_name = packet_name.upcase
         authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
         topic = "#{scope}__DECOM__{#{target_name}}__#{packet_name}"
-        id, _ = Topic.get_newest_message(topic)
+        db_shard = Store.db_shard_for_target(target_name, scope: scope)
+        id, = Topic.get_newest_message(topic, db_shard: db_shard)
         results[topic] = id ? id : '0-0'
       end
       results.to_a.join(SUBSCRIPTION_DELIMITER)
@@ -463,7 +465,20 @@ module OpenC3
       authorize(permission: 'tlm', manual: manual, scope: scope, token: token)
       # Split the list of topic, ID values and turn it into a hash for easy updates
       lookup = Hash[*id.split(SUBSCRIPTION_DELIMITER)]
-      xread = Topic.read_topics(lookup.keys, lookup.values, nil, count) # Always don't block
+      # Group topics by db_shard for multi-shard support
+      db_shard_groups = {}
+      lookup.each do |topic, offset|
+        target_name = topic.match(/__\{?([^}_]+)\}?__/)[1] rescue nil
+        db_shard = Store.db_shard_for_target(target_name, scope: scope)
+        db_shard_groups[db_shard] ||= { topics: [], offsets: [] }
+        db_shard_groups[db_shard][:topics] << topic
+        db_shard_groups[db_shard][:offsets] << offset
+      end
+      xread = {}
+      db_shard_groups.each do |db_shard, group|
+        result = Topic.read_topics(group[:topics], group[:offsets], nil, count, db_shard: db_shard) # Always don't block
+        xread.merge!(result) if result
+      end
       # Return the original ID and and empty array if we didn't get anything
       packets = []
       return [id, packets] if xread.empty?

data/lib/openc3/core_ext/faraday.rb

@@ -1,6 +1,6 @@
 # Remove warnings in CGI
 saved_verbose = $VERBOSE
-$VERBOSE = false
+$VERBOSE = nil
 require 'faraday'
 $VERBOSE = saved_verbose
 

data/lib/openc3/interfaces/interface.rb

@@ -295,17 +295,12 @@ module OpenC3
           else
             data, extra = protocol.read_data(data)
           end
-          protocol.read_protocol_output_base(data, extra) unless blank_test
+          protocol.read_protocol_output_base(data, extra) unless blank_test or data == :STOP or data == :DISCONNECT
           if data == :DISCONNECT
             Logger.info("#{@name}: Protocol #{protocol.class} read_data requested disconnect")
             return nil
           end
           break if data == :STOP
-          if blank_test
-            # This means the blank test returned something so we can log
-            protocol.read_protocol_input_base('', nil)
-            protocol.read_protocol_output_base(data, extra)
-          end
         end
         next if data == :STOP
 

data/lib/openc3/io/json_api.rb

@@ -65,7 +65,7 @@ module OpenC3
 
     def _request(*method_params, **kw_params)
       kw_params[:scope] = $openc3_scope unless kw_params[:scope]
-      kw_params[:json] = true unless kw_params[:json]
+      kw_params[:json] = true # This is JsonApi so should always be speaking json
       @json_api.request(*method_params, **kw_params)
     end
   end

data/lib/openc3/logs/log_writer.rb

@@ -237,7 +237,9 @@ module OpenC3
                   # Now that the file is in S3, trim the Redis stream up until the previous file.
                   # This keeps one minute of data in Redis
                   instance.cleanup_offsets[index].each do |redis_topic, cleanup_offset|
-                    Topic.trim_topic(redis_topic, cleanup_offset)
+                    target_match = redis_topic.match(/__\{?([^}_]+)\}?__/)
+                    db_shard = target_match ? Store.db_shard_for_target(target_match[1]) : 0
+                    Topic.trim_topic(redis_topic, cleanup_offset, db_shard: db_shard)
                   end
                   indexes_to_clear << index
                 end