openc3 7.4.1 → 7.5.1

data/lib/openc3/script/storage.rb

@@ -17,6 +17,8 @@
 
 require 'tempfile'
 require 'net/http'
+require 'base64'
+require 'openc3/utilities/local_mode'
 
 ENV['OPENC3_CLOUD'] ||= 'local'
 
@@ -53,6 +55,15 @@ module OpenC3
 
       upload_path = "#{scope}/targets_modified/#{path}"
 
+      if OpenC3::LocalMode.local_only_name?(path)
+        # The local mode volume is only mounted in the cluster so from outside
+        # write it through the API which never sends it to the bucket
+        return _upload_api_file(upload_path, io_or_string, scope: scope) unless $openc3_in_cluster
+        puts "Writing local #{upload_path}"
+        OpenC3::LocalMode.put_target_file(upload_path, io_or_string, scope: scope)
+        return nil
+      end
+
       if ENV['OPENC3_LOCAL_MODE'] and $openc3_in_cluster
         OpenC3::LocalMode.put_target_file(upload_path, io_or_string, scope: scope)
         io_or_string.rewind if io_or_string.respond_to?(:rewind)
@@ -91,22 +102,22 @@ module OpenC3
     # @param original [Boolean] Whether to get the original or modified file
     # @return [File|nil]
     def get_target_file(path, original: false, scope: $openc3_scope)
+      # Local only targets only exist in the local mode volume so original doesn't apply
+      if OpenC3::LocalMode.local_only_name?(path)
+        # The local mode volume is only mounted in the cluster so from outside
+        # read it through the API which serves it from the volume
+        return _get_api_file("targets_modified/#{path}", scope: scope) unless $openc3_in_cluster
+        return _get_local_file(path, scope: scope)
+      end
+
       part = "targets"
       part += "_modified" unless original
       # Loop to allow redo when switching from modified to original
       loop do
         begin
           if part == "targets_modified" and ENV['OPENC3_LOCAL_MODE']
-            local_file = OpenC3::LocalMode.open_local_file(path, scope: scope)
-            if local_file
-              puts "Reading local #{scope}/#{part}/#{path}"
-              file = Tempfile.new('target', binmode: true)
-              file.filename = path
-              file.write(local_file.read)
-              local_file.close
-              file.rewind
-              return file
-            end
+            file = _get_local_file(path, scope: scope)
+            return file if file
           end
 
           return _get_storage_file("#{part}/#{path}", scope: scope)
@@ -125,7 +136,53 @@ module OpenC3
 
     # These are helper methods ... should not be used directly
 
+    # Write a file through the API instead of a presigned URL
+    def _upload_api_file(path, io_or_string, bucket: 'OPENC3_CONFIG_BUCKET', scope: $openc3_scope)
+      data = String === io_or_string ? io_or_string : io_or_string.read
+      endpoint = "/openc3-api/storage/upload_file/#{path}"
+      puts "Writing #{path}"
+      response = $api_server.request('put', endpoint, query: { bucket: bucket },
+        data: { contents: Base64.strict_encode64(data) }, json: true, scope: scope)
+      if response.nil? || response.status != 200
+        raise "Failed to write #{path}"
+      end
+      nil
+    end
+
+    # Read a file through the API instead of a presigned URL
+    # Returns nil if the file doesn't exist
+    def _get_api_file(path, bucket: 'OPENC3_CONFIG_BUCKET', scope: $openc3_scope)
+      endpoint = "/openc3-api/storage/download_file/#{scope}/#{path}"
+      response = $api_server.request('get', endpoint, query: { bucket: bucket }, scope: scope)
+      return nil if response && response.status == 404
+      if response.nil? || response.status != 200
+        raise "Failed to read #{scope}/#{path}"
+      end
+      puts "Reading #{scope}/#{path}"
+      json = JSON.parse(response.body, allow_nan: true, create_additions: true)
+      file = Tempfile.new('target', binmode: true)
+      file.filename = path
+      file.write(Base64.decode64(json['contents']))
+      file.rewind
+      return file
+    end
+
+    def _get_local_file(path, scope: $openc3_scope)
+      local_file = OpenC3::LocalMode.open_local_file(path, scope: scope)
+      return nil unless local_file
+      puts "Reading local #{scope}/targets_modified/#{path}"
+      file = Tempfile.new('target', binmode: true)
+      file.filename = path
+      file.write(local_file.read)
+      local_file.close
+      file.rewind
+      return file
+    end
+
     def _get_download_url(path, scope: $openc3_scope)
+      if OpenC3::LocalMode.local_only_name?(path)
+        raise "#{path} belongs to a local only target (OPENC3_LOCAL_ONLY_TARGETS) and has no bucket download URL"
+      end
       targets = "targets_modified" # First try targets_modified
       response = $api_server.request('get', "/openc3-api/storage/exists/#{scope}/#{targets}/#{path}", query: { bucket: 'OPENC3_CONFIG_BUCKET' }, scope: scope)
       if response.status != 200
@@ -167,6 +224,20 @@ module OpenC3
       return file
     end
 
+    # Retrieve the file selected in open_bucket_dialog
+    # input is the hash returned by the frontend: { 'bucket', 'path', 'filename' }
+    def _get_bucket_dialog_file(input, scope: $openc3_scope)
+      bucket = input['bucket']
+      # Path from bucket dialog already includes scope prefix (e.g. DEFAULT/targets/...)
+      # _get_storage_file prepends scope, so strip it from the path to avoid doubling
+      path = input['path'].sub(/\A#{Regexp.escape(scope)}\//, '')
+      file = _get_storage_file(path, bucket: bucket, scope: scope)
+      file.filename = input['filename']
+      file.bucket = bucket
+      file.bucket_path = input['path']
+      file
+    end
+
     def _get_uri(url)
       if $openc3_in_cluster
         case ENV.fetch('OPENC3_CLOUD', nil)

data/lib/openc3/system/system.rb

@@ -78,7 +78,8 @@ module OpenC3
     end
 
     # target_version can also be the actual hash used in the target_archives folder
-    def self.setup_targets(target_names, base_dir, target_version: 'current', scope:)
+    # @param descriptions [Boolean] Whether to keep packet and item descriptions
+    def self.setup_targets(target_names, base_dir, target_version: 'current', scope:, descriptions: true)
       # Nothing to do if there are no targets
       return if target_names.nil? or target_names.length == 0
       if @@instance.nil?
@@ -118,7 +119,7 @@ module OpenC3
         end
 
         # Build System from targets
-        System.instance(target_names, targets_path)
+        System.instance(target_names, targets_path, descriptions: descriptions)
       end
     end
 
@@ -137,14 +138,15 @@ module OpenC3
     #
     # @param target_names [Array of target_names]
     # @param target_config_dir Directory where target config folders are
+    # @param descriptions [Boolean] Whether to keep packet and item descriptions
     # @return [System] The System singleton
-    def self.instance(target_names = nil, target_config_dir = nil)
+    def self.instance(target_names = nil, target_config_dir = nil, descriptions: true)
       return @@instance if @@instance
       raise "System.instance parameters are required on first call" unless target_names and target_config_dir
 
       @@instance_mutex.synchronize do
         return @@instance if @@instance
-        @@instance ||= self.new(target_names, target_config_dir)
+        @@instance ||= self.new(target_names, target_config_dir, descriptions: descriptions)
         @@post_instance_callbacks.each do |callback|
           callback.call
         end
@@ -171,10 +173,11 @@ module OpenC3
     #
     # @param target_names [Array of target names]
     # @param target_config_dir Directory where target config folders are
-    def initialize(target_names, target_config_dir)
+    # @param descriptions [Boolean] Whether to keep packet and item descriptions
+    def initialize(target_names, target_config_dir, descriptions: true)
       OpenC3.add_to_search_path(target_config_dir, true) if target_config_dir
       @targets = {}
-      @packet_config = PacketConfig.new
+      @packet_config = PacketConfig.new(descriptions: descriptions)
       @commands = Commands.new(@packet_config)
       @telemetry = Telemetry.new(@packet_config)
       @limits = Limits.new(@packet_config)

data/lib/openc3/topics/command_decom_topic.rb

@@ -58,7 +58,7 @@ module OpenC3
       msg_hash['json_data'] = JSON.generate(json_hash.as_json, allow_nan: true)
       msg_hash['extra'] = JSON.generate(packet.extra.as_json, allow_nan: true) if packet.extra
       db_shard = Store.db_shard_for_target(packet.target_name, scope: scope)
-      EphemeralStoreQueued.instance(db_shard: db_shard).write_topic(topic, msg_hash)
+      EphemeralStoreQueued.instance(db_shard: db_shard).write_topic(topic, msg_hash, minid: Topic.stream_safety_minid)
     end
 
     def self.get_cmd_item(target_name, packet_name, param_name, type: :FORMATTED, scope: $openc3_scope)

data/lib/openc3/topics/command_topic.rb

@@ -35,7 +35,8 @@ module OpenC3
                    buffer: packet.buffer(false) }
       msg_hash[:extra] = JSON.generate(packet.extra.as_json, allow_nan: true) if packet.extra
       db_shard = Store.db_shard_for_target(packet.target_name, scope: scope)
-      EphemeralStoreQueued.instance(db_shard: db_shard).write_topic(topic, msg_hash)
+      minid = Topic.log_stream_safety_minid(packet.target_name, :CMD, scope: scope)
+      EphemeralStoreQueued.instance(db_shard: db_shard).write_topic(topic, msg_hash, minid: minid)
     end
 
     # @param command [Hash] Command hash structure read to be written to a topic

data/lib/openc3/topics/limits_event_topic.rb

@@ -25,6 +25,20 @@ module OpenC3
   # While this isn't a clean separation of topics (streams) and models (key-value)
   # it helps maintain consistency as the topic and model are linked.
   class LimitsEventTopic < Topic
+    # Returns the items in a limits group grouped by target and packet, or nil if the
+    # group is undefined. Reads the store directly to avoid a circular reference to TargetModel.
+    # @return [Hash{String => Hash{String => Array<String>}}, nil]
+    def self.limits_group_items(group_name, scope:)
+      json = Store.hget("#{scope}__limits_groups", group_name)
+      return nil unless json
+
+      items = {}
+      JSON.parse(json, allow_nan: true, create_additions: true).each do |target_name, packet_name, item_name|
+        ((items[target_name] ||= {})[packet_name] ||= []) << item_name
+      end
+      items
+    end
+
     # Collect all unique target db_shards from TargetModel
     def self._active_db_shards(scope:)
       db_shards = Set.new([0])
@@ -85,6 +99,30 @@ module OpenC3
         limits_settings['enabled'] = event[:enabled]
         Store.hset("#{scope}__current_limits_settings", field, JSON.generate(limits_settings, allow_nan: true))
 
+      when :LIMITS_GROUP_ENABLE_STATE
+        # Groups can contain many thousands of items so read and write the settings in bulk
+        fields = []
+        (limits_group_items(event[:group_name], scope: scope) || {}).each do |target_name, packets|
+          packets.each do |packet_name, item_names|
+            item_names.each { |item_name| fields << "#{target_name}__#{packet_name}__#{item_name}" }
+          end
+        end
+        unless fields.empty?
+          all_settings = Store.hmget("#{scope}__current_limits_settings", *fields)
+          updates = {}
+          fields.each_with_index do |field, index|
+            limits_settings = all_settings[index]
+            if limits_settings
+              limits_settings = JSON.parse(limits_settings, allow_nan: true, create_additions: true)
+            else
+              limits_settings = {}
+            end
+            limits_settings['enabled'] = event[:enabled]
+            updates[field] = JSON.generate(limits_settings, allow_nan: true)
+          end
+          Store.hmset("#{scope}__current_limits_settings", *updates.flatten)
+        end
+
       when :LIMITS_STATE_COLOR
         # Persist the state color so it survives a decom microservice restart (applied by sync_system)
         field = "#{event[:target_name]}__#{event[:packet_name]}__#{event[:item_name]}"
@@ -111,8 +149,9 @@ module OpenC3
       end
 
       # Write to all active db_shards so each decom microservice can read limits events inline
+      msg_hash = { event: JSON.generate(event, allow_nan: true) }
       _active_db_shards(scope: scope).each do |db_shard|
-        Topic.write_topic("#{scope}__openc3_limits_events", {event: JSON.generate(event, allow_nan: true)}, '*', 1000, db_shard: db_shard)
+        Topic.write_topic("#{scope}__openc3_limits_events", msg_hash, '*', 1000, db_shard: db_shard)
       end
     end
 
@@ -214,6 +253,26 @@ module OpenC3
       Store.hdel("#{scope}__limits_sets", set_name)
     end
 
+    # Returns true if limits events written after previous_id were trimmed from the
+    # stream before they were read. The stream keeps ~1000 entries so a flood of
+    # LIMITS_CHANGE events can trim events a slow reader hasn't processed yet.
+    # Stream IDs always increase, so if the oldest remaining entry is newer than the
+    # last entry read, the entries in between are gone. This can also report a gap
+    # when only previous_id itself was trimmed, which just causes an extra resync.
+    def self.events_trimmed?(previous_id, scope:, db_shard: 0)
+      # 0-0 means the stream was empty when the reader started
+      return false if previous_id.nil? or previous_id == '0-0'
+      oldest_id = Topic.get_oldest_message("#{scope}__openc3_limits_events", db_shard: db_shard)&.first
+      return false unless oldest_id
+      (oldest_id.split('-').map(&:to_i) <=> previous_id.split('-').map(&:to_i)) > 0
+    end
+
+    # Reload all limits state from the store, used to recover from missed limits events
+    def self.resync_system(scope:)
+      System.limits_set = current_set(scope: scope)
+      sync_system(scope: scope)
+    end
+
     # Update the local System based on overall state
     def self.sync_system(scope:)
       all_limits_settings = Store.hgetall("#{scope}__current_limits_settings")
@@ -251,7 +310,7 @@ module OpenC3
 
     # Process a single limits event hash and update the local System accordingly.
     # Called inline by DecomMicroservice when reading from the limits events topic.
-    def self.process_event(event, telemetry: nil)
+    def self.process_event(event, scope:, telemetry: nil)
       telemetry ||= System.telemetry.all
       case event['type']
       when 'LIMITS_CHANGE'
@@ -289,6 +348,34 @@ module OpenC3
           end
         end
 
+      when 'LIMITS_GROUP_ENABLE_STATE'
+        enabled = ConfigParser.handle_true_false_nil(event['enabled'])
+        (limits_group_items(event['group_name'], scope: scope) || {}).each do |target_name, packets|
+          target = telemetry[target_name]
+          next unless target
+
+          packets.each do |packet_name, item_names|
+            packet = target[packet_name]
+            next unless packet
+
+            item_names.each do |item_name|
+              next unless packet.items[item_name]
+
+              # The stream offset has already advanced past this event so isolate
+              # failures to a single item rather than skipping the rest of the group
+              begin
+                if enabled
+                  System.limits.enable(target_name, packet_name, item_name)
+                else
+                  System.limits.disable(target_name, packet_name, item_name)
+                end
+              rescue => e
+                Logger.error("Error #{enabled ? 'enabling' : 'disabling'} limits for #{target_name} #{packet_name} #{item_name}: #{e.formatted}")
+              end
+            end
+          end
+        end
+
       when 'LIMITS_STATE_COLOR'
         target_name = event['target_name']
         packet_name = event['packet_name']
@@ -313,7 +400,7 @@ module OpenC3
       topics = ["#{scope}__openc3_limits_events"]
       Topic.read_topics(topics, nil, block_ms) do |_topic, _msg_id, event, _redis|
         event = JSON.parse(event['event'], allow_nan: true, create_additions: true)
-        process_event(event, telemetry: telemetry)
+        process_event(event, scope: scope, telemetry: telemetry)
       end
     end
   end

data/lib/openc3/topics/router_topic.rb

@@ -78,13 +78,18 @@ module OpenC3
       end
     end
 
+    # Write to the target's db_shard where its interface reads commands
     def self.route_command(packet, target_names, scope:)
+      # Write to the target's db_shard where its interface reads commands
       if packet.identified?
         topic = "{#{scope}__CMD}TARGET__#{packet.target_name}"
-        Topic.write_topic(topic, { 'target_name' => packet.target_name, 'cmd_name' => packet.packet_name, 'cmd_buffer' => packet.buffer(false) }, '*', 100)
+        db_shard = Store.db_shard_for_target(packet.target_name, scope: scope)
+        Topic.write_topic(topic, { 'target_name' => packet.target_name, 'cmd_name' => packet.packet_name, 'cmd_buffer' => packet.buffer(false) }, '*', 100, db_shard: db_shard)
       elsif target_names.length == 1
         topic = "{#{scope}__CMD}TARGET__#{target_names[0]}"
-        Topic.write_topic(topic, { 'target_name' => packet.target_name ? packet.target_name : 'UNKNOWN', 'cmd_name' => 'UNKNOWN', 'cmd_buffer' => packet.buffer(false) }, '*', 100)
+        db_shard = Store.db_shard_for_target(target_names[0], scope: scope)
+        # Name the routed target so the interface identifies against it (and sends it as UNKNOWN if that fails)
+        Topic.write_topic(topic, { 'target_name' => target_names[0], 'cmd_name' => 'UNKNOWN', 'cmd_buffer' => packet.buffer(false) }, '*', 100, db_shard: db_shard)
       else
         raise "No route for command: #{packet.target_name ? packet.target_name : 'UNKNOWN'} #{packet.packet_name ? packet.packet_name : 'UNKNOWN'}"
       end

data/lib/openc3/topics/system_events_topic.rb

@@ -17,14 +17,17 @@ module OpenC3
   class SystemEventsTopic < Topic
     PRIMARY_KEY = "OPENC3__SYSTEM__EVENTS".freeze
 
-    # Collect all unique target db_shards from TargetModel
+    # Collect all unique target and interface db_shards. Interfaces read system
+    # events on their own db_shard which may not have any targets on it.
     def self._active_db_shards
       db_shards = Set.new([0])
-      # Iterate all scopes to find all target db_shards
-      Store.scan_each(match: '*__openc3_targets', type: 'hash') do |key|
-        Store.hgetall(key).each do |_name, json|
-          parsed = JSON.parse(json, allow_nan: true, create_additions: true)
-          db_shards << (parsed['db_shard'] || 0).to_i
+      # Iterate all scopes to find all target and interface db_shards
+      ['*__openc3_targets', '*__openc3_interfaces'].each do |match|
+        Store.scan_each(match: match, type: 'hash') do |key|
+          Store.hgetall(key).each do |_name, json|
+            parsed = JSON.parse(json, allow_nan: true, create_additions: true)
+            db_shards << (parsed['db_shard'] || 0).to_i
+          end
         end
       end
       db_shards

data/lib/openc3/topics/telemetry_decom_topic.rb

@@ -45,7 +45,7 @@ module OpenC3
         }
         msg_hash[:extra] = JSON.generate(packet.extra.as_json, allow_nan: true) if packet.extra
         db_shard = Store.db_shard_for_target(packet.target_name, scope: scope)
-        Topic.write_topic("#{scope}__DECOM__{#{packet.target_name}}__#{packet.packet_name}", msg_hash, id, db_shard: db_shard)
+        Topic.write_topic("#{scope}__DECOM__{#{packet.target_name}}__#{packet.packet_name}", msg_hash, id, minid: Topic.stream_safety_minid(id), db_shard: db_shard)
 
         unless packet.stored
           # Also update the current value table with the latest decommutated data

data/lib/openc3/topics/telemetry_topic.rb

@@ -33,10 +33,11 @@ module OpenC3
       msg_hash[:extra] = JSON.generate(packet.extra.as_json, allow_nan: true) if packet.extra
       topic = "#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}"
       db_shard = Store.db_shard_for_target(packet.target_name, scope: scope)
+      minid = Topic.log_stream_safety_minid(packet.target_name, :TLM, scope: scope)
       if queued
-        EphemeralStoreQueued.instance(db_shard: db_shard).write_topic(topic, msg_hash)
+        EphemeralStoreQueued.instance(db_shard: db_shard).write_topic(topic, msg_hash, minid: minid)
       else
-        Topic.write_topic(topic, msg_hash, db_shard: db_shard)
+        Topic.write_topic(topic, msg_hash, minid: minid, db_shard: db_shard)
       end
     end
   end

data/lib/openc3/topics/topic.rb

@@ -16,9 +16,48 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/utilities/store'
+require 'openc3/utilities/logger'
 
 module OpenC3
   class Topic
+    # Stream trimming contract for the high rate target streams
+    # (TELEMETRY__, COMMAND__, DECOM__, DECOMCMD__):
+    #
+    # The streams are primarily trimmed by their consumers once the data is safely
+    # persisted elsewhere:
+    #   * TELEMETRY__ / COMMAND__ are trimmed by the log microservice (LogWriter) once a
+    #     log file has been moved to the bucket plus LogWriter::CLEANUP_DELAY.
+    #   * DECOM__ / DECOMCMD__ are trimmed by the TSDB microservice which keeps
+    #     TsdbMicroservice::TRIM_KEEP_MS of data.
+    #
+    # If a trimmer is absent (logging disabled, no TSDB) or falls far behind, the streams
+    # would grow without bound. As a safety net every write to these streams also passes
+    # an approximate XADD MINID so no entry older than the safety max age is retained.
+    # The max age is OPENC3_STREAM_MAX_AGE_SECONDS (default 600s, 0 or empty disables) and is
+    # never less than STREAM_MAX_AGE_MIN_SECONDS. For TELEMETRY__ / COMMAND__ the max age is
+    # never less than two log cycles plus the cleanup delay, because historical streaming reads
+    # data not yet in the bucket from the stream.
+    #
+    # When the minid is calculated from now (auto generated ids) it uses this host's clock while
+    # Redis generates the ids from its own clock. A writer clock ahead of Redis by more than the
+    # max age would trim the entire stream (including the entry being added) so hosts must be
+    # time synchronized (NTP).
+    STREAM_MAX_AGE_DEFAULT_SECONDS = 600
+    # Twice TsdbMicroservice::TRIM_KEEP_MS. Also protects against small clock offsets.
+    STREAM_MAX_AGE_MIN_SECONDS = 120
+    # LogWriter::CLEANUP_DELAY is defined from this
+    LOG_CLEANUP_DELAY_SECONDS = 60
+    # TargetModel default log cycle time
+    DEFAULT_LOG_CYCLE_TIME_SECONDS = 600
+    # How long to wait before retrying a failed log cycle time lookup
+    LOG_CYCLE_LOOKUP_RETRY_SECONDS = 60
+    # These are calculated on the write path of every packet so they are cached per process.
+    # The environment doesn't change while a microservice runs and the log cycle time only
+    # changes when the plugin (and therefore the microservices) are reinstalled.
+    @@stream_max_age_ms = nil
+    @@log_stream_max_age_ms = {}
+    @@log_cycle_lookup_retry_at = {}
+
     # Delegate all unknown class methods to EphemeralStore db_shard 0 (system-level topics)
     def self.method_missing(message, *args, **kwargs, &block)
       EphemeralStore.public_send(message, *args, **kwargs, &block)
@@ -37,8 +76,121 @@ module OpenC3
     # DB_Shard-aware topic methods for target-specific streams.
     # These explicitly route to the correct EphemeralStore db_shard.
 
-    def self.write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true', db_shard: 0)
-      EphemeralStore.instance(db_shard: db_shard).write_topic(topic, msg_hash, id, maxlen, approximate)
+    def self.write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true', minid: nil, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).write_topic(topic, msg_hash, id, maxlen, approximate, minid: minid)
+    end
+
+    # @return [Float] Configured safety max age in seconds (<= 0 means disabled)
+    def self.stream_max_age_seconds
+      value = ENV.fetch('OPENC3_STREAM_MAX_AGE_SECONDS', nil)
+      return STREAM_MAX_AGE_DEFAULT_SECONDS if value.nil?
+      return 0.0 if value.strip.empty?
+      # Strictly parse and reject nan / inf, which would raise when converted to
+      # milliseconds on every write. A typo keeps the safety cap at the default
+      # rather than disabling it.
+      seconds = Float(value, exception: false)
+      return seconds if seconds&.finite?
+      Logger.warn("OPENC3_STREAM_MAX_AGE_SECONDS of '#{value}' is invalid, using #{STREAM_MAX_AGE_DEFAULT_SECONDS}")
+      return STREAM_MAX_AGE_DEFAULT_SECONDS
+    end
+
+    # @return [Integer] Cached safety max age in milliseconds (0 means disabled)
+    def self.stream_max_age_ms
+      @@stream_max_age_ms ||= begin
+        seconds = stream_max_age_seconds
+        if seconds <= 0
+          0
+        elsif seconds < STREAM_MAX_AGE_MIN_SECONDS
+          Logger.warn("OPENC3_STREAM_MAX_AGE_SECONDS of #{seconds} is below the minimum, using #{STREAM_MAX_AGE_MIN_SECONDS}")
+          STREAM_MAX_AGE_MIN_SECONDS * 1000
+        else
+          (seconds * 1000).to_i
+        end
+      end
+    end
+
+    # Calculates the MINID to pass to XADD to cap the age of entries in a target stream.
+    # The cap is anchored to the id being written (or now for auto generated ids) so an
+    # explicitly id'd entry (e.g. decom reusing the raw packet id) is never trimmed by its own add.
+    #
+    # @param id [String] The id being written ('*' or nil for auto generation)
+    # @param max_age_ms [Integer] Max age to retain (<= 0 means disabled)
+    # @return [String|nil] The minid or nil if the safety cap is disabled
+    def self.stream_safety_minid(id = '*', max_age_ms: stream_max_age_ms)
+      return nil if max_age_ms <= 0
+      if id.nil? or id == '*'
+        base_ms = Process.clock_gettime(Process::CLOCK_REALTIME, :millisecond)
+      else
+        base_ms = id.to_s.to_i # Stops at the '-' sequence separator
+      end
+      minid_ms = base_ms - max_age_ms
+      return nil if minid_ms <= 0
+      return minid_ms.to_s
+    end
+
+    # Calculates the MINID for a TELEMETRY__ / COMMAND__ stream write which also never
+    # trims younger than log_stream_min_age_seconds. The resulting max age is cached per stream.
+    #
+    # @param target_name [String] Target name
+    # @param cmd_or_tlm [Symbol] :CMD or :TLM
+    # @return [String|nil] The minid or nil if the safety cap is disabled
+    def self.log_stream_safety_minid(target_name, cmd_or_tlm, scope:)
+      target_cache = ((@@log_stream_max_age_ms[scope] ||= {})[target_name] ||= {})
+      max_age_ms = target_cache[cmd_or_tlm]
+      unless max_age_ms
+        max_age_ms = log_stream_max_age_ms(target_name, cmd_or_tlm, scope: scope)
+        return nil unless max_age_ms # Log cycle time unknown so don't cap yet
+        target_cache[cmd_or_tlm] = max_age_ms
+      end
+      return stream_safety_minid(max_age_ms: max_age_ms)
+    end
+
+    # @param target_name [String] Target name
+    # @param cmd_or_tlm [Symbol] :CMD or :TLM
+    # @return [Integer|nil] Max age in milliseconds for a TELEMETRY__ / COMMAND__ stream (0 means
+    #   disabled) or nil if the log cycle time couldn't be determined. Guessing the log cycle time
+    #   could trim data historical streaming still needs, so the stream isn't capped until a
+    #   lookup (retried every LOG_CYCLE_LOOKUP_RETRY_SECONDS) succeeds.
+    def self.log_stream_max_age_ms(target_name, cmd_or_tlm, scope:)
+      max_age_ms = stream_max_age_ms
+      return max_age_ms if max_age_ms <= 0
+      key = [scope, target_name, cmd_or_tlm]
+      now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      retry_at = @@log_cycle_lookup_retry_at[key]
+      return nil if retry_at and now < retry_at
+      begin
+        min_age_ms = log_stream_min_age_seconds(target_name, cmd_or_tlm, scope: scope) * 1000
+      rescue => e
+        Logger.warn("Unable to determine log cycle time for #{target_name} #{cmd_or_tlm}, stream age not capped: #{e.message}")
+        @@log_cycle_lookup_retry_at[key] = now + LOG_CYCLE_LOOKUP_RETRY_SECONDS
+        return nil
+      end
+      @@log_cycle_lookup_retry_at.delete(key)
+      return [max_age_ms, min_age_ms].max
+    end
+
+    # Minimum age to retain in a TELEMETRY__ / COMMAND__ stream so the log microservice
+    # and historical streaming always have the data not yet moved to the bucket.
+    #
+    # @param target_name [String] Target name
+    # @param cmd_or_tlm [Symbol] :CMD or :TLM
+    # @return [Integer] Minimum age in seconds
+    # @raise If the TargetModel lookup fails
+    def self.log_stream_min_age_seconds(target_name, cmd_or_tlm, scope:)
+      require 'openc3/models/target_model'
+      cycle_time = DEFAULT_LOG_CYCLE_TIME_SECONDS
+      model = TargetModel.get(name: target_name, scope: scope)
+      if model
+        value = model[cmd_or_tlm == :CMD ? 'cmd_log_cycle_time' : 'tlm_log_cycle_time'].to_i
+        cycle_time = value if value > 0
+      end
+      return 2 * (cycle_time + LOG_CLEANUP_DELAY_SECONDS)
+    end
+
+    def self.clear_stream_max_age_cache
+      @@stream_max_age_ms = nil
+      @@log_stream_max_age_ms = {}
+      @@log_cycle_lookup_retry_at = {}
     end
 
     def self.read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil, db_shard: 0, &block)
@@ -69,6 +221,13 @@ module OpenC3
       EphemeralStore.instance(db_shard: db_shard).del(topic)
     end
 
+    # Extract the target name from the {TARGET} hash tag in a topic like
+    # DEFAULT__TELEMETRY__{INST}__HEALTH_STATUS. Returns nil if there is none.
+    def self.target_name_from_topic(topic)
+      match = topic.match(/__\{([^}]+)\}__/)
+      match ? match[1] : nil
+    end
+
     # Group topics by db_shard. Each topic's target name is extracted and looked up.
     # Topics matching target_pattern are db_sharded; others go to db_shard 0.
     # @param topics [Array<String>] List of topic strings
@@ -79,7 +238,7 @@ module OpenC3
       groups = {}
       topics.each do |topic|
         if topic.include?(target_pattern)
-          target_name = topic.match(/__\{?([^}_]+)\}?__/)[1] rescue nil
+          target_name = target_name_from_topic(topic)
           # Handle CMD}TARGET__ pattern where target is after TARGET__
           target_name = topic.split('TARGET__')[1] if target_pattern.include?('TARGET__') && target_name.nil?
           db_shard = (Store.db_shard_for_target(target_name, scope: scope) || 0).to_i

data/lib/openc3/utilities/bucket.rb

@@ -28,6 +28,11 @@ module OpenC3
       klass.new
     end
 
+    # snake_case alias to match the Python Bucket.get_client
+    def self.get_client # rubocop:disable Naming/AccessorMethodName
+      getClient()
+    end
+
     def initialize
       # Setup the client instance
     end