openc3 7.5.0 → 7.5.1

data/lib/openc3/api/tlm_api.rb

@@ -119,6 +119,7 @@ module OpenC3
     # @param type [Symbol] Telemetry type, :RAW, :CONVERTED (default), :FORMATTED
     # @param received_time [Integer, nil] Optional received time as nanoseconds since Unix epoch
     def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, received_time: nil, manual: false, scope: $openc3_scope, token: $openc3_token)
+      _tlm_reject_latest(packet_name.to_s.upcase, 'inject_tlm')
       authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
       type = type.to_s.intern
       target_name = target_name.upcase
@@ -174,12 +175,17 @@ module OpenC3
     #
     # @param args The args must either be a string followed by a value or
     #   three strings followed by a value (see the calling style in the
-    #   description).
+    #   description). A packet name of LATEST overrides the item in every
+    #   packet which contains it.
     # @param type [Symbol] Telemetry type, :ALL (default), :RAW, :CONVERTED, :FORMATTED
     def override_tlm(*args, type: :ALL, manual: false, scope: $openc3_scope, token: $openc3_token)
-      target_name, packet_name, item_name, value = _set_tlm_process_args(args, __method__, scope: scope)
-      authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
-      CvtModel.override(target_name, packet_name, item_name, value, type: type.intern, scope: scope)
+      target_name, packet_names, item_name, value = _set_tlm_process_args_all_packets(args, __method__, scope: scope)
+      packet_names.each do |packet_name|
+        authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
+      end
+      packet_names.each do |packet_name|
+        CvtModel.override(target_name, packet_name, item_name, value, type: type.intern, scope: scope)
+      end
     end
 
     # Get the list of CVT overrides
@@ -198,13 +204,18 @@ module OpenC3
     # Favor the first syntax where possible as it is more succinct.
     #
     # @param args The args must either be a string or three strings
-    #   (see the calling style in the description).
+    #   (see the calling style in the description). A packet name of LATEST
+    #   normalizes the item in every packet which contains it.
     # @param type [Symbol] Telemetry type, :ALL (default), :RAW, :CONVERTED, :FORMATTED
     #   Also takes :ALL which means to normalize all telemetry types
     def normalize_tlm(*args, type: :ALL, manual: false, scope: $openc3_scope, token: $openc3_token)
-      target_name, packet_name, item_name = _tlm_process_args(args, __method__, scope: scope)
-      authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
-      CvtModel.normalize(target_name, packet_name, item_name, type: type.intern, scope: scope)
+      target_name, packet_names, item_name = _tlm_process_args_all_packets(args, __method__, scope: scope)
+      packet_names.each do |packet_name|
+        authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
+      end
+      packet_names.each do |packet_name|
+        CvtModel.normalize(target_name, packet_name, item_name, type: type.intern, scope: scope)
+      end
     end
 
     # Returns the raw buffer for a telemetry packet.
@@ -265,45 +276,25 @@ module OpenC3
         packet_name = orig_packet_name
         raise ArgumentError, "items must be formatted as TGT__PKT__ITEM__TYPE" if target_name.nil? || packet_name.nil? || item_name.nil? || value_type.nil?
         if orig_packet_name == 'LATEST'
-          # TODO: Do we need to lookup ALL the possible packets for this item?
+          # A LATEST item can come from any packet containing it. Report the best value
+          # type and limits of any of them since the CVT falls back to the available
+          # type and historical lookups check the type available in each packet.
           # We can have a large cache_timeout of 1 because all we're trying to do is lookup a packet
           packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: 1, scope: scope)
+          packet_names = _tlm_packet_names(target_name, orig_packet_name, item_name, scope: scope)
+        else
+          packet_names = [packet_name]
+        end
+        # Authorize every packet whose definition is read below, not just the latest one
+        (packet_names | [packet_name]).each do |name|
+          authorize(permission: 'tlm', target_name: target_name, packet_name: name, scope: scope, token: token)
         end
-        authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, scope: scope, token: token)
         begin
-          item = TargetModel.packet_item(target_name, packet_name, item_name, scope: scope)
-          if Packet::RESERVED_ITEM_NAMES.include?(item_name)
-            value_type = 'RAW' # Must request the raw value when dealing with the reserved items
-          end
-
-          # Arrays must be accessed as RAW since there's no conversion
-          if item['array_size']
-            value_type = 'RAW'
-          end
-
-          case value_type
-          when 'FORMATTED', 'WITH_UNITS'
-            if item['format_string'] or item['units']
-              results << [target_name, orig_packet_name, item_name, 'FORMATTED'].join('__')
-            # This logic must match the logic in Packet#decom
-            elsif item['states'] or (item['read_conversion'] and item['data_type'] != 'DERIVED')
-              results << [target_name, orig_packet_name, item_name, 'CONVERTED'].join('__')
-            else
-              results << [target_name, orig_packet_name, item_name, 'RAW'].join('__')
-            end
-          when 'CONVERTED'
-            # This logic must match the logic in Packet#decom
-            if item['states'] or (item['read_conversion'] and item['data_type'] != 'DERIVED')
-              results << [target_name, orig_packet_name, item_name, 'CONVERTED'].join('__')
-            else
-              results << [target_name, orig_packet_name, item_name, 'RAW'].join('__')
-            end
-          else # RAW or unknown
-            results << [target_name, orig_packet_name, item_name, 'RAW'].join('__')
-          end
-
+          packet_items = packet_names.map { |name| TargetModel.packet_item(target_name, name, item_name, scope: scope) }
+          available_type = packet_items.map { |packet_item| _tlm_available_value_type(packet_item, item_name, value_type) }.max_by { |type| VALUE_TYPE_ORDER.index(type) }
+          results << [target_name, orig_packet_name, item_name, available_type].join('__')
           # Tack on __LIMITS to notify that we have an available limits value
-          if item['limits']['DEFAULT']
+          if packet_items.any? { |packet_item| packet_item['limits']['DEFAULT'] }
             results[-1] += '__LIMITS'
           end
         rescue RuntimeError
@@ -341,7 +332,20 @@ module OpenC3
         parts = item.to_s.upcase.split('__')
         raise ArgumentError, "items must be formatted as TGT__PKT__ITEM__TYPE" if parts.length < 4 || parts.length > 5
         target_name, packet_name, item_name, value_type, limits = parts
-        if packet_name == 'LATEST' # Lookup packet_name in case of LATEST
+        if packet_name == 'LATEST'
+          if start_time
+            # The packet received most recently now isn't necessarily the one received most
+            # recently at start_time so keep every candidate packet for _tlm_latest_history
+            # Each packet must be queried for the value type and limits it has or the query fails
+            candidates = _tlm_packet_names(target_name, packet_name, item_name, scope: scope).map do |candidate|
+              packets << [target_name, candidate]
+              candidate_item = TargetModel.packet_item(target_name, candidate, item_name, scope: scope)
+              candidate_limits = (limits and candidate_item['limits']['DEFAULT']) ? limits : nil
+              [target_name, candidate, item_name, _tlm_available_value_type(candidate_item, item_name, value_type), candidate_limits]
+            end
+            cvt_items << [target_name, packet_name, item_name, value_type, limits, candidates]
+            next
+          end
           packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: scope)
         end
         # NOTE: limits is required by the historical (start_time) QuestDB lookup
@@ -352,6 +356,9 @@ module OpenC3
       packets.each do |target_name, packet_name|
         authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
       end
+      if cvt_items.any? { |cvt_item| cvt_item.length == 6 }
+        return _tlm_latest_history(cvt_items, start_time: start_time, end_time: end_time, cache_timeout: cache_timeout, scope: scope)
+      end
       CvtModel.get_tlm_values(cvt_items, stale_time: stale_time, cache_timeout: cache_timeout, start_time: start_time, end_time: end_time, scope: scope)
     end
 
@@ -446,6 +453,7 @@ module OpenC3
       packets.each do |target_name, packet_name|
         target_name = target_name.upcase
         packet_name = packet_name.upcase
+        _tlm_reject_latest(packet_name, 'subscribe_packets')
         authorize(permission: 'tlm', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
         topic = "#{scope}__DECOM__{#{target_name}}__#{packet_name}"
         db_shard = Store.db_shard_for_target(target_name, scope: scope)
@@ -475,7 +483,7 @@ module OpenC3
       # Group topics by db_shard for multi-shard support
       db_shard_groups = {}
       lookup.each do |topic, offset|
-        target_name = topic.match(/__\{?([^}_]+)\}?__/)[1] rescue nil
+        target_name = Topic.target_name_from_topic(topic)
         db_shard = Store.db_shard_for_target(target_name, scope: scope)
         db_shard_groups[db_shard] ||= { topics: [], offsets: [] }
         db_shard_groups[db_shard][:topics] << topic
@@ -551,9 +559,15 @@ module OpenC3
       if target_name.nil? or packet_name.nil?
         raise "ERROR: Both target name and packet name required. Usage: #{method_name}(\"TGT PKT\") or #{method_name}(\"TGT\", \"PKT\")"
       end
+      _tlm_reject_latest(packet_name, method_name)
       return [target_name, packet_name]
     end
 
+    # LATEST picks a packet per item so it can't identify a whole packet
+    def _tlm_reject_latest(packet_name, method_name)
+      raise "ERROR: LATEST is not supported by #{method_name}(), specify a packet name" if packet_name == 'LATEST'
+    end
+
     def _extract_target_packet_item_names(method_name, *args)
       target_name = nil
       packet_name = nil
@@ -589,6 +603,31 @@ module OpenC3
     end
 
     def _tlm_process_args(args, method_name, cache_timeout: nil, scope: $openc3_scope, token: $openc3_token)
+      target_name, packet_name, item_name = _tlm_extract_args(args, method_name)
+      packet_name = _tlm_resolve_packet_name(target_name, packet_name, item_name, cache_timeout: cache_timeout, scope: scope)
+      return [target_name, packet_name, item_name]
+    end
+
+    def _set_tlm_process_args(args, method_name, cache_timeout: nil, scope: $openc3_scope, token: $openc3_token)
+      target_name, packet_name, item_name, value = _set_tlm_extract_args(args, method_name)
+      packet_name = _tlm_resolve_packet_name(target_name, packet_name, item_name, cache_timeout: cache_timeout, scope: scope)
+      return [target_name, packet_name, item_name, value]
+    end
+
+    # Like _tlm_process_args but returns an array of packet names where LATEST
+    # expands to every packet containing the item. Used by APIs which change
+    # an item so the change applies wherever the item appears.
+    def _tlm_process_args_all_packets(args, method_name, scope: $openc3_scope)
+      target_name, packet_name, item_name = _tlm_extract_args(args, method_name)
+      return [target_name, _tlm_packet_names(target_name, packet_name, item_name, scope: scope), item_name]
+    end
+
+    def _set_tlm_process_args_all_packets(args, method_name, scope: $openc3_scope)
+      target_name, packet_name, item_name, value = _set_tlm_extract_args(args, method_name)
+      return [target_name, _tlm_packet_names(target_name, packet_name, item_name, scope: scope), item_name, value]
+    end
+
+    def _tlm_extract_args(args, method_name)
       case args.length
       when 1
         target_name, packet_name, item_name = extract_fields_from_tlm_text(args[0])
@@ -600,21 +639,10 @@ module OpenC3
         # Invalid number of arguments
         raise "ERROR: Invalid number of arguments (#{args.length}) passed to #{method_name}()"
       end
-      target_name = target_name.upcase
-      packet_name = packet_name.upcase
-      item_name = item_name.upcase
-
-      if packet_name == 'LATEST'
-        packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: scope)
-      else
-        # Determine if this item exists, it will raise appropriate errors if not
-        TargetModel.packet_item(target_name, packet_name, item_name, scope: scope)
-      end
-
-      return [target_name, packet_name, item_name]
+      return [target_name.upcase, packet_name.upcase, item_name.upcase]
     end
 
-    def _set_tlm_process_args(args, method_name, cache_timeout: nil, scope: $openc3_scope, token: $openc3_token)
+    def _set_tlm_extract_args(args, method_name)
       case args.length
       when 1
         target_name, packet_name, item_name, value = extract_fields_from_set_tlm_text(args[0])
@@ -627,18 +655,148 @@ module OpenC3
         # Invalid number of arguments
         raise "ERROR: Invalid number of arguments (#{args.length}) passed to #{method_name}()"
       end
-      target_name = target_name.upcase
-      packet_name = packet_name.upcase
-      item_name = item_name.upcase
+      return [target_name.upcase, packet_name.upcase, item_name.upcase, value]
+    end
 
+    # Resolve LATEST to the most recently received packet containing the item
+    def _tlm_resolve_packet_name(target_name, packet_name, item_name, cache_timeout: nil, scope: $openc3_scope)
       if packet_name == 'LATEST'
         packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: scope)
       else
         # Determine if this item exists, it will raise appropriate errors if not
         TargetModel.packet_item(target_name, packet_name, item_name, scope: scope)
       end
+      return packet_name
+    end
 
-      return [target_name, packet_name, item_name, value]
+    VALUE_TYPE_ORDER = %w(RAW CONVERTED FORMATTED).freeze
+
+    # Returns the value type available for an item given the requested value type
+    def _tlm_available_value_type(item, item_name, value_type)
+      # Must request the raw value when dealing with the reserved items
+      return 'RAW' if Packet::RESERVED_ITEM_NAMES.include?(item_name)
+      # Arrays must be accessed as RAW since there's no conversion
+      return 'RAW' if item['array_size']
+
+      # This logic must match the logic in Packet#decom
+      converted = (item['states'] or (item['read_conversion'] and item['data_type'] != 'DERIVED'))
+      case value_type
+      when 'FORMATTED', 'WITH_UNITS'
+        return 'FORMATTED' if item['format_string'] or item['units']
+        return converted ? 'CONVERTED' : 'RAW'
+      when 'CONVERTED'
+        return converted ? 'CONVERTED' : 'RAW'
+      else # RAW or unknown
+        return 'RAW'
+      end
+    end
+
+    # Historical lookup where some items are LATEST. Those items have a 6th element
+    # holding the item to look up in every packet which contains the item.
+    def _tlm_latest_history(cvt_items, start_time:, end_time:, cache_timeout:, scope:)
+      latest = cvt_items.each_index.select { |index| cvt_items[index].length == 6 }
+      regular = cvt_items.each_index.reject { |index| cvt_items[index].length == 6 }
+
+      # The QuestDB lookup ASOF JOINs every packet to the rows of the first, so
+      # a range needs a regular item first to drive the rows. Without one keep
+      # the realtime behavior of using the packet received most recently now.
+      if end_time and regular.all? { |index| cvt_items[index][2].nil? }
+        cvt_items = cvt_items.map do |target_name, packet_name, item_name, value_type, limits, candidates|
+          next [target_name, packet_name, item_name, value_type, limits] unless packet_name == 'LATEST'
+          packet_name = CvtModel.determine_latest_packet_for_item(target_name, item_name, cache_timeout: cache_timeout, scope: scope)
+          # Use the candidate so the query asks for the value type and limits that packet has
+          candidates.find { |candidate| candidate[1] == packet_name } || [target_name, packet_name, item_name, value_type, limits]
+        end
+        return CvtModel.tsdb_lookup(cvt_items, start_time: start_time, end_time: end_time, scope: scope)
+      end
+
+      # A packet without a table fails the query so skip those candidates
+      packets = latest.flat_map { |index| cvt_items[index][5].map { |candidate| candidate[0..1] } }.uniq
+      existing = CvtModel.tsdb_packets_with_tables(packets, scope: scope)
+      cvt_items = cvt_items.each_with_index.map do |cvt_item, index|
+        next cvt_item unless latest.include?(index)
+        cvt_item[0..4] + [cvt_item[5].select { |candidate| existing.include?(candidate[0..1]) }]
+      end
+
+      if end_time
+        query = []
+        query_index = {}
+        add = lambda { |query_item| query_index[query_item] ||= (query << query_item; query.length - 1) }
+        positions = Array.new(cvt_items.length)
+        regular.each { |index| positions[index] = add.call(cvt_items[index]) }
+        latest.each do |index|
+          positions[index] = cvt_items[index][5].map do |candidate|
+            target_name, packet_name = candidate
+            [add.call(candidate), add.call([target_name, packet_name, 'PACKET_TIMESECONDS', 'RAW', nil])]
+          end
+        end
+        rows = CvtModel.tsdb_lookup(query, start_time: start_time, end_time: end_time, scope: scope, flatten: false)
+        return rows unless rows.is_a?(Array) and rows.length > 0
+        rows = rows.map do |row|
+          positions.map do |position|
+            next row[position] if position.is_a?(Integer)
+            # Take the candidate with the newest packet time at this row
+            _tlm_newest(position.map { |value_index, time_index| [row[time_index][0], row[value_index]] }) || [nil, nil]
+          end
+        end
+        # Match tsdb_lookup which returns a flat array for a single row
+        return rows.length == 1 ? rows[0] : rows
+      end
+
+      # A point in time returns the last row of the first packet before start_time
+      # with the other packets ASOF JOINed at that row. A candidate packet received
+      # after that row would be missed so query each candidate packet on its own.
+      found = false
+      row = Array.new(cvt_items.length) { [nil, nil] }
+      unless regular.all? { |index| cvt_items[index][2].nil? }
+        result = CvtModel.tsdb_lookup(regular.map { |index| cvt_items[index] }, start_time: start_time, scope: scope, flatten: false)
+        if result.is_a?(Array) and result.length > 0
+          found = true
+          regular.each_with_index { |index, result_index| row[index] = result[0][result_index] }
+        end
+      end
+      # Group the LATEST items by candidate packet so each packet is queried once
+      by_packet = {}
+      latest.each do |index|
+        cvt_items[index][5].each do |candidate|
+          target_name, packet_name = candidate
+          (by_packet[[target_name, packet_name]] ||= {})[index] = candidate
+        end
+      end
+      found_values = Hash.new { |hash, key| hash[key] = [] }
+      by_packet.each do |(target_name, packet_name), packet_items|
+        query = packet_items.values.uniq
+        query << [target_name, packet_name, 'PACKET_TIMESECONDS', 'RAW', nil]
+        result = CvtModel.tsdb_lookup(query, start_time: start_time, scope: scope, flatten: false)
+        next unless result.is_a?(Array) and result.length > 0
+        values = result[0]
+        packet_items.each { |index, query_item| found_values[index] << [values[-1][0], values[query.index(query_item)]] }
+      end
+      found_values.each do |index, candidates|
+        value = _tlm_newest(candidates)
+        next unless value
+        found = true
+        row[index] = value
+      end
+      return found ? row : {}
+    end
+
+    # Return the value with the newest time from [time, value] pairs, or nil
+    def _tlm_newest(candidates)
+      candidates.select { |time, _| time }.max_by { |time, _| time }&.last
+    end
+
+    # Expand LATEST to every packet containing the item
+    def _tlm_packet_names(target_name, packet_name, item_name, scope: $openc3_scope)
+      if packet_name == 'LATEST'
+        packet_names = TargetModel.get_item_to_packet_map(target_name, scope: scope)[item_name]
+        raise "Item '#{target_name} LATEST #{item_name}' does not exist for scope: #{scope}" unless packet_names
+        return packet_names
+      else
+        # Determine if this item exists, it will raise appropriate errors if not
+        TargetModel.packet_item(target_name, packet_name, item_name, scope: scope)
+        return [packet_name]
+      end
     end
   end
 end

data/lib/openc3/logs/log_writer.rb

@@ -80,14 +80,6 @@ module OpenC3
     # Sleeper used to delay cycle thread
     @@cycle_sleeper = nil
 
-    # Returns the Redis db_shard holding a target stream such as
-    # SCOPE__TELEMETRY__{TARGET}__PACKET, or 0 if no target can be determined
-    def self.db_shard_for_topic(redis_topic)
-      target_match = redis_topic.match(/\{([^{}]+)\}/)
-      return 0 unless target_match
-      Store.db_shard_for_target(target_match[1], scope: redis_topic.split('__')[0])
-    end
-
     # @param remote_log_directory [String] The path to store the log files
     # @param logging_enabled [Boolean] Whether to start with logging enabled
     # @param cycle_time [Integer] The amount of time in seconds before creating
@@ -222,6 +214,15 @@ module OpenC3
       end
     end
 
+    # The db_shard of a target topic like DEFAULT__DECOM__{INST}__HEALTH_STATUS
+    # or db_shard 0 for a topic without a target
+    def self.db_shard_for_topic(redis_topic)
+      scope, _, rest = redis_topic.delete_prefix('{').partition('__')
+      # The target is a {TARGET} part between two others, e.g. DECOM__{INST}__HEALTH_STATUS
+      target = rest.split('__', -1)[1...-1]&.find { |part| part.length > 2 and part.start_with?('{') and part.end_with?('}') }
+      (!scope.empty? and target) ? Store.db_shard_for_target(target[1...-1], scope: scope) : 0
+    end
+
     def cycle_thread_body
       @@cycle_sleeper = Sleeper.new
       while true

data/lib/openc3/microservices/decom_microservice.rb

@@ -99,6 +99,10 @@ module OpenC3
       @limits_event_topic = "#{@scope}__openc3_limits_events"
       @topics << @limits_event_topic
       Topic.update_topic_offsets(@topics, db_shard: @db_shard)
+      # Last limits event read, used to detect events trimmed before they were read.
+      # Read from the stream rather than the thread's cached offsets. The run thread
+      # starts reading at or after this point, so it can only cause an extra resync.
+      @limits_event_offset = Topic.get_last_offset(@limits_event_topic, db_shard: @db_shard)
       # Initialize before assigning limits_change_callback - sync_system below
       # can fire the callback synchronously for any persisted-disabled items.
       @limits_response_queue = Queue.new
@@ -121,14 +125,21 @@ module OpenC3
         break if @cancel_thread
 
         begin
+          limits_events_checked = false
           OpenC3.in_span("read_topics") do
             Topic.read_topics(@topics, db_shard: @db_shard) do |topic, msg_id, msg_hash, redis|
               break if @cancel_thread
               if topic == @microservice_topic
                 microservice_cmd(topic, msg_id, msg_hash, redis)
               elsif topic == @limits_event_topic
+                # Any gap is before the first event in a read so only check once per read
+                unless limits_events_checked
+                  resync_if_limits_events_trimmed()
+                  limits_events_checked = true
+                end
+                @limits_event_offset = msg_id
                 event = JSON.parse(msg_hash['event'], allow_nan: true, create_additions: true)
-                LimitsEventTopic.process_event(event)
+                LimitsEventTopic.process_event(event, scope: @scope)
               elsif topic =~ /__DECOMINTERFACE/
                 if msg_hash.key?('inject_tlm')
                   handle_inject_tlm_with_ack(msg_hash['inject_tlm'], msg_id)
@@ -161,6 +172,17 @@ module OpenC3
       @limits_response_thread = nil
     end
 
+    # Limits events are only read between batches of packets, so a decom microservice
+    # working through a backlog can fall far enough behind that a flood of LIMITS_CHANGE
+    # events trims control events (enable, settings, etc) before it reads them.
+    # Recover by reloading all limits state from the store, which is always current.
+    def resync_if_limits_events_trimmed
+      if LimitsEventTopic.events_trimmed?(@limits_event_offset, scope: @scope, db_shard: @db_shard)
+        @logger.warn("#{@name}: Limits events were trimmed before they could be read, resyncing limits")
+        LimitsEventTopic.resync_system(scope: @scope)
+      end
+    end
+
     def decom_packet(topic, msg_id, msg_hash, _redis)
       OpenC3.in_span("decom_packet") do
         update_topic_lag(topic, msg_id, metric_name: 'decom_topic_delta_seconds', help: 'Delta time between data written to stream and decom start')

data/lib/openc3/microservices/interface_decom_common.rb

@@ -42,6 +42,8 @@ module OpenC3
     def handle_inject_tlm_with_ack(inject_tlm_json, msg_id)
       inject_tlm_hash = JSON.parse(inject_tlm_json, allow_nan: true, create_additions: true)
       target_name = inject_tlm_hash['target_name']
+      # Ack on the db_shard of the target where DecomInterfaceTopic.inject_tlm waits for it
+      db_shard = Store.db_shard_for_target(target_name, scope: @scope)
       ack_topic = "{#{@scope}__ACKCMD}TARGET__#{target_name}"
       begin
         handle_inject_tlm(inject_tlm_json)
@@ -56,7 +58,7 @@ module OpenC3
           result: error.message
         }
       end
-      Topic.write_topic(ack_topic, msg_hash)
+      Topic.write_topic(ack_topic, msg_hash, db_shard: db_shard)
     end
 
     def handle_build_cmd(build_cmd_json, msg_id)

data/lib/openc3/models/cvt_model.rb

@@ -134,8 +134,13 @@ module OpenC3
       end
     end
 
-    def self.tsdb_lookup(items, start_time:, end_time: nil, scope: $openc3_scope)
-      QuestDBClient.tsdb_lookup(items, start_time: start_time, end_time: end_time, scope: scope)
+    def self.tsdb_lookup(items, start_time:, end_time: nil, scope: $openc3_scope, flatten: true)
+      QuestDBClient.tsdb_lookup(items, start_time: start_time, end_time: end_time, scope: scope, flatten: flatten)
+    end
+
+    # Return the [target_name, packet_name] telemetry packets which have a TSDB table
+    def self.tsdb_packets_with_tables(packets, scope: $openc3_scope)
+      QuestDBClient.tlm_packets_with_tables(packets, scope: scope)
     end
 
     # Return all item values and limit state from the CVT

data/lib/openc3/models/target_model.rb

@@ -458,10 +458,13 @@ module OpenC3
     end
 
     def self.get_item_to_packet_map(target_name, scope:)
-      cache_time, item_map = @@item_map_cache[target_name]
-      return item_map if item_map and (Time.now - cache_time) < ITEM_MAP_CACHE_TIMEOUT
-      item_map_key = "#{scope}__#{target_name}__item_to_packet_map"
+      # Normalize before building the key so lowercase lookups share the entry
+      # that update_store_item_map and undeploy refresh and invalidate
       target_name = target_name.upcase
+      # The cache is keyed by item_map_key since the same target can exist in multiple scopes
+      item_map_key = "#{scope}__#{target_name}__item_to_packet_map"
+      cache_time, item_map = @@item_map_cache[item_map_key]
+      return item_map if item_map and (Time.now - cache_time) < ITEM_MAP_CACHE_TIMEOUT
       store = store_for_target(target_name, scope: scope)
       json_data = store.get(item_map_key)
       if json_data
@@ -470,7 +473,7 @@ module OpenC3
         item_map = build_item_to_packet_map(target_name, scope: scope)
         store.set(item_map_key, JSON.generate(item_map, allow_nan: true))
       end
-      @@item_map_cache[target_name] = [Time.now, item_map]
+      @@item_map_cache[item_map_key] = [Time.now, item_map]
       return item_map
     end
 
@@ -917,7 +920,7 @@ module OpenC3
       # Delete item_map
       item_map_key = "#{@scope}__#{@name}__item_to_packet_map"
       db_shard_store.del(item_map_key)
-      @@item_map_cache[@name] = nil
+      @@item_map_cache[item_map_key] = nil
 
       topic = { kind: 'deleted', type: 'target', name: @name }
       # The UNKNOWN target doesn't have an associated plugin
@@ -1101,7 +1104,7 @@ module OpenC3
       item_map_key = "#{@scope}__#{@name}__item_to_packet_map"
       item_map = self.class.build_item_to_packet_map(@name, scope: @scope)
       Store.instance(db_shard: @db_shard).set(item_map_key, JSON.generate(item_map, allow_nan: true))
-      @@item_map_cache[@name] = [Time.now, item_map]
+      @@item_map_cache[item_map_key] = [Time.now, item_map]
     end
 
     def update_store(system, clear_old: true)

data/lib/openc3/packets/packet_item.rb

@@ -398,14 +398,19 @@ module OpenC3
       if @states
         @states.each do |state_name, state_value|
           config << "    STATE #{state_name.to_s.quote_if_necessary} #{state_value.to_s.quote_if_necessary}"
-          if @hazardous and @hazardous[state_name]
-            config << " HAZARDOUS #{@hazardous[state_name].to_s.quote_if_necessary}"
-          end
-          if @messages_disabled and @messages_disabled[state_name]
-            config << " DISABLE_MESSAGES"
-          end
-          if @state_colors and @state_colors[state_name]
-            config << " #{@state_colors[state_name]}"
+          # Only write the attributes STATE accepts for this packet type so the config parses back
+          if cmd_or_tlm == :TELEMETRY
+            if @state_colors and @state_colors[state_name]
+              config << " #{@state_colors[state_name]}"
+            end
+          else
+            if @hazardous and @hazardous[state_name]
+              config << " HAZARDOUS"
+              config << " #{@hazardous[state_name].to_s.quote_if_necessary}" unless @hazardous[state_name].to_s.empty?
+            end
+            if @messages_disabled and @messages_disabled[state_name]
+              config << " DISABLE_MESSAGES"
+            end
           end
           config << "\n"
         end

data/lib/openc3/script/api_shared.rb

@@ -37,7 +37,7 @@ module OpenC3
     # @param args [String|Array<String>] See the description for calling style
     # @param type [Symbol] Telemetry type, :RAW, :CONVERTED (default), :FORMATTED
     def check(*args, type: :CONVERTED, scope: $openc3_scope, token: $openc3_token)
-      _check(*args, scope: scope) { |tgt, pkt, item| tlm(tgt, pkt, item, type: type, scope: scope, token: token) }
+      _check(*args, type: type, scope: scope, token: token) { |tgt, pkt, item| tlm(tgt, pkt, item, type: type, scope: scope, token: token) }
     end
 
     def check_raw(*args, scope: $openc3_scope, token: $openc3_token)
@@ -525,12 +525,12 @@ module OpenC3
     # Implementation of the various check commands. It yields back to the
     # caller to allow the return of the value through various telemetry calls.
     # This method should not be called directly by application code.
-    def _check(*args, scope: $openc3_scope, token: $openc3_token)
+    def _check(*args, type: :CONVERTED, scope: $openc3_scope, token: $openc3_token)
       target_name, packet_name, item_name, comparison_to_eval = _check_process_args(args, 'check')
 
       value = yield(target_name, packet_name, item_name)
       if comparison_to_eval
-        _check_comparison(target_name, packet_name, item_name, comparison_to_eval, value)
+        _check_comparison(target_name, packet_name, item_name, type, comparison_to_eval, value, scope: scope, token: token)
       else
         puts "CHECK: #{_upcase(target_name, packet_name, item_name)} == #{value.nil? ? 'nil' : value.inspect}"
       end
@@ -596,6 +596,8 @@ module OpenC3
                      quiet: false,
                      scope: $openc3_scope, token: $openc3_token)
       type = (check ? 'CHECK' : 'WAIT')
+      # LATEST would compare the RECEIVED_COUNT of whichever packet was received most recently
+      raise "LATEST is not supported by wait_packet, specify a packet name" if packet_name.to_s.upcase == 'LATEST'
       initial_count = tlm(target_name, packet_name, 'RECEIVED_COUNT', scope: scope, token: token)
       # If the packet has not been received the initial_count could be nil
       initial_count = 0 unless initial_count
@@ -768,10 +770,10 @@ module OpenC3
     # Builds a callable which compares a telemetry value against the given comparison string,
     # e.g. "> 1". Returns nil if there is no comparison, e.g. a block based wait_check().
     # Raises if the comparison is invalid, e.g. an unsupported operator or unparsable operand.
-    def _comparison_implementation(comparison_to_eval)
+    def _comparison_implementation(target_name, packet_name, item_name, value_type, comparison_to_eval, scope: $openc3_scope, token: $openc3_token)
       return nil unless comparison_to_eval
 
-      operator, operand = extract_operator_and_operand_from_comparison(comparison_to_eval)
+      operator, operand = extract_comparison(comparison_to_eval, value_type) { get_item(target_name, packet_name, item_name, scope: scope, token: token) }
       return nil unless operator
       lambda { |value| compare_values(value, operator, operand) }
     end
@@ -783,7 +785,7 @@ module OpenC3
         raise "ERROR: Invalid comparison to non-ascii value"
       end
       # A user supplied block is the condition so the comparison text is never parsed
-      comparison = block ? nil : _comparison_implementation(comparison_to_eval)
+      comparison = block ? nil : _comparison_implementation(target_name, packet_name, item_name, value_type, comparison_to_eval, scope: scope, token: token)
       _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, comparison, scope: scope, token: token, &block)
     end
 
@@ -851,14 +853,14 @@ module OpenC3
       end
     end
 
-    def _check_comparison(target_name, packet_name, item_name, comparison_to_eval, value)
+    def _check_comparison(target_name, packet_name, item_name, value_type, comparison_to_eval, value, scope: $openc3_scope, token: $openc3_token)
       check_str = "CHECK: #{_upcase(target_name, packet_name, item_name)} #{comparison_to_eval}"
       # Show user the check against a quoted string
       value_str = value.is_a?(String) ? "'#{value}'" : value
       value_str = 'nil' if value.nil? # Show user nil value as 'nil'
       with_value = "with value == #{value_str}"
 
-      operator, operand = extract_operator_and_operand_from_comparison(comparison_to_eval)
+      operator, operand = extract_comparison(comparison_to_eval, value_type) { get_item(target_name, packet_name, item_name, scope: scope, token: token) }
       unless _valid_comparison?(value, operator, operand)
         message = "Invalid comparison for types"
         if $disconnect