openc3 6.9.2 → 6.10.0

data/lib/openc3/packets/parsers/packet_item_parser.rb

@@ -33,16 +33,17 @@ module OpenC3
     # @param packet [Packet] The packet the item should be added to
     # @param cmd_or_tlm [String] Whether this is a command or telemetry packet
     # @param warnings [Array<String>] Array of warning strings from PacketConfig
-    def self.parse(parser, packet, cmd_or_tlm, warnings)
-      parser = PacketItemParser.new(parser, warnings)
+    def self.parse(parser, packet_config, packet, cmd_or_tlm, warnings)
+      parser = PacketItemParser.new(parser, packet_config, warnings)
       parser.verify_parameters(cmd_or_tlm)
       parser.create_packet_item(packet, cmd_or_tlm)
     end
 
     # @param parser [ConfigParser] Configuration parser
     # @param warnings [Array<String>] Array of warning strings from PacketConfig
-    def initialize(parser, warnings)
+    def initialize(parser, packet_config, warnings)
       @parser = parser
+      @packet_config = packet_config
       @warnings = warnings
       @usage = get_usage()
     end
@@ -57,8 +58,12 @@ module OpenC3
       # The usage is formatted with brackets <XXX> around each option so
       # count the number of open brackets to determine the number of options
       max_options = @usage.count("<")
-      # The last two options (description and endianness) are optional
-      @parser.verify_num_parameters(max_options - 2, max_options, @usage)
+      if @parser.keyword.include?('STRUCTURE')
+        @parser.verify_num_parameters(max_options, max_options, @usage)
+      else
+        # The last two options (description and endianness) are optional
+        @parser.verify_num_parameters(max_options - 2, max_options, @usage)
+      end
       @parser.verify_parameter_naming(1) # Item name is the 1st parameter
     end
 
@@ -69,24 +74,38 @@ module OpenC3
         Logger.instance.warn msg
         @warnings << msg
       end
-      item = PacketItem.new(item_name,
-                            get_bit_offset(),
-                            get_bit_size(),
-                            get_data_type(),
-                            get_endianness(packet),
-                            get_array_size(),
-                            :ERROR) # overflow
-      if cmd_or_tlm == PacketConfig::COMMAND
-        item.range = get_range()
-        item.default = get_default()
+      if @parser.keyword.include?("STRUCTURE")
+        item = PacketItem.new(item_name,
+                              get_bit_offset(),
+                              get_bit_size(true),
+                              :BLOCK,
+                              :BIG_ENDIAN,
+                              nil,
+                              :ERROR) # overflow
+      else
+        item = PacketItem.new(item_name,
+                              get_bit_offset(),
+                              get_bit_size(),
+                              get_data_type(),
+                              get_endianness(packet),
+                              get_array_size(),
+                              :ERROR) # overflow
+        if cmd_or_tlm == PacketConfig::COMMAND
+          item.range = get_range()
+          item.default = get_default()
+        end
+        item.id_value = get_id_value(item)
+        item.description = get_description()
       end
-      item.id_value = get_id_value(item)
-      item.description = get_description()
       if append?
         item = packet.append(item)
       else
         item = packet.define(item)
       end
+      if @parser.keyword.include?("STRUCTURE")
+        structure = lookup_packet(get_cmd_or_tlm(), get_target_name(), get_packet_name())
+        packet.structurize_item(item, structure)
+      end
       item
     end
 
@@ -108,9 +127,15 @@ module OpenC3
       raise @parser.error(e, @usage)
     end
 
-    def get_bit_size
+    def get_bit_size(check_structure = false)
       index = append? ? 1 : 2
-      Integer(@parser.parameters[index])
+      bit_size = @parser.parameters[index]
+      if not check_structure or bit_size.to_s.upcase != 'DEFINED'
+        return Integer(bit_size)
+      else
+        structure = lookup_packet(get_cmd_or_tlm(), get_target_name(), get_packet_name())
+        return structure.defined_length_bits
+      end
     rescue => e
       raise @parser.error(e, @usage)
     end
@@ -152,7 +177,7 @@ module OpenC3
       return nil if @parser.keyword.include?('ARRAY')
 
       data_type = get_data_type()
-      return nil if data_type == :STRING or data_type == :BLOCK
+      return nil unless data_type == :INT or data_type == :UINT or data_type == :FLOAT
 
       index = append? ? 3 : 4
       return nil if @parser.parameters[index] == 'nil'
@@ -165,6 +190,31 @@ module OpenC3
       min..max
     end
 
+    def get_cmd_or_tlm
+      index = append? ? 2 : 3
+      cmd_or_tlm = @parser.parameters[index].to_s.upcase.intern
+      raise ArgumentError, "Unknown type: #{cmd_or_tlm}" unless %i(CMD TLM COMMAND TELEMETRY).include?(cmd_or_tlm)
+      cmd_or_tlm
+    end
+
+    def get_target_name
+      index = append? ? 3 : 4
+      @parser.parameters[index].to_s.upcase
+    end
+
+    def get_packet_name
+      index = append? ? 4 : 5
+      @parser.parameters[index].to_s.upcase
+    end
+
+    def lookup_packet(cmd_or_tlm, target_name, packet_name)
+      if cmd_or_tlm == :CMD or cmd_or_tlm == :COMMAND
+        return @packet_config.commands[target_name][packet_name]
+      else
+        return @packet_config.telemetry[target_name][packet_name]
+      end
+    end
+
     def convert_string_value(index)
       # If the default value is 0x<data> (no quotes), it is treated as
       # binary data.  Otherwise, the default value is considered to be a string.
@@ -182,8 +232,52 @@ module OpenC3
 
       index = append? ? 3 : 4
       data_type = get_data_type()
-      return [] if data_type == :ARRAY
-      return {} if data_type == :OBJECT
+      if data_type == :BOOL
+        value = @parser.parameters[index].to_s.upcase
+        if value == "TRUE" or value == "FALSE"
+          return ConfigParser.handle_true_false(@parser.parameters[index])
+        else
+          raise @parser.error("Default for BOOL data type must be TRUE or FALSE")
+        end
+      end
+      if data_type == :ARRAY
+        value = @parser.parameters[index].to_s
+        begin
+          value = JSON.parse(value, allow_nan: true)
+        rescue Exception
+          raise @parser.error("Unparsable value for ARRAY: #{value}")
+        end
+        if Array === value
+          return value
+        else
+          raise @parser.error("Default for ARRAY data type must be an Array")
+        end
+      end
+      if data_type == :OBJECT
+        value = @parser.parameters[index].to_s
+        begin
+          value = JSON.parse(value, allow_nan: true)
+        rescue Exception
+          raise @parser.error("Unparsable value for OBJECT: #{value}")
+        end
+        if Hash === value
+          return value
+        else
+          raise @parser.error("Default for OBJECT data type must be a Hash")
+        end
+      end
+      if data_type == :ANY
+        value = @parser.parameters[index].to_s
+        if value.length > 0
+          begin
+            return JSON.parse(value, allow_nan: true)
+          rescue Exception
+            return value
+          end
+        else
+          return ""
+        end
+      end
       if data_type == :STRING or data_type == :BLOCK
         return convert_string_value(index)
       else
@@ -209,6 +303,52 @@ module OpenC3
       end
 
       index = append? ? 3 : 4
+      if data_type == :BOOL
+        value = @parser.parameters[index].to_s.upcase
+        if value == "TRUE" or value == "FALSE"
+          return ConfigParser.handle_true_false(@parser.parameters[index])
+        else
+          raise @parser.error("ID Value for BOOL data type must be TRUE or FALSE")
+        end
+      end
+      if data_type == :ARRAY
+        value = @parser.parameters[index].to_s
+        begin
+          value = JSON.parse(value, allow_nan: true)
+        rescue Exception
+          raise @parser.error("Unparsable value for ARRAY: #{value}")
+        end
+        if Array === value
+          return value
+        else
+          raise @parser.error("ID Value for ARRAY data type must be an Array")
+        end
+      end
+      if data_type == :OBJECT
+        value = @parser.parameters[index].to_s
+        begin
+          value = JSON.parse(value, allow_nan: true)
+        rescue Exception
+          raise @parser.error("Unparsable value for OBJECT: #{value}")
+        end
+        if Hash === value
+          return value
+        else
+          raise @parser.error("ID Value for OBJECT data type must be a Hash")
+        end
+      end
+      if data_type == :ANY
+        value = @parser.parameters[index].to_s
+        if value.length > 0
+          begin
+            return JSON.parse(value, allow_nan: true)
+          rescue Exception
+            return value
+          end
+        else
+          return ""
+        end
+      end
       if data_type == :STRING or data_type == :BLOCK
         return convert_string_value(index)
       else
@@ -229,10 +369,14 @@ module OpenC3
       usage = "#{@parser.keyword} <ITEM NAME> "
       usage << "<BIT OFFSET> " unless @parser.keyword.include?("APPEND")
       usage << bit_size_usage()
-      usage << type_usage()
-      usage << "<TOTAL ARRAY BIT SIZE> " if @parser.keyword.include?("ARRAY")
-      usage << id_usage()
-      usage << "<DESCRIPTION (Optional)> <ENDIANNESS (Optional)>"
+      if not @parser.keyword.include?("STRUCTURE")
+        usage << type_usage()
+        usage << "<TOTAL ARRAY BIT SIZE> " if @parser.keyword.include?("ARRAY")
+        usage << id_usage()
+        usage << "<DESCRIPTION (Optional)> <ENDIANNESS (Optional)>"
+      else
+        usage << "<CMD or TLM> <Target Name> <Packet Name>"
+      end
       usage
     end
 
@@ -247,13 +391,13 @@ module OpenC3
     def type_usage
       keyword = @parser.keyword
       # Item type usage is simple so just return it
-      return "<TYPE: INT/UINT/FLOAT/STRING/BLOCK/DERIVED/ARRAY/OBJECT> " if keyword.include?("ITEM")
+      return "<TYPE: INT/UINT/FLOAT/STRING/BLOCK/DERIVED/BOOL/ARRAY/OBJECT/ANY> " if keyword.include?("ITEM")
 
       # Build up the parameter type usage based on the keyword
       usage = "<TYPE: "
       # ARRAY types don't have min or max or default values
       if keyword.include?("ARRAY")
-        usage << "INT/UINT/FLOAT/STRING/BLOCK/OBJECT> "
+        usage << "INT/UINT/FLOAT/STRING/BLOCK/BOOL/OBJECT/ANY> "
       else
         begin
           data_type = get_data_type()
@@ -261,14 +405,13 @@ module OpenC3
           # If the data type could not be determined set something
           data_type = :INT
         end
-        # STRING, BLOCK, ARRAY, OBJECT types do not have min or max values
-        if data_type == :STRING || data_type == :BLOCK
-          usage << "STRING/BLOCK/ARRAY/OBJECT> "
+        if data_type == :INT || data_type == :UINT || data_type == :FLOAT || data_type == :DERIVED
+          usage << "INT/UINT/FLOAT/DERIVED> <MIN VALUE> <MAX VALUE> "
         else
-          usage << "INT/UINT/FLOAT> <MIN VALUE> <MAX VALUE> "
+          usage << "STRING/BLOCK/BOOL/ARRAY/OBJECT/ANY> "
         end
-        # ID Values do not have default values (or ARRAY/OBJECT)
-        unless keyword.include?("ID") or data_type == :ARRAY or data_type == :OBJECT
+        # ID Values do not have default values
+        unless keyword.include?("ID")
           usage << "<DEFAULT_VALUE> "
         end
       end

data/lib/openc3/packets/parsers/xtce_converter.rb

@@ -261,8 +261,8 @@ module OpenC3
         to_xtce_string(item, param_or_arg, xml, 'String')
       when :BLOCK
         to_xtce_string(item, param_or_arg, xml, 'Binary')
-      when :DERIVED
-        raise "DERIVED data type not supported in XTCE"
+      else
+        raise "#{item.data_type} data type not supported in XTCE"
       end
 
       # Handle arrays

data/lib/openc3/packets/structure.rb

@@ -248,32 +248,34 @@ module OpenC3
 
       # Recalculate the overall defined length of the structure
       update_needed = false
-      if item.bit_offset >= 0
-        if item.bit_size > 0
-          if item.array_size
-            if item.array_size >= 0
-              item_defined_length_bits = item.bit_offset + item.array_size
+      if not item.parent_item
+        if item.bit_offset >= 0
+          if item.bit_size > 0
+            if item.array_size
+              if item.array_size >= 0
+                item_defined_length_bits = item.bit_offset + item.array_size
+              else
+                item_defined_length_bits = item.bit_offset
+              end
             else
-              item_defined_length_bits = item.bit_offset
+              item_defined_length_bits = item.bit_offset + item.bit_size
+            end
+            if item_defined_length_bits > @pos_bit_size
+              @pos_bit_size = item_defined_length_bits
+              update_needed = true
             end
           else
-            item_defined_length_bits = item.bit_offset + item.bit_size
-          end
-          if item_defined_length_bits > @pos_bit_size
-            @pos_bit_size = item_defined_length_bits
-            update_needed = true
+            if item.bit_offset > @pos_bit_size
+              @pos_bit_size = item.bit_offset
+              update_needed = true
+            end
           end
         else
-          if item.bit_offset > @pos_bit_size
-            @pos_bit_size = item.bit_offset
+          if item.bit_offset.abs > @neg_bit_size
+            @neg_bit_size = item.bit_offset.abs
             update_needed = true
           end
         end
-      else
-        if item.bit_offset.abs > @neg_bit_size
-          @neg_bit_size = item.bit_offset.abs
-          update_needed = true
-        end
       end
       if update_needed
         @defined_length_bits = @pos_bit_size + @neg_bit_size
@@ -349,7 +351,7 @@ module OpenC3
           elsif item.variable_bit_size['length_value_bit_offset'] > 0
             minimum_data_bits = item.variable_bit_size['length_value_bit_offset'] * item.variable_bit_size['length_bits_per_count']
           end
-          if minimum_data_bits > 0 and item.bit_offset >= 0 and @defined_length_bits == item.bit_offset
+          if minimum_data_bits > 0 and item.bit_offset >= 0 and @defined_length_bits == item.bit_offset and not item.parent_item
             @defined_length_bits += minimum_data_bits
           end
         end
@@ -436,7 +438,11 @@ module OpenC3
     def read_all(value_type = :RAW, buffer = @buffer, top = true)
       item_array = []
       synchronize_allow_reads(top) do
-        @sorted_items.each { |item| item_array << [item.name, read_item(item, value_type, buffer)] }
+        @sorted_items.each do |item|
+          unless item.hidden
+            item_array << [item.name, read_item(item, value_type, buffer)]
+          end
+        end
       end
       return item_array
     end
@@ -454,7 +460,7 @@ module OpenC3
       string = ''
       synchronize_allow_reads(true) do
         @sorted_items.each do |item|
-          next if ignored && ignored.include?(item.name)
+          next if item.hidden || (ignored && ignored.include?(item.name))
 
           if (item.data_type != :BLOCK) ||
              (item.data_type == :BLOCK and value_type != :RAW and
@@ -631,6 +637,8 @@ module OpenC3
     def recalculate_bit_offsets
       adjustment = 0
       @sorted_items.each do |item|
+        # Parented items rely on the parent
+        next if item.parent_item
         # Anything with a negative bit offset should be left alone
         if item.original_bit_offset >= 0
           item.bit_offset = item.original_bit_offset + adjustment

data/lib/openc3/packets/structure_item.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2024, OpenC3, Inc.
+# All changes Copyright 2025, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -31,7 +31,7 @@ module OpenC3
     @@create_index = 0
 
     # Valid data types adds :DERIVED, :ARRAY, :OBJECT to those defined by BinaryAccessor
-    DATA_TYPES = BinaryAccessor::DATA_TYPES << :DERIVED << :ARRAY << :OBJECT
+    DATA_TYPES = [:INT, :UINT, :FLOAT, :STRING, :BLOCK, :BOOL, :OBJECT, :ARRAY, :ANY, :DERIVED]
 
     # Name is used by higher level classes to access the StructureItem.
     # @return [String] Name of the item
@@ -98,6 +98,15 @@ module OpenC3
     # @return [Integer] Incrementing value that shows relative order items are created
     attr_reader :create_index
 
+    # @return [Boolean] Indicates if item should be listed in items and in read all type methods
+    attr_accessor :hidden
+
+    # @return [StructureItem] Parent structure item
+    attr_accessor :parent_item
+
+    # @return [Structure] Structure associated with item
+    attr_accessor :structure
+
     # Create a StructureItem by setting all the attributes. It
     # calls all the setter routines to do the attribute verification and then
     # verifies the overall integrity.
@@ -126,6 +135,9 @@ module OpenC3
       self.overflow = overflow
       self.overlap = false
       self.variable_bit_size = nil
+      self.hidden = false
+      self.parent_item = nil
+      self.structure = nil
       @create_index = @@create_index
       @@create_index += 1
       @structure_item_constructed = true
@@ -198,7 +210,7 @@ module OpenC3
       when *DATA_TYPES
         # Valid data_type
       else
-        raise ArgumentError, "#{@name}: unknown data_type: #{data_type} - Must be :INT, :UINT, :FLOAT, :STRING, :BLOCK, or :DERIVED"
+        raise ArgumentError, "#{@name}: unknown data_type: #{data_type} - Must be #{DATA_TYPES.join(", ")}"
       end
 
       @data_type = data_type
@@ -308,31 +320,31 @@ module OpenC3
     end
     alias dup clone
 
-    def self.from_json(hash)
-      # Convert strings to symbols
-      endianness = hash['endianness'] ? hash['endianness'].intern : nil
-      data_type = hash['data_type'] ? hash['data_type'].intern : nil
-      overflow = hash['overflow'] ? hash['overflow'].intern : nil
-      si = StructureItem.new(hash['name'], hash['bit_offset'], hash['bit_size'], data_type,
-        endianness, hash['array_size'], overflow)
-      si.key = hash['key'] || hash['name']
-      si.variable_bit_size = hash['variable_bit_size']
-      si
-    end
-
     def as_json(*a)
       hash = {}
       hash['name'] = self.name
       hash['key'] = self.key
       hash['bit_offset'] = self.original_bit_offset
       hash['bit_size'] = self.original_bit_size
-      hash['data_type'] = self.data_type
-      hash['endianness'] = self.endianness
-      hash['array_size'] = self.original_array_size
-      hash['overflow'] = self.overflow
+      hash['data_type'] = self.data_type.to_s
+      hash['endianness'] = self.endianness.to_s
+      hash['overflow'] = self.overflow.to_s
+      hash['overlap'] = self.overlap
+      hash['create_index'] = self.create_index
+      hash['hidden'] = self.hidden
+      if self.original_array_size
+        hash['array_size'] = self.original_array_size
+      end
       if @variable_bit_size
         hash['variable_bit_size'] = @variable_bit_size
       end
+      if self.parent_item
+        hash['parent_item'] = self.parent_item.as_json
+      end
+      if self.structure
+        hash['structure'] = self.structure.as_json
+      end
+
       hash
     end
 

data/lib/openc3/packets/telemetry.rb

@@ -264,8 +264,8 @@ module OpenC3
     #   default value of nil means to search all known targets.
     # @return [Packet] The identified packet with its data set to the given
     #   packet_data buffer. Returns nil if no packet could be identified.
-    def identify!(packet_data, target_names = nil)
-      identified_packet = identify(packet_data, target_names)
+    def identify!(packet_data, target_names = nil, subpackets: false)
+      identified_packet = identify(packet_data, target_names, subpackets: subpackets)
       identified_packet.buffer = packet_data if identified_packet
       return identified_packet
     end
@@ -277,7 +277,7 @@ module OpenC3
     # @param target_names [Array<String>] List of target names to limit the search. The
     #   default value of nil means to search all known targets.
     # @return [Packet] The identified packet, Returns nil if no packet could be identified.
-    def identify(packet_data, target_names = nil)
+    def identify(packet_data, target_names = nil, subpackets: false)
       target_names = target_names() unless target_names
 
       target_names.each do |target_name|
@@ -292,18 +292,36 @@ module OpenC3
           next
         end
 
-        target = System.targets[target_name]
-        if target and target.tlm_unique_id_mode
+        if (not subpackets and System.telemetry.tlm_unique_id_mode(target_name)) or (subpackets and System.telemetry.tlm_subpacket_unique_id_mode(target_name))
           # Iterate through the packets and see if any represent the buffer
           target_packets.each do |_packet_name, packet|
-            return packet if packet.identify?(packet_data)
+            if subpackets
+              next unless packet.subpacket
+            else
+              next if packet.subpacket
+            end
+            return packet if packet.identify?(packet_data) # Handles virtual
           end
         else
           # Do a hash lookup to quickly identify the packet
-          if target_packets.length > 0
-            packet = target_packets.first[1]
+          packet = nil
+          target_packets.each do |_packet_name, target_packet|
+            next if target_packet.virtual
+            if subpackets
+              next unless target_packet.subpacket
+            else
+              next if target_packet.subpacket
+            end
+            packet = target_packet
+            break
+          end
+          if packet
             key = packet.read_id_values(packet_data)
-            hash = @config.tlm_id_value_hash[target_name]
+            if subpackets
+              hash = @config.tlm_subpacket_id_value_hash[target_name]
+            else
+              hash = @config.tlm_id_value_hash[target_name]
+            end
             identified_packet = hash[key]
             identified_packet = hash['CATCHALL'.freeze] unless identified_packet
             return identified_packet if identified_packet
@@ -314,9 +332,9 @@ module OpenC3
       return nil
     end
 
-    def identify_and_define_packet(packet, target_names = nil)
+    def identify_and_define_packet(packet, target_names = nil, subpackets: false)
       if !packet.identified?
-        identified_packet = identify(packet.buffer(false), target_names)
+        identified_packet = identify(packet.buffer(false), target_names, subpackets: subpackets)
         return nil unless identified_packet
 
         identified_packet = identified_packet.clone
@@ -425,5 +443,13 @@ module OpenC3
     def dynamic_add_packet(packet, affect_ids: false)
       @config.dynamic_add_packet(packet, :TELEMETRY, affect_ids: affect_ids)
     end
+
+    def tlm_unique_id_mode(target_name)
+      return @config.tlm_unique_id_mode[target_name.upcase]
+    end
+
+    def tlm_subpacket_unique_id_mode(target_name)
+      return @config.tlm_subpacket_unique_id_mode[target_name.upcase]
+    end
   end # class Telemetry
 end

data/lib/openc3/script/suite_results.rb

@@ -172,11 +172,11 @@ module OpenC3
     end
 
     def write(string)
-      @report << (Time.now.sys.formatted + ': ' + string)
+      @report << (Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%6NZ") + ': ' + string)
     end
 
     def puts(string)
-      @report << (Time.now.sys.formatted + ': ' + string)
+      @report << (Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.%6NZ") + ': ' + string)
     end
 
     # def collect_metadata(parent = nil)

data/lib/openc3/subpacketizers/subpacketizer.rb

@@ -0,0 +1,18 @@
+class Subpacketizer
+  attr_reader :args
+
+  def initialize(packet=nil)
+    @packet = packet
+    @args = []
+  end
+
+  # Subclass and implement this method to break packet into array of subpackets
+  # Subpackets should be fully identified and defined
+  def call(packet)
+    return [packet]
+  end
+
+  def as_json(*a)
+    { 'class' => self.class.name, 'args' => @args.as_json(*a) }
+  end
+end