cosmos 4.2.4 → 4.5.0

data/lib/cosmos/io/win32_serial_driver.rb

@@ -24,7 +24,8 @@ module Cosmos
                    read_polling_period = 0.01,
                    read_max_length = 1000,
                    flow_control = :NONE,
-                   data_bits = 8)
+                   data_bits = 8,
+                   struct = [])
 
       # Verify Parameters
       port_name = '\\\\.\\' + port_name if port_name =~ /^COM[0-9]{2,3}$/
@@ -78,10 +79,37 @@ module Cosmos
         # 0x03 - RTS_CONTROL_TOGGLE - Specifies that the RTS line will be high if bytes are available for transmission. After all buffered bytes have been sent, the RTS line will be low.
         dcb.write('fRtsControl', 0x03)
       end
+      # Allow the end user to write arbitrary values into the Windows DCB structure
+      unless struct.empty?
+        struct.each do |key, value|
+          dcb.write(key, value.to_i)
+        end
+      end
       Win32.set_comm_state(@handle, dcb)
 
-      # Configure Timeouts
-      Win32.set_comm_timeouts(@handle, 4294967295, 0, 0, 0, 0)
+      # Configure Timeouts, the WinAPI structure is COMMTIMEOUTS:
+      #   DWORD ReadIntervalTimeout;
+      #   DWORD ReadTotalTimeoutMultiplier;
+      #   DWORD ReadTotalTimeoutConstant;
+      #   DWORD WriteTotalTimeoutMultiplier;
+      #   DWORD WriteTotalTimeoutConstant;
+      # 0xFFFFFFFF, 0, 0 specifies that the read operation is to return immediately
+      # with the bytes that have already been received, even if no bytes have been received.
+      # The WriteTotalTimeoutMultiplier is multiplied by the number of bytes to be written
+      # and the WriteTotalTimeoutConstant is added to that total (both are in milliseconds).
+      bits_per_symbol = data_bits + 1 # 1 start bit
+      case stop_bits
+      when Win32::ONESTOPBIT
+        bits_per_symbol += 1
+      when Win32::TWOSTOPBITS
+        bits_per_symbol += 2
+      end
+      case parity
+      when Win32::ODDPARITY, Win32::EVENPARITY
+        bits_per_symbol += 1
+      end
+      delay = (1000.0 / (baud_rate / bits_per_symbol.to_f)).ceil
+      Win32.set_comm_timeouts(@handle, 0xFFFFFFFF, 0, 0, delay, 1000)
     end
 
     # (see SerialDriver#close)

data/lib/cosmos/packet_logs/packet_log_reader.rb

@@ -35,6 +35,18 @@ module Cosmos
     COSMOS1_LOG_TYPE_RANGE = 6..8
     COSMOS1_CONFIGURATION_NAME_RANGE = 10..41
 
+    # COSMOS 4.3+ log file header definition
+    COSMOS4_MARKER = 'COSMOS4_'
+    COSMOS4_HEADER_LENGTH = COSMOS2_HEADER_LENGTH
+    COSMOS4_MARKER_RANGE = COSMOS2_MARKER_RANGE
+    COSMOS4_LOG_TYPE_RANGE = COSMOS2_LOG_TYPE_RANGE
+    COSMOS4_CONFIGURATION_NAME_RANGE = COSMOS2_CONFIGURATION_NAME_RANGE
+    COSMOS4_HOSTNAME_RANGE = COSMOS2_HOSTNAME_RANGE
+    COSMOS4_STORED_FLAG_MASK = 0x80
+    COSMOS4_EXTRA_FLAG_MASK = 0x40
+
+    MAX_READ_SIZE = 1000000000
+
     # Create a new log file reader
     def initialize
       reset()
@@ -62,10 +74,10 @@ module Cosmos
         packet = read(identify_and_define)
         break unless packet
 
-        received_time = packet.received_time
-        if received_time
-          next if start_time and received_time < start_time
-          break if end_time and received_time > end_time
+        time = packet.packet_time
+        if time
+          next if start_time and time < start_time
+          break if end_time and time > end_time
         end
 
         yield packet
@@ -115,6 +127,8 @@ module Cosmos
       reset()
       @filename = filename
       @file = BufferedFile.open(@filename, 'rb')
+      @max_read_size = @file.size
+      @max_read_size = MAX_READ_SIZE if @max_read_size > MAX_READ_SIZE
       @bytes_read = 0
       return read_file_header()
     rescue => err
@@ -133,12 +147,12 @@ module Cosmos
     # @return [Packet]
     def read(identify_and_define = true)
       # Read the Packet Header
-      success, target_name, packet_name, received_time = read_entry_header()
+      success, target_name, packet_name, received_time, stored, extra = read_entry_header()
       return nil unless success
 
       # Read Packet Data
-      packet_data = @file.read_length_bytes(4)
-      return nil unless packet_data and packet_data.length > 0
+      packet_data = @file.read_length_bytes(4, @max_read_size)
+      return nil unless packet_data and packet_data.length >= 0
 
       if identify_and_define
         packet = identify_and_define_packet_data(target_name, packet_name, received_time, packet_data)
@@ -147,6 +161,8 @@ module Cosmos
         packet = Packet.new(target_name, packet_name, :BIG_ENDIAN, nil, packet_data)
         packet.set_received_time_fast(received_time)
       end
+      packet.stored = stored
+      packet.extra = extra
 
       # Auto change configuration on SYSTEM META
       if packet.target_name == 'SYSTEM'.freeze and packet.packet_name == 'META'.freeze
@@ -231,11 +247,13 @@ module Cosmos
 
     def reset
       @file = nil
+      @max_read_size = MAX_READ_SIZE
       @filename = nil
       @log_type = :TLM
       @configuration_name = nil
       @hostname = nil
-      @file_header_length = COSMOS2_HEADER_LENGTH
+      @mode = 4
+      @file_header_length = COSMOS4_HEADER_LENGTH
     end
 
     # This is best effort. May return unidentified/undefined packets
@@ -257,7 +275,7 @@ module Cosmos
           end
           packet.buffer = packet_data
           packet.set_received_time_fast(received_time)
-        rescue
+        rescue Exception => error
           # Could not find a definition for this packet
           Logger.instance.error "Unknown packet #{target_name} #{packet_name}"
           packet = Packet.new(target_name, packet_name, :BIG_ENDIAN, nil, packet_data)
@@ -269,10 +287,15 @@ module Cosmos
 
     # Should return if successfully switched to requested configuration
     def read_file_header
-      header = @file.read(COSMOS2_HEADER_LENGTH)
-      if header and header.length == COSMOS2_HEADER_LENGTH
-        if header[COSMOS2_MARKER_RANGE] == COSMOS2_MARKER
-          # Found COSMOS 2 File Header
+      header = @file.read(COSMOS4_HEADER_LENGTH)
+      if header and header.length == COSMOS4_HEADER_LENGTH
+        if header[COSMOS4_MARKER_RANGE] == COSMOS4_MARKER or header[COSMOS2_MARKER_RANGE] == COSMOS2_MARKER
+          # Found COSMOS 2/4 File Header
+          if header[COSMOS4_MARKER_RANGE] == COSMOS4_MARKER
+            @mode = 4
+          else
+            @mode = 2
+          end
           @log_type = header[COSMOS2_LOG_TYPE_RANGE].intern
           raise "Unknown log type #{@log_type}" unless [:CMD, :TLM].include? @log_type
           @configuration_name = header[COSMOS2_CONFIGURATION_NAME_RANGE]
@@ -283,6 +306,7 @@ module Cosmos
           return false, error # Did not successfully change to requested configuration name
         elsif header[COSMOS1_MARKER_RANGE] == COSMOS1_MARKER
           # Found COSMOS 1 File Header
+          @mode = 1
           @log_type = header[COSMOS1_LOG_TYPE_RANGE].upcase.intern
           raise "Unknown log type #{@log_type}" unless [:CMD, :TLM].include? @log_type
           @configuration_name = header[COSMOS1_CONFIGURATION_NAME_RANGE]
@@ -297,29 +321,48 @@ module Cosmos
           raise "COSMOS file header not found on packet log"
         end
       else
-        raise "Failed to read at least #{COSMOS2_HEADER_LENGTH} bytes from packet log"
+        raise "Failed to read at least #{COSMOS4_HEADER_LENGTH} bytes from packet log"
       end
     end
 
     def read_entry_header
+      stored = false
+      extra = nil
+
+      if @mode == 4
+        # Read Flags
+        flags = @file.read(1)
+        return [nil, nil, nil, nil, nil, nil] if flags.nil? or flags.length != 1
+        flags = flags.unpack('C')[0]
+
+        stored = true if (flags & COSMOS4_STORED_FLAG_MASK) != 0
+
+        if (flags & COSMOS4_EXTRA_FLAG_MASK) != 0
+          # Read Extra data
+          extra_data = @file.read_length_bytes(4, @max_read_size)
+          return [nil, nil, nil, nil, nil, nil] unless extra_data and extra_data.length > 0
+          extra = JSON.parse(extra_data)
+        end
+      end
+
       # Read Received Time
       time_seconds = @file.read(4)
-      return [nil, nil, nil, nil] if time_seconds.nil? or time_seconds.length != 4
+      return [nil, nil, nil, nil, nil, nil] if time_seconds.nil? or time_seconds.length != 4
       time_seconds = time_seconds.unpack('N')[0]
       time_microseconds = @file.read(4)
-      return [nil, nil, nil, nil] if time_microseconds.nil? or time_microseconds.length != 4
+      return [nil, nil, nil, nil, nil, nil] if time_microseconds.nil? or time_microseconds.length != 4
       time_microseconds = time_microseconds.unpack('N')[0]
       received_time = Time.at(time_seconds, time_microseconds).sys
 
       # Read Target Name
       target_name = @file.read_length_bytes(1)
-      return [nil, nil, nil, nil] unless target_name and target_name.length > 0
+      return [nil, nil, nil, nil, nil, nil] unless target_name and target_name.length > 0
 
       # Read Packet Name
       packet_name = @file.read_length_bytes(1)
-      return [nil, nil, nil, nil] unless packet_name and packet_name.length > 0
+      return [nil, nil, nil, nil, nil, nil] unless packet_name and packet_name.length > 0
 
-      return [true, target_name, packet_name, received_time]
+      return [true, target_name, packet_name, received_time, stored, extra]
     end
 
     def test
@@ -331,7 +374,7 @@ module Cosmos
       begin
         # Try to read the packet header
         # This will fail with file read errors and invalid timestamps
-        success, target_name, packet_name, _ = read_entry_header()
+        success, target_name, packet_name, _, _, _ = read_entry_header()
         if success
           if target_name !~ File::NON_ASCII_PRINTABLE and packet_name !~ File::NON_ASCII_PRINTABLE
             packet_data_length = @file.read(4)
@@ -402,17 +445,17 @@ module Cosmos
           # Read the first packet in the log
           first_packet = first()
           raise "Error reading first packet" unless first
-          raise "First Packet does not contain a packet received time" unless first_packet.received_time
+          raise "First Packet does not contain a packet received time" unless first_packet.packet_time
 
           # Read the last packet in the log
           file_size = size()
           last_packet = last()
           raise "Search failed looking for last packet" unless last_packet
-          raise "Last Packet does not contain a packet received time" unless last_packet.received_time
+          raise "Last Packet does not contain a packet received time" unless last_packet.packet_time
 
-          if time >= first_packet.received_time and time <= last_packet.received_time
+          if time >= first_packet.packet_time and time <= last_packet.packet_time
             # Guess at where to start looking for time in log
-            percentage = (time - first_packet.received_time) / (last_packet.received_time - first_packet.received_time)
+            percentage = (time - first_packet.packet_time) / (last_packet.packet_time - first_packet.packet_time)
             offset = (percentage * file_size.to_f).to_i
             offset = @file_header_length if offset < @file_header_length
             @file.seek(offset, IO::SEEK_SET)
@@ -420,10 +463,10 @@ module Cosmos
             # Move backwards until a packet before the time is found
             while true
               packet = search(-1, :BEFORE)
-              break if !packet or packet.received_time <= time
+              break if !packet or packet.packet_time <= time
 
               # Guess again
-              percentage = 1.0 - ((packet.received_time - time) / (packet.received_time - first_packet.received_time))
+              percentage = 1.0 - ((packet.packet_time - time) / (packet.packet_time - first_packet.packet_time))
               offset = (percentage * @file.pos.to_f).to_i
               offset = @file_header_length if offset < @file_header_length
               @file.seek(offset, IO::SEEK_SET)
@@ -434,7 +477,7 @@ module Cosmos
               position = @file.pos
               packet = read(false)
               raise "Search failed looking for packet after time" unless packet
-              if packet.received_time >= time
+              if packet.packet_time >= time
                 # Back up this packet so the read can get it because we want it
                 @file.seek(position, IO::SEEK_SET)
                 break
@@ -442,7 +485,7 @@ module Cosmos
             end
 
           else
-            if time > last_packet.received_time
+            if time > last_packet.packet_time
               # File is entirely before time, so jump to the end
               @file.seek(0, IO::SEEK_END)
             else

data/lib/cosmos/packet_logs/packet_log_writer.rb

@@ -311,7 +311,7 @@ module Cosmos
 
     def build_file_header(configuration_name = System.configuration_name)
       hostname = Socket.gethostname.to_s
-      file_header = "COSMOS2_#{@log_type}_#{configuration_name.ljust(32, ' ')[0..31]}_"
+      file_header = "COSMOS4_#{@log_type}_#{configuration_name.ljust(32, ' ')[0..31]}_"
       file_header << hostname.ljust(83)
       return file_header
     end
@@ -322,6 +322,18 @@ module Cosmos
       # This is an optimization to avoid creating a new entry_header object
       # each time we create an entry_header which we do a LOT!
       @entry_header.clear
+      flags = 0
+      flags |= PacketLogReader::COSMOS4_STORED_FLAG_MASK if packet.stored
+      extra = packet.extra
+      if extra
+        flags |= PacketLogReader::COSMOS4_EXTRA_FLAG_MASK
+        extra = extra.to_json
+      end
+      @entry_header << [flags].pack('C'.freeze)
+      if extra
+        @entry_header << [extra.length].pack('N'.freeze)
+        @entry_header << extra
+      end
       @entry_header << [received_time.tv_sec].pack('N'.freeze)
       @entry_header << [received_time.tv_usec].pack('N'.freeze)
       target_name = packet.target_name

data/lib/cosmos/packets/commands.rb

@@ -90,6 +90,8 @@ module Cosmos
       target_names = target_names() unless target_names
 
       target_names.each do |target_name|
+        target_name = target_name.to_s.upcase
+        
         target_packets = nil
         begin
           target_packets = packets(target_name)
@@ -98,18 +100,35 @@ module Cosmos
           next
         end
 
-        # Iterate through the packets and see if any represent the buffer
-        target_packets.each do |packet_name, packet|
-          if (packet.identify?(packet_data))
-            identified_packet = packet.clone
-            identified_packet.received_time  = nil
-            identified_packet.received_count = 0
-            identified_packet.buffer = packet_data
-            break
+        target = System.targets[target_name]
+        if target and target.cmd_unique_id_mode
+          # Iterate through the packets and see if any represent the buffer
+          target_packets.each do |packet_name, packet|
+            if packet.identify?(packet_data)
+              identified_packet = packet
+              break
+            end
           end
+        else
+          # Do a hash lookup to quickly identify the packet
+          if target_packets.length > 0
+            packet = target_packets.first[1]
+            key = packet.read_id_values(packet_data)
+            hash = @config.cmd_id_value_hash[target_name]
+            identified_packet = hash[key]
+            identified_packet = hash['CATCHALL'.freeze] unless identified_packet
+          end          
         end
 
-        break if identified_packet
+        if identified_packet
+          identified_packet = identified_packet.clone
+          identified_packet.received_time = nil
+          identified_packet.stored = false
+          identified_packet.extra = nil
+          identified_packet.received_count = 0
+          identified_packet.buffer = packet_data
+          break
+        end
       end
 
       return identified_packet
@@ -141,6 +160,8 @@ module Cosmos
 
       # Set time, parameters, and restore defaults
       command.received_time = Time.now.sys
+      command.stored = false
+      command.extra = nil
       command.given_values = params
       command.restore_defaults(command.buffer(false), params.keys)
       command.raw = raw

data/lib/cosmos/packets/packet.rb

@@ -8,7 +8,7 @@
 # as published by the Free Software Foundation; version 3 with
 # attribution addendums as found in the LICENSE.txt
 
-require 'digest/md5'
+require 'digest'
 require 'cosmos/packets/structure'
 require 'cosmos/packets/packet_item'
 require 'cosmos/ext/packet' if RUBY_ENGINE == 'ruby' and !ENV['COSMOS_NO_EXT']
@@ -19,7 +19,8 @@ module Cosmos
   # Packet adds is the ability to apply formatting to PacketItem values as well
   # as managing PacketItem's limit states.
   class Packet < Structure
-    RESERVED_ITEM_NAMES = ['RECEIVED_TIMESECONDS'.freeze, 'RECEIVED_TIMEFORMATTED'.freeze, 'RECEIVED_COUNT'.freeze]
+    RESERVED_ITEM_NAMES = ['PACKET_TIMESECONDS'.freeze, 'PACKET_TIMEFORMATTED'.freeze, 'RECEIVED_TIMESECONDS'.freeze, 'RECEIVED_TIMEFORMATTED'.freeze, 'RECEIVED_COUNT'.freeze]
+    CATCH_ALL_STATE = 'ANY'
 
     # @return [String] Name of the target this packet is associated with
     attr_reader :target_name
@@ -66,6 +67,12 @@ module Cosmos
     # @return [Boolean] Whether or not this is a 'abstract' packet
     attr_accessor :abstract
 
+    # @return [Boolean] Whether or not this was a stored packet
+    attr_accessor :stored
+
+    # @return [Hash] Extra data to be logged/transferred with packet
+    attr_accessor :extra
+
     # Valid format types
     VALUE_TYPES = [:RAW, :CONVERTED, :FORMATTED, :WITH_UNITS]
 
@@ -100,6 +107,8 @@ module Cosmos
         @meta = nil
         @hidden = false
         @disabled = false
+        @stored = false
+        @extra = nil
       end
 
       # Sets the target name this packet is associated with. Unidentified packets
@@ -177,37 +186,70 @@ module Cosmos
         @read_conversion_cache.clear if @read_conversion_cache
         @received_count
       end
+      
+    end # if RUBY_ENGINE != 'ruby' or ENV['COSMOS_NO_EXT']      
+      
+    # Tries to identify if a buffer represents the currently defined packet. It
+    # does this by iterating over all the packet items that were created with
+    # an ID value and checking whether that ID value is present at the correct
+    # location in the buffer.
+    #
+    # Incorrectly sized buffers will still positively identify if there is
+    # enough data to match the ID values. This is to allow incorrectly sized
+    # packets to still be processed as well as possible given the incorrectly
+    # sized data.
+    #
+    # @param buffer [String] Raw buffer of binary data
+    # @return [Boolean] Whether or not the buffer of data is this packet
+    def identify?(buffer)
+      return false unless buffer
+      return true unless @id_items
 
-      # Tries to identify if a buffer represents the currently defined packet. It
-      # does this by iterating over all the packet items that were created with
-      # an ID value and checking whether that ID value is present at the correct
-      # location in the buffer.
-      #
-      # Incorrectly sized buffers will still positively identify if there is
-      # enough data to match the ID values. This is to allow incorrectly sized
-      # packets to still be processed as well as possible given the incorrectly
-      # sized data.
-      #
-      # @param buffer [String] Raw buffer of binary data
-      # @return [Boolean] Whether or not the buffer of data is this packet
-      def identify?(buffer)
-        return false unless buffer
-        return true unless @id_items
-
-        @id_items.each do |item|
-          begin
-            value = read_item(item, :RAW, buffer)
-          rescue Exception
-            value = nil
-          end
-          return false if item.id_value != value
+      @id_items.each do |item|
+        begin
+          value = read_item(item, :RAW, buffer)
+        rescue Exception
+          value = nil
         end
+        return false if item.id_value != value
+      end
 
-        true
+      true
+    end
+    
+    # Reads the values from a buffer at the position of each id_item defined
+    # in the packet.
+    #
+    # @param buffer [String] Raw buffer of binary data
+    # @return [Array] Array of read id values in order
+    def read_id_values(buffer)
+      return [] unless buffer
+      return [] unless @id_items
+      values = []
+      
+      @id_items.each do |item|
+        begin
+          values << read_item(item, :RAW, buffer)
+        rescue Exception
+          values << nil
+        end
+      end
+      
+      values
+    end
+    
+    # Returns @received_time unless a packet item called PACKET_TIME exists that returns
+    # a Ruby Time object that represents a different timestamp for the packet
+    def packet_time
+      item = @items['PACKET_TIME'.freeze]
+      if item
+        return read_item(item, :CONVERTED, @buffer)
+      else
+        return @received_time
       end
-    end # if RUBY_ENGINE != 'ruby' or ENV['COSMOS_NO_EXT']
+    end
 
-    # Calculates a unique MD5Sum that changes if the parts of the packet configuration change that could affect
+    # Calculates a unique hashing sum that changes if the parts of the packet configuration change that could affect
     # the "shape" of the packet.  This value is cached and that packet should not be changed if this method is being used
     def config_name
       return @config_name if @config_name
@@ -215,7 +257,9 @@ module Cosmos
       @sorted_items.each do |item|
         string << " ITEM #{item.name} #{item.bit_offset} #{item.bit_size} #{item.data_type} #{item.array_size} #{item.endianness} #{item.overflow} #{item.states} #{item.read_conversion ? item.read_conversion.class : 'NO_CONVERSION'}"
       end
-      digest = Digest::MD5.new
+
+      # Use the hashing algorithm established by Cosmos::System
+      digest = Digest.const_get(System.hashing_algorithm).send('new')
       digest << string
       @config_name = digest.hexdigest
       @config_name
@@ -297,7 +341,7 @@ module Cosmos
       previous_item = nil
       warnings = []
       @sorted_items.each do |item|
-        if expected_next_offset and item.bit_offset < expected_next_offset
+        if expected_next_offset and (item.bit_offset < expected_next_offset) and !item.overlap
           msg = "Bit definition overlap at bit offset #{item.bit_offset} for packet #{@target_name} #{@packet_name} items #{item.name} and #{previous_item.name}"
           Logger.instance.warn(msg)
           warnings << msg
@@ -529,6 +573,8 @@ module Cosmos
             value = value.map do |val, index|
               if item.states.key(val)
                 item.states.key(val)
+              elsif item.states.values.include?(CATCH_ALL_STATE)
+                item.states.key(CATCH_ALL_STATE)
               else
                 apply_format_string_and_units(item, val, value_type)
               end
@@ -537,6 +583,8 @@ module Cosmos
             state_value = item.states.key(value)
             if state_value
               value = state_value
+            elsif item.states.values.include?(CATCH_ALL_STATE)
+              value = item.states.key(CATCH_ALL_STATE)
             else
               value = apply_format_string_and_units(item, value, value_type)
             end
@@ -652,9 +700,10 @@ module Cosmos
     # @param value_type (see #read_item)
     # @param indent (see Structure#formatted)
     # @param buffer (see Structure#formatted)
+    # @param ignored (see Structure#ignored)
     # @return (see Structure#formatted)
-    def formatted(value_type = :CONVERTED, indent = 0, buffer = @buffer)
-      return super(value_type, indent, buffer)
+    def formatted(value_type = :CONVERTED, indent = 0, buffer = @buffer, ignored = nil)
+      return super(value_type, indent, buffer, ignored)
     end
 
     # Restore all items in the packet to their default value
@@ -670,6 +719,12 @@ module Cosmos
 
     # Define the reserved items on the current telemetry packet
     def define_reserved_items
+      item = define_item('PACKET_TIMESECONDS', 0, 0, :DERIVED, nil, @default_endianness,
+                         :ERROR, '%0.6f', PacketTimeSecondsConversion.new)
+      item.description = 'COSMOS Packet Time (UTC, Floating point, Unix epoch)'
+      item = define_item('PACKET_TIMEFORMATTED', 0, 0, :DERIVED, nil, @default_endianness,
+                         :ERROR, nil, PacketTimeFormattedConversion.new)
+      item.description = 'COSMOS Packet Time (Local time zone, Formatted string)'
       item = define_item('RECEIVED_TIMESECONDS', 0, 0, :DERIVED, nil, @default_endianness,
                          :ERROR, '%0.6f', ReceivedTimeSecondsConversion.new)
       item.description = 'COSMOS Received Time (UTC, Floating point, Unix epoch)'
@@ -750,7 +805,8 @@ module Cosmos
     def check_limits(limits_set = :DEFAULT, ignore_persistence = false)
       # If check_limits is being called, then a new packet has arrived and
       # this packet is no longer stale
-      if @stale
+      # Stored telemetry doesn't affect the current value table and such doesn't affect stale
+      if @stale and !@stored
         @stale = false
         set_all_limits_states(nil)
       end
@@ -786,6 +842,8 @@ module Cosmos
 
       @received_time = nil
       @received_count = 0
+      @stored = false
+      @extra = nil
       if @read_conversion_cache
         synchronize() do
           @read_conversion_cache.clear
@@ -811,6 +869,7 @@ module Cosmos
         end
       end
       packet.instance_variable_set("@read_conversion_cache".freeze, nil)
+      packet.extra = JSON.parse(packet.extra.to_json) if packet.extra # Deep copy using JSON
       packet
     end
     alias dup clone
@@ -818,7 +877,19 @@ module Cosmos
     def update_id_items(item)
       if item.id_value
         @id_items ||= []
-        @id_items << item
+        # Add to Id Items
+        unless @id_items.empty?
+          last_item = @id_items[-1]
+          @id_items << item
+          # If the current item or last item have a negative offset then we have
+          # to re-sort. We also re-sort if the current item is less than the last
+          # item because we are inserting.
+          if last_item.bit_offset <= 0 or item.bit_offset <= 0 or item.bit_offset < last_item.bit_offset
+            @id_items = @id_items.sort
+          end
+        else
+          @id_items << item
+        end
       end
       item
     end