openc3 7.2.0 → 7.3.0

data/data/xtce_schemas/xml.xsd

@@ -0,0 +1,116 @@
+<?xml version='1.0'?>
+<xs:schema targetNamespace="http://www.w3.org/XML/1998/namespace" xmlns:xs="http://www.w3.org/2001/XMLSchema" xml:lang="en">
+
+ <xs:annotation>
+  <xs:documentation>
+   See http://www.w3.org/XML/1998/namespace.html and
+   http://www.w3.org/TR/REC-xml for information about this namespace.
+
+    This schema document describes the XML namespace, in a form
+    suitable for import by other schema documents.  
+
+    Note that local names in this namespace are intended to be defined
+    only by the World Wide Web Consortium or its subgroups.  The
+    following names are currently defined in this namespace and should
+    not be used with conflicting semantics by any Working Group,
+    specification, or document instance:
+
+    base (as an attribute name): denotes an attribute whose value
+         provides a URI to be used as the base for interpreting any
+         relative URIs in the scope of the element on which it
+         appears; its value is inherited.  This name is reserved
+         by virtue of its definition in the XML Base specification.
+
+    lang (as an attribute name): denotes an attribute whose value
+         is a language code for the natural language of the content of
+         any element; its value is inherited.  This name is reserved
+         by virtue of its definition in the XML specification.
+  
+    space (as an attribute name): denotes an attribute whose
+         value is a keyword indicating what whitespace processing
+         discipline is intended for the content of the element; its
+         value is inherited.  This name is reserved by virtue of its
+         definition in the XML specification.
+
+    Father (in any context at all): denotes Jon Bosak, the chair of 
+         the original XML Working Group.  This name is reserved by 
+         the following decision of the W3C XML Plenary and 
+         XML Coordination groups:
+
+             In appreciation for his vision, leadership and dedication
+             the W3C XML Plenary on this 10th day of February, 2000
+             reserves for Jon Bosak in perpetuity the XML name
+             xml:Father
+  </xs:documentation>
+ </xs:annotation>
+
+ <xs:annotation>
+  <xs:documentation>This schema defines attributes and an attribute group
+        suitable for use by
+        schemas wishing to allow xml:base, xml:lang or xml:space attributes
+        on elements they define.
+
+        To enable this, such a schema must import this schema
+        for the XML namespace, e.g. as follows:
+        &lt;schema . . .>
+         . . .
+         &lt;import namespace="http://www.w3.org/XML/1998/namespace"
+                    schemaLocation="http://www.w3.org/2001/03/xml.xsd"/>
+
+        Subsequently, qualified reference to any of the attributes
+        or the group defined below will have the desired effect, e.g.
+
+        &lt;type . . .>
+         . . .
+         &lt;attributeGroup ref="xml:specialAttrs"/>
+ 
+         will define a type which will schema-validate an instance
+         element with any of those attributes</xs:documentation>
+ </xs:annotation>
+
+ <xs:annotation>
+  <xs:documentation>In keeping with the XML Schema WG's standard versioning
+   policy, this schema document will persist at
+   http://www.w3.org/2001/03/xml.xsd.
+   At the date of issue it can also be found at
+   http://www.w3.org/2001/xml.xsd.
+   The schema document at that URI may however change in the future,
+   in order to remain compatible with the latest version of XML Schema
+   itself.  In other words, if the XML Schema namespace changes, the version
+   of this document at
+   http://www.w3.org/2001/xml.xsd will change
+   accordingly; the version at
+   http://www.w3.org/2001/03/xml.xsd will not change.
+  </xs:documentation>
+ </xs:annotation>
+
+ <xs:attribute name="lang" type="xs:language">
+  <xs:annotation>
+   <xs:documentation>In due course, we should install the relevant ISO 2- and 3-letter
+         codes as the enumerated possible values . . .</xs:documentation>
+  </xs:annotation>
+ </xs:attribute>
+
+ <xs:attribute name="space" default="preserve">
+  <xs:simpleType>
+   <xs:restriction base="xs:NCName">
+    <xs:enumeration value="default"/>
+    <xs:enumeration value="preserve"/>
+   </xs:restriction>
+  </xs:simpleType>
+ </xs:attribute>
+
+ <xs:attribute name="base" type="xs:anyURI">
+  <xs:annotation>
+   <xs:documentation>See http://www.w3.org/TR/xmlbase/ for
+                     information about this attribute.</xs:documentation>
+  </xs:annotation>
+ </xs:attribute>
+
+ <xs:attributeGroup name="specialAttrs">
+  <xs:attribute ref="xml:base"/>
+  <xs:attribute ref="xml:lang"/>
+  <xs:attribute ref="xml:space"/>
+ </xs:attributeGroup>
+
+</xs:schema>

data/ext/openc3/ext/telemetry/telemetry.c

@@ -56,7 +56,7 @@ static VALUE packets(VALUE self, VALUE target_name)
 
   if (!(RTEST(target_packets)))
   {
-    rb_raise(rb_eRuntimeError, "Telemetry target '%s' does not exist", RSTRING_PTR(upcase_target_name));
+    rb_raise(rb_eRuntimeError, "Telemetry target '%s' does not exist (packets lookup)", RSTRING_PTR(upcase_target_name));
   }
 
   return target_packets;

data/lib/openc3/accessors/binary_accessor.rb

@@ -108,6 +108,10 @@ module OpenC3
 
     def handle_read_variable_bit_size(item, _buffer)
       length_value = @packet.read(item.variable_bit_size['length_item_name'], :CONVERTED)
+      # length_value can be nil when reading an undersized packet where the length
+      # item falls outside the buffer bounds
+      raise "Length value #{item.variable_bit_size['length_item_name']} for item #{item.name} is nil" if length_value.nil?
+
       if item.array_size
         item.array_size = (length_value * item.variable_bit_size['length_bits_per_count']) + item.variable_bit_size['length_value_bit_offset']
       else

data/lib/openc3/accessors/template_accessor.rb

@@ -109,7 +109,7 @@ module OpenC3
       configure()
 
       success = buffer.gsub!("#{@left_char}#{item.key}#{@right_char}", value.to_s)
-      raise "Key #{item.key} not found in template" unless success
+      raise "Key '#{item.key}' not found in template (write_item)" unless success
       return value
     end
 
@@ -118,7 +118,7 @@ module OpenC3
       items.each_with_index do |item, index|
         next if item.data_type == :DERIVED
         success = buffer.gsub!("#{@left_char}#{item.key}#{@right_char}", values[index].to_s)
-        raise "Key #{item.key} not found in template" unless success
+        raise "Key '#{item.key}' not found in template (write_items)" unless success
       end
       return values
     end

data/lib/openc3/api/cmd_api.rb

@@ -450,7 +450,7 @@ module OpenC3
 
     # NOTE: When adding new keywords to this method, make sure to update script/commands.rb
     def _cmd_implementation(method_name, *args, range_check:, hazardous_check:, raw:, timeout: nil, log_message: nil, manual: false, validate: true, queue: nil,
-                            scope: $openc3_scope, token: $openc3_token, **kwargs)
+                            queue_username: nil, scope: $openc3_scope, token: $openc3_token, **kwargs)
       extract_string_kwargs_to_args(args, kwargs)
       unless [nil, true, false].include?(log_message)
         raise "Invalid log_message parameter: #{log_message}. Must be true or false."
@@ -535,6 +535,10 @@ module OpenC3
         'log_message' => log_message.to_s,
         'obfuscated_items' => packet['obfuscated_items'].to_s
       }
+      # Record the original queuing user (author) separately from 'username' (the
+      # user or process that actually executed the command). Command History shows
+      # 'username' as "Executed By" and queue_username as "Queued By".
+      command['queue_username'] = queue_username if queue_username
       # Users have to explicitly opt into a default queue by setting the OPENC3_DEFAULT_QUEUE
       # At which point ALL commands will go to that queue unless they specifically opt out with queue: false
       if ENV['OPENC3_DEFAULT_QUEUE'] && queue.nil?

data/lib/openc3/api/interface_api.rb

@@ -47,7 +47,7 @@ module OpenC3
     def get_interface(interface_name, manual: false, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'system', interface_name: interface_name, manual: manual, scope: scope, token: token)
       interface = InterfaceModel.get(name: interface_name, scope: scope)
-      raise "Interface '#{interface_name}' does not exist" unless interface
+      raise "Interface '#{interface_name}' does not exist (get_interface)" unless interface
 
       interface.merge(InterfaceStatusModel.get(name: interface_name, scope: scope))
     end
@@ -160,7 +160,7 @@ module OpenC3
       # TODO: Check if they have command authority for the targets mapped to this interface
       authorize(permission: 'system_set', interface_name: interface_name, manual: manual, scope: scope, token: token)
       interface = InterfaceModel.get_model(name: interface_name, scope: scope)
-      raise "Interface '#{interface_name}' does not exist" unless interface
+      raise "Interface '#{interface_name}' does not exist (unmap_target_from_interface)" unless interface
 
       if Array === target_name
         target_names = target_name

data/lib/openc3/api/limits_api.rb

@@ -17,6 +17,7 @@
 
 require 'openc3/api/target_api'
 require 'openc3/models/cvt_model'
+require 'openc3/packets/packet_item'
 
 module OpenC3
   module Api
@@ -29,12 +30,14 @@ module OpenC3
                        'disable_limits',
                        'get_limits',
                        'set_limits',
+                       'set_state_color',
                        'get_limits_groups',
                        'enable_limits_group',
                        'disable_limits_group',
                        'get_limits_sets',
                        'set_limits_set',
                        'get_limits_set',
+                       'delete_limits_set',
                        'get_limits_events',
                      ])
 
@@ -131,7 +134,7 @@ module OpenC3
           break
         end
       end
-      raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist" unless found_item
+      raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist while enabling limits" unless found_item
 
       TargetModel.set_packet(target_name, packet_name, packet, scope: scope)
 
@@ -164,7 +167,7 @@ module OpenC3
           break
         end
       end
-      raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist" unless found_item
+      raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist while disabling limits" unless found_item
 
       TargetModel.set_packet(target_name, packet_name, packet, scope: scope)
 
@@ -236,7 +239,7 @@ module OpenC3
           end
         end
       end
-      raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist" unless found_item
+      raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist while setting limits" unless found_item
       message = "Setting '#{target_name} #{packet_name} #{item_name}' limits to #{red_low} #{yellow_low} #{yellow_high} #{red_high}"
       message << " #{green_low} #{green_high}" if green_low && green_high
       message << " in set #{limits_set} with persistence #{persistence} as enabled #{enabled}"
@@ -251,6 +254,64 @@ module OpenC3
       LimitsEventTopic.write(event, scope: scope)
     end
 
+    # Change the color associated with a telemetry item state in realtime.
+    # Items with states use the state color (GREEN, YELLOW, RED) to determine
+    # their limits state rather than numeric red/yellow/green limits.
+    #
+    # @param target_name [String] Target name
+    # @param packet_name [String] Packet name
+    # @param item_name [String] Item name
+    # @param state_name [String] Name of the state to change (e.g. 'CONNECTED')
+    # @param color [String, nil] New color for the state. Must be GREEN, YELLOW,
+    #   or RED. Pass nil to clear (remove) the state color. Setting a color also
+    #   enables limits for the item, but clearing the color (nil) does not change
+    #   whether limits are enabled.
+    def set_state_color(target_name, packet_name, item_name, state_name, color,
+                        manual: false, scope: $openc3_scope, token: $openc3_token)
+      authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, manual: manual, scope: scope, token: token)
+      state_name = state_name.to_s.upcase
+      unless color.nil?
+        color = color.to_s.upcase
+        unless PacketItem::STATE_COLORS.include?(color.intern)
+          raise "Invalid state color '#{color}'. Must be one of #{PacketItem::STATE_COLORS.join(', ')}."
+        end
+      end
+      packet = TargetModel.packet(target_name, packet_name, scope: scope)
+      found_item = nil
+      packet['items'].each do |item|
+        if item['name'] == item_name
+          unless item['states'] && item['states'][state_name]
+            raise "State '#{state_name}' does not exist for item '#{target_name} #{packet_name} #{item_name}'"
+          end
+          if color.nil?
+            # Clear the state color
+            item['states'][state_name].delete('color')
+          else
+            item['states'][state_name]['color'] = color
+            # Defining a state color implies limits are enabled for the item
+            item['limits'] ||= {}
+            item['limits']['enabled'] = true
+          end
+          found_item = item
+          break
+        end
+      end
+      raise "Item '#{target_name} #{packet_name} #{item_name}' does not exist while setting state color" unless found_item
+      if color.nil?
+        message = "Clearing '#{target_name} #{packet_name} #{item_name}' state #{state_name} color"
+      else
+        message = "Setting '#{target_name} #{packet_name} #{item_name}' state #{state_name} color to #{color}"
+      end
+      Logger.info(message, scope: scope)
+
+      TargetModel.set_packet(target_name, packet_name, packet, scope: scope)
+
+      event = { type: :LIMITS_STATE_COLOR, target_name: target_name, packet_name: packet_name,
+                item_name: item_name, state_name: state_name, color: color,
+                time_nsec: Time.now.to_nsec_from_epoch, message: message }
+      LimitsEventTopic.write(event, scope: scope)
+    end
+
     # Returns all limits_groups and their members
     # @since 5.0.0 Returns hash with values
     # @return [Hash{String => Array<Array<String, String, String>>]
@@ -292,6 +353,31 @@ module OpenC3
         time_nsec: Time.now.to_nsec_from_epoch, message: message }, scope: scope)
     end
 
+    # Deletes a limits set. The DEFAULT limits set and the currently active
+    # limits set cannot be deleted. Use set_limits_set to change the active set
+    # before deleting it. Note that the limits set is not removed from the
+    # TargetModel packet definitions; that is cleaned up on the next plugin
+    # install.
+    #
+    # @param limits_set [String] The name of the limits set to delete
+    def delete_limits_set(limits_set, manual: false, scope: $openc3_scope, token: $openc3_token)
+      authorize(permission: 'tlm_set', manual: manual, scope: scope, token: token)
+      limits_set = limits_set.to_s
+      raise "Cannot delete the DEFAULT limits set" if limits_set == 'DEFAULT'
+      if limits_set == LimitsEventTopic.current_set(scope: scope)
+        raise "Cannot delete the current limits set '#{limits_set}'. Use set_limits_set to change the current set first."
+      end
+      unless LimitsEventTopic.sets(scope: scope).key?(limits_set)
+        raise "Limits set '#{limits_set}' does not exist"
+      end
+
+      # Remove the limits set from Redis (limits_sets and current_limits_settings)
+      LimitsEventTopic.delete_set(limits_set, scope: scope)
+
+      message = "Deleting Limits Set: #{limits_set}"
+      Logger.info(message, scope: scope)
+    end
+
     # Returns the active limits set that applies to all telemetry
     #
     # @return [String] The current limits set

data/lib/openc3/api/router_api.rb

@@ -46,7 +46,7 @@ module OpenC3
     def get_router(router_name, manual: false, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'system', router_name: router_name, manual: manual, scope: scope, token: token)
       router = RouterModel.get(name: router_name, scope: scope)
-      raise "Router '#{router_name}' does not exist" unless router
+      raise "Router '#{router_name}' does not exist (get_router)" unless router
 
       router.merge(RouterStatusModel.get(name: router_name, scope: scope))
     end
@@ -144,7 +144,7 @@ module OpenC3
       # TODO: Check if they have command authority for the targets mapped to this router
       authorize(permission: 'system_set', router_name: router_name, manual: manual, scope: scope, token: token)
       router = RouterModel.get_model(name: router_name, scope: scope)
-      raise "Router '#{router_name}' does not exist" unless router
+      raise "Router '#{router_name}' does not exist (map_target_to_router)" unless router
 
       if Array === target_name
         target_names = target_name
@@ -167,7 +167,7 @@ module OpenC3
       # TODO: Check if they have command authority for the targets mapped to this router
       authorize(permission: 'system_set', router_name: router_name, manual: manual, scope: scope, token: token)
       router = RouterModel.get_model(name: router_name, scope: scope)
-      raise "Router '#{router_name}' does not exist" unless router
+      raise "Router '#{router_name}' does not exist (unmap_target_from_router)" unless router
 
       if Array === target_name
         target_names = target_name

data/lib/openc3/api/tlm_api.rb

@@ -123,7 +123,7 @@ module OpenC3
       target_name = target_name.upcase
       packet_name = packet_name.upcase
       unless CvtModel::VALUE_TYPES.include?(type)
-        raise "Unknown type '#{type}' for #{target_name} #{packet_name}"
+        raise "Unknown type '#{type}' for #{target_name} #{packet_name} (inject_tlm)"
       end
 
       if item_hash

data/lib/openc3/interfaces/http_client_interface.rb

@@ -237,7 +237,7 @@ module OpenC3
       result['read_timeout'] = @read_timeout
       result['connect_timeout'] = @connect_timeout
       result['include_request_in_response'] = @include_request_in_response
-      result['request_queue_length'] = @request_queue.length
+      result['response_queue_length'] = @response_queue.length
       return result
     end
   end

data/lib/openc3/io/json_drb.rb

@@ -17,6 +17,7 @@
 
 require 'socket'
 require 'json'
+require 'set'
 # require 'drb/acl'
 require 'drb/drb'
 require 'openc3/io/json_rpc'
@@ -47,8 +48,9 @@ module OpenC3
   # JsonDRb implements the JSON-RPC 2.0 Specification to provide an interface
   # for both internal and external tools to access the OpenC3 server. It
   # provides methods to install an access control list to control access to the
-  # API. It also limits the available methods to a known list of allowable API
-  # methods.
+  # API. Access is controlled by a required method_whitelist which names every
+  # method that may be invoked. There is no permissive fallback: any method not
+  # explicitly listed is rejected.
   class JsonDRb
     # Minimum amount of time in seconds to receive the JSON request,
     # process it, and send the response. Requests for less than this amount
@@ -62,18 +64,21 @@ module OpenC3
 
     # @return [Integer] The number of JSON-RPC requests processed
     attr_accessor :request_count
-    # @return [Array<String>] List of methods that should be allowed
-    attr_accessor :method_whitelist
+    # @return [Set<String>] Set of downcased method names which may be called
+    attr_reader :method_whitelist
     # @return [ACL] The access control list
     # attr_accessor :acl
 
     attr_accessor :object
 
-    def initialize
+    # @param method_whitelist [Enumerable<String>] Every method name which may
+    #   be called over JSON-RPC. Required and must not be empty since there is
+    #   no permissive fallback.
+    def initialize(method_whitelist:)
+      self.method_whitelist = method_whitelist
       @thread = nil
       # @acl = nil
       @object = nil
-      @method_whitelist = nil
       @request_count = 0
       @request_times = []
       @request_times_index = 0
@@ -82,6 +87,18 @@ module OpenC3
       @server_mutex = Mutex.new
     end
 
+    # Replaces the set of methods which may be called over JSON-RPC. Names are
+    # downcased to match the dispatch in {#process_request}.
+    #
+    # @param whitelist [Enumerable<String>] Every method name which may be called
+    def method_whitelist=(whitelist)
+      if whitelist.nil? or whitelist.to_a.empty?
+        raise ArgumentError, "method_whitelist is required and must not be empty"
+      end
+
+      @method_whitelist = whitelist.map { |name| name.to_s.downcase }.to_set
+    end
+
     # Returns the number of connected clients
     # @return [Integer] The number of connected clients
     def num_clients
@@ -254,8 +271,7 @@ module OpenC3
         error_code = nil
         response_data = nil
 
-        if (@method_whitelist and @method_whitelist.include?(request.method.downcase())) or
-           (!@method_whitelist and !JsonRpcRequest::DANGEROUS_METHODS.include?(request.method.downcase()))
+        if @method_whitelist.include?(request.method.downcase())
           begin
             if request.keyword_params
               result = @object.public_send(request.method.downcase().intern, *request.params, **request.keyword_params)

data/lib/openc3/io/json_rpc.rb

@@ -227,8 +227,6 @@ module OpenC3
 
   # Represents a JSON Remote Procedure Call Request
   class JsonRpcRequest < JsonRpc
-    DANGEROUS_METHODS = Set['__send__', 'send', 'instance_eval', 'instance_exec']
-
     # @param method_name [String] The name of the method to call
     # @param method_params [Array<String>] Array of strings which represent the
     #   parameters to send to the method

data/lib/openc3/logs/log_writer.rb

@@ -113,6 +113,7 @@ module OpenC3
       @cycle_minute = Integer(@cycle_minute) if @cycle_minute
       @enforce_time_order = ConfigParser.handle_true_false(enforce_time_order)
       @out_of_order = false
+      @bad_time = false
       @mutex = Mutex.new
       @file = nil
       @file_size = 0
@@ -125,6 +126,13 @@ module OpenC3
       @cleanup_offsets = []
       @cleanup_times = []
       @previous_time_nsec_since_epoch = nil
+      # Stored packets older than 20 years are treated as invalid (e.g. 1970 before
+      # clock/GPS sync). They never drive file segmentation nor become the ordering
+      # baseline, so a late-syncing clock doesn't thrash log files.
+      @min_valid_time_nsec = (@start_time - (20 * 365 * 24 * 60 * 60)).to_nsec_from_epoch
+      # Validity (valid/invalid time) of the packets in the currently open file. Used to
+      # segregate invalid-time (1970) packets into their own file. nil = no packets yet.
+      @current_file_valid = nil
       @tmp_dir = Dir.mktmpdir
       @wait_threads = []
 
@@ -280,6 +288,7 @@ module OpenC3
       @first_time = nil
       @last_time = nil
       @previous_time_nsec_since_epoch = nil
+      @current_file_valid = nil
       Logger.debug "Log File Opened : #{@filename}"
     rescue => e
       Logger.error "Error starting new log file: #{e.formatted}"
@@ -289,7 +298,14 @@ module OpenC3
 
     # @enforce_time_order requires the timestamps on each write to be greater than the previous
     # process_out_of_order ignores the timestamps for the current entry (used to ignore timestamps on metadata entries, vs actual packets)
-    def prepare_write(time_nsec_since_epoch, data_length, redis_topic = nil, redis_offset = nil, allow_new_file: true, process_out_of_order: true)
+    # stored: stored packets roll a new file when GOOD times move backward (a new /
+    #   overlapping replay run). Packets with an invalid time (e.g. 1970 before clock/GPS
+    #   sync) - realtime or stored - are segregated into their own file: the first valid
+    #   packet after a run of 1970s rolls a fresh, properly-named file (and vice-versa).
+    def prepare_write(time_nsec_since_epoch, data_length, redis_topic = nil, redis_offset = nil, allow_new_file: true, process_out_of_order: true, stored: false)
+      # An invalid time is a packet from before the clock/GPS synced (e.g. 1970). This
+      # applies to both realtime and stored packets so neither taints a file's filename.
+      packet_valid = !(process_out_of_order and time_nsec_since_epoch < @min_valid_time_nsec)
       # This check includes logging_enabled again because it might have changed since we acquired the mutex
       # Ensures new files based on size, and ensures always increasing time order in files
       if @logging_enabled
@@ -299,17 +315,37 @@ module OpenC3
         elsif @cycle_size and ((@file_size + data_length) > @cycle_size)
           Logger.debug("Log writer start new file due to cycle size #{@cycle_size}")
           start_new_file() if allow_new_file
+        elsif process_out_of_order and !@current_file_valid.nil? and packet_valid != @current_file_valid and allow_new_file
+          # Time validity changed (1970 <-> valid): segregate invalid-time packets into
+          # their own file so valid files keep a clean, correctly-timed filename.
+          Logger.debug("Log writer start new file due to time validity change")
+          start_new_file()
         elsif process_out_of_order and @enforce_time_order and @previous_time_nsec_since_epoch and (@previous_time_nsec_since_epoch > time_nsec_since_epoch)
-          # Warning: Creating new files here can cause lots of files to be created if packets make it through out of order
-          # Changed to just a error to prevent file thrashing
-          unless @out_of_order
-            Logger.error("Log writer out of order time detected (increase buffer depth?): #{Time.from_nsec_from_epoch(@previous_time_nsec_since_epoch)} #{Time.from_nsec_from_epoch(time_nsec_since_epoch)}")
-            @out_of_order = true
+          if stored and packet_valid and allow_new_file
+            # Good stored time went backward => new/overlapping replay run. Roll a new
+            # file so each stored file stays monotonic (start_new_file resets @previous_time).
+            Logger.debug("Log writer start new file due to out of order stored time")
+            start_new_file()
+          elsif !stored
+            # Warning: Creating new files here can cause lots of files to be created if packets make it through out of order
+            # Changed to just a warning to prevent file thrashing
+            unless @out_of_order
+              Logger.warn("Log writer out of order time detected (increase buffer depth?): #{Time.from_nsec_from_epoch(@previous_time_nsec_since_epoch)} #{Time.from_nsec_from_epoch(time_nsec_since_epoch)}")
+              @out_of_order = true
+            end
           end
         end
       end
       @last_offsets[redis_topic] = redis_offset if redis_topic and redis_offset # This is needed for the redis offset marker entry at the end of the log file
-      @previous_time_nsec_since_epoch = time_nsec_since_epoch if process_out_of_order
+      if process_out_of_order
+        @current_file_valid = packet_valid
+        if packet_valid
+          @previous_time_nsec_since_epoch = time_nsec_since_epoch
+        elsif !@bad_time
+          Logger.warn("Log writer segregating invalid packet time (before clock sync?): #{Time.from_nsec_from_epoch(time_nsec_since_epoch)}")
+          @bad_time = true
+        end
+      end
     end
 
     # Closing a log file isn't critical so we just log an error. NOTE: This also trims the Redis stream

data/lib/openc3/logs/packet_log_writer.rb

@@ -96,13 +96,12 @@ module OpenC3
       @mutex.lock if take_mutex
       begin
         if entry_type == :RAW_PACKET or entry_type == :JSON_PACKET
-          # Only care about the timestamps on the real packets being in order
           process_out_of_order = true
         else
           # Metadata timestamps don't matter
           process_out_of_order = false
         end
-        prepare_write(time_nsec_since_epoch, data.length, redis_topic, redis_offset, allow_new_file: allow_new_file, process_out_of_order: process_out_of_order)
+        prepare_write(time_nsec_since_epoch, data.length, redis_topic, redis_offset, allow_new_file: allow_new_file, process_out_of_order: process_out_of_order, stored: stored)
         write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: extra) if @file
       ensure
         @mutex.unlock if take_mutex

data/lib/openc3/microservices/decom_common.rb

@@ -1,4 +1,4 @@
-# encoding: ascii-8bit
+# encoding: utf-8
 
 # Copyright 2026 OpenC3, Inc.
 # All Rights Reserved.
@@ -35,13 +35,19 @@ module OpenC3
     #   (microservice name, or "REINGEST:<job_id>").
     # @param check_limits [Boolean] When false, skips the Packet#check_limits call.
     #   Reingest passes false so historical data does not re-fire limits events.
+    # @param stored_limits_mode [String] Controls limits handling for stored packets.
+    #   'PROCESS' (default) processes limits normally.
+    #   'LOG' still evaluates limits (callback handles suppressing reactions).
+    #   'DISABLE' skips limits evaluation entirely for stored packets and omits
+    #   limits states from the decommutated output.
     # @param metric [Metric, nil] Optional; when set, records decom_duration_seconds.
     # @param error_callback [Proc, nil] Called as error_callback.call(exception) when
     #   Packet#process or Packet#check_limits raises. The microservice uses this to
     #   bump its decom_error_total metric.
     # @return [Integer] Number of (sub)packets published.
     def decom_and_publish(packet, scope:, target_names:, logger:, name:,
-                          check_limits: true, metric: nil, error_callback: nil)
+                          check_limits: true, stored_limits_mode: 'PROCESS',
+                          metric: nil, error_callback: nil)
       start = Process.clock_gettime(Process::CLOCK_MONOTONIC) if metric
       published = 0
 
@@ -62,9 +68,14 @@ module OpenC3
           logger.error e.message
         end
 
-        packet_or_subpacket.check_limits(System.limits_set) if check_limits
+        disable_stored_limits = packet_or_subpacket.stored && stored_limits_mode == 'DISABLE'
+        if check_limits && !disable_stored_limits
+          packet_or_subpacket.check_limits(System.limits_set)
+        end
 
-        TelemetryDecomTopic.write_packet(packet_or_subpacket, scope: scope)
+        TelemetryDecomTopic.write_packet(packet_or_subpacket,
+                                         include_limits_states: !disable_stored_limits,
+                                         scope: scope)
         published += 1
       end
 

data/lib/openc3/microservices/decom_microservice.rb

@@ -100,6 +100,8 @@ module OpenC3
       @limits_response_thread = nil
       System.telemetry.limits_change_callback = method(:limits_change_callback)
       LimitsEventTopic.sync_system(scope: @scope)
+      target_model = TargetModel.get_model(name: @target_names[0], scope: @scope)
+      @stored_limits_mode = target_model ? (target_model.stored_limits_mode || 'PROCESS') : 'PROCESS'
       @error_count = 0
       @metric.set(name: 'decom_total', value: @count, type: 'counter')
       @metric.set(name: 'decom_error_total', value: @error_count, type: 'counter')
@@ -184,6 +186,7 @@ module OpenC3
           logger: @logger,
           name: @name,
           check_limits: true,
+          stored_limits_mode: @stored_limits_mode,
           metric: @metric,
           error_callback: ->(e) {
             @error_count += 1
@@ -253,9 +256,11 @@ module OpenC3
       event = { type: :LIMITS_CHANGE, target_name: packet.target_name, packet_name: packet.packet_name,
                 item_name: item.name, old_limits_state: old_limits_state.to_s, new_limits_state: item.limits.state.to_s,
                 time_nsec: packet_time.to_nsec_from_epoch, message: message.to_s }
+      suppress_stored = packet.stored && @stored_limits_mode != 'PROCESS'
+      event[:suppress_stored] = true if suppress_stored
       LimitsEventTopic.write(event, scope: @scope)
 
-      if item.limits.response
+      if item.limits.response && !suppress_stored
         copied_packet = packet.deep_copy
         copied_item = packet.items[item.name]
         @limits_response_queue << [copied_packet, copied_item, old_limits_state]