openc3 7.4.1 → 7.5.1

data/lib/openc3/interfaces/read_queue_interface.rb

@@ -0,0 +1,331 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'timeout' # For Timeout::Error
+require 'openc3/interfaces/interface'
+require 'openc3/core_ext/exception'
+require 'openc3/core_ext/time'
+require 'openc3/utilities/logger'
+
+module OpenC3
+  # Interface base class which reads an underlying socket / stream (the reader)
+  # in a dedicated thread and buffers the results on a queue. This keeps the
+  # operating system receive buffers drained as fast as possible so data isn't
+  # dropped while the packet reading thread is busy processing protocols and
+  # writing to the database.
+  #
+  # Subclasses call start_read_queue_thread with the object they read from
+  # (anything with a read method) once it is connected and stop_read_queue_thread
+  # once it is disconnected. While the thread is running the reader's read
+  # method is redirected to the queue, so the subclass (and anything else such
+  # as derived classes or protocols) keeps calling reader.read exactly as before
+  # and receives the data in the order it was read rather than competing with
+  # the read thread for it.
+  #
+  # Setting READ_QUEUE_MAX_SIZE to 0 disables the read thread so the reader is
+  # read inline, which is how interfaces read before the queue existed.
+  class ReadQueueInterface < Interface
+    # Maximum number of bytes buffered on the queue before the read thread
+    # blocks and lets the operating system do the buffering instead
+    DEFAULT_READ_QUEUE_MAX_SIZE = 20 * 1024 * 1024 # 20MB
+
+    # Approximate memory each queued read costs on top of the data itself (the
+    # String object plus the queue entry). Small reads are dominated by this so
+    # it is charged against the budget, otherwise a stream delivering a few
+    # bytes at a time would hold many times the budget in actual memory.
+    READ_QUEUE_ENTRY_OVERHEAD = 64
+
+    # How long to wait before re-checking whether the interface has been
+    # disconnected while blocked waiting for room on the queue
+    QUEUE_POLL_TIMEOUT = 0.1
+
+    # How long to wait for the read thread to exit when disconnecting
+    THREAD_JOIN_TIMEOUT = 2
+
+    # @return [Integer] Maximum number of bytes buffered on the queue
+    attr_accessor :read_queue_max_size
+
+    # @return [Time, nil] When the read thread read the data most recently
+    #   returned by the reader (nil when the read queue is disabled)
+    attr_reader :read_queue_data_time
+
+    def initialize
+      super()
+      initialize_read_queue()
+    end
+
+    # Initialize (or reset) the read queue attributes. Called by initialize.
+    def initialize_read_queue(max_size = DEFAULT_READ_QUEUE_MAX_SIZE)
+      @raw_read_queue = nil
+      @raw_read_thread = nil
+      @raw_reader = nil
+      @raw_reader_read = nil
+      @raw_read_bytes = 0
+      @raw_read_budget = 0
+      @raw_read_timeout_queued = false
+      @read_queue_data_time = nil
+      # Guards @raw_read_queue / @raw_read_bytes / @raw_read_budget and wakes the
+      # read thread when the bytes it queued are consumed and there is room to
+      # queue more. Each read thread is cancelled by closing its own queue under
+      # this mutex, so a thread which outlives stop_read_queue_thread (and a new
+      # thread starting) keeps seeing its old queue closed and can never touch
+      # the byte counters or read again.
+      @raw_read_mutex = Mutex.new
+      @raw_read_condition = ConditionVariable.new
+      @read_queue_max_size = max_size
+    end
+
+    # Supported Options
+    # READ_QUEUE_MAX_SIZE - Maximum number of bytes buffered on the read queue
+    #   or 0 to disable the read thread and read inline (see Interface#set_option)
+    def set_option(option_name, option_values)
+      super(option_name, option_values)
+      if option_name.upcase == 'READ_QUEUE_MAX_SIZE'
+        max_size = Integer(option_values[0])
+        raise "READ_QUEUE_MAX_SIZE must be 0 (disabled) or a positive integer but was #{max_size}" if max_size < 0
+
+        @read_queue_max_size = max_size
+      end
+    end
+
+    # @return [Boolean] Whether reads are buffered by a read thread (false when
+    #   READ_QUEUE_MAX_SIZE is 0 and reads happen inline)
+    def read_queue_enabled?
+      @read_queue_max_size > 0
+    end
+
+    # @return [Boolean] Whether a read thread has been started (and not stopped)
+    def read_queue_started?
+      !@raw_read_queue.nil?
+    end
+
+    # @return [Integer] The number of reads waiting on the read queue
+    def read_queue_size
+      queue = @raw_read_queue
+      queue ? queue.size : 0
+    end
+
+    # @return [Integer] The number of bytes waiting on the read queue
+    def read_queue_bytes
+      return 0 unless @raw_read_queue
+
+      @raw_read_mutex.synchronize { @raw_read_bytes }
+    end
+
+    # Start the thread which continuously reads and queues data from the reader
+    # and redirect the reader's read method to the queue. Does nothing beyond
+    # stopping any existing thread if the read queue is disabled.
+    #
+    # @param reader [#read] Object to read from, e.g. a Stream or socket
+    # @param read_args [Array] Arguments the read thread passes to reader.read.
+    #   Arguments passed to the redirected read are ignored.
+    def start_read_queue_thread(reader, *read_args)
+      stop_read_queue_thread()
+      return unless read_queue_enabled?
+
+      queue = Queue.new
+      @raw_read_mutex.synchronize do
+        @raw_read_bytes = 0
+        @raw_read_budget = 0
+        @raw_read_timeout_queued = false
+        @raw_read_queue = queue
+      end
+      begin
+        raw_read = redirect_reader(reader)
+      rescue FrozenError, TypeError
+        Logger.warn "#{@name}: Can't redirect #{reader.class}#read so disabling the read queue"
+        @raw_read_mutex.synchronize { @raw_read_queue = nil }
+        @read_queue_max_size = 0
+        return
+      end
+      @raw_read_thread = Thread.new do
+        read_queue_thread_body(queue, raw_read, read_args)
+      rescue Exception => e
+        Logger.error "#{@name}: Read queue thread unexpectedly died: #{e.formatted}"
+      end
+    end
+
+    # Stop the read thread, restore the reader's read method and discard anything
+    # left on the queue. The reader should be disconnected first so the thread
+    # isn't blocked reading.
+    def stop_read_queue_thread
+      # Allow subclasses to set their reader before initialize_read_queue
+      return unless @raw_read_mutex
+
+      @raw_read_mutex.synchronize do
+        # Closing the queue under the mutex cancels the read thread (it either
+        # already charged its bytes, which are reset below, or never will) and
+        # unblocks read_queue_pop
+        @raw_read_queue.close if @raw_read_queue
+        @raw_read_queue = nil
+        @raw_read_bytes = 0
+        @raw_read_budget = 0
+        @raw_read_timeout_queued = false
+        @read_queue_data_time = nil
+        # Unblock the read thread if it is waiting for room on the queue
+        @raw_read_condition.broadcast
+      end
+      restore_reader()
+      thread = @raw_read_thread
+      @raw_read_thread = nil
+      # The read source should already be disconnected which unblocks the read
+      if thread and thread != Thread.current and !thread.join(THREAD_JOIN_TIMEOUT)
+        OpenC3.kill_thread(nil, thread)
+      end
+    end
+
+    # Retrieve the next data off the read queue, blocking until it is available.
+    # This is what the reader's read method returns while it is redirected.
+    #
+    # @return [String, nil] Data read or nil if the read source is done
+    def read_queue_pop
+      queue = @raw_read_queue
+      return nil unless queue
+
+      loop do
+        data = queue.pop
+        if data.kind_of?(Array)
+          data, time = data
+          @raw_read_mutex.synchronize do
+            # Counters were reset if the queue was stopped / replaced
+            if queue.equal?(@raw_read_queue)
+              @raw_read_bytes -= data.length
+              @raw_read_budget -= data.length + READ_QUEUE_ENTRY_OVERHEAD
+              @read_queue_data_time = time
+              # Tell the read thread there is room for more data
+              @raw_read_condition.broadcast
+            end
+          end
+          return data
+        end
+
+        if data.kind_of?(Timeout::Error)
+          @raw_read_mutex.synchronize { @raw_read_timeout_queued = false if queue.equal?(@raw_read_queue) }
+          # Data queued behind the timeout arrived while nobody was waiting on
+          # it, so an inline read would have returned it rather than timing out
+          next unless queue.empty?
+        end
+        # Exceptions raised by the read thread are re-raised here so they are
+        # handled exactly as they were when the read happened inline
+        raise data if data.kind_of?(Exception)
+
+        return data
+      end
+    end
+
+    # @param data [String, nil] Data returned by reader.read
+    # @return [Boolean] Whether the data indicates the reader is done (closed) so
+    #   the read thread should stop. Subclasses can override this.
+    def read_queue_done?(data)
+      data.nil? or data.length <= 0
+    end
+
+    # protected
+
+    def read_queue_thread_body(queue, raw_read, read_args)
+      until queue.closed?
+        begin
+          data = raw_read.call(*read_args)
+          data_time = Time.now.sys
+        rescue Timeout::Error => e
+          # A read timeout doesn't end the reader so pass it along and keep
+          # reading. Only one is queued at a time so a reader which times out
+          # quickly can't fill the queue with them.
+          queue_timeout = @raw_read_mutex.synchronize do
+            next false if queue.closed? or @raw_read_timeout_queued
+
+            @raw_read_timeout_queued = true
+          end
+          queue.push(e) if queue_timeout
+          next
+        rescue Exception => e
+          queue.push(e)
+          break
+        end
+        # Tell whoever reads next that the reader is done and stop reading
+        if read_queue_done?(data)
+          queue.push(data)
+          break
+        end
+        # Charge the bytes against the budget before the push so
+        # read_queue_bytes never goes negative if the data is dequeued
+        # before we get back here
+        break unless reserve_read_queue_bytes(queue, data.length)
+
+        # Queue the time of the read so packets are timestamped when the data
+        # arrived rather than when it was processed
+        queue.push([data, data_time])
+      end
+    rescue ClosedQueueError
+      # Interface disconnected while we were pushing
+    ensure
+      # Closing tells read_queue_pop there will never be more data so it
+      # returns nil rather than blocking forever
+      queue.close
+    end
+
+    # Wait until the data fits in the byte budget and charge it against the
+    # budget. A read larger than the entire budget is queued on its own rather
+    # than never fitting, so the budget can be exceeded by at most one read.
+    #
+    # @return [Boolean] Whether the bytes were reserved (false if disconnected)
+    def reserve_read_queue_bytes(queue, length)
+      cost = length + READ_QUEUE_ENTRY_OVERHEAD
+      @raw_read_mutex.synchronize do
+        while @raw_read_budget > 0 and (@raw_read_budget + cost) > @read_queue_max_size
+          return false if queue.closed?
+
+          @raw_read_condition.wait(@raw_read_mutex, QUEUE_POLL_TIMEOUT)
+        end
+        return false if queue.closed?
+
+        @raw_read_bytes += length
+        @raw_read_budget += cost
+      end
+      return true
+    end
+
+    # Redirect reader.read to the read queue so everything which reads the
+    # reader directly gets the queued data in order.
+    #
+    # @return [Method] The original read method for the read thread to call
+    def redirect_reader(reader)
+      raw_read = reader.method(:read)
+      # Remember a read already defined on the reader itself (e.g. a test stub)
+      # so it can be put back rather than removed
+      reader_read = reader.singleton_methods(false).include?(:read) ? raw_read : nil
+      interface = self
+      reader.define_singleton_method(:read) { |*_args| interface.read_queue_pop }
+      @raw_reader = reader
+      @raw_reader_read = reader_read
+      return raw_read
+    end
+
+    # Undo redirect_reader so the reader reads inline again
+    def restore_reader
+      reader = @raw_reader
+      return unless reader
+
+      if @raw_reader_read
+        reader.define_singleton_method(:read, @raw_reader_read)
+      else
+        reader.singleton_class.send(:remove_method, :read)
+      end
+    rescue NameError
+      # The redirect was already removed
+    ensure
+      @raw_reader = nil
+      @raw_reader_read = nil
+    end
+  end
+end

data/lib/openc3/interfaces/stream_interface.rb

@@ -15,12 +15,12 @@
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
-require 'openc3/interfaces/interface'
+require 'openc3/interfaces/read_queue_interface'
 
 module OpenC3
   # Base class for interfaces that act read and write from a stream
-  class StreamInterface < Interface
-    attr_accessor :stream
+  class StreamInterface < ReadQueueInterface
+    attr_reader :stream
 
     def initialize(protocol_type = nil, protocol_args = [])
       super()
@@ -34,9 +34,20 @@ module OpenC3
       end
     end
 
+    # Replacing the stream stops the read thread which is reading the old one
+    def stream=(stream)
+      stop_read_queue_thread()
+      @stream = stream
+    end
+
     def connect
       super() # Reset the protocols
       @stream.connect if @stream
+      # The read thread continuously drains the stream so the operating system
+      # buffers don't fill up while we're busy processing the previous read.
+      # @stream.read is redirected to the queue so everything reading the
+      # stream (including subclasses and protocols) still sees all the data.
+      start_read_queue_thread(@stream) if @stream and read_allowed?
     end
 
     def connected?
@@ -48,11 +59,19 @@ module OpenC3
     end
 
     def disconnect
+      # Disconnect the stream first to unblock the read thread
       @stream.disconnect if @stream
+      stop_read_queue_thread()
       super()
     end
 
     def read_interface
+      # The read thread is normally started by connect but interfaces can also
+      # be handed an already connected stream so start it on demand
+      if @stream and !read_queue_started? and read_queue_enabled? and read_allowed? and connected?
+        start_read_queue_thread(@stream)
+      end
+
       timeout = false
       begin
         data = @stream.read

data/lib/openc3/interfaces/tcpip_server_interface.rb

@@ -257,6 +257,15 @@ module OpenC3
       @read_queue ? @read_queue.size : 0
     end
 
+    # @return [Integer] The number of bytes waiting on the read queues of the
+    #   connected clients. The server itself has no stream so the raw reads are
+    #   buffered by each client connection.
+    def read_queue_bytes
+      @connection_mutex.synchronize do
+        @read_interface_infos.sum { |rii| rii.interface.read_queue_bytes }
+      end
+    end
+
     # @return [Integer] The number of packets waiting on the write queue
     def write_queue_size
       @write_queue ? @write_queue.size : 0
@@ -400,6 +409,10 @@ module OpenC3
       end
 
       interface = StreamInterface.new
+      # Only the read side of the connection needs a read thread
+      interface.read_allowed = listen_read
+      # Each connection buffers its own reads so give it the configured limit
+      interface.read_queue_max_size = @read_queue_max_size
       interface.target_names = @target_names
       interface.cmd_target_names = @cmd_target_names
       interface.tlm_target_names = @tlm_target_names

data/lib/openc3/interfaces/udp_interface.rb

@@ -15,13 +15,13 @@
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
-require 'openc3/interfaces/interface'
+require 'openc3/interfaces/read_queue_interface'
 require 'openc3/io/udp_sockets'
 require 'openc3/config/config_parser'
 
 module OpenC3
   # Base class for interfaces that send and receive messages over UDP
-  class UdpInterface < Interface
+  class UdpInterface < ReadQueueInterface
     HOST_127_0_0_1 = '127.0.0.1'
     HOST_0_0_0_0 = '0.0.0.0'
 
@@ -140,6 +140,11 @@ module OpenC3
       end
       @thread_sleeper = nil
       super()
+      # The read thread continuously drains the socket so the operating system
+      # buffers don't fill up (and drop datagrams) while we're busy processing
+      # the previous read. @read_socket.read is redirected to the queue so
+      # everything reading the socket (including subclasses) still sees all the data.
+      start_read_queue_thread(@read_socket, @read_timeout) if @read_socket
     end
 
     # @return [Boolean] Whether the active ports (read and/or write) have
@@ -163,6 +168,8 @@ module OpenC3
       OpenC3.close_socket(@read_socket)
       @write_socket = nil
       @read_socket = nil
+      # The sockets are closed above which unblocks the read thread
+      stop_read_queue_thread()
       @thread_sleeper.cancel if @thread_sleeper
       @thread_sleeper = nil
       super()
@@ -177,9 +184,16 @@ module OpenC3
       return nil
     end
 
+    # Zero length datagrams are valid so only nil means the socket is done
+    def read_queue_done?(data)
+      data.nil?
+    end
+
     # Reads from the socket if the read_port is defined
     def read_interface
       data = @read_socket.read(@read_timeout)
+      return nil unless data # Disconnected
+
       Logger.info "#{@name}: Udp read returned 0 bytes (stream closed)" if data.length <= 0
       extra = nil
       read_interface_base(data, extra)

data/lib/openc3/interfaces.rb

@@ -22,6 +22,7 @@ module OpenC3
   autoload(:Interface, 'openc3/interfaces/interface.rb')
   autoload(:MqttInterface, 'openc3/interfaces/mqtt_interface.rb')
   autoload(:MqttStreamInterface, 'openc3/interfaces/mqtt_stream_interface.rb')
+  autoload(:ReadQueueInterface, 'openc3/interfaces/read_queue_interface.rb')
   autoload(:SerialInterface, 'openc3/interfaces/serial_interface.rb')
   autoload(:SimulatedTargetInterface, 'openc3/interfaces/simulated_target_interface.rb')
   autoload(:StreamInterface, 'openc3/interfaces/stream_interface.rb')

data/lib/openc3/logs/log_writer.rb

@@ -63,7 +63,10 @@ module OpenC3
     CYCLE_TIME_INTERVAL = 10
 
     # Delay in seconds before trimming Redis streams
-    CLEANUP_DELAY = 60
+    # This is the primary trimmer of the TELEMETRY__ and COMMAND__ streams. Writers
+    # also apply a safety MINID cap (see Topic.stream_safety_minid) which depends on this
+    # value, so it is defined in Topic.
+    CLEANUP_DELAY = Topic::LOG_CLEANUP_DELAY_SECONDS
 
     # Mutex protecting class variables
     @@mutex = Mutex.new
@@ -211,6 +214,15 @@ module OpenC3
       end
     end
 
+    # The db_shard of a target topic like DEFAULT__DECOM__{INST}__HEALTH_STATUS
+    # or db_shard 0 for a topic without a target
+    def self.db_shard_for_topic(redis_topic)
+      scope, _, rest = redis_topic.delete_prefix('{').partition('__')
+      # The target is a {TARGET} part between two others, e.g. DECOM__{INST}__HEALTH_STATUS
+      target = rest.split('__', -1)[1...-1]&.find { |part| part.length > 2 and part.start_with?('{') and part.end_with?('}') }
+      (!scope.empty? and target) ? Store.db_shard_for_target(target[1...-1], scope: scope) : 0
+    end
+
     def cycle_thread_body
       @@cycle_sleeper = Sleeper.new
       while true
@@ -245,9 +257,7 @@ module OpenC3
                   # Now that the file is in S3, trim the Redis stream up until the previous file.
                   # This keeps one minute of data in Redis
                   instance.cleanup_offsets[index].each do |redis_topic, cleanup_offset|
-                    target_match = redis_topic.match(/__\{?([^}_]+)\}?__/)
-                    db_shard = target_match ? Store.db_shard_for_target(target_match[1]) : 0
-                    Topic.trim_topic(redis_topic, cleanup_offset, db_shard: db_shard)
+                    Topic.trim_topic(redis_topic, cleanup_offset, db_shard: LogWriter.db_shard_for_topic(redis_topic))
                   end
                   indexes_to_clear << index
                 end

data/lib/openc3/microservices/decom_microservice.rb

@@ -84,6 +84,11 @@ module OpenC3
     include InterfaceDecomCommon
     LIMITS_STATE_INDEX = { RED_LOW: 0, YELLOW_LOW: 1, YELLOW_HIGH: 2, RED_HIGH: 3, GREEN_LOW: 4, GREEN_HIGH: 5 }
 
+    # Decom never uses descriptions so don't keep them in memory
+    def load_descriptions?
+      false
+    end
+
     def initialize(*args)
       super(*args)
       # Should only be one target, but there might be multiple decom microservices for a given target
@@ -94,6 +99,10 @@ module OpenC3
       @limits_event_topic = "#{@scope}__openc3_limits_events"
       @topics << @limits_event_topic
       Topic.update_topic_offsets(@topics, db_shard: @db_shard)
+      # Last limits event read, used to detect events trimmed before they were read.
+      # Read from the stream rather than the thread's cached offsets. The run thread
+      # starts reading at or after this point, so it can only cause an extra resync.
+      @limits_event_offset = Topic.get_last_offset(@limits_event_topic, db_shard: @db_shard)
       # Initialize before assigning limits_change_callback - sync_system below
       # can fire the callback synchronously for any persisted-disabled items.
       @limits_response_queue = Queue.new
@@ -116,14 +125,21 @@ module OpenC3
         break if @cancel_thread
 
         begin
+          limits_events_checked = false
           OpenC3.in_span("read_topics") do
             Topic.read_topics(@topics, db_shard: @db_shard) do |topic, msg_id, msg_hash, redis|
               break if @cancel_thread
               if topic == @microservice_topic
                 microservice_cmd(topic, msg_id, msg_hash, redis)
               elsif topic == @limits_event_topic
+                # Any gap is before the first event in a read so only check once per read
+                unless limits_events_checked
+                  resync_if_limits_events_trimmed()
+                  limits_events_checked = true
+                end
+                @limits_event_offset = msg_id
                 event = JSON.parse(msg_hash['event'], allow_nan: true, create_additions: true)
-                LimitsEventTopic.process_event(event)
+                LimitsEventTopic.process_event(event, scope: @scope)
               elsif topic =~ /__DECOMINTERFACE/
                 if msg_hash.key?('inject_tlm')
                   handle_inject_tlm_with_ack(msg_hash['inject_tlm'], msg_id)
@@ -156,11 +172,20 @@ module OpenC3
       @limits_response_thread = nil
     end
 
-    def decom_packet(_topic, msg_id, msg_hash, _redis)
+    # Limits events are only read between batches of packets, so a decom microservice
+    # working through a backlog can fall far enough behind that a flood of LIMITS_CHANGE
+    # events trims control events (enable, settings, etc) before it reads them.
+    # Recover by reloading all limits state from the store, which is always current.
+    def resync_if_limits_events_trimmed
+      if LimitsEventTopic.events_trimmed?(@limits_event_offset, scope: @scope, db_shard: @db_shard)
+        @logger.warn("#{@name}: Limits events were trimmed before they could be read, resyncing limits")
+        LimitsEventTopic.resync_system(scope: @scope)
+      end
+    end
+
+    def decom_packet(topic, msg_id, msg_hash, _redis)
       OpenC3.in_span("decom_packet") do
-        msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
-        delta = Time.now.to_f - msgid_seconds_from_epoch
-        @metric.set(name: 'decom_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and decom start')
+        update_topic_lag(topic, msg_id, metric_name: 'decom_topic_delta_seconds', help: 'Delta time between data written to stream and decom start')
 
         #######################################
         # Build packet object from topic data

data/lib/openc3/microservices/interface_decom_common.rb

@@ -42,6 +42,8 @@ module OpenC3
     def handle_inject_tlm_with_ack(inject_tlm_json, msg_id)
       inject_tlm_hash = JSON.parse(inject_tlm_json, allow_nan: true, create_additions: true)
       target_name = inject_tlm_hash['target_name']
+      # Ack on the db_shard of the target where DecomInterfaceTopic.inject_tlm waits for it
+      db_shard = Store.db_shard_for_target(target_name, scope: @scope)
       ack_topic = "{#{@scope}__ACKCMD}TARGET__#{target_name}"
       begin
         handle_inject_tlm(inject_tlm_json)
@@ -56,7 +58,7 @@ module OpenC3
           result: error.message
         }
       end
-      Topic.write_topic(ack_topic, msg_hash)
+      Topic.write_topic(ack_topic, msg_hash, db_shard: db_shard)
     end
 
     def handle_build_cmd(build_cmd_json, msg_id)