openc3 7.4.1 → 7.5.0

data/lib/openc3/microservices/interface_microservice.rb

@@ -42,7 +42,7 @@ module OpenC3
   class InterfaceCmdHandlerThread
     include InterfaceDecomCommon
 
-    def initialize(interface, tlm, logger: nil, metric: nil, db_shard: 0, scope:)
+    def initialize(interface, tlm, logger: nil, metric: nil, lag_monitor: nil, db_shard: 0, scope:)
       @interface = interface
       @tlm = tlm
       @scope = scope
@@ -56,6 +56,7 @@ module OpenC3
       @logger = logger
       @logger = Logger unless @logger
       @metric = metric
+      @lag_monitor = lag_monitor || TopicLagMonitor.new(name: "#{@scope}__INTERFACE__#{@interface.name}", logger: @logger, metric: @metric)
       @count = 0
       @directive_count = 0
       @metric.set(name: 'interface_directive_total', value: @directive_count, type: 'counter') if @metric
@@ -86,9 +87,7 @@ module OpenC3
         OpenC3.with_context(msg_hash) do
           release_critical = false
           critical_model = nil
-          msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
-          delta = Time.now.to_f - msgid_seconds_from_epoch
-          @metric.set(name: 'interface_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and interface cmd start') if @metric
+          @lag_monitor.record(topic, msg_id, metric_name: 'interface_topic_delta_seconds', help: 'Delta time between data written to stream and interface cmd start')
 
           if topic == "OPENC3__SYSTEM__EVENTS"
             msg = JSON.parse(msg_hash['event'])
@@ -113,7 +112,13 @@ module OpenC3
               @logger.info "#{@interface.name}: Connect requested"
               params = []
               if msg_hash['params']
-                params = JSON.parse(msg_hash['params'], allow_nan: true, create_additions: true)
+                begin
+                  params = JSON.parse(msg_hash['params'], allow_nan: true, create_additions: true)
+                rescue => e
+                  # Reject malformed params rather than killing the handler thread
+                  @logger.error "#{@interface.name}: Invalid connect params: #{e.message}"
+                  next e.message
+                end
               end
               @interface = @tlm.attempting(*params)
               next 'SUCCESS'
@@ -156,8 +161,8 @@ module OpenC3
               next 'SUCCESS'
             end
             if msg_hash.key?('interface_cmd')
-              params = JSON.parse(msg_hash['interface_cmd'], allow_nan: true, create_additions: true)
               begin
+                params = JSON.parse(msg_hash['interface_cmd'], allow_nan: true, create_additions: true)
                 @logger.info "#{@interface.name}: interface_cmd: #{params['cmd_name']} #{params['cmd_params'].join(' ')}"
                 @interface.interface_cmd(params['cmd_name'], *params['cmd_params'])
                 InterfaceStatusModel.set(@interface.as_json(), queued: true, scope: @scope)
@@ -168,8 +173,8 @@ module OpenC3
               next 'SUCCESS'
             end
             if msg_hash.key?('protocol_cmd')
-              params = JSON.parse(msg_hash['protocol_cmd'], allow_nan: true, create_additions: true)
               begin
+                params = JSON.parse(msg_hash['protocol_cmd'], allow_nan: true, create_additions: true)
                 @logger.info "#{@interface.name}: protocol_cmd: #{params['cmd_name']} #{params['cmd_params'].join(' ')} read_write: #{params['read_write']} index: #{params['index']}"
                 @interface.protocol_cmd(params['cmd_name'], *params['cmd_params'], read_write: params['read_write'], index: params['index'])
                 InterfaceStatusModel.set(@interface.as_json(), queued: true, scope: @scope)
@@ -232,21 +237,28 @@ module OpenC3
 
           target_name = msg_hash['target_name']
           if target_name and not @interface.cmd_target_enabled[target_name]
+            # A released critical command was already acked as pending so it must get a result
+            next "Target #{target_name} is disabled" if release_critical
             next nil # Return and don't ack given target_name if disabled
           end
 
           cmd_name = msg_hash['cmd_name']
-          manual = ConfigParser.handle_true_false(msg_hash['manual'])
+          manual = ConfigParser.handle_true_false(msg_hash['manual'], default: false)
           cmd_params = nil
           range_check = true
           raw = false
           cmd_buffer = nil
           hazardous_check = nil
           if msg_hash['cmd_params']
-            cmd_params = JSON.parse(msg_hash['cmd_params'], allow_nan: true, create_additions: true)
-            range_check = ConfigParser.handle_true_false(msg_hash['range_check'])
-            raw = ConfigParser.handle_true_false(msg_hash['raw'])
-            hazardous_check = ConfigParser.handle_true_false(msg_hash['hazardous_check'])
+            begin
+              cmd_params = JSON.parse(msg_hash['cmd_params'], allow_nan: true, create_additions: true)
+            rescue => e
+              @logger.error "#{@interface.name}: Invalid cmd_params: #{e.message}"
+              next "Invalid cmd_params: #{e.message}"
+            end
+            range_check = ConfigParser.handle_true_false(msg_hash['range_check'], default: true)
+            raw = ConfigParser.handle_true_false(msg_hash['raw'], default: false)
+            hazardous_check = ConfigParser.handle_true_false(msg_hash['hazardous_check'], default: true)
           elsif msg_hash['cmd_buffer']
             cmd_buffer = msg_hash['cmd_buffer']
           end
@@ -269,12 +281,16 @@ module OpenC3
                 raise "Invalid command received:\n #{msg_hash}"
               end
 
-              if @interface.cmd_target_enabled[command.target_name]
+              # An unidentified buffer is sent as UNKNOWN on behalf of the target it was sent to
+              enabled_target_name = command.target_name
+              enabled_target_name = target_name if enabled_target_name == 'UNKNOWN' and target_name
+              if @interface.cmd_target_enabled[enabled_target_name]
                 orig_command = System.commands.packet(command.target_name, command.packet_name)
                 orig_command.received_count = TargetModel.increment_command_count(command.target_name, command.packet_name, 1, scope: @scope)
                 command.received_count = orig_command.received_count
                 command.received_time = Time.now
               else
+                next "Target #{command.target_name} is disabled" if release_critical
                 next nil # Don't ack disabled targets
               end
             rescue RangeError => e
@@ -322,7 +338,7 @@ module OpenC3
               end
             end
 
-            validate = ConfigParser.handle_true_false(msg_hash['validate'])
+            validate = ConfigParser.handle_true_false(msg_hash['validate'], default: true)
             begin
               if @interface.connected?
                 result = true
@@ -344,7 +360,7 @@ module OpenC3
                 @count += 1
                 @metric.set(name: 'interface_cmd_total', value: @count, type: 'counter') if @metric
 
-                log_message = ConfigParser.handle_true_false(msg_hash['log_message'])
+                log_message = ConfigParser.handle_true_false(msg_hash['log_message'], default: true)
                 if log_message
                   @logger.info(msg_hash['cmd_string'], user: msg_hash['username'], scope: @scope)
                 end
@@ -393,7 +409,7 @@ module OpenC3
   end
 
   class RouterTlmHandlerThread
-    def initialize(router, tlm, logger: nil, metric: nil, db_shard: 0, scope:)
+    def initialize(router, tlm, logger: nil, metric: nil, lag_monitor: nil, db_shard: 0, scope:)
       @router = router
       @tlm = tlm
       @scope = scope
@@ -401,6 +417,7 @@ module OpenC3
       @logger = logger
       @logger = Logger unless @logger
       @metric = metric
+      @lag_monitor = lag_monitor || TopicLagMonitor.new(name: "#{@scope}__ROUTER__#{@router.name}", logger: @logger, metric: @metric)
       @count = 0
       @directive_count = 0
       @metric.set(name: 'router_directive_total', value: @directive_count, type: 'counter') if @metric
@@ -428,9 +445,7 @@ module OpenC3
 
     def run
       RouterTopic.receive_telemetry(@router, scope: @scope, db_shard: @db_shard) do |topic, msg_id, msg_hash, _redis|
-        msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
-        delta = Time.now.to_f - msgid_seconds_from_epoch
-        @metric.set(name: 'router_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and router tlm start') if @metric
+        @lag_monitor.record(topic, msg_id, metric_name: 'router_topic_delta_seconds', help: 'Delta time between data written to stream and router tlm start')
 
         # Check for commands to the router itself
         if /CMD}ROUTER/.match?(topic)
@@ -446,7 +461,13 @@ module OpenC3
             @logger.info "#{@router.name}: Connect requested"
             params = []
             if msg_hash['params']
-              params = JSON.parse(msg_hash['params'], allow_nan: true, create_additions: true)
+              begin
+                params = JSON.parse(msg_hash['params'], allow_nan: true, create_additions: true)
+              rescue => e
+                # Reject malformed params rather than killing the handler thread
+                @logger.error "#{@router.name}: Invalid connect params: #{e.message}"
+                next e.message
+              end
             end
             @router = @tlm.attempting(*params)
             next 'SUCCESS'
@@ -555,6 +576,14 @@ module OpenC3
 
   class InterfaceMicroservice < Microservice
     UNKNOWN_BYTES_TO_PRINT = 16
+    DEFAULT_UPDATE_INTERVAL = 0.1 # seconds
+    # Most distinct unmapped target / packet names to remember warning about
+    UNMAPPED_TLM_WARNED_MAX = 1000
+
+    # Interfaces never use descriptions so don't keep them in memory
+    def load_descriptions?
+      false
+    end
 
     def initialize(name)
       @mutex = Mutex.new
@@ -591,28 +620,35 @@ module OpenC3
         RouterStatusModel.set(@interface.as_json(), scope: @scope)
       end
 
-      @queued = false
+      # Insertion ordered so the oldest key can be evicted (see handle_packet)
+      @unmapped_tlm_warned = {}
+      # Queue Redis writes by default and flush them every update_interval seconds
+      update_interval = DEFAULT_UPDATE_INTERVAL
       @interface.options.each do |option_name, option_values|
         # OPTIMIZE_THROUGHPUT was changed to UPDATE_INTERVAL to better represent the setting
         if option_name.upcase == 'UPDATE_INTERVAL' or option_name.upcase == 'OPTIMIZE_THROUGHPUT'
-          @queued = true
           update_interval = option_values[0].to_f
-          EphemeralStoreQueued.instance.set_update_interval(update_interval)
-          StoreQueued.instance.set_update_interval(update_interval)
         end
         if option_name.upcase == 'SYNC_PACKET_COUNT_DELAY_SECONDS'
           TargetModel.sync_packet_count_delay_seconds = option_values[0].to_f
         end
       end
+      # An UPDATE_INTERVAL of 0 (or less) disables queuing and writes directly to Redis
+      @queued = update_interval > 0.0
+      if @queued
+        # Apply to every db_shard since writes are routed to the shard of each target
+        EphemeralStoreQueued.set_update_interval_all_shards(update_interval)
+        StoreQueued.set_update_interval_all_shards(update_interval)
+      end
 
       @interface_thread_sleeper = Sleeper.new
       @cancel_thread = false
       @connection_failed_messages = []
       @connection_lost_messages = []
       if @interface_or_router == 'INTERFACE'
-        @handler_thread = InterfaceCmdHandlerThread.new(@interface, self, logger: @logger, metric: @metric, db_shard: @db_shard, scope: @scope)
+        @handler_thread = InterfaceCmdHandlerThread.new(@interface, self, logger: @logger, metric: @metric, lag_monitor: topic_lag_monitor, db_shard: @db_shard, scope: @scope)
       else
-        @handler_thread = RouterTlmHandlerThread.new(@interface, self, logger: @logger, metric: @metric, db_shard: @db_shard, scope: @scope)
+        @handler_thread = RouterTlmHandlerThread.new(@interface, self, logger: @logger, metric: @metric, lag_monitor: topic_lag_monitor, db_shard: @db_shard, scope: @scope)
       end
       @handler_thread.start
     end
@@ -630,9 +666,17 @@ module OpenC3
         return @interface
       end
 
+      # A user requested (re)connect starts fresh so backtraces are logged again
+      clear_connection_messages()
       attempt_connection(*params)
     end
 
+    # Forget which errors have already had their backtrace logged
+    def clear_connection_messages
+      @connection_failed_messages.clear
+      @connection_lost_messages.clear
+    end
+
     # Sets the state to 'ATTEMPTING', first rebuilding the interface/router if
     # parameters are given, so the run method performs the actual connection.
     # Unlike attempting() this always transitions. The reconnect path in
@@ -720,6 +764,7 @@ module OpenC3
                   else
                     @metric.set(name: 'router_cmd_total', value: @count, type: 'counter')
                   end
+                  set_read_queue_bytes_metric()
                 else
                   @logger.info "#{@interface.name}: Internal disconnect requested (returned nil)"
                   handle_connection_lost()
@@ -753,31 +798,56 @@ module OpenC3
       @logger.info "#{@interface.name}: Stopped packet reading"
     end
 
+    # Report the bytes buffered on the read queue. Called after each packet and
+    # after a disconnect (which discards the queue) so the gauge doesn't stay
+    # stuck at its last value while no packets are flowing.
+    def set_read_queue_bytes_metric
+      if @interface_or_router == 'INTERFACE'
+        @metric.set(name: 'interface_read_queue_bytes', value: @interface.read_queue_bytes, type: 'gauge',
+                    unit: 'bytes', help: 'Bytes buffered on the interface read queue waiting to be processed')
+      else
+        @metric.set(name: 'router_read_queue_bytes', value: @interface.read_queue_bytes, type: 'gauge',
+                    unit: 'bytes', help: 'Bytes buffered on the router read queue waiting to be processed')
+      end
+    end
+
     def handle_packet(packet)
       # Skip status update if stop() has been called to avoid re-creating the status model
       InterfaceStatusModel.set(@interface.as_json(), queued: true, scope: @scope) unless @cancel_thread
       packet.received_time = Time.now.sys unless packet.received_time
 
+      if packet.identified?
+        unless @interface.tlm_target_names.include?(packet.target_name)
+          # Trust the identity but drop it because the target isn't mapped to this interface
+          key = "#{packet.target_name} #{packet.packet_name}"
+          unless @unmapped_tlm_warned.key?(key)
+            # The names come from the wire so bound how many are remembered.
+            # Evict the oldest rather than stop adding so every new name still warns.
+            @unmapped_tlm_warned.delete(@unmapped_tlm_warned.first[0]) if @unmapped_tlm_warned.length >= UNMAPPED_TLM_WARNED_MAX
+            @unmapped_tlm_warned[key] = true
+            @logger.warn "#{@interface.name}: Dropping telemetry for unmapped target: #{key}"
+          end
+          return
+        end
+        unless known_tlm_packet?(packet)
+          # Packet identified but we don't know about it
+          # Clear packet_name and target_name and try to identify
+          @logger.warn "#{@interface.name}: Received unknown identified telemetry: #{packet.target_name} #{packet.packet_name}"
+          packet.target_name = nil
+          packet.packet_name = nil
+        end
+      end
+
       if packet.stored
         # Stored telemetry does not update the current value table
         identified_packet = System.telemetry.identify_and_define_packet(packet, @interface.tlm_target_names)
       else
         # Identify and update packet
         if packet.identified?
-          begin
-            # Preidentifed packet - place it into the current value table
-            identified_packet = System.telemetry.update!(packet.target_name,
-                                                         packet.packet_name,
-                                                         packet.buffer)
-          rescue RuntimeError
-            # Packet identified but we don't know about it
-            # Clear packet_name and target_name and try to identify
-            @logger.warn "#{@interface.name}: Received unknown identified telemetry: #{packet.target_name} #{packet.packet_name}"
-            packet.target_name = nil
-            packet.packet_name = nil
-            identified_packet = System.telemetry.identify!(packet.buffer,
-                                                           @interface.tlm_target_names)
-          end
+          # Preidentifed packet - place it into the current value table
+          identified_packet = System.telemetry.update!(packet.target_name,
+                                                       packet.packet_name,
+                                                       packet.buffer)
         else
           # Packet needs to be identified
           identified_packet = System.telemetry.identify!(packet.buffer,
@@ -791,13 +861,21 @@ module OpenC3
         identified_packet.extra = packet.extra
         packet = identified_packet
       else
-        unknown_packet = System.telemetry.update!('UNKNOWN', 'UNKNOWN', packet.buffer)
+        if packet.stored
+          # Stored telemetry does not update the current value table
+          unknown_packet = System.telemetry.packet('UNKNOWN', 'UNKNOWN').clone
+          unknown_packet.buffer = packet.buffer
+        else
+          unknown_packet = System.telemetry.update!('UNKNOWN', 'UNKNOWN', packet.buffer)
+        end
         unknown_packet.received_time = packet.received_time
         unknown_packet.stored = packet.stored
         unknown_packet.extra = packet.extra
         packet = unknown_packet
-        json_hash = CvtModel.build_json_from_packet(packet)
-        CvtModel.set(json_hash, target_name: packet.target_name, packet_name: packet.packet_name, queued: @queued, scope: @scope)
+        unless packet.stored
+          json_hash = CvtModel.build_json_from_packet(packet)
+          CvtModel.set(json_hash, target_name: packet.target_name, packet_name: packet.packet_name, queued: @queued, scope: @scope)
+        end
         num_bytes_to_print = [UNKNOWN_BYTES_TO_PRINT, packet.length].min
         data = packet.buffer(false)[0..(num_bytes_to_print - 1)]
         prefix = data.each_byte.map { | byte | sprintf("%02X", byte) }.join()
@@ -811,6 +889,14 @@ module OpenC3
       end
     end
 
+    # @return [Boolean] Whether a pre-identified packet names a defined packet
+    def known_tlm_packet?(packet)
+      System.telemetry.packet(packet.target_name, packet.packet_name)
+      true
+    rescue RuntimeError
+      false
+    end
+
     def handle_connection_failed(connection, connect_error)
       @error = connect_error
       @logger.error "#{@interface.name}: Connection #{connection} failed due to #{connect_error.formatted(false, false)}"
@@ -824,9 +910,11 @@ module OpenC3
         if RuntimeError === connect_error and (connect_error.message =~ /canceled/ or connect_error.message =~ /timeout/)
           # Do not write an exception file for these extremely common cases
         else
-          @logger.error "#{@interface.name}: #{connect_error.formatted}"
-          unless @connection_failed_messages.include?(connect_error.message)
-            @connection_failed_messages << connect_error.message
+          # Only log the backtrace the first time we see each distinct error
+          key = [connect_error.class, connect_error.message]
+          unless @connection_failed_messages.include?(key)
+            @logger.error "#{@interface.name}: #{connect_error.formatted}"
+            @connection_failed_messages << key
           end
         end
       end
@@ -844,9 +932,11 @@ module OpenC3
         when Errno::ECONNABORTED, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EBADF, Errno::ENOTSOCK, IOError
           # Do not write an exception file for these extremely common cases
         else
-          @logger.error "#{@interface.name}: #{err.formatted}"
-          unless @connection_lost_messages.include?(err.message)
-            @connection_lost_messages << err.message
+          # Only log the backtrace the first time we see each distinct error
+          key = [err.class, err.message]
+          unless @connection_lost_messages.include?(key)
+            @logger.error "#{@interface.name}: #{err.formatted}"
+            @connection_lost_messages << key
           end
         end
       else
@@ -905,6 +995,7 @@ module OpenC3
         rescue => e
           @logger.error "Disconnect: #{@interface.name}: #{e.formatted}"
         end
+        set_read_queue_bytes_metric() if @metric
 
         # If the interface is set to auto_reconnect then delay so the thread
         # can come back around and allow the interface a chance to reconnect.
@@ -940,7 +1031,12 @@ module OpenC3
         @handler_thread.stop if @handler_thread
         @interface_thread_sleeper.cancel if @interface_thread_sleeper
         if @interface
-          @interface.disconnect
+          begin
+            @interface.disconnect
+          rescue => e
+            # Still clean up the status and stream logs if disconnect fails
+            @logger.error "#{@interface.name}: Disconnect failed during stop: #{e.formatted}"
+          end
           if @interface_or_router == 'INTERFACE'
             valid_interface = InterfaceStatusModel.get_model(name: @interface.name, scope: @scope)
           else

data/lib/openc3/microservices/log_microservice.rb

@@ -95,9 +95,7 @@ module OpenC3
     end
 
     def log_data(topic, msg_id, msg_hash, redis)
-      msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
-      delta = Time.now.to_f - msgid_seconds_from_epoch
-      @metric.set(name: 'log_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and log start')
+      update_topic_lag(topic, msg_id, metric_name: 'log_topic_delta_seconds', help: 'Delta time between data written to stream and log start')
 
       topic_split = topic.gsub(/{|}/, '').split("__") # Remove the redis hashtag curly braces
       target_name = topic_split[2]

data/lib/openc3/microservices/microservice.rb

@@ -26,6 +26,7 @@ OpenC3.require_file 'openc3/utilities/secrets'
 OpenC3.require_file 'openc3/utilities/sleeper'
 OpenC3.require_file 'openc3/utilities/open_telemetry'
 OpenC3.require_file 'openc3/utilities/thread_manager'
+OpenC3.require_file 'openc3/utilities/topic_lag_monitor'
 OpenC3.require_file 'openc3/models/microservice_model'
 OpenC3.require_file 'openc3/models/microservice_status_model'
 OpenC3.require_file 'tmpdir'
@@ -133,7 +134,7 @@ module OpenC3
       @target_names = @config["target_names"]
       @target_names ||= []
       # NOTE: setup_targets doesn't do anything if @target_names is empty
-      System.setup_targets(@target_names, @temp_dir, scope: @scope) unless is_plugin
+      System.setup_targets(@target_names, @temp_dir, scope: @scope, descriptions: load_descriptions?) unless is_plugin
 
       # Use at_exit to shutdown cleanly no matter how we die
       at_exit do
@@ -231,6 +232,12 @@ module OpenC3
       shutdown()
     end
 
+    # Whether System keeps packet and item descriptions. Subclasses that never
+    # use them override this to return false to reduce memory.
+    def load_descriptions?
+      true
+    end
+
     def shutdown(state = 'STOPPED')
       return if @shutdown_complete
       begin
@@ -263,6 +270,23 @@ module OpenC3
       topic_offsets[@microservice_topic] = "0-0" # Always get all available
     end
 
+    # Update the lag metric for a message read from a topic and notify users
+    # if this microservice is falling behind
+    #
+    # @param topic [String] Redis stream the message was read from
+    # @param msg_id [String] Redis stream id of the message
+    # @param metric_name [String] Gauge metric to update with the lag
+    # @param help [String] Help text for the gauge metric
+    # @return [Float] The lag in seconds
+    def update_topic_lag(topic, msg_id, metric_name:, help:)
+      topic_lag_monitor.record(topic, msg_id, metric_name: metric_name, help: help)
+    end
+
+    # @return [TopicLagMonitor] Lag monitor shared by this microservice's topic readers
+    def topic_lag_monitor
+      @topic_lag_monitor ||= TopicLagMonitor.new(name: @name, logger: @logger || Logger, metric: @metric)
+    end
+
     # Returns if the command was handled
     def microservice_cmd(topic, msg_id, msg_hash, _redis)
       command = msg_hash['command']

data/lib/openc3/microservices/plugin_microservice.rb

@@ -35,6 +35,38 @@ module OpenC3
         # Its ok if this fails
       end
 
+      # exec() below replaces this process image, so the Metric update thread
+      # this bootstrap started dies without ever running shutdown. Whatever it
+      # managed to publish would then sit in Redis untouched until it expired,
+      # looking for all the world like a live microservice pinned at whatever
+      # cpu the bootstrap itself burned loading the OpenC3 gem. Clear it out and
+      # stop the thread before handing the process over to the real cmd. A cmd
+      # that is itself an OpenC3::Microservice builds a new Metric and starts
+      # publishing again; one that isn't gets its cpu and memory reported by the
+      # operator instead (see Operator#publish_process_metrics).
+      begin
+        @metric.shutdown
+        # Stop and drain pending writes before deleting the row, otherwise a
+        # queued bootstrap update can recreate it before exec. Any later write
+        # through this StoreQueued goes straight to Redis instead of the queue.
+        MetricModel.store_queued.instance(db_shard: @db_shard).shutdown
+        MetricModel.new(name: @name, scope: @scope, db_shard: @db_shard).destroy
+      rescue Exception => e
+        @logger.warn("Failed to clear bootstrap metrics for #{@name}: #{e.message}")
+      end
+
+      # Microservice.run published INITIALIZED and then set @state to RUNNING
+      # in memory only. A plugin has no status thread, and exec() below means
+      # nothing here gets another chance to write it, so a cmd that isn't
+      # itself an OpenC3::Microservice (Rails app, python script, any other
+      # binary) would show INITIALIZED forever. Publish RUNNING now. A cmd that
+      # is an OpenC3::Microservice overwrites this with its own status.
+      begin
+        MicroserviceStatusModel.set(as_json(), scope: @scope)
+      rescue Exception => e
+        @logger.warn("Failed to publish status for #{@name}: #{e.message}")
+      end
+
       # Fortify: Process Control
       # This is dangerous! However, plugins need to be able to run whatever they want.
       # Only admins can install plugins and they need to be vetted for content.

data/lib/openc3/microservices/text_log_microservice.rb

@@ -82,9 +82,7 @@ module OpenC3
     end
 
     def log_data(topic, msg_id, msg_hash, redis)
-      msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
-      delta = Time.now.to_f - msgid_seconds_from_epoch
-      @metric.set(name: 'text_log_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and text log start')
+      update_topic_lag(topic, msg_id, metric_name: 'text_log_topic_delta_seconds', help: 'Delta time between data written to stream and text log start')
       @tlws[topic].write(msg_hash["time"].to_i, msg_hash.as_json(allow_nan: true).to_json(allow_nan: true), topic, msg_id)
       @count += 1
     rescue => err

data/lib/openc3/models/interface_model.rb

@@ -291,7 +291,7 @@ module OpenC3
         elsif enabled == 'DISABLED'
           return false
         else
-          raise parser.error("MAP_TARGET enabled state must be ENABLED or DISABLED.", usage)
+          raise parser.error("#{parser.keyword} enabled state must be ENABLED or DISABLED.", "#{parser.keyword} <Target Name> <ENABLED/DISABLED>")
         end
       end
     end

data/lib/openc3/models/interface_status_model.rb

@@ -62,6 +62,12 @@ module OpenC3
       _db_sharded_get("#{scope}__#{_get_key}", name: name, scope: scope)
     end
 
+    # Like get but returns default instead of nil when no status exists
+    def self.get_or_default(name:, scope:, default: { 'state' => 'UNKNOWN' })
+      status = get(name: name, scope: scope)
+      status.nil? ? default : status
+    end
+
     def self.names(scope:)
       _db_sharded_names("#{scope}__#{_get_key}", scope: scope)
     end

data/lib/openc3/models/metric_model.rb

@@ -23,6 +23,13 @@ module OpenC3
     include DbShardedModel
 
     PRIMARY_KEY = '__openc3__metric'.freeze
+    # Process level metrics (cpu, memory) reported by the operator on behalf of
+    # the microservices it spawned. These live in their own hash so that the
+    # operator and the microservice are never two writers of one field - each
+    # write replaces the whole values hash, so sharing a field would have them
+    # clobbering each other every cycle. The two are merged back together on
+    # read by get/names/all, so callers see a single row per microservice.
+    PROCESS_PRIMARY_KEY = '__openc3__process_metric'.freeze
     METRIC_EXPIRE_SECONDS = 3600 # Expire metrics after 1 hour
 
     attr_accessor :values
@@ -47,17 +54,48 @@ module OpenC3
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
     def self.get(name:, scope:)
-      _db_sharded_get("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
+      _merge_entries(
+        _db_sharded_get("#{scope}#{PRIMARY_KEY}", name: name, scope: scope),
+        _db_sharded_get("#{scope}#{PROCESS_PRIMARY_KEY}", name: name, scope: scope)
+      )
     end
 
     def self.names(scope:)
-      _db_sharded_names("#{scope}#{PRIMARY_KEY}", scope: scope)
+      names = _db_sharded_names("#{scope}#{PRIMARY_KEY}", scope: scope)
+      names.concat(_db_sharded_names("#{scope}#{PROCESS_PRIMARY_KEY}", scope: scope))
+      names.uniq.sort
     end
 
     def self.all(scope:)
+      result = _db_sharded_all("#{scope}#{PRIMARY_KEY}", scope: scope)
+      _db_sharded_all("#{scope}#{PROCESS_PRIMARY_KEY}", scope: scope).each do |name, process_entry|
+        result[name] = _merge_entries(result[name], process_entry)
+      end
+      result
+    end
+
+    # Only the metrics each microservice reported about itself, without the
+    # operator's process metrics. Their updated_at is when the microservice
+    # last published, which the merged rows from all can't tell you because
+    # the operator keeps refreshing them while the process is alive.
+    def self.all_self_reported(scope:)
       _db_sharded_all("#{scope}#{PRIMARY_KEY}", scope: scope)
     end
 
+    # Combine a microservice's self reported metrics with the process metrics
+    # the operator reported for it. The microservice's own values win on a key
+    # collision - a process that can measure itself knows better than the
+    # operator watching from outside.
+    def self._merge_entries(entry, process_entry)
+      return entry unless process_entry
+      return process_entry unless entry
+
+      entry['values'] = (process_entry['values'] || {}).merge(entry['values'] || {})
+      updated_at = [entry['updated_at'], process_entry['updated_at']].compact.max
+      entry['updated_at'] = updated_at if updated_at
+      entry
+    end
+
     # Sets (updates) the redis hash of this model
     # Queued defaults to true for MetricModel
     def self.set(json, scope:, queued: true)
@@ -66,13 +104,23 @@ module OpenC3
       self.new(**json).create(force: true, queued: queued, expire_seconds: METRIC_EXPIRE_SECONDS)
     end
 
+    # Sets (updates) the process metrics reported by the operator for a
+    # microservice it spawned. See PROCESS_PRIMARY_KEY.
+    def self.set_process(json, scope:, queued: true)
+      json[:scope] = scope
+      json.transform_keys!(&:to_sym)
+      json[:primary_key] = PROCESS_PRIMARY_KEY
+      self.new(**json).create(force: true, queued: queued, expire_seconds: METRIC_EXPIRE_SECONDS)
+    end
+
     def self.destroy(scope:, name:)
       db_shard = _db_shard_for_name(name, scope: scope)
       store.instance(db_shard: db_shard).hdel("#{scope}#{PRIMARY_KEY}", name)
+      store.instance(db_shard: db_shard).hdel("#{scope}#{PROCESS_PRIMARY_KEY}", name)
     end
 
-    def initialize(name:, values: {}, db_shard: 0, scope:)
-      super("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
+    def initialize(name:, values: {}, db_shard: 0, scope:, primary_key: PRIMARY_KEY)
+      super("#{scope}#{primary_key}", name: name, scope: scope)
       @values = values
       @db_shard = db_shard.to_i
     end