openc3 5.2.0 → 5.4.0

data/lib/openc3/interfaces.rb

@@ -17,11 +17,12 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 module OpenC3
   autoload(:Interface, 'openc3/interfaces/interface.rb')
+  autoload(:MqttInterface, 'openc3/interfaces/mqtt_interface.rb')
   autoload(:StreamInterface, 'openc3/interfaces/stream_interface.rb')
   autoload(:SerialInterface, 'openc3/interfaces/serial_interface.rb')
   autoload(:SimulatedTargetInterface, 'openc3/interfaces/simulated_target_interface.rb')
@@ -31,7 +32,6 @@ module OpenC3
   autoload(:LincInterface, 'openc3/interfaces/linc_interface.rb')
   autoload(:LincHandshakeCommand, 'openc3/interfaces/linc_interface.rb')
   autoload(:LincHandshake, 'openc3/interfaces/linc_interface.rb')
-  autoload(:DartStatusInterface, 'openc3/interfaces/dart_status_interface.rb')
 
   autoload(:Protocol, 'openc3/interfaces/protocols/protocol.rb')
   autoload(:BurstProtocol, 'openc3/interfaces/protocols/burst_protocol.rb')
@@ -41,7 +41,6 @@ module OpenC3
   autoload(:TemplateProtocol, 'openc3/interfaces/protocols/template_protocol.rb')
   autoload(:TerminatedProtocol, 'openc3/interfaces/protocols/terminated_protocol.rb')
 
-  autoload(:OverrideProtocol, 'openc3/interfaces/protocols/override_protocol.rb')
   autoload(:CrcProtocol, 'openc3/interfaces/protocols/crc_protocol.rb')
   autoload(:IgnorePacketProtocol, 'openc3/interfaces/protocols/ignore_packet_protocol.rb')
 end

data/lib/openc3/logs/buffered_packet_log_reader.rb

@@ -31,8 +31,8 @@ module OpenC3
       @buffer_depth = buffer_depth
     end
 
-    def next_packet_time
-      fill_buffer()
+    def next_packet_time(identify_and_define = true)
+      fill_buffer(identify_and_define)
       packet = @buffer[0]
       return packet.packet_time if packet
       return nil

data/lib/openc3/microservices/cleanup_microservice.rb

@@ -27,6 +27,14 @@ require 'openc3/utilities/bucket_utilities'
 
 module OpenC3
   class CleanupMicroservice < Microservice
+    def initialize(*args)
+      super(*args)
+      @metric.set(name: 'cleanup_total', value: @count, type: 'counter')
+      @delete_count = 0
+      @metric.set(name: 'cleanup_delete_total', value: @delete_count, type: 'counter')
+      @sleeper = Sleeper.new
+    end
+
     def run
       split_name = @name.split("__")
       target_name = split_name[-1]
@@ -55,15 +63,23 @@ module OpenC3
             oldest_list.each_slice(1000) do |slice|
               bucket.delete_objects(bucket: ENV['OPENC3_LOGS_BUCKET'], keys: slice)
               @logger.info("Deleted #{slice.length} #{target_name} log files")
+              @delete_count += slice.length
+              @metric.set(name: 'cleanup_delete_total', value: @delete_count, type: 'counter')
             end
           end
         end
 
         @count += 1
+        @metric.set(name: 'cleanup_total', value: @count, type: 'counter')
         @state = 'SLEEPING'
-        break if @microservice_sleeper.sleep(target.cleanup_poll_time)
+        break if @sleeper.sleep(target.cleanup_poll_time)
       end
     end
+
+    def shutdown
+      @sleeper.cancel
+      super()
+    end
   end
 end
 

data/lib/openc3/microservices/decom_microservice.rb

@@ -28,15 +28,14 @@ require 'openc3/models/notification_model'
 
 module OpenC3
   class DecomMicroservice < Microservice
-    DECOM_METRIC_NAME = "decom_packet_duration_seconds"
-    LIMIT_METRIC_NAME = "limits_change_callback_duration_seconds"
-    NS_PER_MSEC = 1000000
-
     def initialize(*args)
       super(*args)
       Topic.update_topic_offsets(@topics)
       System.telemetry.limits_change_callback = method(:limits_change_callback)
       LimitsEventTopic.sync_system(scope: @scope)
+      @error_count = 0
+      @metric.set(name: 'decom_total', value: @count, type: 'counter')
+      @metric.set(name: 'decom_error_total', value: @error_count, type: 'counter')
     end
 
     def run
@@ -50,18 +49,25 @@ module OpenC3
 
               decom_packet(topic, msg_id, msg_hash, redis)
               @count += 1
+              @metric.set(name: 'decom_total', value: @count, type: 'counter')
             end
           end
           LimitsEventTopic.sync_system_thread_body(scope: @scope)
         rescue => e
+          @error_count += 1
+          @metric.set(name: 'decom_error_total', value: @error_count, type: 'counter')
           @error = e
           @logger.error("Decom error: #{e.formatted}")
         end
       end
     end
 
-    def decom_packet(topic, _msg_id, msg_hash, _redis)
+    def decom_packet(topic, msg_id, msg_hash, _redis)
       OpenC3.in_span("decom_packet") do
+        msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
+        delta = Time.now.to_f - msgid_seconds_from_epoch
+        @metric.set(name: 'decom_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and decom start')
+
         start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         target_name = msg_hash["target_name"]
         packet_name = msg_hash["packet_name"]
@@ -75,8 +81,7 @@ module OpenC3
 
         TelemetryDecomTopic.write_packet(packet, scope: @scope)
         diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-        metric_labels = { "packet" => packet_name, "target" => target_name }
-        @metric.add_sample(name: DECOM_METRIC_NAME, value: diff, labels: metric_labels)
+        @metric.set(name: 'decom_duration_seconds', value: diff, type: 'gauge', unit: 'seconds')
       end
     end
 
@@ -133,8 +138,6 @@ module OpenC3
       end
 
       diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-      labels = { "packet" => packet.packet_name, "target" => packet.target_name }
-      @metric.add_sample(name: LIMIT_METRIC_NAME, value: diff, labels: labels)
     end
   end
 end

data/lib/openc3/microservices/interface_microservice.rb

@@ -33,12 +33,17 @@ require 'openc3/topics/router_topic'
 
 module OpenC3
   class InterfaceCmdHandlerThread
-    def initialize(interface, tlm, logger: nil, scope:)
+    def initialize(interface, tlm, logger: nil, metric: nil, scope:)
       @interface = interface
       @tlm = tlm
       @scope = scope
       @logger = logger
       @logger = Logger unless @logger
+      @metric = metric
+      @count = 0
+      @directive_count = 0
+      @metric.set(name: 'interface_directive_total', value: @directive_count, type: 'counter') if @metric
+      @metric.set(name: 'interface_cmd_total', value: @count, type: 'counter') if @metric
     end
 
     def start
@@ -59,10 +64,16 @@ module OpenC3
     end
 
     def run
-      InterfaceTopic.receive_commands(@interface, scope: @scope) do |topic, msg_hash|
+      InterfaceTopic.receive_commands(@interface, scope: @scope) do |topic, msg_id, msg_hash, redis|
         OpenC3.with_context(msg_hash) do
+          msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
+          delta = Time.now.to_f - msgid_seconds_from_epoch
+          @metric.set(name: 'interface_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and interface cmd start') if @metric
+
           # Check for a raw write to the interface
           if topic =~ /CMD}INTERFACE/
+            @directive_count += 1
+            @metric.set(name: 'interface_directive_total', value: @directive_count, type: 'counter') if @metric
             if msg_hash['shutdown']
               @logger.info "#{@interface.name}: Shutdown requested"
               return
@@ -107,6 +118,28 @@ module OpenC3
               end
               next 'SUCCESS'
             end
+            if msg_hash.key?('interface_cmd')
+              params = JSON.parse(msg_hash['interface_cmd'], allow_nan: true, create_additions: true)
+              begin
+                @logger.info "#{@interface.name}: interface_cmd: #{params['cmd_name']} #{params['cmd_params'].join(' ')}"
+                @interface.interface_cmd(params['cmd_name'], params['cmd_params'])
+              rescue => e
+                @logger.error "#{@interface.name}: interface_cmd: #{e.formatted}"
+                next e.message
+              end
+              next 'SUCCESS'
+            end
+            if msg_hash.key?('protocol_cmd')
+              params = JSON.parse(msg_hash['protocol_cmd'], allow_nan: true, create_additions: true)
+              begin
+                @logger.info "#{@interface.name}: protocol_cmd: #{params['cmd_name']} #{params['cmd_params'].join(' ')} read_write: #{params['read_write']} index: #{params['index']}"
+                @interface.protocol_cmd(params['cmd_name'], params['cmd_params'], read_write: params['read_write'], index: params['index'])
+              rescue => e
+                @logger.error "#{@interface.name}: protocol_cmd: #{e.formatted}"
+                next e.message
+              end
+              next 'SUCCESS'
+            end
           end
 
           target_name = msg_hash['target_name']
@@ -158,6 +191,9 @@ module OpenC3
 
             begin
               if @interface.connected?
+                @count += 1
+                @metric.set(name: 'interface_cmd_total', value: @count, type: 'counter') if @metric
+
                 @interface.write(command)
                 CommandTopic.write_packet(command, scope: @scope)
                 CommandDecomTopic.write_packet(command, scope: @scope)
@@ -180,12 +216,17 @@ module OpenC3
   end
 
   class RouterTlmHandlerThread
-    def initialize(router, tlm, logger: nil, scope:)
+    def initialize(router, tlm, logger: nil, metric: nil, scope:)
       @router = router
       @tlm = tlm
       @scope = scope
       @logger = logger
       @logger = Logger unless @logger
+      @metric = metric
+      @count = 0
+      @directive_count = 0
+      @metric.set(name: 'router_directive_total', value: @directive_count, type: 'counter') if @metric
+      @metric.set(name: 'router_tlm_total', value: @count, type: 'counter') if @metric
     end
 
     def start
@@ -206,9 +247,16 @@ module OpenC3
     end
 
     def run
-      RouterTopic.receive_telemetry(@router, scope: @scope) do |topic, msg_hash|
+      RouterTopic.receive_telemetry(@router, scope: @scope) do |topic, msg_id, msg_hash, redis|
+        msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
+        delta = Time.now.to_f - msgid_seconds_from_epoch
+        @metric.set(name: 'router_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and router tlm start') if @metric
+
         # Check for commands to the router itself
         if /CMD}ROUTER/.match?(topic)
+          @directive_count += 1
+          @metric.set(name: 'router_directive_total', value: @directive_count, type: 'counter') if @metric
+
           if msg_hash['shutdown']
             @logger.info "#{@router.name}: Shutdown requested"
             return
@@ -234,10 +282,35 @@ module OpenC3
               @router.stop_raw_logging
             end
           end
+          if msg_hash.key?('router_cmd')
+            params = JSON.parse(msg_hash['router_cmd'], allow_nan: true, create_additions: true)
+            begin
+              @logger.info "#{@router.name}: router_cmd: #{params['cmd_name']} #{params['cmd_params'].join(' ')}"
+              @router.interface_cmd(params['cmd_name'], params['cmd_params'])
+            rescue => e
+              @logger.error "#{@router.name}: router_cmd: #{e.formatted}"
+              next e.message
+            end
+            next 'SUCCESS'
+          end
+          if msg_hash.key?('protocol_cmd')
+            params = JSON.parse(msg_hash['protocol_cmd'], allow_nan: true, create_additions: true)
+            begin
+              @logger.info "#{@router.name}: protocol_cmd: #{params['cmd_name']} #{params['cmd_params'].join(' ')} read_write: #{params['read_write']} index: #{params['index']}"
+              @router.protocol_cmd(params['cmd_name'], params['cmd_params'], read_write: params['read_write'], index: params['index'])
+            rescue => e
+              @logger.error "#{@router.name}: protoco_cmd: #{e.formatted}"
+              next e.message
+            end
+            next 'SUCCESS'
+          end
           next 'SUCCESS'
         end
 
         if @router.connected?
+          @count += 1
+          @metric.set(name: 'router_tlm_total', value: @count, type: 'counter') if @metric
+
           target_name = msg_hash["target_name"]
           packet_name = msg_hash["packet_name"]
 
@@ -264,7 +337,14 @@ module OpenC3
     UNKNOWN_BYTES_TO_PRINT = 16
 
     def initialize(name)
+      @mutex = Mutex.new
       super(name)
+      if @interface_or_router == 'INTERFACE'
+        @metric.set(name: 'interface_tlm_total', value: @count, type: 'counter')
+      else
+        @metric.set(name: 'router_cmd_total', value: @count, type: 'counter')
+      end
+
       @interface_or_router = self.class.name.to_s.split("Microservice")[0].upcase.split("::")[-1]
       @scope = name.split("__")[0]
       interface_name = name.split("__")[2]
@@ -307,11 +387,10 @@ module OpenC3
       @cancel_thread = false
       @connection_failed_messages = []
       @connection_lost_messages = []
-      @mutex = Mutex.new
       if @interface_or_router == 'INTERFACE'
-        @handler_thread = InterfaceCmdHandlerThread.new(@interface, self, logger: @logger, scope: @scope)
+        @handler_thread = InterfaceCmdHandlerThread.new(@interface, self, logger: @logger, metric: @metric, scope: @scope)
       else
-        @handler_thread = RouterTlmHandlerThread.new(@interface, self, logger: @logger, scope: @scope)
+        @handler_thread = RouterTlmHandlerThread.new(@interface, self, logger: @logger, metric: @metric, scope: @scope)
       end
       @handler_thread.start
     end
@@ -382,6 +461,11 @@ module OpenC3
                 if packet
                   handle_packet(packet)
                   @count += 1
+                  if @interface_or_router == 'INTERFACE'
+                    @metric.set(name: 'interface_tlm_total', value: @count, type: 'counter')
+                  else
+                    @metric.set(name: 'router_cmd_total', value: @count, type: 'counter')
+                  end
                 else
                   @logger.info "#{@interface.name}: Internal disconnect requested (returned nil)"
                   handle_connection_lost()
@@ -560,7 +644,7 @@ module OpenC3
 
     # Disconnect from the interface and stop the thread
     def stop
-      @logger.info "#{@interface.name}: stop requested"
+      @logger.info "#{@interface ? @interface.name : @name}: stop requested"
       @mutex.synchronize do
         # Need to make sure that @cancel_thread is set and the interface disconnected within
         # mutex to ensure that connect() is not called when we want to stop()
@@ -578,7 +662,7 @@ module OpenC3
     end
 
     def shutdown(sig = nil)
-      @logger.info "#{@interface.name}: shutdown requested"
+      @logger.info "#{@interface ? @interface.name : @name}: shutdown requested"
       stop()
       super()
     end

data/lib/openc3/microservices/log_microservice.rb

@@ -55,6 +55,9 @@ module OpenC3
       @cycle_size = 50_000_000 unless @cycle_size # ~50 MB
 
       @buffer_depth = DEFAULT_BUFFER_DEPTH unless @buffer_depth
+      @error_count = 0
+      @metric.set(name: 'log_total', value: @count, type: 'counter')
+      @metric.set(name: 'log_error_total', value: @error_count, type: 'counter')
     end
 
     def run
@@ -65,6 +68,8 @@ module OpenC3
         Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
           break if @cancel_thread
           log_data(topic, msg_id, msg_hash, redis)
+          @count += 1
+          @metric.set(name: 'log_total', value: @count, type: 'counter')
         end
       end
     end
@@ -88,7 +93,10 @@ module OpenC3
     end
 
     def log_data(topic, msg_id, msg_hash, redis)
-      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
+      delta = Time.now.to_f - msgid_seconds_from_epoch
+      @metric.set(name: 'log_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and log start')
+
       topic_split = topic.gsub(/{|}/, '').split("__") # Remove the redis hashtag curly braces
       target_name = topic_split[2]
       packet_name = topic_split[3]
@@ -103,13 +111,11 @@ module OpenC3
         data_key = "json_data"
       end
       @plws[target_name][rt_or_stored].buffered_write(packet_type, @cmd_or_tlm, target_name, packet_name, msg_hash["time"].to_i, rt_or_stored == :STORED, msg_hash[data_key], nil, topic, msg_id)
-      @count += 1
-      diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-      metric_labels = { "packet" => packet_name, "target" => target_name, "raw_or_decom" => @raw_or_decom.to_s, "cmd_or_tlm" => @cmd_or_tlm.to_s }
-      @metric.add_sample(name: "log_duration_seconds", value: diff, labels: metric_labels)
     rescue => err
       @error = err
       @logger.error("#{@name} error: #{err.formatted}")
+      @error_count += 1
+      @metric.set(name: 'log_error_total', value: @error_count, type: 'counter')
     end
 
     def shutdown

data/lib/openc3/microservices/microservice.rb

@@ -27,6 +27,7 @@ OpenC3.require_file 'fileutils'
 OpenC3.require_file 'openc3/utilities/zip'
 OpenC3.require_file 'openc3/utilities/store'
 OpenC3.require_file 'openc3/utilities/bucket'
+OpenC3.require_file 'openc3/utilities/secrets'
 OpenC3.require_file 'openc3/utilities/sleeper'
 OpenC3.require_file 'openc3/utilities/open_telemetry'
 OpenC3.require_file 'openc3/models/microservice_model'
@@ -43,6 +44,7 @@ module OpenC3
     attr_accessor :custom
     attr_accessor :scope
     attr_accessor :logger
+    attr_accessor :secrets
 
     def self.run(name = nil)
       name = ENV['OPENC3_MICROSERVICE_NAME'] unless name
@@ -93,6 +95,7 @@ module OpenC3
       @logger = Logger.new
       @logger.scope = @scope
       @logger.microservice_name = @name
+      @secrets = Secrets.getClient
 
       OpenC3.setup_open_telemetry(@name, false)
 
@@ -104,6 +107,9 @@ module OpenC3
       if @config
         @topics = @config['topics']
         @plugin = @config['plugin']
+        if @config['secrets']
+          @secrets.setup(@config['secrets'])
+        end
       else
         @config = {}
         @plugin = nil
@@ -125,7 +131,6 @@ module OpenC3
       @error = nil
       @custom = nil
       @state = 'INITIALIZED'
-      metric_name = "metric_output_duration_seconds"
       @work_dir = @config["work_dir"]
 
       if is_plugin
@@ -174,16 +179,12 @@ module OpenC3
           end
         end
       else
-        @microservice_sleeper = Sleeper.new
+        @microservice_status_sleeper = Sleeper.new
         @microservice_status_period_seconds = 5
         @microservice_status_thread = Thread.new do
           until @cancel_thread
-            start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-            @metric.output
-            diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-            @metric.add_sample(name: metric_name, value: diff, labels: {})
             MicroserviceStatusModel.set(as_json(:allow_nan => true), scope: @scope) unless @cancel_thread
-            break if @microservice_sleeper.sleep(@microservice_status_period_seconds)
+            break if @microservice_status_sleeper.sleep(@microservice_status_period_seconds)
           end
         rescue Exception => err
           @logger.error "#{@name} status thread died: #{err.formatted}"
@@ -200,10 +201,10 @@ module OpenC3
     def shutdown
       @logger.info("Shutting down microservice: #{@name}")
       @cancel_thread = true
-      @microservice_sleeper.cancel if @microservice_sleeper
+      @microservice_status_sleeper.cancel if @microservice_status_sleeper
       MicroserviceStatusModel.set(as_json(:allow_nan => true), scope: @scope)
       FileUtils.remove_entry(@temp_dir) if File.exist?(@temp_dir)
-      @metric.destroy
+      @metric.shutdown
       @logger.info("Shutting down microservice complete: #{@name}")
     end
   end

data/lib/openc3/microservices/periodic_microservice.rb

@@ -24,6 +24,11 @@ module OpenC3
     STARTUP_DELAY_SECONDS = 2 * 60 # Two Minutes
     SLEEP_PERIOD_SECONDS = 24 * 60 * 60 # Run once per day
 
+    def initialize(*args)
+      super(*args)
+      @metric.set(name: 'periodic_total', value: @count, type: 'counter')
+    end
+
     def run
       @run_sleeper = Sleeper.new
       return if @run_sleeper.sleep(STARTUP_DELAY_SECONDS)
@@ -43,6 +48,8 @@ module OpenC3
             model.update
           end
         end
+        @count += 1
+        @metric.set(name: 'periodic_total', value: @count, type: 'counter')
         break if @cancel_thread
         break if @run_sleeper.sleep(SLEEP_PERIOD_SECONDS)
       end

data/lib/openc3/microservices/reaction_microservice.rb

@@ -222,8 +222,6 @@ module OpenC3
   # queues a trigger to evaluate against the reactions. The worker will check
   # the reactions to see if it needs to fire any reactions.
   class ReactionWorker
-    REACTION_METRIC_NAME = 'reaction_duration_seconds'.freeze
-
     attr_reader :name, :scope, :share
 
     def initialize(name:, logger:, scope:, share:, ident:)
@@ -232,8 +230,6 @@ module OpenC3
       @scope = scope
       @share = share
       @ident = ident
-      @metric_output_time = 0
-      @metric = Metric.new(microservice: @name, scope: @scope)
     end
 
     def get_token(username)
@@ -269,10 +265,6 @@ module OpenC3
             process_enabled_trigger(data: data)
           end
           current_time = Time.now.to_i
-          if @metric_output_time < current_time
-            @metric.output
-            @metric_output_time = current_time + 120
-          end
         rescue StandardError => e
           @logger.error "ReactionWorker-#{@ident} failed to evaluate kind: #{kind} data: #{data}\n#{e.formatted}"
         end
@@ -281,24 +273,16 @@ module OpenC3
     end
 
     def process_enabled_trigger(data:)
-      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       @share.reaction_base.get_reactions(trigger_name: data['name']).each do | reaction |
         run_reaction(reaction: reaction)
       end
-      diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-      metric_labels = { 'type' => 'trigger', 'thread' => "worker-#{@ident}" }
-      @metric.add_sample(name: REACTION_METRIC_NAME, value: diff, labels: metric_labels)
     end
 
     def run_reaction(reaction:)
-      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       reaction.actions.each do |action|
         run_action(reaction: reaction, action: action)
       end
       @share.reaction_base.sleep(name: reaction.name)
-      diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-      metric_labels = { 'type' => 'reaction', 'thread' => "worker-#{@ident}" }
-      @metric.add_sample(name: REACTION_METRIC_NAME, value: diff, labels: metric_labels)
     end
 
     def run_action(reaction:, action:)
@@ -354,8 +338,6 @@ module OpenC3
   # The reaction snooze manager starts a thread pool and keeps track of when a
   # reaction is activated and to evalute triggers when the snooze is complete.
   class ReactionSnoozeManager
-    SNOOZE_METRIC_NAME = 'snooze_manager_duration_seconds'.freeze
-
     attr_reader :name, :scope, :share, :thread_pool
 
     def initialize(name:, logger:, scope:, share:)
@@ -366,8 +348,6 @@ module OpenC3
       @worker_count = 3
       @thread_pool = nil
       @cancel_thread = false
-      @metric = Metric.new(microservice: @name, scope: @scope)
-      @metric_output_time = 0
     end
 
     def generate_thread_pool()
@@ -385,15 +365,7 @@ module OpenC3
       loop do
         begin
           current_time = Time.now.to_i
-          start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
           manage_snoozed_reactions(current_time: current_time)
-          diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-          metric_labels = { 'type' => 'snooze', 'thread' => "manager" }
-          @metric.add_sample(name: SNOOZE_METRIC_NAME, value: diff, labels: metric_labels)
-          if @metric_output_time < current_time
-            @metric.output
-            @metric_output_time = current_time + 120
-          end
         rescue StandardError => e
           @logger.error "ReactionSnoozeManager failed to snooze reactions.\n#{e.formatted}"
         end
@@ -443,8 +415,6 @@ module OpenC3
   # reactions and triggers from redis. It then monitors the
   # AutonomicTopic for changes.
   class ReactionMicroservice < Microservice
-    ACTION_METRIC_NAME = 'reactions_duration_seconds'.freeze
-
     attr_reader :name, :scope, :share, :manager, :manager_thread
 
     def initialize(*args)
@@ -459,11 +429,8 @@ module OpenC3
       @logger.info "ReactionMicroservice running"
       @manager_thread = Thread.new { @manager.run }
       loop do
-        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         reactions = ReactionModel.all(scope: @scope)
         @share.reaction_base.setup(reactions: reactions)
-        diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-        @metric.add_sample(name: ACTION_METRIC_NAME, value: diff, labels: { 'thread' => 'microservice' })
         break if @cancel_thread
 
         block_for_updates()

data/lib/openc3/microservices/reducer_microservice.rb

@@ -73,9 +73,9 @@ module OpenC3
   end
 
   class ReducerMicroservice < Microservice
-    MINUTE_METRIC = 'reducer_minute_duration'
-    HOUR_METRIC = 'reducer_hour_duration'
-    DAY_METRIC = 'reducer_day_duration'
+    MINUTE_METRIC = 'reducer_minute_processing_seconds'
+    HOUR_METRIC = 'reducer_hour_processing_seconds'
+    DAY_METRIC = 'reducer_day_processing_seconds'
 
     # How long to wait for any currently running jobs to complete before killing them
     SHUTDOWN_DELAY_SECS = 5
@@ -105,6 +105,10 @@ module OpenC3
       @buffer_depth = 10 unless @buffer_depth
       @target_name = name.split('__')[-1]
       @packet_logs = {}
+
+      @error_count = 0
+      @metric.set(name: 'reducer_total', value: @count, type: 'counter')
+      @metric.set(name: 'reducer_error_total', value: @error_count, type: 'counter')
     end
 
     def run
@@ -148,13 +152,7 @@ module OpenC3
       start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       yield
       elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-      @metric.add_sample(
-        name: name,
-        value: elapsed,
-        labels: {
-          'target' => @target_name,
-        },
-      )
+      @metric.set(name: name, value: elapsed, type: 'gauge', unit: 'seconds')
     end
 
     def reduce_minute
@@ -366,6 +364,10 @@ module OpenC3
       file.delete # Remove the local copy
 
       write_all_entries(reducer_state, plw, type, target_name, stored)
+
+      @count += 1
+      @metric.set(name: 'reducer_total', value: @count, type: 'counter')
+
       true
     rescue => e
       if file.local_path and File.exist?(file.local_path)
@@ -373,6 +375,8 @@ module OpenC3
       else
         @logger.error("Reducer Error: #{filename}: \n#{e.formatted}")
       end
+      @error_count += 1
+      @metric.set(name: 'reducer_error_total', value: @error_count, type: 'counter')
       false
     end