openc3 5.2.0 → 5.4.0

data/lib/openc3/microservices/text_log_microservice.rb

@@ -41,6 +41,10 @@ module OpenC3
       # These settings limit the log file to 10 minutes or 50MB of data, whichever comes first
       @cycle_time = 600 unless @cycle_time # 10 minutes
       @cycle_size = 50_000_000 unless @cycle_size # ~50 MB
+
+      @error_count = 0
+      @metric.set(name: 'text_log_total', value: @count, type: 'counter')
+      @metric.set(name: 'text_error_total', value: @error_count, type: 'counter')
     end
 
     def run
@@ -52,6 +56,8 @@ module OpenC3
           break if @cancel_thread
 
           log_data(topic, msg_id, msg_hash, redis)
+          @count += 1
+          @metric.set(name: 'text_log_total', value: @count, type: 'counter')
         end
       end
     end
@@ -68,17 +74,20 @@ module OpenC3
     end
 
     def log_data(topic, msg_id, msg_hash, redis)
-      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
+      delta = Time.now.to_f - msgid_seconds_from_epoch
+      @metric.set(name: 'text_log_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and text log start')
+
       keys = msg_hash.keys
       keys.delete("time")
       entry = keys.reduce("") { |data, key| data + "#{key}: #{msg_hash[key]}\t" }
       @tlws[topic].write(msg_hash["time"].to_i, entry, topic, msg_id)
       @count += 1
-      diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-      @metric.add_sample(name: "log_duration_seconds", value: diff, labels: {})
     rescue => err
       @error = err
       @logger.error("#{@name} error: #{err.formatted}")
+      @error_count += 1
+      @metric.set(name: 'text_log_error_total', value: @error_count, type: 'counter')
     end
 
     def shutdown

data/lib/openc3/microservices/timeline_microservice.rb

@@ -278,8 +278,6 @@ module OpenC3
   # manager. Timeline will then wait for an update on the timeline
   # stream this will trigger an update again to the schedule.
   class TimelineMicroservice < Microservice
-    TIMELINE_METRIC_NAME = 'timeline_activities_duration_seconds'.freeze
-
     def initialize(name)
       super(name)
       @timeline_name = name.split('__')[2]
@@ -293,12 +291,8 @@ module OpenC3
       @logger.info "#{@name} timeine running"
       @manager_thread = Thread.new { @manager.run }
       loop do
-        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         current_activities = ActivityModel.activities(name: @timeline_name, scope: @scope)
         @schedule.update(current_activities)
-        diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-        metric_labels = { 'timeline' => @timeline_name, 'thread' => 'microservice' }
-        @metric.add_sample(name: TIMELINE_METRIC_NAME, value: diff, labels: metric_labels)
         break if @cancel_thread
 
         block_for_updates()

data/lib/openc3/microservices/trigger_group_microservice.rb

@@ -244,8 +244,6 @@ module OpenC3
   # these workers will evaluate the triggers in the kit and
   # evaluate triggers for that packet.
   class TriggerGroupWorker
-    TRIGGER_METRIC_NAME = 'trigger_eval_duration_seconds'.freeze
-
     TYPE = 'type'.freeze
     ITEM_RAW = 'raw'.freeze
     ITEM_TARGET = 'target'.freeze
@@ -266,8 +264,6 @@ module OpenC3
       @queue = queue
       @share = share
       @ident = ident
-      @metric = Metric.new(microservice: @name, scope: @scope)
-      @metric_output_time = 0
     end
 
     def run
@@ -277,11 +273,6 @@ module OpenC3
         break if topic.nil?
         begin
           evaluate_wrapper(topic: topic)
-          current_time = Time.now.to_i
-          if @metric_output_time < current_time
-            @metric.output
-            @metric_output_time = current_time + 120
-          end
         rescue StandardError => e
           @logger.error "TriggerGroupWorker-#{@ident} failed to evaluate data packet from topic: #{topic}\n#{e.formatted}"
         end
@@ -289,13 +280,8 @@ module OpenC3
       @logger.info "TriggerGroupWorker-#{@ident} exiting"
     end
 
-    # time how long each packet takes to eval and produce a metric to public
     def evaluate_wrapper(topic:)
-      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       evaluate_data_packet(topic: topic, triggers: @share.trigger_base.triggers)
-      diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-      metric_labels = { 'trigger_group' => @group, 'thread' => "worker-#{@ident}" }
-      @metric.add_sample(name: TRIGGER_METRIC_NAME, value: diff, labels: metric_labels)
     end
 
     # Each packet will be evaluated to all triggers and use the result to send
@@ -546,8 +532,6 @@ module OpenC3
   # manager. Timeline will then wait for an update on the timeline
   # stream this will trigger an update again to the schedule.
   class TriggerGroupMicroservice < Microservice
-    TRIGGER_METRIC_NAME = 'update_triggers_duration_seconds'.freeze
-
     attr_reader :name, :scope, :share, :group, :manager, :manager_thread
 
     def initialize(*args)
@@ -563,12 +547,8 @@ module OpenC3
       @logger.info "TriggerGroupMicroservice running"
       @manager_thread = Thread.new { @manager.run }
       loop do
-        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         triggers = TriggerModel.all(scope: @scope, group: @group)
         @share.trigger_base.update(triggers: triggers)
-        diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-        metric_labels = { 'trigger_group' => @group, 'thread' => 'microservice' }
-        @metric.add_sample(name: TRIGGER_METRIC_NAME, value: diff, labels: metric_labels)
         break if @cancel_thread
 
         block_for_updates()

data/lib/openc3/models/cvt_model.rb

@@ -25,30 +25,29 @@ require 'openc3/utilities/store'
 module OpenC3
   class CvtModel
     VALUE_TYPES = [:RAW, :CONVERTED, :FORMATTED, :WITH_UNITS]
-    # Stores telemetry item overrides which are returned on every request to get_item
-    @overrides = {}
-
     def self.build_json_from_packet(packet)
       packet.decom
     end
 
     # Delete the current value table for a target
-    def self.del(target_name:, packet_name:, scope:)
+    def self.del(target_name:, packet_name:, scope: $openc3_scope)
       Store.hdel("#{scope}__tlm__#{target_name}", packet_name)
     end
 
     # Set the current value table for a target, packet
-    def self.set(hash, target_name:, packet_name:, scope:)
+    def self.set(hash, target_name:, packet_name:, scope: $openc3_scope)
       Store.hset("#{scope}__tlm__#{target_name}", packet_name, JSON.generate(hash.as_json(:allow_nan => true)))
     end
 
     # Set an item in the current value table
-    def self.set_item(target_name, packet_name, item_name, value, type:, scope:)
+    def self.set_item(target_name, packet_name, item_name, value, type:, scope: $openc3_scope)
       case type
       when :WITH_UNITS
         field = "#{item_name}__U"
+        value = value.to_s # WITH_UNITS should always be a string
       when :FORMATTED
         field = "#{item_name}__F"
+        value = value.to_s # FORMATTED should always be a string
       when :CONVERTED
         field = "#{item_name}__C"
       when :RAW
@@ -62,27 +61,37 @@ module OpenC3
     end
 
     # Get an item from the current value table
-    def self.get_item(target_name, packet_name, item_name, type:, scope:)
-      if @overrides["#{target_name}__#{packet_name}__#{item_name}__#{type}"]
-        return @overrides["#{target_name}__#{packet_name}__#{item_name}__#{type}"]
-      end
-
+    def self.get_item(target_name, packet_name, item_name, type:, scope: $openc3_scope)
+      override_key = item_name
       types = []
       case type
       when :WITH_UNITS
         types = ["#{item_name}__U", "#{item_name}__F", "#{item_name}__C", item_name]
+        override_key = "#{item_name}__U"
       when :FORMATTED
         types = ["#{item_name}__F", "#{item_name}__C", item_name]
+        override_key = "#{item_name}__F"
       when :CONVERTED
         types = ["#{item_name}__C", item_name]
+        override_key = "#{item_name}__C"
       when :RAW
         types = [item_name]
       else
         raise "Unknown type '#{type}' for #{target_name} #{packet_name} #{item_name}"
       end
+      overrides = Store.hget("#{scope}__override__#{target_name}", packet_name)
+      if overrides
+        result = JSON.parse(overrides, :allow_nan => true, :create_additions => true)[override_key]
+        return result if result
+      end
       hash = JSON.parse(Store.hget("#{scope}__tlm__#{target_name}", packet_name), :allow_nan => true, :create_additions => true)
       hash.values_at(*types).each do |result|
-        return result if result
+        if result
+          if type == :FORMATTED or type == :WITH_UNITS
+            return result.to_s
+          end
+          return result
+        end
       end
       return nil
     end
@@ -97,10 +106,11 @@ module OpenC3
       results = []
       lookups = []
       packet_lookup = {}
+      overrides = {}
       # First generate a lookup hash of all the items represented so we can query the CVT
-      items.each { |item| _parse_item(lookups, item) }
+      items.each { |item| _parse_item(lookups, overrides, item, scope: scope) }
 
-      lookups.each do |target_packet_key, target_name, packet_name, packet_values|
+      lookups.each do |target_packet_key, target_name, packet_name, value_keys|
         unless packet_lookup[target_packet_key]
           packet = Store.hget("#{scope}__tlm__#{target_name}", packet_name)
           raise "Packet '#{target_name} #{packet_name}' does not exist" unless packet
@@ -108,23 +118,26 @@ module OpenC3
         end
         hash = packet_lookup[target_packet_key]
         item_result = []
-        packet_values.each do |value|
-          item_result[0] = hash[value]
-          break if item_result[0] # We want the first value
-        end
-        # If we were able to find a value, try to get the limits state
-        if item_result[0]
-          if now - hash['RECEIVED_TIMESECONDS'] > stale_time
-            item_result[1] = :STALE
+        if value_keys.is_a?(Hash) # Set in _parse_item to indicate override
+          item_result[0] = value_keys['value']
+        else
+          value_keys.each do |key|
+            item_result[0] = hash[key]
+            break if item_result[0] # We want the first value
+          end
+          # If we were able to find a value, try to get the limits state
+          if item_result[0]
+            if now - hash['RECEIVED_TIMESECONDS'] > stale_time
+              item_result[1] = :STALE
+            else
+              # The last key is simply the name (RAW) so we can append __L
+              # If there is no limits then it returns nil which is acceptable
+              item_result[1] = hash["#{value_keys[-1]}__L"]
+              item_result[1] = item_result[1].intern if item_result[1] # Convert to symbol
+            end
           else
-            # The last key is simply the name (RAW) so we can append __L
-            # If there is no limits then it returns nil which is acceptable
-            item_result[1] = hash["#{packet_values[-1]}__L"]
-            item_result[1] = item_result[1].intern if item_result[1] # Convert to symbol
+            raise "Item '#{target_name} #{packet_name} #{value_keys[-1]}' does not exist" unless hash.key?(value_keys[-1])
           end
-        else
-          raise "Item '#{target_name} #{packet_name} #{packet_values[-1]}' does not exist" unless hash.key?(packet_values[-1])
-          item_result[1] = nil
         end
         results << item_result
       end
@@ -133,35 +146,64 @@ module OpenC3
 
     # Override a current value table item such that it always returns the same value
     # for the given type
-    def self.override(target_name, packet_name, item_name, value, type:, scope: $openc3_scope)
-      if VALUE_TYPES.include?(type)
-        @overrides["#{target_name}__#{packet_name}__#{item_name}__#{type}"] = value
+    def self.override(target_name, packet_name, item_name, value, type: :ALL, scope: $openc3_scope)
+      hash = Store.hget("#{scope}__override__#{target_name}", packet_name)
+      hash = JSON.parse(hash, :allow_nan => true, :create_additions => true) if hash
+      hash ||= {} # In case the above didn't create anything
+      case type
+      when :ALL
+        hash[item_name] = value
+        hash["#{item_name}__C"] = value
+        hash["#{item_name}__F"] = value.to_s
+        hash["#{item_name}__U"] = value.to_s
+      when :RAW
+        hash[item_name] = value
+      when :CONVERTED
+        hash["#{item_name}__C"] = value
+      when :FORMATTED
+        hash["#{item_name}__F"] = value.to_s # Always a String
+      when :WITH_UNITS
+        hash["#{item_name}__U"] = value.to_s # Always a String
       else
         raise "Unknown type '#{type}' for #{target_name} #{packet_name} #{item_name}"
       end
+      Store.hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json(:allow_nan => true)))
     end
 
     # Normalize a current value table item such that it returns the actual value
     def self.normalize(target_name, packet_name, item_name, type: :ALL, scope: $openc3_scope)
-      if type == :ALL
-        VALUE_TYPES.each do |type|
-          @overrides.delete("#{target_name}__#{packet_name}__#{item_name}__#{type}")
-        end
+      hash = Store.hget("#{scope}__override__#{target_name}", packet_name)
+      hash = JSON.parse(hash, :allow_nan => true, :create_additions => true) if hash
+      hash ||= {} # In case the above didn't create anything
+      case type
+      when :ALL
+        hash.delete(item_name)
+        hash.delete("#{item_name}__C")
+        hash.delete("#{item_name}__F")
+        hash.delete("#{item_name}__U")
+      when :RAW
+        hash.delete(item_name)
+      when :CONVERTED
+        hash.delete("#{item_name}__C")
+      when :FORMATTED
+        hash.delete("#{item_name}__F")
+      when :WITH_UNITS
+        hash.delete("#{item_name}__U")
       else
-        if VALUE_TYPES.include?(type)
-          @overrides.delete("#{target_name}__#{packet_name}__#{item_name}__#{type}")
-        else
-          raise "Unknown type '#{type}' for #{target_name} #{packet_name} #{item_name}"
-        end
+        raise "Unknown type '#{type}' for #{target_name} #{packet_name} #{item_name}"
+      end
+      if hash.empty?
+        Store.hdel("#{scope}__override__#{target_name}", packet_name)
+      else
+        Store.hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json(:allow_nan => true)))
       end
     end
 
     # PRIVATE METHODS
 
-    def self._parse_item(lookups, item)
-      # parse item and update lookups with packet_name and target_name and keys
-      #
-      # return an ordered array of hash with keys
+    # parse item and update lookups with packet_name and target_name and keys
+    # return an ordered array of hash with keys
+    def self._parse_item(lookups, overrides, item, scope:)
       target_name, packet_name, item_name, value_type = item.split('__')
       raise ArgumentError, "items must be formatted as TGT__PKT__ITEM__TYPE" if target_name.nil? || packet_name.nil? || item_name.nil? || value_type.nil?
 
@@ -177,9 +219,23 @@ module OpenC3
       when 'WITH_UNITS'
         keys = ["#{item_name}__U", "#{item_name}__F", "#{item_name}__C", item_name]
       else
-        raise "Unknown value type #{value_type}"
+        raise "Unknown value type '#{value_type}'"
+      end
+      tgt_pkt_key = "#{target_name}__#{packet_name}"
+      # Check the overrides cache for this target / packet
+      unless overrides[tgt_pkt_key]
+        override_data = Store.hget("#{scope}__override__#{target_name}", packet_name)
+        if override_data
+          overrides[tgt_pkt_key] = JSON.parse(override_data, :allow_nan => true, :create_additions => true)
+        else
+          overrides[tgt_pkt_key] = {}
+        end
+      end
+      if overrides[tgt_pkt_key][keys[0]]
+        # Set the result as a Hash to distingish it from the key array and from an overridden Array value
+        keys = {'value' => overrides[tgt_pkt_key][keys[0]]}
       end
-      lookups << ["#{target_name}__#{packet_name}", target_name, packet_name, keys]
+      lookups << [tgt_pkt_key, target_name, packet_name, keys]
     end
   end
 end

data/lib/openc3/models/interface_model.rb

@@ -38,11 +38,13 @@ module OpenC3
     attr_accessor :reconnect_delay
     attr_accessor :disable_disconnect
     attr_accessor :options
+    attr_accessor :secret_options
     attr_accessor :protocols
     attr_accessor :interfaces
     attr_accessor :log
     attr_accessor :log_raw
     attr_accessor :needs_dependencies
+    attr_accessor :secrets
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
@@ -101,12 +103,14 @@ module OpenC3
       reconnect_delay: 5.0,
       disable_disconnect: false,
       options: [],
+      secret_options: [],
       protocols: [],
       log: true,
       log_raw: false,
       updated_at: nil,
       plugin: nil,
       needs_dependencies: false,
+      secrets: [],
       scope:
     )
       if self.class._get_type == 'INTERFACE'
@@ -123,10 +127,12 @@ module OpenC3
       @reconnect_delay = reconnect_delay
       @disable_disconnect = disable_disconnect
       @options = options
+      @secret_options = secret_options
       @protocols = protocols
       @log = log
       @log_raw = log_raw
       @needs_dependencies = needs_dependencies
+      @secrets = secrets
     end
 
     # Called by InterfaceMicroservice to instantiate the Interface defined
@@ -139,6 +145,7 @@ module OpenC3
       else
         interface_or_router = klass.new
       end
+      interface_or_router.secrets.setup(@secrets)
       interface_or_router.target_names = @target_names.dup
       interface_or_router.cmd_target_names = @cmd_target_names.dup
       interface_or_router.tlm_target_names = @tlm_target_names.dup
@@ -149,6 +156,11 @@ module OpenC3
       @options.each do |option|
         interface_or_router.set_option(option[0], option[1..-1])
       end
+      @secret_options.each do |option|
+        secret_name = option[1]
+        secret_value = interface_or_router.secrets.get(secret_name, scope: @scope)
+        interface_or_router.set_option(option[0], [secret_value])
+      end
       @protocols.each do |protocol|
         klass = OpenC3.require_class(protocol[1])
         interface_or_router.add_protocol(klass, protocol[2..-1], protocol[0].upcase.intern)
@@ -168,11 +180,13 @@ module OpenC3
         'reconnect_delay' => @reconnect_delay,
         'disable_disconnect' => @disable_disconnect,
         'options' => @options,
+        'secret_options' => @secret_options,
         'protocols' => @protocols,
         'log' => @log,
         'log_raw' => @log_raw,
         'plugin' => @plugin,
         'needs_dependencies' => @needs_dependencies,
+        'secrets' => @secrets.as_json(*a),
         'updated_at' => @updated_at
       }
     end
@@ -242,6 +256,14 @@ module OpenC3
         parser.verify_num_parameters(0, 0, "#{keyword}")
         @log_raw = true
 
+      when 'SECRET'
+        parser.verify_num_parameters(3, 4, "#{keyword} <Secret Type: ENV or FILE> <Secret Name> <Environment Variable Name or File Path> <Option Name (Optional)>")
+        @secrets << parameters[0..2]
+        if parameters[3]
+          # Option Name, Secret Name
+          @secret_options << [parameters[3], parameters[1]]
+        end
+
       else
         raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Interface/Router: #{keyword} #{parameters.join(" ")}")
 
@@ -261,6 +283,7 @@ module OpenC3
         target_names: @target_names,
         plugin: @plugin,
         needs_dependencies: @needs_dependencies,
+        secrets: @secrets,
         scope: @scope
       )
       unless validate_only

data/lib/openc3/models/metric_model.rb

@@ -17,7 +17,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'openc3/models/model'
@@ -26,6 +26,8 @@ module OpenC3
   class MetricModel < EphemeralModel
     PRIMARY_KEY = '__openc3__metric'.freeze
 
+    attr_accessor :values
+
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
     def self.get(name:, scope:)
@@ -44,19 +46,64 @@ module OpenC3
       EphemeralStore.hdel("#{scope}#{PRIMARY_KEY}", name)
     end
 
-    def initialize(name:, scope:, metric_name:, label_list:)
+    def initialize(name:, values: {}, scope:)
       super("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
-      @metric_name = metric_name
-      @label_list = label_list
+      @values = values
     end
 
     def as_json(*a)
       {
         'name' => @name,
         'updated_at' => @updated_at,
-        'metric_name' => @metric_name,
-        'label_list' => @label_list
+        'values' => @values.as_json(*a)
       }
     end
+
+    def self.redis_extract_p50_and_p99_seconds(value)
+      if value
+        split_value = value.to_s.split(',')
+        p50 = split_value[0].split('=')[-1].to_f / 1_000_000
+        p99 = split_value[-1].split('=')[-1].to_f / 1_000_000
+        return p50, p99
+      else
+        return 0.0, 0.0
+      end
+    end
+
+    def self.redis_metrics
+      result = {}
+
+      metrics = OpenC3::Store.info("all")
+      result['redis_connected_clients_total'] = metrics['connected_clients']
+      result['redis_used_memory_rss_total'] = metrics['used_memory_rss']
+      result['redis_commands_processed_total'] = metrics['total_commands_processed']
+      result['redis_iops'] = metrics['instantaneous_ops_per_sec']
+      result['redis_instantaneous_input_kbps'] = metrics['instantaneous_input_kbps']
+      result['redis_instantaneous_output_kbps'] = metrics['instantaneous_output_kbps']
+      result['redis_hget_p50_seconds'], result['redis_hget_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
+      result['redis_hgetall_p50_seconds'], result['redis_hgetall_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
+      result['redis_hset_p50_seconds'], result['redis_hset_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
+      result['redis_xadd_p50_seconds'], result['redis_xadd_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
+      result['redis_xread_p50_seconds'], result['redis_xread_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
+      result['redis_xrevrange_p50_seconds'], result['redis_xrevrange_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
+      result['redis_xtrim_p50_seconds'], result['redis_xtrim_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
+
+      metrics = OpenC3::EphemeralStore.info("all")
+      result['redis_ephemeral_connected_clients_total'] = metrics['connected_clients']
+      result['redis_ephemeral_used_memory_rss_total'] = metrics['used_memory_rss']
+      result['redis_ephemeral_commands_processed_total'] = metrics['total_commands_processed']
+      result['redis_ephemeral_iops'] = metrics['instantaneous_ops_per_sec']
+      result['redis_ephemeral_instantaneous_input_kbps'] = metrics['instantaneous_input_kbps']
+      result['redis_ephemeral_instantaneous_output_kbps'] = metrics['instantaneous_output_kbps']
+      result['redis_ephemeral_hget_p50_seconds'], result['redis_ephemeral_hget_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
+      result['redis_ephemeral_hgetall_p50_seconds'], result['redis_ephemeral_hgetall_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
+      result['redis_ephemeral_hset_p50_seconds'], result['redis_ephemeral_hset_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
+      result['redis_ephemeral_xadd_p50_seconds'], result['redis_ephemeral_xadd_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
+      result['redis_ephemeral_xread_p50_seconds'], result['redis_ephemeral_xread_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
+      result['redis_ephemeral_xrevrange_p50_seconds'], result['redis_ephemeral_xrevrange_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
+      result['redis_ephemeral_xtrim_p50_seconds'], result['redis_ephemeral_xtrim_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
+
+      return result
+    end
   end
 end

data/lib/openc3/models/microservice_model.rb

@@ -22,6 +22,7 @@
 
 require 'openc3/top_level'
 require 'openc3/models/model'
+require 'openc3/models/metric_model'
 require 'openc3/utilities/bucket'
 
 module OpenC3
@@ -39,6 +40,7 @@ module OpenC3
     attr_accessor :work_dir
     attr_accessor :ports
     attr_accessor :parent
+    attr_accessor :secrets
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
@@ -96,6 +98,7 @@ module OpenC3
       updated_at: nil,
       plugin: nil,
       needs_dependencies: false,
+      secrets: [],
       scope:
     )
       parts = name.split("__")
@@ -118,6 +121,7 @@ module OpenC3
       @parent = parent
       @container = container
       @needs_dependencies = needs_dependencies
+      @secrets = secrets
       @bucket = Bucket.getClient()
     end
 
@@ -137,6 +141,7 @@ module OpenC3
         'updated_at' => @updated_at,
         'plugin' => @plugin,
         'needs_dependencies' => @needs_dependencies,
+        'secrets' => @secrets.as_json(*a)
       }
     end
 
@@ -182,6 +187,9 @@ module OpenC3
       when 'CONTAINER'
         parser.verify_num_parameters(1, 1, "#{keyword} <Container Image Name>")
         @container = parameters[0]
+      when 'SECRET'
+        parser.verify_num_parameters(3, 3, "#{keyword} <Secret Type: ENV or FILE> <Secret Name> <Environment Variable Name or File Path>")
+        @secrets << parameters.dup
       else
         raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Microservice: #{keyword} #{parameters.join(" ")}")
       end
@@ -202,7 +210,7 @@ module OpenC3
         # Load microservice files
         data = File.read(filename, mode: "rb")
         OpenC3.set_working_dir(File.dirname(filename)) do
-          data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable?
+          data = ERB.new(data, trim_mode: "-").result(binding.set_variables(variables)) if data.is_printable? and File.basename(filename)[0] != '_'
         end
         unless validate_only
           @bucket.put_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: key, body: data)
@@ -220,5 +228,11 @@ module OpenC3
     rescue Exception => error
       Logger.error("Error undeploying microservice model #{@name} in scope #{@scope} due to #{error}")
     end
+
+    def cleanup
+      # Cleanup metrics
+      metric_model = MetricModel.new(name: @name, scope: @scope)
+      metric_model.destroy
+    end
   end
 end

data/lib/openc3/models/model.rb

@@ -154,7 +154,7 @@ module OpenC3
         end
       end
       @updated_at = Time.now.to_nsec_from_epoch
-      self.class.store.hset(@primary_key, @name, JSON.generate(self.as_json(:allow_nan => true)))
+      self.class.store.hset(@primary_key, @name, JSON.generate(self.as_json(:allow_nan => true), :allow_nan => true))
     end
 
     # Alias for create(update: true)

data/lib/openc3/models/plugin_model.rb

@@ -269,7 +269,8 @@ module OpenC3
     # Undeploy all models associated with this plugin
     def undeploy
       microservice_count = 0
-      MicroserviceModel.find_all_by_plugin(plugin: @name, scope: @scope).each do |name, model_instance|
+      microservices = MicroserviceModel.find_all_by_plugin(plugin: @name, scope: @scope)
+      microservices.each do |name, model_instance|
         model_instance.destroy
         microservice_count += 1
       end
@@ -282,6 +283,10 @@ module OpenC3
           model_instance.destroy
         end
       end
+      # Cleanup Redis stuff that might have been left by microservices
+      microservices.each do |name, model_instance|
+        model_instance.cleanup
+      end
     end
 
     # Reinstall