openc3 7.0.1 → 7.1.0

data/lib/openc3/microservices/decom_microservice.rb

@@ -18,6 +18,7 @@
 require 'time'
 require 'thread'
 require 'openc3/microservices/microservice'
+require 'openc3/microservices/decom_common'
 require 'openc3/microservices/interface_decom_common'
 require 'openc3/microservices/interface_microservice'
 require 'openc3/topics/telemetry_decom_topic'
@@ -90,7 +91,9 @@ module OpenC3
       if @name =~ /__DECOM__/
         @topics << "#{@scope}__DECOMINTERFACE__{#{@target_names[0]}}"
       end
-      Topic.update_topic_offsets(@topics)
+      @limits_event_topic = "#{@scope}__openc3_limits_events"
+      @topics << @limits_event_topic
+      Topic.update_topic_offsets(@topics, db_shard: @db_shard)
       System.telemetry.limits_change_callback = method(:limits_change_callback)
       LimitsEventTopic.sync_system(scope: @scope)
       @error_count = 0
@@ -110,10 +113,13 @@ module OpenC3
 
         begin
           OpenC3.in_span("read_topics") do
-            Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
+            Topic.read_topics(@topics, db_shard: @db_shard) do |topic, msg_id, msg_hash, redis|
               break if @cancel_thread
               if topic == @microservice_topic
                 microservice_cmd(topic, msg_id, msg_hash, redis)
+              elsif topic == @limits_event_topic
+                event = JSON.parse(msg_hash['event'], allow_nan: true, create_additions: true)
+                LimitsEventTopic.process_event(event)
               elsif topic =~ /__DECOMINTERFACE/
                 if msg_hash.key?('inject_tlm')
                   handle_inject_tlm_with_ack(msg_hash['inject_tlm'], msg_id)
@@ -134,7 +140,6 @@ module OpenC3
               @count += 1
             end
           end
-          LimitsEventTopic.sync_system_thread_body(scope: @scope)
         rescue => e
           @error_count += 1
           @metric.set(name: 'decom_error_total', value: @error_count, type: 'counter')
@@ -153,8 +158,6 @@ module OpenC3
         delta = Time.now.to_f - msgid_seconds_from_epoch
         @metric.set(name: 'decom_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and decom start')
 
-        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
         #######################################
         # Build packet object from topic data
         #######################################
@@ -172,98 +175,21 @@ module OpenC3
         end
         packet.buffer = msg_hash["buffer"]
 
-        ################################################################################
-        # Break packet into subpackets (if necessary)
-        # Subpackets are typically channelized data
-        ################################################################################
-        packet_and_subpackets = packet.subpacketize
-
-        packet_and_subpackets.each do |packet_or_subpacket|
-          if packet_or_subpacket.subpacket
-            packet_or_subpacket = handle_subpacket(packet, packet_or_subpacket)
-          end
-
-          #####################################################################################
-          # Run Processors
-          # This must be before the full decom so that processor derived values are available
-          #####################################################################################
-          begin
-            packet_or_subpacket.process # Run processors
-          rescue Exception => e
+        DecomCommon.decom_and_publish(
+          packet,
+          scope: @scope,
+          target_names: @target_names,
+          logger: @logger,
+          name: @name,
+          check_limits: true,
+          metric: @metric,
+          error_callback: ->(e) {
             @error_count += 1
             @metric.set(name: 'decom_error_total', value: @error_count, type: 'counter')
             @error = e
-            @logger.error e.message
-          end
-
-          #############################################################################
-          # Process all the limits and call the limits_change_callback (as necessary)
-          # This must be before the full decom so that limits states are available
-          #############################################################################
-          packet_or_subpacket.check_limits(System.limits_set)
-
-          # This is what actually decommutates the packet and updates the CVT
-          TelemetryDecomTopic.write_packet(packet_or_subpacket, scope: @scope)
-        end
-
-        diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-        @metric.set(name: 'decom_duration_seconds', value: diff, type: 'gauge', unit: 'seconds')
-      end
-    end
-
-    def handle_subpacket(packet, subpacket)
-      # Subpacket received time always = packet.received_time
-      # Use packet_time appropriately if another timestamp is needed
-      subpacket.received_time = packet.received_time
-      subpacket.stored = packet.stored
-      subpacket.extra = packet.extra
-
-      if subpacket.stored
-        # Stored telemetry does not update the current value table
-        identified_subpacket = System.telemetry.identify_and_define_packet(subpacket, @target_names, subpackets: true)
-      else
-        # Identify and update subpacket
-        if subpacket.identified?
-          begin
-            # Preidentifed subpacket - place it into the current value table
-            identified_subpacket = System.telemetry.update!(subpacket.target_name,
-                                                            subpacket.packet_name,
-                                                            subpacket.buffer)
-          rescue RuntimeError
-            # Subpacket identified but we don't know about it
-            # Clear packet_name and target_name and try to identify
-            @logger.warn "#{@name}: Received unknown identified subpacket: #{subpacket.target_name} #{subpacket.packet_name}"
-            subpacket.target_name = nil
-            subpacket.packet_name = nil
-            identified_subpacket = System.telemetry.identify!(subpacket.buffer,
-                                                              @target_names, subpackets: true)
-          end
-        else
-          # Packet needs to be identified
-          identified_subpacket = System.telemetry.identify!(subpacket.buffer,
-                                                            @target_names, subpackets: true)
-        end
+          },
+        )
       end
-
-      if identified_subpacket
-        identified_subpacket.received_time = subpacket.received_time
-        identified_subpacket.stored = subpacket.stored
-        identified_subpacket.extra = subpacket.extra
-        subpacket = identified_subpacket
-      else
-        unknown_subpacket = System.telemetry.update!('UNKNOWN', 'UNKNOWN', subpacket.buffer)
-        unknown_subpacket.received_time = subpacket.received_time
-        unknown_subpacket.stored = subpacket.stored
-        unknown_subpacket.extra = subpacket.extra
-        subpacket = unknown_subpacket
-        num_bytes_to_print = [InterfaceMicroservice::UNKNOWN_BYTES_TO_PRINT, subpacket.length].min
-        data = subpacket.buffer(false)[0..(num_bytes_to_print - 1)]
-        prefix = data.each_byte.map { | byte | sprintf("%02X", byte) }.join()
-        @logger.warn "#{@name} #{subpacket.target_name} packet length: #{subpacket.length} starting with: #{prefix}"
-      end
-
-      TargetModel.sync_tlm_packet_counts(subpacket, @target_names, scope: @scope)
-      return subpacket
     end
 
     # Called when an item in any packet changes limits states.
@@ -285,7 +211,12 @@ module OpenC3
       if value
         message = "#{packet.target_name} #{packet.packet_name} #{item.name} = #{value} is #{item.limits.state}"
         if item.limits.values
-          values = item.limits.values[System.limits_set]
+          selected_limits_set = if item.limits.values.has_key?(System.limits_set)
+            System.limits_set
+          else
+            :DEFAULT
+          end
+          values = item.limits.values[selected_limits_set]
           # Check if the state is RED_LOW, YELLOW_LOW, YELLOW_HIGH, RED_HIGH, GREEN_LOW, GREEN_HIGH
           if LIMITS_STATE_INDEX[item.limits.state]
             # Directly index into the values and return the value
@@ -335,4 +266,4 @@ if __FILE__ == $0
   OpenC3::DecomMicroservice.run
   OpenC3::ThreadManager.instance.shutdown
   OpenC3::ThreadManager.instance.join
-end
+end

data/lib/openc3/microservices/interface_decom_common.rb

@@ -31,6 +31,8 @@ module OpenC3
           packet.write(name.to_s, value, type)
         end
       end
+      stored = inject_tlm_hash.fetch('stored', false)
+      packet.stored = stored.to_s.downcase == 'true'
       packet.received_time = Time.now.sys
       packet.received_count = TargetModel.increment_telemetry_count(packet.target_name, packet.packet_name, 1, scope: @scope)
       TelemetryTopic.write_packet(packet, scope: @scope)
@@ -63,6 +65,7 @@ module OpenC3
       cmd_params = build_cmd_hash['cmd_params']
       range_check = build_cmd_hash['range_check']
       raw = build_cmd_hash['raw']
+      db_shard = Store.db_shard_for_target(target_name, scope: @scope)
       ack_topic = "{#{@scope}__ACKCMD}TARGET__#{target_name}"
       begin
         command = System.commands.build_cmd(target_name, cmd_name, cmd_params, range_check, raw)
@@ -84,13 +87,14 @@ module OpenC3
           result: error.message
         }
       end
-      Topic.write_topic(ack_topic, msg_hash)
+      Topic.write_topic(ack_topic, msg_hash, db_shard: db_shard)
     end
 
     def handle_get_tlm_buffer(get_tlm_buffer_json, msg_id)
       get_tlm_buffer_hash = JSON.parse(get_tlm_buffer_json, allow_nan: true, create_additions: true)
       target_name = get_tlm_buffer_hash['target_name']
       packet_name = get_tlm_buffer_hash['packet_name']
+      db_shard = Store.db_shard_for_target(target_name, scope: @scope)
       ack_topic = "{#{@scope}__ACKCMD}TARGET__#{target_name}"
       begin
         packet = System.telemetry.packet(target_name, packet_name)
@@ -115,7 +119,7 @@ module OpenC3
           result: error.message
         }
       end
-      Topic.write_topic(ack_topic, msg_hash)
+      Topic.write_topic(ack_topic, msg_hash, db_shard: db_shard)
     end
   end
 end

data/lib/openc3/microservices/interface_microservice.rb

@@ -42,10 +42,11 @@ module OpenC3
   class InterfaceCmdHandlerThread
     include InterfaceDecomCommon
 
-    def initialize(interface, tlm, logger: nil, metric: nil, scope:)
+    def initialize(interface, tlm, logger: nil, metric: nil, db_shard: 0, scope:)
       @interface = interface
       @tlm = tlm
       @scope = scope
+      @db_shard = db_shard.to_i
       scope_model = ScopeModel.get_model(name: @scope)
       if scope_model
         @critical_commanding = scope_model.critical_commanding
@@ -81,7 +82,7 @@ module OpenC3
     end
 
     def run
-      InterfaceTopic.receive_commands(@interface, scope: @scope) do |topic, msg_id, msg_hash, _redis|
+      InterfaceTopic.receive_commands(@interface, scope: @scope, db_shard: @db_shard) do |topic, msg_id, msg_hash, _redis|
         OpenC3.with_context(msg_hash) do
           release_critical = false
           critical_model = nil
@@ -105,7 +106,7 @@ module OpenC3
             @metric.set(name: 'interface_directive_total', value: @directive_count, type: 'counter') if @metric
             if msg_hash['shutdown']
               @logger.info "#{@interface.name}: Shutdown requested"
-              InterfaceTopic.clear_topics(InterfaceTopic.topics(@interface, scope: @scope))
+              InterfaceTopic.clear_topics(InterfaceTopic.topics(@interface, scope: @scope), db_shard: @db_shard)
               return
             end
             if msg_hash['connect']
@@ -379,10 +380,11 @@ module OpenC3
   end
 
   class RouterTlmHandlerThread
-    def initialize(router, tlm, logger: nil, metric: nil, scope:)
+    def initialize(router, tlm, logger: nil, metric: nil, db_shard: 0, scope:)
       @router = router
       @tlm = tlm
       @scope = scope
+      @db_shard = db_shard.to_i
       @logger = logger
       @logger = Logger unless @logger
       @metric = metric
@@ -412,7 +414,7 @@ module OpenC3
     end
 
     def run
-      RouterTopic.receive_telemetry(@router, scope: @scope) do |topic, msg_id, msg_hash, _redis|
+      RouterTopic.receive_telemetry(@router, scope: @scope, db_shard: @db_shard) do |topic, msg_id, msg_hash, _redis|
         msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
         delta = Time.now.to_f - msgid_seconds_from_epoch
         @metric.set(name: 'router_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and router tlm start') if @metric
@@ -424,7 +426,7 @@ module OpenC3
 
           if msg_hash['shutdown']
             @logger.info "#{@router.name}: Shutdown requested"
-            RouterTopic.clear_topics(RouterTopic.topics(@router, scope: @scope))
+            RouterTopic.clear_topics(RouterTopic.topics(@router, scope: @scope), db_shard: @db_shard)
             return
           end
           if msg_hash['connect']
@@ -590,9 +592,9 @@ module OpenC3
       @connection_failed_messages = []
       @connection_lost_messages = []
       if @interface_or_router == 'INTERFACE'
-        @handler_thread = InterfaceCmdHandlerThread.new(@interface, self, logger: @logger, metric: @metric, scope: @scope)
+        @handler_thread = InterfaceCmdHandlerThread.new(@interface, self, logger: @logger, metric: @metric, db_shard: @db_shard, scope: @scope)
       else
-        @handler_thread = RouterTlmHandlerThread.new(@interface, self, logger: @logger, metric: @metric, scope: @scope)
+        @handler_thread = RouterTlmHandlerThread.new(@interface, self, logger: @logger, metric: @metric, db_shard: @db_shard, scope: @scope)
       end
       @handler_thread.start
     end

data/lib/openc3/microservices/log_microservice.rb

@@ -63,7 +63,7 @@ module OpenC3
       while true
         break if @cancel_thread
 
-        Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
+        Topic.read_topics(@topics, db_shard: @db_shard) do |topic, msg_id, msg_hash, redis|
           break if @cancel_thread
           if topic == @microservice_topic
             microservice_cmd(topic, msg_id, msg_hash, redis)

data/lib/openc3/microservices/microservice.rb

@@ -127,6 +127,7 @@ module OpenC3
       @logger.info("Microservice initialized with config:\n#{@config}")
       @topics ||= []
       @microservice_topic = "MICROSERVICE__#{@name}"
+      @db_shard = (@config['db_shard'] || 0).to_i
 
       # Get configuration for any targets
       @target_names = @config["target_names"]
@@ -259,10 +260,10 @@ module OpenC3
         else
           raise "Invalid topics given to microservice_cmd: #{topics}"
         end
-        Topic.trim_topic(topic, msg_id)
+        Topic.trim_topic(topic, msg_id, db_shard: @db_shard)
         return true
       end
-      Topic.trim_topic(topic, msg_id)
+      Topic.trim_topic(topic, msg_id, db_shard: @db_shard)
       return false
     end
   end

data/lib/openc3/microservices/queue_microservice.rb

@@ -19,7 +19,7 @@ require 'openc3/api/api'
 
 module OpenC3
   saved_verbose = $VERBOSE
-  $VERBOSE = false
+  $VERBOSE = nil
   module Script
     private
     # Override the prompt_for_hazardous method to always return true since there is no user to prompt

data/lib/openc3/microservices/scope_cleanup_microservice.rb

@@ -12,6 +12,7 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/models/scope_model'
+require 'openc3/models/target_model'
 require 'openc3/microservices/cleanup_microservice'
 
 module OpenC3
@@ -62,63 +63,76 @@ ORDER BY
         super(areas, bucket)
       end
 
-      # Always check TSDB health
+      # Always check TSDB health across all db_shards
       if @scope == 'DEFAULT'
+        # Collect all unique db_shard numbers from targets
+        db_shards = Set.new([0]) # Always check db_shard 0
         begin
-          conn = OpenC3::QuestDBClient.connection
-          result = conn.exec(TSDB_HEALTH_QUERY)
-          columns = result.fields
-          rows = result.values
-
-          table_name_column = columns.index("table_name")
-          wal_pending_row_count_column = columns.index("wal_pending_row_count")
-          status_column = columns.index("status")
-          lag_txns_column = columns.index("lag_txns")
-
-          rows.each do |values|
-            table_name = values[table_name_column]
-            wal_pending_row_count = values[wal_pending_row_count_column].to_i
-            status = values[status_column]
-            lag_txns = values[lag_txns_column].to_i
-
-            if status != 'OK'
-              @logger.error("QuestDB: #{table_name} in bad state: #{status}")
-
-              if status == 'SUSPENDED'
-                # Try to automatically unsuspend
-                @logger.info("QuestDB: Attempting to unsuspend: #{table_name}")
-                conn.exec("ALTER TABLE #{table_name} RESUME WAL;")
+          targets = OpenC3::TargetModel.all(scope: @scope)
+          targets.each_value { |target| db_shards << target['db_shard'].to_i if target['db_shard'] }
+        rescue => e
+          @logger.error("QuestDB: Error getting target db_shards: #{e.formatted}")
+        end
+
+        db_shards.each do |db_shard|
+          begin
+            conn = OpenC3::QuestDBClient.connection(db_shard: db_shard)
+            result = conn.exec(TSDB_HEALTH_QUERY)
+            columns = result.fields
+            rows = result.values
+
+            table_name_column = columns.index("table_name")
+            wal_pending_row_count_column = columns.index("wal_pending_row_count")
+            status_column = columns.index("status")
+            lag_txns_column = columns.index("lag_txns")
+
+            rows.each do |values|
+              table_name = values[table_name_column]
+              # Prefix with db_shard to avoid key collisions across db_shards
+              tracking_key = "s#{db_shard}__#{table_name}"
+              wal_pending_row_count = values[wal_pending_row_count_column].to_i
+              status = values[status_column]
+              lag_txns = values[lag_txns_column].to_i
+
+              if status != 'OK'
+                @logger.error("QuestDB db_shard #{db_shard}: #{table_name} in bad state: #{status}")
+
+                if status == 'SUSPENDED'
+                  # Try to automatically unsuspend
+                  @logger.info("QuestDB db_shard #{db_shard}: Attempting to unsuspend: #{table_name}")
+                  conn.exec("ALTER TABLE \"#{table_name}\" RESUME WAL;")
+                end
               end
-            end
 
-            @wal_pending_row_count[table_name] ||= []
-            @wal_pending_row_count[table_name] << wal_pending_row_count
-            @lag_txns[table_name] ||= []
-            @lag_txns[table_name] << lag_txns
+              @wal_pending_row_count[tracking_key] ||= []
+              @wal_pending_row_count[tracking_key] << wal_pending_row_count
+              @lag_txns[tracking_key] ||= []
+              @lag_txns[tracking_key] << lag_txns
 
-            if @wal_pending_row_count[table_name].length > GROWTH_NUM_SAMPLE_PERIODS
-              if detect_growth(@wal_pending_row_count[table_name], GROWTH_NUM_SAMPLE_PERIODS)
-                # Crossed threshold of sample periods of growth
-                @logger.error("QuestDB: #{table_name} has growing wal_pending_row_count: #{wal_pending_row_count}")
+              if @wal_pending_row_count[tracking_key].length > GROWTH_NUM_SAMPLE_PERIODS
+                if detect_growth(@wal_pending_row_count[tracking_key], GROWTH_NUM_SAMPLE_PERIODS)
+                  # Crossed threshold of sample periods of growth
+                  @logger.error("QuestDB db_shard #{db_shard}: #{table_name} has growing wal_pending_row_count: #{wal_pending_row_count}")
+                end
+
+                # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
+                @wal_pending_row_count[tracking_key] = @wal_pending_row_count[tracking_key][-GROWTH_NUM_SAMPLE_PERIODS..-1]
               end
 
-              # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
-              @wal_pending_row_count[table_name] = @wal_pending_row_count[table_name][-GROWTH_NUM_SAMPLE_PERIODS..-1]
-            end
+              if @lag_txns[tracking_key].length > GROWTH_NUM_SAMPLE_PERIODS
+                if detect_growth(@lag_txns[tracking_key], GROWTH_NUM_SAMPLE_PERIODS)
+                  # Crossed threshold of sample periods of growth
+                  @logger.error("QuestDB db_shard #{db_shard}: #{table_name} has growing lag_txns: #{lag_txns}")
+                end
 
-            if @lag_txns[table_name].length > GROWTH_NUM_SAMPLE_PERIODS
-              if detect_growth(@lag_txns[table_name], GROWTH_NUM_SAMPLE_PERIODS)
-                # Crossed threshold of sample periods of growth
-                @logger.error("QuestDB: #{table_name} has growing lag_txns: #{lag_txns}")
+                # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
+                @lag_txns[tracking_key] = @lag_txns[tracking_key][-GROWTH_NUM_SAMPLE_PERIODS..-1]
               end
-
-              # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
-              @lag_txns[table_name] = @lag_txns[table_name][-GROWTH_NUM_SAMPLE_PERIODS..-1]
             end
+          rescue => e
+            OpenC3::QuestDBClient.disconnect(db_shard: db_shard)
+            @logger.error("QuestDB db_shard #{db_shard} Error: #{e.formatted}")
           end
-        rescue => e
-          OpenC3::QuestDBClient.disconnect
-          @logger.error("QuestDB Error: #{e.formatted}")
         end
       end
     end

data/lib/openc3/microservices/text_log_microservice.rb

@@ -55,8 +55,7 @@ module OpenC3
       while true
         break if @cancel_thread
 
-        # Read each topic separately to support multiple redis shards
-        # This is needed with Redis cluster because the topics will likely be on different shards and we don't use {} in the topic names
+        # Read each topic separately to support multiple redis db_shards
         individual_topics.each do |individual_topic|
           break if @cancel_thread
           # 500ms timeout - To support completing within 1 second with two topics (DEFAULT and NOSCOPE)

data/lib/openc3/models/cvt_model.rb

@@ -30,12 +30,22 @@ module OpenC3
       packet.decom
     end
 
+    # Get a Store instance routed to the correct db_shard for a target
+    def self.store_for_target(target_name, scope:)
+      Store.instance(db_shard: Store.db_shard_for_target(target_name, scope: scope))
+    end
+
+    # Get a StoreQueued instance routed to the correct db_shard for a target
+    def self.store_queued_for_target(target_name, scope:)
+      StoreQueued.instance(db_shard: Store.db_shard_for_target(target_name, scope: scope))
+    end
+
     # Delete the current value table for a target
     def self.del(target_name:, packet_name:, scope: $openc3_scope)
       key = "#{scope}__tlm__#{target_name}"
       tgt_pkt_key = key + "__#{packet_name}"
       @@packet_cache[tgt_pkt_key] = nil
-      Store.hdel(key, packet_name)
+      store_for_target(target_name, scope: scope).hdel(key, packet_name)
     end
 
     # Set the current value table for a target, packet
@@ -45,9 +55,9 @@ module OpenC3
       tgt_pkt_key = key + "__#{packet_name}"
       @@packet_cache[tgt_pkt_key] = [Time.now, hash]
       if queued
-        StoreQueued.hset(key, packet_name, packet_json)
+        store_queued_for_target(target_name, scope: scope).hset(key, packet_name, packet_json)
       else
-        Store.hset(key, packet_name, packet_json)
+        store_for_target(target_name, scope: scope).hset(key, packet_name, packet_json)
       end
     end
 
@@ -59,9 +69,9 @@ module OpenC3
       tgt_pkt_key = key + "__#{packet_name}"
       @@packet_cache[tgt_pkt_key] = [Time.now, hash]
       if queued
-        StoreQueued.hset(key, packet_name, packet_json)
+        store_queued_for_target(target_name, scope: scope).hset(key, packet_name, packet_json)
       else
-        Store.hset(key, packet_name, packet_json)
+        store_for_target(target_name, scope: scope).hset(key, packet_name, packet_json)
       end
     end
 
@@ -75,7 +85,7 @@ module OpenC3
         cache_time, hash = @@packet_cache[tgt_pkt_key]
         return hash if hash and (now - cache_time) < cache_timeout
       end
-      packet = Store.hget(key, packet_name)
+      packet = store_for_target(target_name, scope: scope).hget(key, packet_name)
       raise "Packet '#{target_name} #{packet_name}' does not exist" unless packet
       hash = JSON.parse(packet, allow_nan: true, create_additions: true)
       @@packet_cache[tgt_pkt_key] = [now, hash]
@@ -198,7 +208,7 @@ module OpenC3
     def self.overrides(scope: $openc3_scope)
       overrides = []
       TargetModel.names(scope: scope).each do |target_name|
-        all = Store.hgetall("#{scope}__override__#{target_name}")
+        all = store_for_target(target_name, scope: scope).hgetall("#{scope}__override__#{target_name}")
         next if all.nil? or all.empty?
         all.each do |packet_name, hash|
           items = JSON.parse(hash, allow_nan: true, create_additions: true)
@@ -227,7 +237,7 @@ module OpenC3
     # Override a current value table item such that it always returns the same value
     # for the given type
     def self.override(target_name, packet_name, item_name, value, type: :ALL, scope: $openc3_scope)
-      hash = Store.hget("#{scope}__override__#{target_name}", packet_name)
+      hash = store_for_target(target_name, scope: scope).hget("#{scope}__override__#{target_name}", packet_name)
       hash = JSON.parse(hash, allow_nan: true, create_additions: true) if hash
       hash ||= {} # In case the above didn't create anything
       case type
@@ -247,12 +257,12 @@ module OpenC3
 
       tgt_pkt_key = "#{scope}__tlm__#{target_name}__#{packet_name}"
       @@override_cache[tgt_pkt_key] = [Time.now, hash]
-      Store.hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json, allow_nan: true))
+      store_for_target(target_name, scope: scope).hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json, allow_nan: true))
     end
 
     # Normalize a current value table item such that it returns the actual value
     def self.normalize(target_name, packet_name, item_name, type: :ALL, scope: $openc3_scope)
-      hash = Store.hget("#{scope}__override__#{target_name}", packet_name)
+      hash = store_for_target(target_name, scope: scope).hget("#{scope}__override__#{target_name}", packet_name)
       hash = JSON.parse(hash, allow_nan: true, create_additions: true) if hash
       hash ||= {} # In case the above didn't create anything
       case type
@@ -271,12 +281,13 @@ module OpenC3
       end
 
       tgt_pkt_key = "#{scope}__tlm__#{target_name}__#{packet_name}"
+      store = store_for_target(target_name, scope: scope)
       if hash.empty?
         @@override_cache.delete(tgt_pkt_key)
-        Store.hdel("#{scope}__override__#{target_name}", packet_name)
+        store.hdel("#{scope}__override__#{target_name}", packet_name)
       else
         @@override_cache[tgt_pkt_key] = [Time.now, hash]
-        Store.hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json, allow_nan: true))
+        store.hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json, allow_nan: true))
       end
     end
 
@@ -334,7 +345,7 @@ module OpenC3
           return hash
         end
       end
-      override_data = Store.hget("#{scope}__override__#{target_name}", packet_name)
+      override_data = store_for_target(target_name, scope: scope).hget("#{scope}__override__#{target_name}", packet_name)
       if override_data
         hash = JSON.parse(override_data, allow_nan: true, create_additions: true)
         overrides[tgt_pkt_key] = hash