openc3 7.0.0 → 7.1.0

data/lib/openc3/microservices/decom_common.rb

@@ -0,0 +1,128 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/system/system'
+require 'openc3/microservices/interface_microservice'
+require 'openc3/topics/telemetry_decom_topic'
+require 'openc3/models/target_model'
+
+module OpenC3
+  # Shared decom pipeline used by DecomMicroservice (live telemetry) and
+  # ReingestJob (historical raw log replay). The reingest path passes
+  # check_limits: false so historical data does not re-fire limits events.
+  module DecomCommon
+    extend self
+
+    # Decommutate a Packet and publish it on the TelemetryDecomTopic. This is the
+    # step that lands data in the CVT and in the Python TsdbMicroservice → QuestDB.
+    #
+    # @param packet [Packet] A fully buffered Packet. Caller sets received_time,
+    #   received_count, stored, extra, buffer.
+    # @param scope [String] Scope name.
+    # @param target_names [Array<String>] Used when a subpacket must be re-identified.
+    # @param logger [Logger] Destination for warnings.
+    # @param name [String] Identifier used in subpacket warning messages
+    #   (microservice name, or "REINGEST:<job_id>").
+    # @param check_limits [Boolean] When false, skips the Packet#check_limits call.
+    #   Reingest passes false so historical data does not re-fire limits events.
+    # @param metric [Metric, nil] Optional; when set, records decom_duration_seconds.
+    # @param error_callback [Proc, nil] Called as error_callback.call(exception) when
+    #   Packet#process or Packet#check_limits raises. The microservice uses this to
+    #   bump its decom_error_total metric.
+    # @return [Integer] Number of (sub)packets published.
+    def decom_and_publish(packet, scope:, target_names:, logger:, name:,
+                          check_limits: true, metric: nil, error_callback: nil)
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC) if metric
+      published = 0
+
+      packet_and_subpackets = packet.subpacketize
+      packet_and_subpackets.each do |packet_or_subpacket|
+        if packet_or_subpacket.subpacket
+          packet_or_subpacket = handle_subpacket(packet, packet_or_subpacket,
+                                                 target_names: target_names,
+                                                 scope: scope,
+                                                 logger: logger,
+                                                 name: name)
+        end
+
+        begin
+          packet_or_subpacket.process
+        rescue Exception => e
+          error_callback&.call(e)
+          logger.error e.message
+        end
+
+        packet_or_subpacket.check_limits(System.limits_set) if check_limits
+
+        TelemetryDecomTopic.write_packet(packet_or_subpacket, scope: scope)
+        published += 1
+      end
+
+      if metric
+        diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+        metric.set(name: 'decom_duration_seconds', value: diff, type: 'gauge', unit: 'seconds')
+      end
+
+      published
+    end
+
+    # Identify a subpacket and (except for stored telemetry) update the CVT.
+    # Extracted from DecomMicroservice so reingest can handle subpackets too.
+    def handle_subpacket(packet, subpacket, target_names:, scope:, logger:, name:)
+      subpacket.received_time = packet.received_time
+      subpacket.stored = packet.stored
+      subpacket.extra = packet.extra
+
+      if subpacket.stored
+        identified_subpacket = System.telemetry.identify_and_define_packet(subpacket, target_names, subpackets: true)
+      else
+        if subpacket.identified?
+          begin
+            identified_subpacket = System.telemetry.update!(subpacket.target_name,
+                                                            subpacket.packet_name,
+                                                            subpacket.buffer)
+          rescue RuntimeError
+            logger.warn "#{name}: Received unknown identified subpacket: #{subpacket.target_name} #{subpacket.packet_name}"
+            subpacket.target_name = nil
+            subpacket.packet_name = nil
+            identified_subpacket = System.telemetry.identify!(subpacket.buffer,
+                                                              target_names, subpackets: true)
+          end
+        else
+          identified_subpacket = System.telemetry.identify!(subpacket.buffer,
+                                                            target_names, subpackets: true)
+        end
+      end
+
+      if identified_subpacket
+        identified_subpacket.received_time = subpacket.received_time
+        identified_subpacket.stored = subpacket.stored
+        identified_subpacket.extra = subpacket.extra
+        subpacket = identified_subpacket
+      else
+        unknown_subpacket = System.telemetry.update!('UNKNOWN', 'UNKNOWN', subpacket.buffer)
+        unknown_subpacket.received_time = subpacket.received_time
+        unknown_subpacket.stored = subpacket.stored
+        unknown_subpacket.extra = subpacket.extra
+        subpacket = unknown_subpacket
+        num_bytes_to_print = [InterfaceMicroservice::UNKNOWN_BYTES_TO_PRINT, subpacket.length].min
+        data = subpacket.buffer(false)[0..(num_bytes_to_print - 1)]
+        prefix = data.each_byte.map { |byte| sprintf("%02X", byte) }.join()
+        logger.warn "#{name} #{subpacket.target_name} packet length: #{subpacket.length} starting with: #{prefix}"
+      end
+
+      TargetModel.sync_tlm_packet_counts(subpacket, target_names, scope: scope)
+      subpacket
+    end
+  end
+end

data/lib/openc3/microservices/decom_microservice.rb

@@ -18,6 +18,7 @@
 require 'time'
 require 'thread'
 require 'openc3/microservices/microservice'
+require 'openc3/microservices/decom_common'
 require 'openc3/microservices/interface_decom_common'
 require 'openc3/microservices/interface_microservice'
 require 'openc3/topics/telemetry_decom_topic'
@@ -90,7 +91,9 @@ module OpenC3
       if @name =~ /__DECOM__/
         @topics << "#{@scope}__DECOMINTERFACE__{#{@target_names[0]}}"
       end
-      Topic.update_topic_offsets(@topics)
+      @limits_event_topic = "#{@scope}__openc3_limits_events"
+      @topics << @limits_event_topic
+      Topic.update_topic_offsets(@topics, db_shard: @db_shard)
       System.telemetry.limits_change_callback = method(:limits_change_callback)
       LimitsEventTopic.sync_system(scope: @scope)
       @error_count = 0
@@ -110,13 +113,16 @@ module OpenC3
 
         begin
           OpenC3.in_span("read_topics") do
-            Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
+            Topic.read_topics(@topics, db_shard: @db_shard) do |topic, msg_id, msg_hash, redis|
               break if @cancel_thread
               if topic == @microservice_topic
                 microservice_cmd(topic, msg_id, msg_hash, redis)
+              elsif topic == @limits_event_topic
+                event = JSON.parse(msg_hash['event'], allow_nan: true, create_additions: true)
+                LimitsEventTopic.process_event(event)
               elsif topic =~ /__DECOMINTERFACE/
                 if msg_hash.key?('inject_tlm')
-                  handle_inject_tlm(msg_hash['inject_tlm'])
+                  handle_inject_tlm_with_ack(msg_hash['inject_tlm'], msg_id)
                   next
                 end
                 if msg_hash.key?('build_cmd')
@@ -134,7 +140,6 @@ module OpenC3
               @count += 1
             end
           end
-          LimitsEventTopic.sync_system_thread_body(scope: @scope)
         rescue => e
           @error_count += 1
           @metric.set(name: 'decom_error_total', value: @error_count, type: 'counter')
@@ -153,8 +158,6 @@ module OpenC3
         delta = Time.now.to_f - msgid_seconds_from_epoch
         @metric.set(name: 'decom_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and decom start')
 
-        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-
         #######################################
         # Build packet object from topic data
         #######################################
@@ -172,98 +175,21 @@ module OpenC3
         end
         packet.buffer = msg_hash["buffer"]
 
-        ################################################################################
-        # Break packet into subpackets (if necessary)
-        # Subpackets are typically channelized data
-        ################################################################################
-        packet_and_subpackets = packet.subpacketize
-
-        packet_and_subpackets.each do |packet_or_subpacket|
-          if packet_or_subpacket.subpacket
-            packet_or_subpacket = handle_subpacket(packet, packet_or_subpacket)
-          end
-
-          #####################################################################################
-          # Run Processors
-          # This must be before the full decom so that processor derived values are available
-          #####################################################################################
-          begin
-            packet_or_subpacket.process # Run processors
-          rescue Exception => e
+        DecomCommon.decom_and_publish(
+          packet,
+          scope: @scope,
+          target_names: @target_names,
+          logger: @logger,
+          name: @name,
+          check_limits: true,
+          metric: @metric,
+          error_callback: ->(e) {
             @error_count += 1
             @metric.set(name: 'decom_error_total', value: @error_count, type: 'counter')
             @error = e
-            @logger.error e.message
-          end
-
-          #############################################################################
-          # Process all the limits and call the limits_change_callback (as necessary)
-          # This must be before the full decom so that limits states are available
-          #############################################################################
-          packet_or_subpacket.check_limits(System.limits_set)
-
-          # This is what actually decommutates the packet and updates the CVT
-          TelemetryDecomTopic.write_packet(packet_or_subpacket, scope: @scope)
-        end
-
-        diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-        @metric.set(name: 'decom_duration_seconds', value: diff, type: 'gauge', unit: 'seconds')
-      end
-    end
-
-    def handle_subpacket(packet, subpacket)
-      # Subpacket received time always = packet.received_time
-      # Use packet_time appropriately if another timestamp is needed
-      subpacket.received_time = packet.received_time
-      subpacket.stored = packet.stored
-      subpacket.extra = packet.extra
-
-      if subpacket.stored
-        # Stored telemetry does not update the current value table
-        identified_subpacket = System.telemetry.identify_and_define_packet(subpacket, @target_names, subpackets: true)
-      else
-        # Identify and update subpacket
-        if subpacket.identified?
-          begin
-            # Preidentifed subpacket - place it into the current value table
-            identified_subpacket = System.telemetry.update!(subpacket.target_name,
-                                                            subpacket.packet_name,
-                                                            subpacket.buffer)
-          rescue RuntimeError
-            # Subpacket identified but we don't know about it
-            # Clear packet_name and target_name and try to identify
-            @logger.warn "#{@name}: Received unknown identified subpacket: #{subpacket.target_name} #{subpacket.packet_name}"
-            subpacket.target_name = nil
-            subpacket.packet_name = nil
-            identified_subpacket = System.telemetry.identify!(subpacket.buffer,
-                                                              @target_names, subpackets: true)
-          end
-        else
-          # Packet needs to be identified
-          identified_subpacket = System.telemetry.identify!(subpacket.buffer,
-                                                            @target_names, subpackets: true)
-        end
+          },
+        )
       end
-
-      if identified_subpacket
-        identified_subpacket.received_time = subpacket.received_time
-        identified_subpacket.stored = subpacket.stored
-        identified_subpacket.extra = subpacket.extra
-        subpacket = identified_subpacket
-      else
-        unknown_subpacket = System.telemetry.update!('UNKNOWN', 'UNKNOWN', subpacket.buffer)
-        unknown_subpacket.received_time = subpacket.received_time
-        unknown_subpacket.stored = subpacket.stored
-        unknown_subpacket.extra = subpacket.extra
-        subpacket = unknown_subpacket
-        num_bytes_to_print = [InterfaceMicroservice::UNKNOWN_BYTES_TO_PRINT, subpacket.length].min
-        data = subpacket.buffer(false)[0..(num_bytes_to_print - 1)]
-        prefix = data.each_byte.map { | byte | sprintf("%02X", byte) }.join()
-        @logger.warn "#{@name} #{subpacket.target_name} packet length: #{subpacket.length} starting with: #{prefix}"
-      end
-
-      TargetModel.sync_tlm_packet_counts(subpacket, @target_names, scope: @scope)
-      return subpacket
     end
 
     # Called when an item in any packet changes limits states.
@@ -285,7 +211,12 @@ module OpenC3
       if value
         message = "#{packet.target_name} #{packet.packet_name} #{item.name} = #{value} is #{item.limits.state}"
         if item.limits.values
-          values = item.limits.values[System.limits_set]
+          selected_limits_set = if item.limits.values.has_key?(System.limits_set)
+            System.limits_set
+          else
+            :DEFAULT
+          end
+          values = item.limits.values[selected_limits_set]
           # Check if the state is RED_LOW, YELLOW_LOW, YELLOW_HIGH, RED_HIGH, GREEN_LOW, GREEN_HIGH
           if LIMITS_STATE_INDEX[item.limits.state]
             # Directly index into the values and return the value
@@ -335,4 +266,4 @@ if __FILE__ == $0
   OpenC3::DecomMicroservice.run
   OpenC3::ThreadManager.instance.shutdown
   OpenC3::ThreadManager.instance.join
-end
+end

data/lib/openc3/microservices/interface_decom_common.rb

@@ -31,13 +31,31 @@ module OpenC3
           packet.write(name.to_s, value, type)
         end
       end
+      stored = inject_tlm_hash.fetch('stored', false)
+      packet.stored = stored.to_s.downcase == 'true'
       packet.received_time = Time.now.sys
       packet.received_count = TargetModel.increment_telemetry_count(packet.target_name, packet.packet_name, 1, scope: @scope)
       TelemetryTopic.write_packet(packet, scope: @scope)
-    # If the inject_tlm parameters are bad we rescue so
-    # interface_microservice and decom_microservice can continue
-    rescue => e
-      @logger.error "inject_tlm error due to #{e.message}"
+    end
+
+    def handle_inject_tlm_with_ack(inject_tlm_json, msg_id)
+      inject_tlm_hash = JSON.parse(inject_tlm_json, allow_nan: true, create_additions: true)
+      target_name = inject_tlm_hash['target_name']
+      ack_topic = "{#{@scope}__ACKCMD}TARGET__#{target_name}"
+      begin
+        handle_inject_tlm(inject_tlm_json)
+        msg_hash = {
+          id: msg_id,
+          result: 'SUCCESS'
+        }
+      rescue => error
+        @logger.error "inject_tlm error due to #{error.message}"
+        msg_hash = {
+          id: msg_id,
+          result: error.message
+        }
+      end
+      Topic.write_topic(ack_topic, msg_hash)
     end
 
     def handle_build_cmd(build_cmd_json, msg_id)
@@ -47,6 +65,7 @@ module OpenC3
       cmd_params = build_cmd_hash['cmd_params']
       range_check = build_cmd_hash['range_check']
       raw = build_cmd_hash['raw']
+      db_shard = Store.db_shard_for_target(target_name, scope: @scope)
       ack_topic = "{#{@scope}__ACKCMD}TARGET__#{target_name}"
       begin
         command = System.commands.build_cmd(target_name, cmd_name, cmd_params, range_check, raw)
@@ -65,17 +84,17 @@ module OpenC3
       rescue => error
         msg_hash = {
           id: msg_id,
-          result: 'ERROR',
-          message: error.message
+          result: error.message
         }
       end
-      Topic.write_topic(ack_topic, msg_hash)
+      Topic.write_topic(ack_topic, msg_hash, db_shard: db_shard)
     end
 
     def handle_get_tlm_buffer(get_tlm_buffer_json, msg_id)
       get_tlm_buffer_hash = JSON.parse(get_tlm_buffer_json, allow_nan: true, create_additions: true)
       target_name = get_tlm_buffer_hash['target_name']
       packet_name = get_tlm_buffer_hash['packet_name']
+      db_shard = Store.db_shard_for_target(target_name, scope: @scope)
       ack_topic = "{#{@scope}__ACKCMD}TARGET__#{target_name}"
       begin
         packet = System.telemetry.packet(target_name, packet_name)
@@ -97,11 +116,10 @@ module OpenC3
       rescue => error
         msg_hash = {
           id: msg_id,
-          result: 'ERROR',
-          message: error.message
+          result: error.message
         }
       end
-      Topic.write_topic(ack_topic, msg_hash)
+      Topic.write_topic(ack_topic, msg_hash, db_shard: db_shard)
     end
   end
 end

data/lib/openc3/microservices/interface_microservice.rb

@@ -42,10 +42,11 @@ module OpenC3
   class InterfaceCmdHandlerThread
     include InterfaceDecomCommon
 
-    def initialize(interface, tlm, logger: nil, metric: nil, scope:)
+    def initialize(interface, tlm, logger: nil, metric: nil, db_shard: 0, scope:)
       @interface = interface
       @tlm = tlm
       @scope = scope
+      @db_shard = db_shard.to_i
       scope_model = ScopeModel.get_model(name: @scope)
       if scope_model
         @critical_commanding = scope_model.critical_commanding
@@ -81,7 +82,7 @@ module OpenC3
     end
 
     def run
-      InterfaceTopic.receive_commands(@interface, scope: @scope) do |topic, msg_id, msg_hash, _redis|
+      InterfaceTopic.receive_commands(@interface, scope: @scope, db_shard: @db_shard) do |topic, msg_id, msg_hash, _redis|
         OpenC3.with_context(msg_hash) do
           release_critical = false
           critical_model = nil
@@ -105,7 +106,7 @@ module OpenC3
             @metric.set(name: 'interface_directive_total', value: @directive_count, type: 'counter') if @metric
             if msg_hash['shutdown']
               @logger.info "#{@interface.name}: Shutdown requested"
-              InterfaceTopic.clear_topics(InterfaceTopic.topics(@interface, scope: @scope))
+              InterfaceTopic.clear_topics(InterfaceTopic.topics(@interface, scope: @scope), db_shard: @db_shard)
               return
             end
             if msg_hash['connect']
@@ -179,7 +180,12 @@ module OpenC3
               next 'SUCCESS'
             end
             if msg_hash.key?('inject_tlm')
-              handle_inject_tlm(msg_hash['inject_tlm'])
+              begin
+                handle_inject_tlm(msg_hash['inject_tlm'])
+              rescue => e
+                @logger.error "#{@interface.name}: inject_tlm: #{e.formatted}"
+                next e.message
+              end
               next 'SUCCESS'
             end
             if msg_hash.key?('release_critical')
@@ -374,10 +380,11 @@ module OpenC3
   end
 
   class RouterTlmHandlerThread
-    def initialize(router, tlm, logger: nil, metric: nil, scope:)
+    def initialize(router, tlm, logger: nil, metric: nil, db_shard: 0, scope:)
       @router = router
       @tlm = tlm
       @scope = scope
+      @db_shard = db_shard.to_i
       @logger = logger
       @logger = Logger unless @logger
       @metric = metric
@@ -407,7 +414,7 @@ module OpenC3
     end
 
     def run
-      RouterTopic.receive_telemetry(@router, scope: @scope) do |topic, msg_id, msg_hash, _redis|
+      RouterTopic.receive_telemetry(@router, scope: @scope, db_shard: @db_shard) do |topic, msg_id, msg_hash, _redis|
         msgid_seconds_from_epoch = msg_id.split('-')[0].to_i / 1000.0
         delta = Time.now.to_f - msgid_seconds_from_epoch
         @metric.set(name: 'router_topic_delta_seconds', value: delta, type: 'gauge', unit: 'seconds', help: 'Delta time between data written to stream and router tlm start') if @metric
@@ -419,7 +426,7 @@ module OpenC3
 
           if msg_hash['shutdown']
             @logger.info "#{@router.name}: Shutdown requested"
-            RouterTopic.clear_topics(RouterTopic.topics(@router, scope: @scope))
+            RouterTopic.clear_topics(RouterTopic.topics(@router, scope: @scope), db_shard: @db_shard)
             return
           end
           if msg_hash['connect']
@@ -585,9 +592,9 @@ module OpenC3
       @connection_failed_messages = []
       @connection_lost_messages = []
       if @interface_or_router == 'INTERFACE'
-        @handler_thread = InterfaceCmdHandlerThread.new(@interface, self, logger: @logger, metric: @metric, scope: @scope)
+        @handler_thread = InterfaceCmdHandlerThread.new(@interface, self, logger: @logger, metric: @metric, db_shard: @db_shard, scope: @scope)
       else
-        @handler_thread = RouterTlmHandlerThread.new(@interface, self, logger: @logger, metric: @metric, scope: @scope)
+        @handler_thread = RouterTlmHandlerThread.new(@interface, self, logger: @logger, metric: @metric, db_shard: @db_shard, scope: @scope)
       end
       @handler_thread.start
     end

data/lib/openc3/microservices/log_microservice.rb

@@ -63,7 +63,7 @@ module OpenC3
       while true
         break if @cancel_thread
 
-        Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
+        Topic.read_topics(@topics, db_shard: @db_shard) do |topic, msg_id, msg_hash, redis|
           break if @cancel_thread
           if topic == @microservice_topic
             microservice_cmd(topic, msg_id, msg_hash, redis)

data/lib/openc3/microservices/microservice.rb

@@ -127,6 +127,7 @@ module OpenC3
       @logger.info("Microservice initialized with config:\n#{@config}")
       @topics ||= []
       @microservice_topic = "MICROSERVICE__#{@name}"
+      @db_shard = (@config['db_shard'] || 0).to_i
 
       # Get configuration for any targets
       @target_names = @config["target_names"]
@@ -259,10 +260,10 @@ module OpenC3
         else
           raise "Invalid topics given to microservice_cmd: #{topics}"
         end
-        Topic.trim_topic(topic, msg_id)
+        Topic.trim_topic(topic, msg_id, db_shard: @db_shard)
         return true
       end
-      Topic.trim_topic(topic, msg_id)
+      Topic.trim_topic(topic, msg_id, db_shard: @db_shard)
       return false
     end
   end

data/lib/openc3/microservices/queue_microservice.rb

@@ -19,7 +19,7 @@ require 'openc3/api/api'
 
 module OpenC3
   saved_verbose = $VERBOSE
-  $VERBOSE = false
+  $VERBOSE = nil
   module Script
     private
     # Override the prompt_for_hazardous method to always return true since there is no user to prompt

data/lib/openc3/microservices/scope_cleanup_microservice.rb

@@ -12,6 +12,7 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/models/scope_model'
+require 'openc3/models/target_model'
 require 'openc3/microservices/cleanup_microservice'
 
 module OpenC3
@@ -62,63 +63,76 @@ ORDER BY
         super(areas, bucket)
       end
 
-      # Always check TSDB health
+      # Always check TSDB health across all db_shards
       if @scope == 'DEFAULT'
+        # Collect all unique db_shard numbers from targets
+        db_shards = Set.new([0]) # Always check db_shard 0
         begin
-          conn = OpenC3::QuestDBClient.connection
-          result = conn.exec(TSDB_HEALTH_QUERY)
-          columns = result.fields
-          rows = result.values
-
-          table_name_column = columns.index("table_name")
-          wal_pending_row_count_column = columns.index("wal_pending_row_count")
-          status_column = columns.index("status")
-          lag_txns_column = columns.index("lag_txns")
-
-          rows.each do |values|
-            table_name = values[table_name_column]
-            wal_pending_row_count = values[wal_pending_row_count_column].to_i
-            status = values[status_column]
-            lag_txns = values[lag_txns_column].to_i
-
-            if status != 'OK'
-              @logger.error("QuestDB: #{table_name} in bad state: #{status}")
-
-              if status == 'SUSPENDED'
-                # Try to automatically unsuspend
-                @logger.info("QuestDB: Attempting to unsuspend: #{table_name}")
-                conn.exec("ALTER TABLE #{table_name} RESUME WAL;")
+          targets = OpenC3::TargetModel.all(scope: @scope)
+          targets.each_value { |target| db_shards << target['db_shard'].to_i if target['db_shard'] }
+        rescue => e
+          @logger.error("QuestDB: Error getting target db_shards: #{e.formatted}")
+        end
+
+        db_shards.each do |db_shard|
+          begin
+            conn = OpenC3::QuestDBClient.connection(db_shard: db_shard)
+            result = conn.exec(TSDB_HEALTH_QUERY)
+            columns = result.fields
+            rows = result.values
+
+            table_name_column = columns.index("table_name")
+            wal_pending_row_count_column = columns.index("wal_pending_row_count")
+            status_column = columns.index("status")
+            lag_txns_column = columns.index("lag_txns")
+
+            rows.each do |values|
+              table_name = values[table_name_column]
+              # Prefix with db_shard to avoid key collisions across db_shards
+              tracking_key = "s#{db_shard}__#{table_name}"
+              wal_pending_row_count = values[wal_pending_row_count_column].to_i
+              status = values[status_column]
+              lag_txns = values[lag_txns_column].to_i
+
+              if status != 'OK'
+                @logger.error("QuestDB db_shard #{db_shard}: #{table_name} in bad state: #{status}")
+
+                if status == 'SUSPENDED'
+                  # Try to automatically unsuspend
+                  @logger.info("QuestDB db_shard #{db_shard}: Attempting to unsuspend: #{table_name}")
+                  conn.exec("ALTER TABLE \"#{table_name}\" RESUME WAL;")
+                end
               end
-            end
 
-            @wal_pending_row_count[table_name] ||= []
-            @wal_pending_row_count[table_name] << wal_pending_row_count
-            @lag_txns[table_name] ||= []
-            @lag_txns[table_name] << lag_txns
+              @wal_pending_row_count[tracking_key] ||= []
+              @wal_pending_row_count[tracking_key] << wal_pending_row_count
+              @lag_txns[tracking_key] ||= []
+              @lag_txns[tracking_key] << lag_txns
 
-            if @wal_pending_row_count[table_name].length > GROWTH_NUM_SAMPLE_PERIODS
-              if detect_growth(@wal_pending_row_count[table_name], GROWTH_NUM_SAMPLE_PERIODS)
-                # Crossed threshold of sample periods of growth
-                @logger.error("QuestDB: #{table_name} has growing wal_pending_row_count: #{wal_pending_row_count}")
+              if @wal_pending_row_count[tracking_key].length > GROWTH_NUM_SAMPLE_PERIODS
+                if detect_growth(@wal_pending_row_count[tracking_key], GROWTH_NUM_SAMPLE_PERIODS)
+                  # Crossed threshold of sample periods of growth
+                  @logger.error("QuestDB db_shard #{db_shard}: #{table_name} has growing wal_pending_row_count: #{wal_pending_row_count}")
+                end
+
+                # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
+                @wal_pending_row_count[tracking_key] = @wal_pending_row_count[tracking_key][-GROWTH_NUM_SAMPLE_PERIODS..-1]
               end
 
-              # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
-              @wal_pending_row_count[table_name] = @wal_pending_row_count[table_name][-GROWTH_NUM_SAMPLE_PERIODS..-1]
-            end
+              if @lag_txns[tracking_key].length > GROWTH_NUM_SAMPLE_PERIODS
+                if detect_growth(@lag_txns[tracking_key], GROWTH_NUM_SAMPLE_PERIODS)
+                  # Crossed threshold of sample periods of growth
+                  @logger.error("QuestDB db_shard #{db_shard}: #{table_name} has growing lag_txns: #{lag_txns}")
+                end
 
-            if @lag_txns[table_name].length > GROWTH_NUM_SAMPLE_PERIODS
-              if detect_growth(@lag_txns[table_name], GROWTH_NUM_SAMPLE_PERIODS)
-                # Crossed threshold of sample periods of growth
-                @logger.error("QuestDB: #{table_name} has growing lag_txns: #{lag_txns}")
+                # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
+                @lag_txns[tracking_key] = @lag_txns[tracking_key][-GROWTH_NUM_SAMPLE_PERIODS..-1]
               end
-
-              # Leave the last GROWTH_NUM_SAMPLE_PERIODS samples
-              @lag_txns[table_name] = @lag_txns[table_name][-GROWTH_NUM_SAMPLE_PERIODS..-1]
             end
+          rescue => e
+            OpenC3::QuestDBClient.disconnect(db_shard: db_shard)
+            @logger.error("QuestDB db_shard #{db_shard} Error: #{e.formatted}")
           end
-        rescue => e
-          OpenC3::QuestDBClient.disconnect
-          @logger.error("QuestDB Error: #{e.formatted}")
         end
       end
     end

data/lib/openc3/microservices/text_log_microservice.rb

@@ -55,8 +55,7 @@ module OpenC3
       while true
         break if @cancel_thread
 
-        # Read each topic separately to support multiple redis shards
-        # This is needed with Redis cluster because the topics will likely be on different shards and we don't use {} in the topic names
+        # Read each topic separately to support multiple redis db_shards
         individual_topics.each do |individual_topic|
           break if @cancel_thread
           # 500ms timeout - To support completing within 1 second with two topics (DEFAULT and NOSCOPE)