openc3 7.0.1 → 7.1.0

data/lib/openc3/topics/router_topic.rb

@@ -21,6 +21,12 @@ module OpenC3
   class RouterTopic < Topic
     COMMAND_ACK_TIMEOUT_S = 30
 
+    # Look up db_shard from RouterModel
+    def self._db_shard_for_router(router_name, scope:)
+      json = Store.hget("#{scope}__openc3_routers", router_name)
+      json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
+    end
+
     # Generate a list of topics for this router. This includes the router itself
     # and all the targets which are assigned to this router.
     def self.topics(router, scope:)
@@ -34,15 +40,39 @@ module OpenC3
       topics
     end
 
-    def self.receive_telemetry(router, scope:)
+    def self.receive_telemetry(router, scope:, db_shard: 0)
+      db_shard = db_shard.to_i
+      router_cmd_topic = "{#{scope}__CMD}ROUTER__#{router.name}"
+
+      target_topics = []
+      router.tlm_target_names.each do |target_name|
+        System.telemetry.packets(target_name).each do |_packet_name, packet|
+          target_topics << "#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}"
+        end
+      end
+
+      # Group telemetry topics by db_shard; include router cmd topic on db_shard
+      db_shard_groups = Topic.group_topics_by_db_shard(target_topics, target_pattern: '__TELEMETRY__', scope: scope)
+      db_shard_groups[db_shard] ||= []
+      db_shard_groups[db_shard] << router_cmd_topic
+
+      all_same_db_shard = Topic.all_same_db_shard?(db_shard_groups)
+
       while true
-        Topic.read_topics(RouterTopic.topics(router, scope: scope)) do |topic, msg_id, msg_hash, redis|
-          result = yield topic, msg_id, msg_hash, redis
-          if result and /CMD}ROUTER/.match?(topic)
-            ack_topic = topic.split("__")
-            ack_topic[1] = 'ACK' + ack_topic[1]
-            ack_topic = ack_topic.join("__")
-            Topic.write_topic(ack_topic, { 'result' => result, 'id' => msg_id }, msg_id, 100)
+        if all_same_db_shard
+          # Fast path: everything on one db_shard, single read
+          db_shard = db_shard_groups.keys.first || 0
+          Topic.read_topics(db_shard_groups[db_shard], db_shard: db_shard) do |topic, msg_id, msg_hash, redis|
+            result = yield topic, msg_id, msg_hash, redis
+            Topic.write_ack(topic, result, msg_id, db_shard: db_shard) if result and /CMD}ROUTER/.match?(topic)
+          end
+        else
+          timeout_per_db_shard = [1000 / [db_shard_groups.length, 1].max, 100].max
+          db_shard_groups.each do |db_shard, topics|
+            Topic.read_topics(topics, nil, timeout_per_db_shard, db_shard: db_shard) do |topic, msg_id, msg_hash, redis|
+              result = yield topic, msg_id, msg_hash, redis
+              Topic.write_ack(topic, result, msg_id, db_shard: db_shard) if result and /CMD}ROUTER/.match?(topic)
+            end
           end
         end
       end
@@ -61,74 +91,84 @@ module OpenC3
     end
 
     def self.connect_router(router_name, *router_params, scope:)
+      db_shard = _db_shard_for_router(router_name, scope: scope)
       if router_params && !router_params.empty?
-        Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'connect' => 'true', 'params' => JSON.generate(router_params, allow_nan: true) }, '*', 100)
+        Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'connect' => 'true', 'params' => JSON.generate(router_params, allow_nan: true) }, '*', 100, db_shard: db_shard)
       else
-        Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'connect' => 'true' }, '*', 100)
+        Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'connect' => 'true' }, '*', 100, db_shard: db_shard)
       end
     end
 
     def self.disconnect_router(router_name, scope:)
-      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'disconnect' => 'true' }, '*', 100)
+      db_shard = _db_shard_for_router(router_name, scope: scope)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'disconnect' => 'true' }, '*', 100, db_shard: db_shard)
     end
 
     def self.start_raw_logging(router_name, scope:)
-      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_stream' => 'true' }, '*', 100)
+      db_shard = _db_shard_for_router(router_name, scope: scope)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_stream' => 'true' }, '*', 100, db_shard: db_shard)
     end
 
     def self.stop_raw_logging(router_name, scope:)
-      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_stream' => 'false' }, '*', 100)
+      db_shard = _db_shard_for_router(router_name, scope: scope)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'log_stream' => 'false' }, '*', 100, db_shard: db_shard)
     end
 
     def self.shutdown(router, scope:)
-      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router.name}", { 'shutdown' => 'true' }, '*', 100)
+      db_shard = _db_shard_for_router(router.name, scope: scope)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router.name}", { 'shutdown' => 'true' }, '*', 100, db_shard: db_shard)
     end
 
     def self.router_cmd(router_name, cmd_name, *cmd_params, scope:)
+      db_shard = _db_shard_for_router(router_name, scope: scope)
       data = {}
       data['cmd_name'] = cmd_name
       data['cmd_params'] = cmd_params
-      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'router_cmd' => JSON.generate(data, allow_nan: true) }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'router_cmd' => JSON.generate(data, allow_nan: true) }, '*', 100, db_shard: db_shard)
     end
 
     def self.protocol_cmd(router_name, cmd_name, *cmd_params, read_write: :READ_WRITE, index: -1, scope:)
+      db_shard = _db_shard_for_router(router_name, scope: scope)
       data = {}
       data['cmd_name'] = cmd_name
       data['cmd_params'] = cmd_params
       data['read_write'] = read_write.to_s.upcase
       data['index'] = index
-      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'protocol_cmd' => JSON.generate(data, allow_nan: true) }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'protocol_cmd' => JSON.generate(data, allow_nan: true) }, '*', 100, db_shard: db_shard)
     end
 
     def self.router_target_enable(router_name, target_name, cmd_only: false, tlm_only: false, scope:)
+      db_shard = _db_shard_for_router(router_name, scope: scope)
       data = {}
       data['target_name'] = target_name.to_s.upcase
       data['cmd_only'] = cmd_only
       data['tlm_only'] = tlm_only
       data['action'] = 'enable'
-      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'target_control' => JSON.generate(data, allow_nan: true) }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'target_control' => JSON.generate(data, allow_nan: true) }, '*', 100, db_shard: db_shard)
     end
 
     def self.router_target_disable(router_name, target_name, cmd_only: false, tlm_only: false, scope:)
+      db_shard = _db_shard_for_router(router_name, scope: scope)
       data = {}
       data['target_name'] = target_name.to_s.upcase
       data['cmd_only'] = cmd_only
       data['tlm_only'] = tlm_only
       data['action'] = 'disable'
-      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'target_control' => JSON.generate(data, allow_nan: true) }, '*', 100)
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'target_control' => JSON.generate(data, allow_nan: true) }, '*', 100, db_shard: db_shard)
     end
 
     def self.router_details(router_name, timeout: nil, scope:)
       router_name = router_name.upcase
+      db_shard = _db_shard_for_router(router_name, scope: scope)
 
       timeout = COMMAND_ACK_TIMEOUT_S unless timeout
       ack_topic = "{#{scope}__ACKCMD}ROUTER__#{router_name}"
-      Topic.update_topic_offsets([ack_topic])
+      Topic.update_topic_offsets([ack_topic], db_shard: db_shard)
 
-      cmd_id = Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'router_details' => 'true' }, '*', 100)
+      cmd_id = Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'router_details' => 'true' }, '*', 100, db_shard: db_shard)
       time = Time.now
       while (Time.now - time) < timeout
-        Topic.read_topics([ack_topic]) do |_topic, _msg_id, msg_hash, _redis|
+        Topic.read_topics([ack_topic], db_shard: db_shard) do |_topic, _msg_id, msg_hash, _redis|
           if msg_hash["id"] == cmd_id
             return JSON.parse(msg_hash["result"], :allow_nan => true, :create_additions => true)
           end

data/lib/openc3/topics/system_events_topic.rb

@@ -17,13 +17,30 @@ module OpenC3
   class SystemEventsTopic < Topic
     PRIMARY_KEY = "OPENC3__SYSTEM__EVENTS".freeze
 
+    # Collect all unique target db_shards from TargetModel
+    def self._active_db_shards
+      db_shards = Set.new([0])
+      # Iterate all scopes to find all target db_shards
+      Store.scan_each(match: '*__openc3_targets', type: 'hash') do |key|
+        Store.hgetall(key).each do |_name, json|
+          parsed = JSON.parse(json, allow_nan: true, create_additions: true)
+          db_shards << (parsed['db_shard'] || 0).to_i
+        end
+      end
+      db_shards
+    end
+
     def self.update_topic_offsets()
       Topic.update_topic_offsets([PRIMARY_KEY])
     end
 
     def self.write(type, event)
       event['type'] = type
-      Topic.write_topic(PRIMARY_KEY, {event: JSON.generate(event, allow_nan: true)}, '*', 1000)
+      msg = {event: JSON.generate(event, allow_nan: true)}
+      # Write to all active db_shards so every interface microservice can read system events inline
+      _active_db_shards.each do |db_shard|
+        Topic.write_topic(PRIMARY_KEY, msg, '*', 1000, db_shard: db_shard)
+      end
     end
 
     def self.read()

data/lib/openc3/topics/telemetry_decom_topic.rb

@@ -43,7 +43,8 @@ module OpenC3
           :received_count => packet.received_count,
           :json_data => json_data,
         }
-        Topic.write_topic("#{scope}__DECOM__{#{packet.target_name}}__#{packet.packet_name}", msg_hash, id)
+        db_shard = Store.db_shard_for_target(packet.target_name, scope: scope)
+        Topic.write_topic("#{scope}__DECOM__{#{packet.target_name}}__#{packet.packet_name}", msg_hash, id, db_shard: db_shard)
 
         unless packet.stored
           # Also update the current value table with the latest decommutated data

data/lib/openc3/topics/telemetry_topic.rb

@@ -31,10 +31,12 @@ module OpenC3
         :buffer => packet.buffer(false)
       }
       msg_hash[:extra] = JSON.generate(packet.extra.as_json, allow_nan: true) if packet.extra
+      topic = "#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}"
+      db_shard = Store.db_shard_for_target(packet.target_name, scope: scope)
       if queued
-        EphemeralStoreQueued.write_topic("#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}", msg_hash)
+        EphemeralStoreQueued.instance(db_shard: db_shard).write_topic(topic, msg_hash)
       else
-        Topic.write_topic("#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}", msg_hash)
+        Topic.write_topic(topic, msg_hash, db_shard: db_shard)
       end
     end
   end

data/lib/openc3/topics/topic.rb

@@ -19,18 +19,90 @@ require 'openc3/utilities/store'
 
 module OpenC3
   class Topic
-    # Delegate all unknown class methods to delegate to the EphemeralStore
+    # Delegate all unknown class methods to EphemeralStore db_shard 0 (system-level topics)
     def self.method_missing(message, *args, **kwargs, &block)
       EphemeralStore.public_send(message, *args, **kwargs, &block)
     end
 
-    def self.clear_topics(topics, maxlen = 0)
-      topics.each { |topic| EphemeralStore.xtrim(topic, maxlen) }
+    def self.clear_topics(topics, maxlen = 0, db_shard: 0)
+      store = EphemeralStore.instance(db_shard: db_shard)
+      topics.each { |topic| store.xtrim(topic, maxlen) }
     end
 
-    def self.get_cnt(topic)
-      _, packet = EphemeralStore.get_newest_message(topic)
+    def self.get_cnt(topic, db_shard: 0)
+      _, packet = EphemeralStore.instance(db_shard: db_shard).get_newest_message(topic)
       packet ? packet["received_count"].to_i : 0
     end
+
+    # DB_Shard-aware topic methods for target-specific streams.
+    # These explicitly route to the correct EphemeralStore db_shard.
+
+    def self.write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true', db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).write_topic(topic, msg_hash, id, maxlen, approximate)
+    end
+
+    def self.read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil, db_shard: 0, &block)
+      EphemeralStore.instance(db_shard: db_shard).read_topics(topics, offsets, timeout_ms, count, &block)
+    end
+
+    def self.get_newest_message(topic, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).get_newest_message(topic)
+    end
+
+    def self.get_oldest_message(topic, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).get_oldest_message(topic)
+    end
+
+    def self.get_last_offset(topic, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).get_last_offset(topic)
+    end
+
+    def self.update_topic_offsets(topics, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).update_topic_offsets(topics)
+    end
+
+    def self.trim_topic(topic, minid, approximate = true, limit: 0, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).trim_topic(topic, minid, approximate, limit: limit)
+    end
+
+    def self.del(topic, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).del(topic)
+    end
+
+    # Group topics by db_shard. Each topic's target name is extracted and looked up.
+    # Topics matching target_pattern are db_sharded; others go to db_shard 0.
+    # @param topics [Array<String>] List of topic strings
+    # @param target_pattern [String] Substring to identify target-specific topics (e.g. 'CMD}TARGET__', '__TELEMETRY__')
+    # @param scope [String] Scope name for db_shard lookup
+    # @return [Hash] { db_shard => [topic, ...] }
+    def self.group_topics_by_db_shard(topics, target_pattern:, scope:)
+      groups = {}
+      topics.each do |topic|
+        if topic.include?(target_pattern)
+          target_name = topic.match(/__\{?([^}_]+)\}?__/)[1] rescue nil
+          # Handle CMD}TARGET__ pattern where target is after TARGET__
+          target_name = topic.split('TARGET__')[1] if target_pattern.include?('TARGET__') && target_name.nil?
+          db_shard = (Store.db_shard_for_target(target_name, scope: scope) || 0).to_i
+        else
+          db_shard = 0
+        end
+        groups[db_shard] ||= []
+        groups[db_shard] << topic
+      end
+      groups
+    end
+
+    # Check if all db_shard groups resolve to a single db_shard (fast path).
+    def self.all_same_db_shard?(db_shard_groups)
+      db_shard_groups.length <= 1
+    end
+
+    # Build the ACK topic from a command/router topic and write the ack.
+    def self.write_ack(topic, result, msg_id, db_shard: 0)
+      ack_topic = topic.split("__")
+      ack_topic[1] = 'ACK' + ack_topic[1]
+      ack_topic = ack_topic.join("__")
+      Topic.write_topic(ack_topic, { 'result' => result, 'id' => msg_id }, '*', 100, db_shard: db_shard)
+    end
   end
 end

data/lib/openc3/utilities/aws_bucket.rb

@@ -177,6 +177,7 @@ module OpenC3
         @client.put_bucket_policy(options)
       rescue Aws::S3::Errors::NotImplemented, Aws::S3::Errors::ServiceError, Aws::S3::Errors::InternalError => e
         Logger.warn("put_bucket_policy for #{config_bucket} not supported by S3 backend: #{e.message}")
+        Logger.warn("Policy applied:\n#{config_policy}")
       end
 
       begin
@@ -186,6 +187,7 @@ module OpenC3
         @client.put_bucket_policy(options)
       rescue Aws::S3::Errors::NotImplemented, Aws::S3::Errors::ServiceError, Aws::S3::Errors::InternalError => e
         Logger.warn("put_bucket_policy for #{logs_bucket} not supported by S3 backend: #{e.message}")
+        Logger.warn("Policy applied:\n#{logs_policy}")
       end
     end
 

data/lib/openc3/utilities/cli_generator.rb

@@ -141,8 +141,9 @@ module OpenC3
         abort("Usage: cli generate #{args[0]} <NAME> (--ruby or --python)")
       end
 
-      # Create the local variables
-      plugin = args[1].downcase.gsub(/_+|-+/, '-')
+      # Create the local variables that are used in process_template below (see openc3/templates/plugin/plugin.gemspec as an example)
+      plugin_orig = args[1]
+      plugin = plugin_orig.downcase.gsub(/_+|-+/, '-')
       plugin_name = "openc3-cosmos-#{plugin}"
       if File.exist?(plugin_name)
         abort("Plugin #{plugin_name} already exists!")

data/lib/openc3/utilities/metric.rb

@@ -37,15 +37,28 @@ module OpenC3
 
     attr_reader :microservice
     attr_reader :scope
+    attr_reader :db_shard
     attr_reader :data
     attr_reader :mutex
 
-    def initialize(microservice:, scope:)
+    def initialize(microservice:, scope:, db_shard: nil)
       @scope = scope
       @microservice = microservice
       @data = {}
       @mutex = Mutex.new
 
+      if db_shard
+        @db_shard = db_shard
+      else
+        # Look up db_shard from MicroserviceModel
+        begin
+          json = Store.hget('openc3_microservices', microservice)
+          @db_shard = json ? JSON.parse(json)['db_shard'].to_i : 0
+        rescue
+          @db_shard = 0
+        end
+      end
+
       # Always make sure there is a update thread
       @@mutex.synchronize do
         @@instances << self
@@ -90,6 +103,7 @@ module OpenC3
             instance.mutex.synchronize do
               json = {}
               json['name'] = instance.microservice
+              json['db_shard'] = instance.db_shard
               values = instance.data
               json['values'] = values
               MetricModel.set(json, scope: instance.scope) if values.length > 0