openc3 7.0.0 → 7.1.0

data/lib/openc3/models/cvt_model.rb

@@ -30,12 +30,22 @@ module OpenC3
       packet.decom
     end
 
+    # Get a Store instance routed to the correct db_shard for a target
+    def self.store_for_target(target_name, scope:)
+      Store.instance(db_shard: Store.db_shard_for_target(target_name, scope: scope))
+    end
+
+    # Get a StoreQueued instance routed to the correct db_shard for a target
+    def self.store_queued_for_target(target_name, scope:)
+      StoreQueued.instance(db_shard: Store.db_shard_for_target(target_name, scope: scope))
+    end
+
     # Delete the current value table for a target
     def self.del(target_name:, packet_name:, scope: $openc3_scope)
       key = "#{scope}__tlm__#{target_name}"
       tgt_pkt_key = key + "__#{packet_name}"
       @@packet_cache[tgt_pkt_key] = nil
-      Store.hdel(key, packet_name)
+      store_for_target(target_name, scope: scope).hdel(key, packet_name)
     end
 
     # Set the current value table for a target, packet
@@ -45,9 +55,9 @@ module OpenC3
       tgt_pkt_key = key + "__#{packet_name}"
       @@packet_cache[tgt_pkt_key] = [Time.now, hash]
       if queued
-        StoreQueued.hset(key, packet_name, packet_json)
+        store_queued_for_target(target_name, scope: scope).hset(key, packet_name, packet_json)
       else
-        Store.hset(key, packet_name, packet_json)
+        store_for_target(target_name, scope: scope).hset(key, packet_name, packet_json)
       end
     end
 
@@ -59,9 +69,9 @@ module OpenC3
       tgt_pkt_key = key + "__#{packet_name}"
       @@packet_cache[tgt_pkt_key] = [Time.now, hash]
       if queued
-        StoreQueued.hset(key, packet_name, packet_json)
+        store_queued_for_target(target_name, scope: scope).hset(key, packet_name, packet_json)
       else
-        Store.hset(key, packet_name, packet_json)
+        store_for_target(target_name, scope: scope).hset(key, packet_name, packet_json)
       end
     end
 
@@ -75,7 +85,7 @@ module OpenC3
         cache_time, hash = @@packet_cache[tgt_pkt_key]
         return hash if hash and (now - cache_time) < cache_timeout
       end
-      packet = Store.hget(key, packet_name)
+      packet = store_for_target(target_name, scope: scope).hget(key, packet_name)
       raise "Packet '#{target_name} #{packet_name}' does not exist" unless packet
       hash = JSON.parse(packet, allow_nan: true, create_additions: true)
       @@packet_cache[tgt_pkt_key] = [now, hash]
@@ -198,7 +208,7 @@ module OpenC3
     def self.overrides(scope: $openc3_scope)
       overrides = []
       TargetModel.names(scope: scope).each do |target_name|
-        all = Store.hgetall("#{scope}__override__#{target_name}")
+        all = store_for_target(target_name, scope: scope).hgetall("#{scope}__override__#{target_name}")
         next if all.nil? or all.empty?
         all.each do |packet_name, hash|
           items = JSON.parse(hash, allow_nan: true, create_additions: true)
@@ -227,7 +237,7 @@ module OpenC3
     # Override a current value table item such that it always returns the same value
     # for the given type
     def self.override(target_name, packet_name, item_name, value, type: :ALL, scope: $openc3_scope)
-      hash = Store.hget("#{scope}__override__#{target_name}", packet_name)
+      hash = store_for_target(target_name, scope: scope).hget("#{scope}__override__#{target_name}", packet_name)
       hash = JSON.parse(hash, allow_nan: true, create_additions: true) if hash
       hash ||= {} # In case the above didn't create anything
       case type
@@ -247,12 +257,12 @@ module OpenC3
 
       tgt_pkt_key = "#{scope}__tlm__#{target_name}__#{packet_name}"
       @@override_cache[tgt_pkt_key] = [Time.now, hash]
-      Store.hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json, allow_nan: true))
+      store_for_target(target_name, scope: scope).hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json, allow_nan: true))
     end
 
     # Normalize a current value table item such that it returns the actual value
     def self.normalize(target_name, packet_name, item_name, type: :ALL, scope: $openc3_scope)
-      hash = Store.hget("#{scope}__override__#{target_name}", packet_name)
+      hash = store_for_target(target_name, scope: scope).hget("#{scope}__override__#{target_name}", packet_name)
       hash = JSON.parse(hash, allow_nan: true, create_additions: true) if hash
       hash ||= {} # In case the above didn't create anything
       case type
@@ -271,12 +281,13 @@ module OpenC3
       end
 
       tgt_pkt_key = "#{scope}__tlm__#{target_name}__#{packet_name}"
+      store = store_for_target(target_name, scope: scope)
       if hash.empty?
         @@override_cache.delete(tgt_pkt_key)
-        Store.hdel("#{scope}__override__#{target_name}", packet_name)
+        store.hdel("#{scope}__override__#{target_name}", packet_name)
       else
         @@override_cache[tgt_pkt_key] = [Time.now, hash]
-        Store.hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json, allow_nan: true))
+        store.hset("#{scope}__override__#{target_name}", packet_name, JSON.generate(hash.as_json, allow_nan: true))
       end
     end
 
@@ -334,7 +345,7 @@ module OpenC3
           return hash
         end
       end
-      override_data = Store.hget("#{scope}__override__#{target_name}", packet_name)
+      override_data = store_for_target(target_name, scope: scope).hget("#{scope}__override__#{target_name}", packet_name)
       if override_data
         hash = JSON.parse(override_data, allow_nan: true, create_additions: true)
         overrides[tgt_pkt_key] = hash

data/lib/openc3/models/db_sharded_model.rb

@@ -0,0 +1,110 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+# Mixin that provides db_shard-aware Redis operations with hard caching.
+# Including classes must define two class methods:
+#   _lookup_db_shard(name, scope:) -> Integer
+#   _collect_db_shards(scope:) -> Set
+module OpenC3
+  module DbShardedModel
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    module ClassMethods
+      # Lookup of db_shard for a given name.
+      # Hard-cached only when use_cache: true (intended for the set/create path
+      # where the db_shard won't change within the process lifetime).
+      def _db_shard_for_name(name, scope:, use_cache: false)
+        cache = nil
+
+        if use_cache
+          cache = (@db_shard_cache ||= {})
+          cache_key = "#{scope}__#{name}"
+          cached = cache[cache_key]
+          return cached unless cached.nil?
+        end
+
+        db_shard = _lookup_db_shard(name, scope: scope)
+
+        if use_cache
+          cache[cache_key] = db_shard
+        end
+
+        db_shard
+      end
+
+      # Collect all active db_shards (always fresh lookup, no cache).
+      def _active_db_shards(scope:)
+        _collect_db_shards(scope: scope)
+      end
+
+      # DB_Shard-aware get: looks up the db_shard for name, reads from the correct store instance.
+      def _db_sharded_get(key, name:, scope:)
+        db_shard = _db_shard_for_name(name, scope: scope)
+        json = store.instance(db_shard: db_shard).hget(key, name)
+        json ? JSON.parse(json, allow_nan: true, create_additions: true) : nil
+      end
+
+      # DB_Shard-aware names: iterates all active db_shards and collects keys.
+      def _db_sharded_names(key, scope:)
+        result = []
+        _active_db_shards(scope: scope).each do |db_shard|
+          result.concat(store.instance(db_shard: db_shard).hkeys(key))
+        end
+        result.uniq.sort
+      end
+
+      # DB_Shard-aware all: iterates all active db_shards and collects all values.
+      def _db_sharded_all(key, scope:)
+        result = {}
+        _active_db_shards(scope: scope).each do |db_shard|
+          hash = store.instance(db_shard: db_shard).hgetall(key)
+          hash.each do |k, value|
+            result[k] = JSON.parse(value, allow_nan: true, create_additions: true)
+          end
+        end
+        result
+      end
+    end
+
+    # DB_Shard-aware create: writes to the store instance for the given db_shard.
+    def _db_sharded_create(db_shard, update: false, force: false, queued: false, isoformat: false, expire_seconds: nil)
+      db_shard_store = self.class.store.instance(db_shard: db_shard)
+      unless force
+        existing = db_shard_store.hget(@primary_key, @name)
+        if existing
+          raise RuntimeError.new("#{@primary_key}:#{@name} already exists at create") unless update
+        else
+          raise RuntimeError.new("#{@primary_key}:#{@name} doesn't exist at update") if update
+        end
+      end
+      @updated_at = isoformat ? Time.now.utc.iso8601 : Time.now.utc.to_nsec_from_epoch
+
+      if queued
+        store = self.class.store_queued.instance(db_shard: db_shard)
+        store.hset(@primary_key, @name, JSON.generate(self.as_json(), allow_nan: true))
+        store.call(:hexpire, @primary_key, expire_seconds, 'FIELDS', 1, @name) if expire_seconds
+      else
+        db_shard_store.hset(@primary_key, @name, JSON.generate(self.as_json(), allow_nan: true))
+        db_shard_store.call(:hexpire, @primary_key, expire_seconds, 'FIELDS', 1, @name) if expire_seconds
+      end
+    end
+
+    # DB_Shard-aware destroy: deletes from the store instance for the given db_shard.
+    def _db_sharded_destroy(db_shard)
+      @destroyed = true
+      self.class.store.instance(db_shard: db_shard).hdel(@primary_key, @name)
+    end
+  end
+end

data/lib/openc3/models/interface_model.rb

@@ -48,6 +48,7 @@ module OpenC3
     attr_accessor :ports
     attr_accessor :prefix
     attr_accessor :shard
+    attr_accessor :db_shard
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
@@ -122,6 +123,7 @@ module OpenC3
       container: nil,
       prefix: nil,
       shard: 0,
+      db_shard: 0,
       scope:
     )
       if self.class._get_type == 'INTERFACE'
@@ -177,6 +179,7 @@ module OpenC3
       @container = container
       @prefix = prefix
       @shard = shard.to_i # to_i to handle nil
+      @db_shard = db_shard.to_i # to_i to handle nil
       @secrets = secrets
     end
 
@@ -246,6 +249,7 @@ module OpenC3
         'container' => @container,
         'prefix' => @prefix,
         'shard' => @shard,
+        'db_shard' => @db_shard,
         'updated_at' => @updated_at
       }
     end
@@ -398,6 +402,10 @@ module OpenC3
       when 'SHARD'
         parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
         @shard = Integer(parameters[0])
+
+      when 'DB_SHARD'
+        parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
+        @db_shard = Integer(parameters[0])
       else
         raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Interface/Router: #{keyword} #{parameters.join(" ")}")
 
@@ -423,6 +431,7 @@ module OpenC3
         secrets: @secrets,
         prefix: @prefix,
         shard: @shard,
+        db_shard: @db_shard,
         scope: @scope
       )
       unless validate_only

data/lib/openc3/models/interface_status_model.rb

@@ -16,12 +16,15 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/models/model'
+require 'openc3/models/db_sharded_model'
 
 module OpenC3
   # Stores the status about an interface. This class also implements logic
   # to handle status for a router since the functionality is identical
   # (only difference is the Redis key used).
   class InterfaceStatusModel < Model
+    include DbShardedModel
+
     INTERFACES_PRIMARY_KEY = 'openc3_interface_status'
     ROUTERS_PRIMARY_KEY = 'openc3_router_status'
 
@@ -34,18 +37,37 @@ module OpenC3
     attr_accessor :txcnt
     attr_accessor :rxcnt
 
+    # Look up db_shard from the corresponding InterfaceModel or RouterModel.
+    def self._lookup_db_shard(name, scope:)
+      type = _get_type
+      key = type == 'INTERFACESTATUS' ? "#{scope}__openc3_interfaces" : "#{scope}__openc3_routers"
+      json = Store.hget(key, name)
+      json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
+    end
+
+    # Collect all unique db_shard values from InterfaceModels or RouterModels.
+    def self._collect_db_shards(scope:)
+      db_shards = Set.new([0])
+      type = _get_type
+      key = type == 'INTERFACESTATUS' ? "#{scope}__openc3_interfaces" : "#{scope}__openc3_routers"
+      Store.hgetall(key).each do |_name, json|
+        db_shards << (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i
+      end
+      db_shards
+    end
+
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
     def self.get(name:, scope:)
-      super("#{scope}__#{_get_key}", name: name)
+      _db_sharded_get("#{scope}__#{_get_key}", name: name, scope: scope)
     end
 
     def self.names(scope:)
-      super("#{scope}__#{_get_key}")
+      _db_sharded_names("#{scope}__#{_get_key}", scope: scope)
     end
 
     def self.all(scope:)
-      super("#{scope}__#{_get_key}")
+      _db_sharded_all("#{scope}__#{_get_key}", scope: scope)
     end
     # END NOTE
 
@@ -96,6 +118,14 @@ module OpenC3
       @rxcnt = rxcnt
     end
 
+    def create(update: false, force: false, queued: false, isoformat: false)
+      _db_sharded_create(self.class._db_shard_for_name(@name, scope: @scope, use_cache: true), update: update, force: force, queued: queued, isoformat: isoformat)
+    end
+
+    def destroy
+      _db_sharded_destroy(self.class._db_shard_for_name(@name, scope: @scope))
+    end
+
     def as_json(*a)
       {
         'name' => @name,

data/lib/openc3/models/metric_model.rb

@@ -16,25 +16,46 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/models/model'
+require 'openc3/models/db_sharded_model'
 
 module OpenC3
   class MetricModel < EphemeralModel
+    include DbShardedModel
+
     PRIMARY_KEY = '__openc3__metric'.freeze
+    METRIC_EXPIRE_SECONDS = 3600 # Expire metrics after 1 hour
 
     attr_accessor :values
+    attr_accessor :db_shard
+
+    # Look up db_shard from the corresponding MicroserviceModel.
+    def self._lookup_db_shard(name, scope:) # NOSONAR
+      json = Store.hget('openc3_microservices', name)
+      json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
+    end
+
+    # Collect all unique db_shard values from MicroserviceModels.
+    def self._collect_db_shards(scope:)
+      db_shards = Set.new([0])
+      Store.hgetall('openc3_microservices').each do |name, json|
+        next if scope and name.split("__")[0] != scope
+        db_shards << (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i
+      end
+      db_shards
+    end
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
     def self.get(name:, scope:)
-      super("#{scope}#{PRIMARY_KEY}", name: name)
+      _db_sharded_get("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
     end
 
     def self.names(scope:)
-      super("#{scope}#{PRIMARY_KEY}")
+      _db_sharded_names("#{scope}#{PRIMARY_KEY}", scope: scope)
     end
 
     def self.all(scope:)
-      super("#{scope}#{PRIMARY_KEY}")
+      _db_sharded_all("#{scope}#{PRIMARY_KEY}", scope: scope)
     end
 
     # Sets (updates) the redis hash of this model
@@ -42,23 +63,34 @@ module OpenC3
     def self.set(json, scope:, queued: true)
       json[:scope] = scope
       json.transform_keys!(&:to_sym)
-      self.new(**json).create(force: true, queued: queued)
+      self.new(**json).create(force: true, queued: queued, expire_seconds: METRIC_EXPIRE_SECONDS)
     end
 
     def self.destroy(scope:, name:)
-      EphemeralStore.hdel("#{scope}#{PRIMARY_KEY}", name)
+      db_shard = _db_shard_for_name(name, scope: scope)
+      store.instance(db_shard: db_shard).hdel("#{scope}#{PRIMARY_KEY}", name)
     end
 
-    def initialize(name:, values: {}, scope:)
+    def initialize(name:, values: {}, db_shard: 0, scope:)
       super("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
       @values = values
+      @db_shard = db_shard.to_i
+    end
+
+    def create(update: false, force: false, queued: false, isoformat: false, expire_seconds: nil)
+      _db_sharded_create(@db_shard, update: update, force: force, queued: queued, isoformat: isoformat, expire_seconds: expire_seconds)
+    end
+
+    def destroy
+      _db_sharded_destroy(@db_shard)
     end
 
     def as_json(*a)
       {
         'name' => @name,
         'updated_at' => @updated_at,
-        'values' => @values.as_json(*a)
+        'values' => @values.as_json(*a),
+        'db_shard' => @db_shard,
       }
     end
 
@@ -74,37 +106,64 @@ module OpenC3
     end
 
     def self.redis_metrics
-      result = {}
+      # This prevents a circular dependency
+      require 'openc3/models/scope_model' # NOSONAR
+      require 'openc3/models/target_model' # NOSONAR
+
+      db_shards = Set.new
+      OpenC3::ScopeModel.names.each do |scope|
+        targets = OpenC3::TargetModel.all(scope: scope)
+        targets.each do |_target_name, target_hash|
+          db_shards << target_hash['db_shard'].to_i
+        end
+      end
 
-      metrics = OpenC3::Store.info("all")
-      result['redis_connected_clients_total'] = metrics['connected_clients']
-      result['redis_used_memory_rss_total'] = metrics['used_memory_rss']
-      result['redis_commands_processed_total'] = metrics['total_commands_processed']
-      result['redis_iops'] = metrics['instantaneous_ops_per_sec']
-      result['redis_instantaneous_input_kbps'] = metrics['instantaneous_input_kbps']
-      result['redis_instantaneous_output_kbps'] = metrics['instantaneous_output_kbps']
-      result['redis_hget_p50_seconds'], result['redis_hget_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
-      result['redis_hgetall_p50_seconds'], result['redis_hgetall_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
-      result['redis_hset_p50_seconds'], result['redis_hset_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
-      result['redis_xadd_p50_seconds'], result['redis_xadd_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
-      result['redis_xread_p50_seconds'], result['redis_xread_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
-      result['redis_xrevrange_p50_seconds'], result['redis_xrevrange_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
-      result['redis_xtrim_p50_seconds'], result['redis_xtrim_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
-
-      metrics = OpenC3::EphemeralStore.info("all")
-      result['redis_ephemeral_connected_clients_total'] = metrics['connected_clients']
-      result['redis_ephemeral_used_memory_rss_total'] = metrics['used_memory_rss']
-      result['redis_ephemeral_commands_processed_total'] = metrics['total_commands_processed']
-      result['redis_ephemeral_iops'] = metrics['instantaneous_ops_per_sec']
-      result['redis_ephemeral_instantaneous_input_kbps'] = metrics['instantaneous_input_kbps']
-      result['redis_ephemeral_instantaneous_output_kbps'] = metrics['instantaneous_output_kbps']
-      result['redis_ephemeral_hget_p50_seconds'], result['redis_ephemeral_hget_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
-      result['redis_ephemeral_hgetall_p50_seconds'], result['redis_ephemeral_hgetall_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
-      result['redis_ephemeral_hset_p50_seconds'], result['redis_ephemeral_hset_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
-      result['redis_ephemeral_xadd_p50_seconds'], result['redis_ephemeral_xadd_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
-      result['redis_ephemeral_xread_p50_seconds'], result['redis_ephemeral_xread_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
-      result['redis_ephemeral_xrevrange_p50_seconds'], result['redis_ephemeral_xrevrange_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
-      result['redis_ephemeral_xtrim_p50_seconds'], result['redis_ephemeral_xtrim_p99_seconds'] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
+      result = {}
+      db_shards.each do |index|
+        db_shard_result = {}
+        metrics = OpenC3::Store.instance(db_shard: index).info("all")
+        db_shard_result["redis_connected_clients_total"] = metrics['connected_clients']
+        db_shard_result["redis_used_memory_rss_total"] = metrics['used_memory_rss']
+        db_shard_result["redis_commands_processed_total"] = metrics['total_commands_processed']
+        db_shard_result["redis_iops"] = metrics['instantaneous_ops_per_sec']
+        db_shard_result["redis_instantaneous_input_kbps"] = metrics['instantaneous_input_kbps']
+        db_shard_result["redis_instantaneous_output_kbps"] = metrics['instantaneous_output_kbps']
+        db_shard_result["redis_instantaneous_eventloop_cps"] = metrics['instantaneous_eventloop_cycles_per_sec']
+        db_shard_result["redis_instantaneous_eventloop_duration_usec"] = metrics['instantaneous_eventloop_duration_usec']
+        db_shard_result["redis_cpu_sys"] = metrics['used_cpu_sys']
+        db_shard_result["redis_cpu_user"] = metrics['used_cpu_user']
+        db_shard_result["redis_error_noauth_total"] = metrics['errorstat_NOAUTH'].to_s.split("count=")[-1].to_i
+        db_shard_result["redis_error_noperm_total"] = metrics['errorstat_NOPERM'].to_s.split("count=")[-1].to_i
+        db_shard_result["redis_hget_p50_seconds"], db_shard_result["redis_hget_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
+        db_shard_result["redis_hgetall_p50_seconds"], db_shard_result["redis_hgetall_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
+        db_shard_result["redis_hset_p50_seconds"], db_shard_result["redis_hset_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
+        db_shard_result["redis_xadd_p50_seconds"], db_shard_result["redis_xadd_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
+        db_shard_result["redis_xread_p50_seconds"], db_shard_result["redis_xread_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
+        db_shard_result["redis_xrevrange_p50_seconds"], db_shard_result["redis_xrevrange_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
+        db_shard_result["redis_xtrim_p50_seconds"], db_shard_result["redis_xtrim_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
+
+        metrics = OpenC3::EphemeralStore.instance(db_shard: index).info("all")
+        db_shard_result["redis_ephemeral_connected_clients_total"] = metrics['connected_clients']
+        db_shard_result["redis_ephemeral_used_memory_rss_total"] = metrics['used_memory_rss']
+        db_shard_result["redis_ephemeral_commands_processed_total"] = metrics['total_commands_processed']
+        db_shard_result["redis_ephemeral_iops"] = metrics['instantaneous_ops_per_sec']
+        db_shard_result["redis_ephemeral_instantaneous_input_kbps"] = metrics['instantaneous_input_kbps']
+        db_shard_result["redis_ephemeral_instantaneous_output_kbps"] = metrics['instantaneous_output_kbps']
+        db_shard_result["redis_ephemeral_instantaneous_eventloop_cps"] = metrics['instantaneous_eventloop_cycles_per_sec']
+        db_shard_result["redis_ephemeral_instantaneous_eventloop_duration_usec"] = metrics['instantaneous_eventloop_duration_usec']
+        db_shard_result["redis_ephemeral_cpu_sys"] = metrics['used_cpu_sys']
+        db_shard_result["redis_ephemeral_cpu_user"] = metrics['used_cpu_user']
+        db_shard_result["redis_ephemeral_error_noauth_total"] = metrics['errorstat_NOAUTH'].to_s.split("count=")[-1].to_i
+        db_shard_result["redis_ephemeral_error_noperm_total"] = metrics['errorstat_NOPERM'].to_s.split("count=")[-1].to_i
+        db_shard_result["redis_ephemeral_hget_p50_seconds"], db_shard_result["redis_ephemeral_hget_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hget'])
+        db_shard_result["redis_ephemeral_hgetall_p50_seconds"], db_shard_result["redis_ephemeral_hgetall_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hgetall'])
+        db_shard_result["redis_ephemeral_hset_p50_seconds"], db_shard_result["redis_ephemeral_hset_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_hset'])
+        db_shard_result["redis_ephemeral_xadd_p50_seconds"], db_shard_result["redis_ephemeral_xadd_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xadd'])
+        db_shard_result["redis_ephemeral_xread_p50_seconds"], db_shard_result["redis_ephemeral_xread_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xread'])
+        db_shard_result["redis_ephemeral_xrevrange_p50_seconds"], db_shard_result["redis_ephemeral_xrevrange_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xrevrange'])
+        db_shard_result["redis_ephemeral_xtrim_p50_seconds"], db_shard_result["redis_ephemeral_xtrim_p99_seconds"] = redis_extract_p50_and_p99_seconds(metrics['latency_percentiles_usec_xtrim'])
+        result[index] = db_shard_result
+      end
 
       return result
     end

data/lib/openc3/models/microservice_model.rb

@@ -41,6 +41,7 @@ module OpenC3
     attr_accessor :disable_erb
     attr_accessor :ignore_changes
     attr_accessor :shard
+    attr_accessor :db_shard
     attr_accessor :enabled
 
     # NOTE: The following three class methods are used by the ModelController
@@ -104,6 +105,7 @@ module OpenC3
       disable_erb: nil,
       ignore_changes: nil,
       shard: 0,
+      db_shard: 0,
       enabled: true,
       scope:
     )
@@ -132,6 +134,7 @@ module OpenC3
       @disable_erb = disable_erb
       @ignore_changes = ignore_changes
       @shard = shard.to_i # to_i to handle nil
+      @db_shard = db_shard.to_i # to_i to handle nil
       @enabled = enabled
       @enabled = true if @enabled.nil?
       @bucket = Bucket.getClient()
@@ -158,6 +161,7 @@ module OpenC3
         'disable_erb' => @disable_erb,
         'ignore_changes' => @ignore_changes,
         'shard' => @shard,
+        'db_shard' => @db_shard,
         'enabled' => @enabled,
       }
     end
@@ -236,6 +240,9 @@ module OpenC3
       when 'SHARD'
         parser.verify_num_parameters(1, 1, "#{keyword} <Shard Number Starting from 0>")
         @shard = Integer(parameters[0])
+      when 'DB_SHARD'
+        parser.verify_num_parameters(1, 1, "#{keyword} <DB_Shard Number Starting from 0>")
+        @db_shard = Integer(parameters[0])
       when 'STOPPED'
         parser.verify_num_parameters(0, 0, "#{keyword}")
         @enabled = false

data/lib/openc3/models/microservice_status_model.rb

@@ -16,9 +16,12 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/models/model'
+require 'openc3/models/db_sharded_model'
 
 module OpenC3
   class MicroserviceStatusModel < Model
+    include DbShardedModel
+
     PRIMARY_KEY = 'openc3_microservice_status'
 
     attr_accessor :state
@@ -26,18 +29,34 @@ module OpenC3
     attr_accessor :error
     attr_accessor :custom
 
+    # Look up db_shard from the corresponding MicroserviceModel.
+    def self._lookup_db_shard(name, scope:) # NOSONAR
+      json = Store.hget('openc3_microservices', name)
+      json ? (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i : 0
+    end
+
+    # Collect all unique db_shard values from MicroserviceModels.
+    def self._collect_db_shards(scope:)
+      db_shards = Set.new([0])
+      Store.hgetall('openc3_microservices').each do |name, json|
+        next if scope and name.split("__")[0] != scope
+        db_shards << (JSON.parse(json, allow_nan: true, create_additions: true)['db_shard'] || 0).to_i
+      end
+      db_shards
+    end
+
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
     def self.get(name:, scope:)
-      super("#{scope}__#{PRIMARY_KEY}", name: name)
+      _db_sharded_get("#{scope}__#{PRIMARY_KEY}", name: name, scope: scope)
     end
 
     def self.names(scope:)
-      super("#{scope}__#{PRIMARY_KEY}")
+      _db_sharded_names("#{scope}__#{PRIMARY_KEY}", scope: scope)
     end
 
     def self.all(scope:)
-      super("#{scope}__#{PRIMARY_KEY}")
+      _db_sharded_all("#{scope}__#{PRIMARY_KEY}", scope: scope)
     end
 
     def initialize(
@@ -57,6 +76,14 @@ module OpenC3
       @custom = custom
     end
 
+    def create(update: false, force: false, queued: false, isoformat: false)
+      _db_sharded_create(self.class._db_shard_for_name(@name, scope: @scope, use_cache: true), update: update, force: force, queued: queued, isoformat: isoformat)
+    end
+
+    def destroy
+      _db_sharded_destroy(self.class._db_shard_for_name(@name, scope: @scope))
+    end
+
     def as_json(*a)
       {
         'name' => @name,

data/lib/openc3/models/plugin_model.rb

@@ -280,7 +280,15 @@ module OpenC3
                 pip_args = "-i #{pypi_url} --trusted-host #{URI.parse(pypi_url).host} -r #{requirements_path}"
               end
             end
-            puts `/openc3/bin/pipinstall #{pip_args}`
+            # Capture output and check exit code so failures surface as a warning
+            # rather than silently succeeding. pipinstall is non-fatal: the plugin
+            # continues to install even if Python packages fail so that non-Python
+            # functionality still works.
+            output = `/openc3/bin/pipinstall #{pip_args}`
+            puts output
+            unless $?.success?
+              Logger.warn "Python package installation failed. Plugin Python microservices may not function correctly."
+            end
           end
           needs_dependencies = true
         end