openc3 7.0.1 → 7.1.0

data/lib/openc3/utilities/store_autoload.rb

@@ -20,27 +20,16 @@ require 'hiredis-client'
 require 'json'
 require 'connection_pool'
 
-if ENV['OPENC3_REDIS_CLUSTER']
-  require 'openc3-enterprise/utilities/store'
-  $openc3_redis_cluster = true
-else
-  $openc3_redis_cluster = false
-end
-
 module OpenC3
   class StoreConnectionPool < ConnectionPool
     def pipelined
-      if $openc3_redis_cluster
-        yield # TODO: Update keys to support pipelining in cluster
-      else
-        with do |redis|
-          redis.pipelined do |pipeline|
-            Thread.current[:pipeline] = pipeline
-            begin
-              yield
-            ensure
-              Thread.current[:pipeline] = nil
-            end
+      with do |redis|
+        redis.pipelined do |pipeline|
+          Thread.current[:pipeline] = pipeline
+          begin
+            yield
+          ensure
+            Thread.current[:pipeline] = nil
           end
         end
       end
@@ -57,8 +46,13 @@ module OpenC3
   end
 
   class Store
-    # Variable that holds the singleton instance
-    @instance = nil
+    # Variable that holds the singleton instances per db_shard
+    @instances = []
+
+    # DB_Shard cache: { "scope__target_name" => [db_shard_number, Time] }
+    @@db_shard_cache = {}
+    @@db_shard_cache_mutex = Mutex.new
+    DB_SHARD_CACHE_TIMEOUT = 60 # seconds
 
     # Mutex used to ensure that only one instance is created
     @@instance_mutex = Mutex.new
@@ -66,14 +60,48 @@ module OpenC3
     attr_reader :redis_url
     attr_reader :redis_pool
 
+    # Look up the db_shard number for a target with a 1-minute cache.
+    # Reads directly from Redis db_shard 0 to avoid circular deps with TargetModel.
+    # Non-target-specific data (nil target_name) always returns db_shard 0.
+    def self.db_shard_for_target(target_name, scope: "DEFAULT")
+      return 0 unless target_name
+
+      cache_key = "#{scope}__#{target_name}"
+      now = Time.now
+
+      @@db_shard_cache_mutex.synchronize do
+        cached = @@db_shard_cache[cache_key]
+        if cached
+          db_shard, cached_at = cached
+          return db_shard if (now - cached_at) < DB_SHARD_CACHE_TIMEOUT
+        end
+      end
+
+      begin
+        json = Store.instance(db_shard: 0).hget("#{scope}__openc3_targets", target_name)
+        db_shard = json ? JSON.parse(json)['db_shard'].to_i : 0
+      rescue
+        db_shard = 0
+      end
+
+      @@db_shard_cache_mutex.synchronize do
+        @@db_shard_cache[cache_key] = [db_shard, now]
+      end
+
+      db_shard
+    end
+
     # Get the singleton instance
-    def self.instance(pool_size = 100)
+    def self.instance(pool_size = 100, db_shard: 0)
       # Logger.level = Logger::DEBUG
-      return @instance if @instance
+      @instances ||= []
+      the_instance = @instances[db_shard]
+      return the_instance if the_instance
 
       @@instance_mutex.synchronize do
-        @instance ||= self.new(pool_size)
-        return @instance
+        @instances ||= []
+        @instances[db_shard] ||= self.new(pool_size, db_shard: db_shard)
+        return @instances[db_shard]
       end
     end
 
@@ -87,17 +115,16 @@ module OpenC3
       @redis_pool.with { |redis| redis.public_send(message, *args, **kwargs, &block) }
     end
 
-    def initialize(pool_size = 10)
+    def initialize(pool_size = 10, db_shard: 0)
       @redis_username = ENV['OPENC3_REDIS_USERNAME']
       @redis_key = ENV['OPENC3_REDIS_PASSWORD']
-      @redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME']}:#{ENV['OPENC3_REDIS_PORT']}"
+      hostname = ENV['OPENC3_REDIS_HOSTNAME'].to_s.gsub("SHARDNUM", db_shard.to_s)
+      @redis_url = "redis://#{hostname}:#{ENV.fetch('OPENC3_REDIS_PORT', 6379)}"
       @redis_pool = StoreConnectionPool.new(size: pool_size) { build_redis() }
     end
 
-    unless $openc3_redis_cluster
-      def build_redis
-        return Redis.new(url: @redis_url, username: @redis_username, password: @redis_key)
-      end
+    def build_redis
+      return Redis.new(url: @redis_url, username: @redis_username, password: @redis_key)
     end
 
     ###########################################################################
@@ -160,30 +187,28 @@ module OpenC3
       return offsets
     end
 
-    unless $openc3_redis_cluster
-      def read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil)
-        return {} if topics.empty?
-        Thread.current[:topic_offsets] ||= {}
-        topic_offsets = Thread.current[:topic_offsets]
-        begin
-          # Logger.debug "read_topics: #{topics}, #{offsets} pool:#{@redis_pool}"
-          @redis_pool.with do |redis|
-            offsets = update_topic_offsets(topics) unless offsets
-            result = redis.xread(topics, offsets, block: timeout_ms, count: count)
-            if result and result.length > 0
-              result.each do |topic, messages|
-                messages.each do |msg_id, msg_hash|
-                  topic_offsets[topic] = msg_id
-                  yield topic, msg_id, msg_hash, redis if block_given?
-                end
+    def read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil)
+      return {} if topics.empty?
+      Thread.current[:topic_offsets] ||= {}
+      topic_offsets = Thread.current[:topic_offsets]
+      begin
+        # Logger.debug "read_topics: #{topics}, #{offsets} pool:#{@redis_pool}"
+        @redis_pool.with do |redis|
+          offsets = update_topic_offsets(topics) unless offsets
+          result = redis.xread(topics, offsets, block: timeout_ms, count: count)
+          if result and result.length > 0
+            result.each do |topic, messages|
+              messages.each do |msg_id, msg_hash|
+                topic_offsets[topic] = msg_id
+                yield topic, msg_id, msg_hash, redis if block_given?
               end
             end
-            # Logger.debug "result:#{result}" if result and result.length > 0
-            return result
           end
-        rescue Redis::TimeoutError
-          return {} # Should return an empty hash not array - xread returns a hash
+          # Logger.debug "result:#{result}" if result and result.length > 0
+          return result
         end
+      rescue Redis::TimeoutError
+        return {} # Should return an empty hash not array - xread returns a hash
       end
     end
 
@@ -233,9 +258,10 @@ module OpenC3
   end
 
   class EphemeralStore < Store
-    def initialize(pool_size = 10)
+    def initialize(pool_size = 10, db_shard: 0)
       super(pool_size)
-      @redis_url = "redis://#{ENV['OPENC3_REDIS_EPHEMERAL_HOSTNAME']}:#{ENV['OPENC3_REDIS_EPHEMERAL_PORT']}"
+      hostname = ENV['OPENC3_REDIS_EPHEMERAL_HOSTNAME'].to_s.gsub("SHARDNUM", db_shard.to_s)
+      @redis_url = "redis://#{hostname}:#{ENV.fetch('OPENC3_REDIS_EPHEMERAL_PORT', 6380)}"
       @redis_pool = StoreConnectionPool.new(size: pool_size) { build_redis() }
     end
   end

data/lib/openc3/utilities/store_queued.rb

@@ -18,30 +18,34 @@ module OpenC3
   class StoreQueued
     attr_reader :update_interval
 
-    # Variable that holds the singleton instance
-    @instance = nil
+    # Variable that holds the singleton instances per db_shard
+    @instances = []
 
     # Mutex used to ensure that only one instance is created
     @@instance_mutex = Mutex.new
 
-    # Get the singleton instance
+    # Get the singleton instance for the given db_shard
     # Sets the update interval to 1 second by default
-    def self.instance(update_interval = 1) # seconds
-      return @instance if @instance
+    def self.instance(update_interval = 1, db_shard: 0) # seconds
+      @instances ||= []
+      the_instance = @instances[db_shard]
+      return the_instance if the_instance
 
       @@instance_mutex.synchronize do
-        @instance ||= self.new(update_interval)
-        return @instance
+        @instances ||= []
+        @instances[db_shard] ||= self.new(update_interval, db_shard: db_shard)
+        return @instances[db_shard]
       end
     end
 
-    # Delegate all unknown class methods to delegate to the instance
+    # Delegate all unknown class methods to delegate to the instance (db_shard 0)
     def self.method_missing(message, *args, **kwargs, &)
       self.instance.public_send(message, *args, **kwargs, &)
     end
 
-    def initialize(update_interval)
+    def initialize(update_interval, db_shard: 0)
       @update_interval = update_interval
+      @db_shard = db_shard
       @store = store_instance()
       # Queue to hold the store requests
       @store_queue = Queue.new
@@ -80,7 +84,11 @@ module OpenC3
       while true
         start_time = Time.now
 
-        process_queue()
+        begin
+          process_queue()
+        rescue => e
+          puts "StoreQueued thread error (db_shard=#{@db_shard}):\n#{e.formatted}"
+        end
 
         # Only check whether to update at a set interval
         run_time = Time.now - start_time
@@ -107,7 +115,7 @@ module OpenC3
 
     # Returns the store we're working with
     def store_instance
-      Store.instance
+      Store.instance(db_shard: @db_shard)
     end
 
     def graceful_kill
@@ -117,7 +125,7 @@ module OpenC3
 
   class EphemeralStoreQueued < StoreQueued
     def store_instance
-      EphemeralStore.instance
+      EphemeralStore.instance(db_shard: @db_shard)
     end
   end
 end

data/lib/openc3/version.rb

@@ -1,14 +1,14 @@
 # encoding: ascii-8bit
 
-OPENC3_VERSION = '7.0.1'
+OPENC3_VERSION = '7.1.0'
 module OpenC3
   module Version
     MAJOR = '7'
-    MINOR = '0'
-    PATCH = '1'
+    MINOR = '1'
+    PATCH = '0'
     OTHER = ''
-    BUILD = '665de79cef884b8d470b08817fd4a1fb538fb187'
+    BUILD = '1074049d7a87d4b4d8cdc31e3512ab495b7492bb'
   end
-  VERSION = '7.0.1'
-  GEM_VERSION = '7.0.1'
+  VERSION = '7.1.0'
+  GEM_VERSION = '7.1.0'
 end

data/templates/plugin/plugin.gemspec

@@ -7,7 +7,7 @@ Gem::Specification.new do |s|
   s.description = <<-EOF
     <%= plugin_name %> plugin for deployment to OpenC3
   EOF
-  s.license = 'MIT'
+  s.licenses = 'MIT'
   s.authors = ['Anonymous']
   s.email = ['name@domain.com']
   s.homepage = 'https://github.com/OpenC3/cosmos'
@@ -21,4 +21,16 @@ Gem::Specification.new do |s|
     s.version = '0.0.0' + ".#{time}"
   end
   s.files = Dir.glob("{targets,lib,public,tools,microservices}/**/*") + %w(Rakefile README.md LICENSE.md plugin.txt)
+
+  s.metadata = {
+    # These fields are used when you submit your plugin to our App Store at store.openc3.com
+    # See this help page for more detail: https://store.openc3.com/help/guidelines
+    "source_code_uri" => "https://github.com/your-github/plugin-repo",
+    "openc3_store_title" => "<%= plugin_orig %>",
+    "openc3_store_description" => "Describe what your plugin does here.",
+    "openc3_store_keywords" => "some, comma-delimited, search terms",
+    "openc3_store_image" => "public/store_img.png",
+    "openc3_cosmos_minimum_version" => "6.0.0", # OPTIONAL
+    "openc3_store_access_type" => "public" # OPTIONAL
+  }
 end

data/templates/tool_angular/package.json

@@ -1,6 +1,6 @@
 {
   "name": "<%= tool_name %>",
-  "version": "7.0.1",
+  "version": "7.1.0",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
@@ -23,7 +23,7 @@
     "@angular/platform-browser-dynamic": "^18.2.6",
     "@angular/router": "^18.2.6",
     "@astrouxds/astro-web-components": "^7.24.0",
-    "@openc3/js-common": "7.0.1",
+    "@openc3/js-common": "7.1.0",
     "rxjs": "~7.8.0",
     "single-spa": "^5.9.5",
     "single-spa-angular": "^9.2.0",

data/templates/tool_react/package.json

@@ -16,7 +16,7 @@
     "@emotion/react": "^11.13.3",
     "@emotion/styled": "^11.11.0",
     "@mui/material": "^6.1.1",
-    "@openc3/js-common": "7.0.1",
+    "@openc3/js-common": "7.1.0",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "single-spa-react": "^5.1.4"

data/templates/tool_svelte/package.json

@@ -12,7 +12,7 @@
   },
   "dependencies": {
     "@astrouxds/astro-web-components": "^7.24.0",
-    "@openc3/js-common": "7.0.1",
+    "@openc3/js-common": "7.1.0",
     "@smui/button": "^7.0.0",
     "@smui/common": "^7.0.0",
     "@smui/card": "^7.0.0",

data/templates/tool_vue/package.json

@@ -1,6 +1,6 @@
 {
   "name": "<%= tool_name %>",
-  "version": "7.0.1",
+  "version": "7.1.0",
   "private": true,
   "type": "module",
   "scripts": {
@@ -11,8 +11,8 @@
   },
   "dependencies": {
     "@astrouxds/astro-web-components": "^7.24.0",
-    "@openc3/js-common": "7.0.1",
-    "@openc3/vue-common": "7.0.1",
+    "@openc3/js-common": "7.1.0",
+    "@openc3/vue-common": "7.1.0",
     "axios": "^1.7.7",
     "date-fns": "^4.1.0",
     "lodash": "^4.17.21",

data/templates/tool_vue/src/router.js

@@ -1,5 +1,5 @@
 /*
-# Copyright 2023 OpenC3, Inc.
+# Copyright 2026 OpenC3, Inc.
 # All Rights Reserved.
 #
 # This file may also be used under the terms of a commercial license
@@ -22,7 +22,7 @@ const routes = [
     component: NotFound,
   },
 ]
-routes.forEach(prependBasePath)
+routes.forEach((route) => prependBasePath(route))
 
 export default createRouter({
   history: createWebHistory(),

data/templates/widget/package.json

@@ -1,6 +1,6 @@
 {
   "name": "<%= widget_name %>",
-  "version": "7.0.1",
+  "version": "7.1.0",
   "private": true,
   "type": "module",
   "scripts": {
@@ -8,7 +8,7 @@
   },
   "dependencies": {
     "@astrouxds/astro-web-components": "^7.24.0",
-    "@openc3/vue-common": "7.0.1",
+    "@openc3/vue-common": "7.1.0",
     "vuetify": "^3.7.1"
   },
   "devDependencies": {

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: openc3
 version: !ruby/object:Gem::Version
-  version: 7.0.1
+  version: 7.1.0
 platform: ruby
 authors:
 - Ryan Melton
@@ -184,14 +184,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '8.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '7.0'
+        version: '8.0'
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
@@ -1081,6 +1081,7 @@ files:
 - lib/openc3/logs/stream_log_pair.rb
 - lib/openc3/logs/text_log_writer.rb
 - lib/openc3/microservices/cleanup_microservice.rb
+- lib/openc3/microservices/decom_common.rb
 - lib/openc3/microservices/decom_microservice.rb
 - lib/openc3/microservices/interface_decom_common.rb
 - lib/openc3/microservices/interface_microservice.rb
@@ -1107,6 +1108,7 @@ files:
 - lib/openc3/models/activity_model.rb
 - lib/openc3/models/auth_model.rb
 - lib/openc3/models/cvt_model.rb
+- lib/openc3/models/db_sharded_model.rb
 - lib/openc3/models/environment_model.rb
 - lib/openc3/models/gem_model.rb
 - lib/openc3/models/info_model.rb
@@ -1128,6 +1130,7 @@ files:
 - lib/openc3/models/python_package_model.rb
 - lib/openc3/models/queue_model.rb
 - lib/openc3/models/reaction_model.rb
+- lib/openc3/models/reingest_job_model.rb
 - lib/openc3/models/router_model.rb
 - lib/openc3/models/router_status_model.rb
 - lib/openc3/models/scope_model.rb
@@ -1259,6 +1262,7 @@ files:
 - lib/openc3/utilities/quaternion.rb
 - lib/openc3/utilities/questdb_client.rb
 - lib/openc3/utilities/redis_secrets.rb
+- lib/openc3/utilities/reingest_job.rb
 - lib/openc3/utilities/ruby_lex_utils.rb
 - lib/openc3/utilities/running_script.rb
 - lib/openc3/utilities/s3_autoload.rb