openc3 7.0.0 → 7.1.0

data/lib/openc3/topics/topic.rb

@@ -19,18 +19,90 @@ require 'openc3/utilities/store'
 
 module OpenC3
   class Topic
-    # Delegate all unknown class methods to delegate to the EphemeralStore
+    # Delegate all unknown class methods to EphemeralStore db_shard 0 (system-level topics)
     def self.method_missing(message, *args, **kwargs, &block)
       EphemeralStore.public_send(message, *args, **kwargs, &block)
     end
 
-    def self.clear_topics(topics, maxlen = 0)
-      topics.each { |topic| EphemeralStore.xtrim(topic, maxlen) }
+    def self.clear_topics(topics, maxlen = 0, db_shard: 0)
+      store = EphemeralStore.instance(db_shard: db_shard)
+      topics.each { |topic| store.xtrim(topic, maxlen) }
     end
 
-    def self.get_cnt(topic)
-      _, packet = EphemeralStore.get_newest_message(topic)
+    def self.get_cnt(topic, db_shard: 0)
+      _, packet = EphemeralStore.instance(db_shard: db_shard).get_newest_message(topic)
       packet ? packet["received_count"].to_i : 0
     end
+
+    # DB_Shard-aware topic methods for target-specific streams.
+    # These explicitly route to the correct EphemeralStore db_shard.
+
+    def self.write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true', db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).write_topic(topic, msg_hash, id, maxlen, approximate)
+    end
+
+    def self.read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil, db_shard: 0, &block)
+      EphemeralStore.instance(db_shard: db_shard).read_topics(topics, offsets, timeout_ms, count, &block)
+    end
+
+    def self.get_newest_message(topic, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).get_newest_message(topic)
+    end
+
+    def self.get_oldest_message(topic, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).get_oldest_message(topic)
+    end
+
+    def self.get_last_offset(topic, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).get_last_offset(topic)
+    end
+
+    def self.update_topic_offsets(topics, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).update_topic_offsets(topics)
+    end
+
+    def self.trim_topic(topic, minid, approximate = true, limit: 0, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).trim_topic(topic, minid, approximate, limit: limit)
+    end
+
+    def self.del(topic, db_shard: 0)
+      EphemeralStore.instance(db_shard: db_shard).del(topic)
+    end
+
+    # Group topics by db_shard. Each topic's target name is extracted and looked up.
+    # Topics matching target_pattern are db_sharded; others go to db_shard 0.
+    # @param topics [Array<String>] List of topic strings
+    # @param target_pattern [String] Substring to identify target-specific topics (e.g. 'CMD}TARGET__', '__TELEMETRY__')
+    # @param scope [String] Scope name for db_shard lookup
+    # @return [Hash] { db_shard => [topic, ...] }
+    def self.group_topics_by_db_shard(topics, target_pattern:, scope:)
+      groups = {}
+      topics.each do |topic|
+        if topic.include?(target_pattern)
+          target_name = topic.match(/__\{?([^}_]+)\}?__/)[1] rescue nil
+          # Handle CMD}TARGET__ pattern where target is after TARGET__
+          target_name = topic.split('TARGET__')[1] if target_pattern.include?('TARGET__') && target_name.nil?
+          db_shard = (Store.db_shard_for_target(target_name, scope: scope) || 0).to_i
+        else
+          db_shard = 0
+        end
+        groups[db_shard] ||= []
+        groups[db_shard] << topic
+      end
+      groups
+    end
+
+    # Check if all db_shard groups resolve to a single db_shard (fast path).
+    def self.all_same_db_shard?(db_shard_groups)
+      db_shard_groups.length <= 1
+    end
+
+    # Build the ACK topic from a command/router topic and write the ack.
+    def self.write_ack(topic, result, msg_id, db_shard: 0)
+      ack_topic = topic.split("__")
+      ack_topic[1] = 'ACK' + ack_topic[1]
+      ack_topic = ack_topic.join("__")
+      Topic.write_topic(ack_topic, { 'result' => result, 'id' => msg_id }, '*', 100, db_shard: db_shard)
+    end
   end
 end

data/lib/openc3/utilities/aws_bucket.rb

@@ -177,6 +177,7 @@ module OpenC3
         @client.put_bucket_policy(options)
       rescue Aws::S3::Errors::NotImplemented, Aws::S3::Errors::ServiceError, Aws::S3::Errors::InternalError => e
         Logger.warn("put_bucket_policy for #{config_bucket} not supported by S3 backend: #{e.message}")
+        Logger.warn("Policy applied:\n#{config_policy}")
       end
 
       begin
@@ -186,6 +187,7 @@ module OpenC3
         @client.put_bucket_policy(options)
       rescue Aws::S3::Errors::NotImplemented, Aws::S3::Errors::ServiceError, Aws::S3::Errors::InternalError => e
         Logger.warn("put_bucket_policy for #{logs_bucket} not supported by S3 backend: #{e.message}")
+        Logger.warn("Policy applied:\n#{logs_policy}")
       end
     end
 

data/lib/openc3/utilities/cli_generator.rb

@@ -141,8 +141,9 @@ module OpenC3
         abort("Usage: cli generate #{args[0]} <NAME> (--ruby or --python)")
       end
 
-      # Create the local variables
-      plugin = args[1].downcase.gsub(/_+|-+/, '-')
+      # Create the local variables that are used in process_template below (see openc3/templates/plugin/plugin.gemspec as an example)
+      plugin_orig = args[1]
+      plugin = plugin_orig.downcase.gsub(/_+|-+/, '-')
       plugin_name = "openc3-cosmos-#{plugin}"
       if File.exist?(plugin_name)
         abort("Plugin #{plugin_name} already exists!")

data/lib/openc3/utilities/ctrf.rb

@@ -0,0 +1,231 @@
+# encoding: ascii-8bit
+
+# Copyright 2026 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+# See LICENSE.md for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/version'
+require 'date'
+
+module OpenC3
+  # Utility class for converting COSMOS script reports to CTRF (Common Test Report Format)
+  # See https://ctrf.io/docs/category/specification
+  class Ctrf
+    # Convert a COSMOS plain text script report to CTRF JSON format
+    # @param report_content [String] Plain text script report
+    # @param version [String] Version string to include in CTRF output (defaults to OpenC3::VERSION)
+    # @return [Hash] CTRF formatted report as a Ruby hash
+    def self.convert_report(report_content, version: OpenC3::VERSION)
+      lines = report_content.split("\n")
+      tests = []
+      summary = {}
+      settings = {}
+      in_settings = false
+      last_result = nil
+      in_summary = false
+
+      lines.each do |line|
+        next if line.nil?
+        line_clean = line.strip
+
+        if line_clean == 'Settings:'
+          in_settings = true
+          next
+        end
+
+        if in_settings
+          if line_clean.include?('Manual')
+            parts = line.split('=')
+            settings[:manual] = parts[1].strip if parts[1]
+            next
+          elsif line_clean.include?('Pause on Error')
+            parts = line.split('=')
+            settings[:pauseOnError] = parts[1].strip if parts[1]
+            next
+          elsif line_clean.include?('Continue After Error')
+            parts = line.split('=')
+            settings[:continueAfterError] = parts[1].strip if parts[1]
+            next
+          elsif line_clean.include?('Abort After Error')
+            parts = line.split('=')
+            settings[:abortAfterError] = parts[1].strip if parts[1]
+            next
+          elsif line_clean.include?('Loop =')
+            parts = line.split('=')
+            settings[:loop] = parts[1].strip if parts[1]
+            next
+          elsif line_clean.include?('Break Loop On Error')
+            parts = line.split('=')
+            settings[:breakLoopOnError] = parts[1].strip if parts[1]
+            in_settings = false
+            next
+          end
+        end
+
+        if line_clean == 'Results:'
+          last_result = line_clean
+          next
+        end
+
+        if last_result
+          # The first line should always have a timestamp and what it is executing
+          # Format: "2026-04-02T19:45:41.228209Z: Executing MySuite:ExampleGroup:script_2"
+          if last_result == 'Results:' and line_clean.include?("Executing")
+            # Split on first ': ' to separate timestamp from message
+            timestamp_and_msg = line_clean.split(': ', 2)
+            if timestamp_and_msg.length >= 2
+              timestamp = timestamp_and_msg[0]
+              begin
+                summary[:startTime] = DateTime.parse(timestamp).to_time.to_f * 1000
+              rescue Date::Error
+                # Skip malformed timestamps
+              end
+            end
+            last_result = line_clean
+            next
+          end
+
+          # Format: "2026-04-02T19:45:44.041472Z: ExampleGroup:script_2:PASS"
+          # Check if line contains a test result (but not Executing or Completed)
+          if !line_clean.include?("Executing") && !line_clean.include?("Completed")
+            # Try to parse as a test result line
+            timestamp_and_msg = line_clean.split(': ', 2)
+            if timestamp_and_msg.length >= 2
+              result_string = timestamp_and_msg[1]
+              # Must have at least 2 colons for group:name:status format
+              result_parts = result_string.split(':')
+              if result_parts.length >= 3
+                # Get start time from last_result - could be Executing line or previous test result
+                start_time = nil
+                if last_result
+                  last_timestamp_and_msg = last_result.split(': ', 2)
+                  if last_timestamp_and_msg.length >= 2
+                    begin
+                      start_time = DateTime.parse(last_timestamp_and_msg[0]).to_time.to_f * 1000
+                    rescue Date::Error
+                      # Skip malformed timestamps
+                    end
+                  end
+                end
+
+                # Parse current line timestamp
+                timestamp = timestamp_and_msg[0]
+                begin
+                  end_time = DateTime.parse(timestamp).to_time.to_f * 1000
+                rescue Date::Error
+                  last_result = line_clean
+                  next # Skip lines with malformed timestamps
+                end
+
+                # Parse the test result: ExampleGroup:script_2:PASS
+                suite_group = result_parts[0]
+                name = result_parts[1]
+                status = result_parts[2]
+
+                format_status = case status
+                when 'PASS'
+                  'passed'
+                when 'SKIP'
+                  'skipped'
+                when 'FAIL'
+                  'failed'
+                else
+                  'unknown'
+                end
+
+                tests << {
+                  name: "#{suite_group}:#{name}",
+                  status: format_status,
+                  duration: start_time ? (end_time - start_time) : 0,
+                }
+                last_result = line_clean
+                next
+              end
+            end
+          end
+
+          # Format: "2026-04-02T19:45:44.044982Z: Completed MySuite:ExampleGroup:script_2"
+          if line_clean.include?("Completed")
+            timestamp_and_msg = line_clean.split(': ', 2)
+            if timestamp_and_msg.length >= 2
+              timestamp = timestamp_and_msg[0]
+              begin
+                summary[:stopTime] = DateTime.parse(timestamp).to_time.to_f * 1000
+              rescue Date::Error
+                # Skip malformed timestamps
+              end
+            end
+            last_result = nil
+            next
+          end
+        end
+
+        if line_clean == '--- Test Summary ---'
+          in_summary = true
+          next
+        end
+
+        if in_summary
+          if line_clean.include?("Total Tests")
+            parts = line_clean.split(':')
+            summary[:total] = parts[1].to_i if parts[1]
+          end
+          if line_clean.include?("Pass:")
+            parts = line_clean.split(':')
+            summary[:passed] = parts[1].to_i if parts[1]
+          end
+          if line_clean.include?("Skip:")
+            parts = line_clean.split(':')
+            summary[:skipped] = parts[1].to_i if parts[1]
+          end
+          if line_clean.include?("Fail:")
+            parts = line_clean.split(':')
+            summary[:failed] = parts[1].to_i if parts[1]
+          end
+        end
+      end
+
+      # Build CTRF report
+      # See https://ctrf.io/docs/specification/root
+      return {
+        reportFormat: "CTRF",
+        results: {
+          # See https://ctrf.io/docs/specification/tool
+          tool: {
+            name: "COSMOS Script Runner",
+            version: version,
+          },
+          # See https://ctrf.io/docs/specification/summary
+          summary: {
+            tests: summary[:total],
+            passed: summary[:passed],
+            failed: summary[:failed],
+            pending: 0,
+            skipped: summary[:skipped],
+            other: 0,
+            start: summary[:startTime],
+            stop: summary[:stopTime],
+          },
+          # See https://ctrf.io/docs/specification/tests
+          tests: tests,
+          # See https://ctrf.io/docs/specification/extra
+          extra: {
+            manual: settings[:manual],
+            pauseOnError: settings[:pauseOnError],
+            continueAfterError: settings[:continueAfterError],
+            abortAfterError: settings[:abortAfterError],
+            loop: settings[:loop],
+            breakLoopOnError: settings[:breakLoopOnError],
+          },
+        },
+      }
+    end
+  end
+end

data/lib/openc3/utilities/metric.rb

@@ -37,15 +37,28 @@ module OpenC3
 
     attr_reader :microservice
     attr_reader :scope
+    attr_reader :db_shard
     attr_reader :data
     attr_reader :mutex
 
-    def initialize(microservice:, scope:)
+    def initialize(microservice:, scope:, db_shard: nil)
       @scope = scope
       @microservice = microservice
       @data = {}
       @mutex = Mutex.new
 
+      if db_shard
+        @db_shard = db_shard
+      else
+        # Look up db_shard from MicroserviceModel
+        begin
+          json = Store.hget('openc3_microservices', microservice)
+          @db_shard = json ? JSON.parse(json)['db_shard'].to_i : 0
+        rescue
+          @db_shard = 0
+        end
+      end
+
       # Always make sure there is a update thread
       @@mutex.synchronize do
         @@instances << self
@@ -90,6 +103,7 @@ module OpenC3
             instance.mutex.synchronize do
               json = {}
               json['name'] = instance.microservice
+              json['db_shard'] = instance.db_shard
               values = instance.data
               json['values'] = values
               MetricModel.set(json, scope: instance.scope) if values.length > 0