openc3 5.14.2 → 5.15.0

data/lib/openc3/models/target_model.rb

@@ -97,7 +97,7 @@ module OpenC3
     # All targets with indication of modified targets
     def self.all_modified(scope:)
       targets = self.all(scope: scope)
-      targets.each { |target_name, target| target['modified'] = false }
+      targets.each { |_target_name, target| target['modified'] = false }
 
       if ENV['OPENC3_LOCAL_MODE']
         modified_targets = OpenC3::LocalMode.modified_targets(scope: scope)
@@ -191,13 +191,6 @@ module OpenC3
       return result
     end
 
-    # @return [Array] Array of all the packet names
-    def self.packet_names(target_name, type: :TLM, scope:)
-      raise "Unknown type #{type} for #{target_name}" unless VALID_TYPES.include?(type)
-      # If the key doesn't exist or if there are no packets we return empty array
-      Store.hkeys("#{scope}__openc3#{type.to_s.downcase}__#{target_name}").sort
-    end
-
     # @return [Hash] Packet hash or raises an exception
     def self.packet(target_name, packet_name, type: :TLM, scope:)
       raise "Unknown type #{type} for #{target_name} #{packet_name}" unless VALID_TYPES.include?(type)
@@ -216,7 +209,7 @@ module OpenC3
 
       result = []
       packets = Store.hgetall("#{scope}__openc3#{type.to_s.downcase}__#{target_name}")
-      packets.sort.each do |packet_name, packet_json|
+      packets.sort.each do |_packet_name, packet_json|
         result << JSON.parse(packet_json, :allow_nan => true, :create_additions => true)
       end
       result
@@ -232,9 +225,9 @@ module OpenC3
 
       begin
         Store.hset("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name, JSON.generate(packet.as_json(:allow_nan => true)))
-      rescue JSON::GeneratorError => err
+      rescue JSON::GeneratorError => e
         Logger.error("Invalid text present in #{target_name} #{packet_name} #{type.to_s.downcase} packet")
-        raise err
+        raise e
       end
     end
 
@@ -403,7 +396,7 @@ module OpenC3
       @children = []
     end
 
-    def as_json(*a)
+    def as_json(*_a)
       {
         'name' => @name,
         'folder_name' => @folder_name,
@@ -586,13 +579,13 @@ module OpenC3
                 end
               end
             end
-          rescue => error
+          rescue => e
             # ERB error parsing a screen is just a logger error because life can go on
             # With cmd/tlm or scripts this is a serious error and we raise
             if (filename.include?('/screens/'))
-              Logger.error("ERB error parsing #{key} due to #{error.message}")
+              Logger.error("ERB error parsing #{key} due to #{e.message}")
             else
-              raise "ERB error parsing #{key} due to #{error.message}"
+              raise "ERB error parsing #{key} due to #{e.message}"
             end
           end
           local_path = File.join(temp_dir, @name, target_folder_path)
@@ -667,8 +660,8 @@ module OpenC3
       @@item_map_cache[@name] = nil
 
       ConfigTopic.write({ kind: 'deleted', type: 'target', name: @name, plugin: @plugin }, scope: @scope)
-    rescue Exception => error
-      Logger.error("Error undeploying target model #{@name} in scope #{@scope} due to #{error}")
+    rescue Exception => e
+      Logger.error("Error undeploying target model #{@name} in scope #{@scope} due to #{e}")
     end
 
     ##################################################
@@ -702,8 +695,8 @@ module OpenC3
             return ERB.new(data.force_encoding("UTF-8").comment_erb(), trim_mode: "-").result(b)
           end
         end
-      rescue => error
-        raise "ERB error parsing: #{path}: #{error.formatted}"
+      rescue => e
+        raise "ERB error parsing: #{path}: #{e.formatted}"
       end
     end
 
@@ -763,9 +756,9 @@ module OpenC3
           Logger.info "Configuring tlm packet: #{target_name} #{packet_name}"
           begin
             Store.hset("#{@scope}__openc3tlm__#{target_name}", packet_name, JSON.generate(packet.as_json(:allow_nan => true)))
-          rescue JSON::GeneratorError => err
+          rescue JSON::GeneratorError => e
             Logger.error("Invalid text present in #{target_name} #{packet_name} tlm packet")
-            raise err
+            raise e
           end
           json_hash = Hash.new
           packet.sorted_items.each do |item|
@@ -783,9 +776,9 @@ module OpenC3
           Logger.info "Configuring cmd packet: #{target_name} #{packet_name}"
           begin
             Store.hset("#{@scope}__openc3cmd__#{target_name}", packet_name, JSON.generate(packet.as_json(:allow_nan => true)))
-          rescue JSON::GeneratorError => err
+          rescue JSON::GeneratorError => e
             Logger.error("Invalid text present in #{target_name} #{packet_name} cmd packet")
-            raise err
+            raise e
           end
         end
       end
@@ -795,9 +788,9 @@ module OpenC3
       system.limits.groups.each do |group, items|
         begin
           Store.hset("#{@scope}__limits_groups", group, JSON.generate(items))
-        rescue JSON::GeneratorError => err
+        rescue JSON::GeneratorError => e
           Logger.error("Invalid text present in #{group} limits group")
-          raise err
+          raise e
         end
       end
     end
@@ -1107,7 +1100,7 @@ module OpenC3
 
           # Figure out if there are individual packets assigned to this microservice
           target_microservices.sort! {|a, b| a.length <=> b.length}
-          target_microservices.each_with_index do |packet_names, index|
+          target_microservices.each_with_index do |packet_names, _index|
             topics = []
             packet_names.each do |packet_name|
               topics << "#{topic_prefix}__#{packet_name}"
@@ -1146,7 +1139,7 @@ module OpenC3
       packet_topic_list = []
       decom_topic_list = []
       begin
-        system.commands.packets(@name).each do |packet_name, packet|
+        system.commands.packets(@name).each do |packet_name, _packet|
           command_topic_list << "#{@scope}__COMMAND__{#{@name}}__#{packet_name}"
           decom_command_topic_list << "#{@scope}__DECOMCMD__{#{@name}}__#{packet_name}"
         end
@@ -1154,7 +1147,7 @@ module OpenC3
         # No command packets for this target
       end
       begin
-        system.telemetry.packets(@name).each do |packet_name, packet|
+        system.telemetry.packets(@name).each do |packet_name, _packet|
           packet_topic_list << "#{@scope}__TELEMETRY__{#{@name}}__#{packet_name}"
           decom_topic_list  << "#{@scope}__DECOM__{#{@name}}__#{packet_name}"
         end

data/lib/openc3/models/tool_model.rb

@@ -157,7 +157,7 @@ module OpenC3
       @disable_erb = disable_erb
     end
 
-    def create(update: false, force: false)
+    def create(update: false, force: false, queued: false)
       tools = self.class.all(scope: @scope)
 
       # Make sure a tool with this folder_name doesn't already exist
@@ -181,7 +181,7 @@ module OpenC3
         end
       end
 
-      super(update: update, force: force)
+      super(update: update, force: force, queued: queued)
     end
 
     def as_json(*a)

data/lib/openc3/packets/json_packet.rb

@@ -67,7 +67,9 @@ module OpenC3
     def read(name, value_type = :CONVERTED, reduced_type = nil)
       value = nil
       array_index = nil
-      if name[-1] == ']'
+      # Check for array index to handle array items but also make sure there
+      # isn't a REAL item that has brackets in the name
+      if name[-1] == ']' and !@json_hash.include?(name)
         open_bracket_index = name.index('[')
         if open_bracket_index
           array_index = name[(open_bracket_index + 1)..-2].to_i
@@ -212,12 +214,12 @@ module OpenC3
         else
           postfix = nil if value_type == :RAW
         end
-        @json_hash.each do |key, value|
+        @json_hash.each do |key, _value|
           key_split = key.split("__")
           result[key_split[0]] = true if key_split[1] == postfix
         end
       else
-        @json_hash.each { |key, value| result[key.split("__")[0]] = true }
+        @json_hash.each { |key, _value| result[key.split("__")[0]] = true }
       end
       return result.keys
     end

data/lib/openc3/script/suite_results.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2024, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -22,7 +22,7 @@
 
 module OpenC3
   class SuiteResults
-    attr_accessor :metadata
+    attr_accessor :metadata, :context
 
     def initialize
       @report = nil
@@ -41,13 +41,13 @@ module OpenC3
       @report = []
 
       if test_case
-        # Executing a single test case
+        # Executing a script
         @context = "#{test_suite_class.name}:#{test_class.name}:#{test_case} #{test_type}"
       elsif test_class
-        # Executing an entire test
+        # Executing a group
         @context = "#{test_suite_class.name}:#{test_class.name} #{test_type}"
       else
-        # Executing a test suite
+        # Executing a suite
         @context = "#{test_suite_class.name} #{test_type}"
       end
 
@@ -75,7 +75,7 @@ module OpenC3
               line.chomp!
               line = line.inspect.remove_quotes
             end
-            @report << '  ' + line.strip
+            @report << ('  ' + line.strip)
           end
         end
         if result.exceptions
@@ -90,7 +90,7 @@ module OpenC3
                 line.chomp!
                 line = line.inspect.remove_quotes
               end
-              @report << '    ' + line.strip
+              @report << ('    ' + line.strip)
             end
             @report << '' if index != (result.exceptions.length - 1)
           end
@@ -172,11 +172,11 @@ module OpenC3
     end
 
     def write(string)
-      @report << Time.now.sys.formatted + ': ' + string
+      @report << (Time.now.sys.formatted + ': ' + string)
     end
 
     def puts(string)
-      @report << Time.now.sys.formatted + ': ' + string
+      @report << (Time.now.sys.formatted + ': ' + string)
     end
 
     # def collect_metadata(parent = nil)

data/lib/openc3/topics/command_decom_topic.rb

@@ -21,6 +21,7 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/topics/topic'
+require 'openc3/utilities/store_queued'
 
 module OpenC3
   class CommandDecomTopic < Topic
@@ -39,7 +40,7 @@ module OpenC3
         json_hash[item.name + "__U"] = packet.read_item(item, :WITH_UNITS) if item.units
       end
       msg_hash['json_data'] = JSON.generate(json_hash.as_json(:allow_nan => true))
-      Topic.write_topic(topic, msg_hash)
+      EphemeralStoreQueued.write_topic(topic, msg_hash)
     end
 
     def self.get_cmd_item(target_name, packet_name, param_name, type: :WITH_UNITS, scope: $openc3_scope)

data/lib/openc3/topics/command_topic.rb

@@ -21,6 +21,7 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/topics/topic'
+require 'openc3/utilities/store_queued'
 require 'openc3/utilities/open_telemetry'
 
 module OpenC3
@@ -36,7 +37,7 @@ module OpenC3
                    received_count: packet.received_count,
                    stored: packet.stored.to_s,
                    buffer: packet.buffer(false) }
-      Topic.write_topic(topic, msg_hash)
+      EphemeralStoreQueued.write_topic(topic, msg_hash)
     end
 
     # @param command [Hash] Command hash structure read to be written to a topic

data/lib/openc3/topics/telemetry_topic.rb

@@ -21,10 +21,11 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/topics/topic'
+require 'openc3/utilities/store_queued'
 
 module OpenC3
   class TelemetryTopic < Topic
-    def self.write_packet(packet, scope:)
+    def self.write_packet(packet, queued: false, scope:)
       msg_hash = {
         :time => packet.packet_time.to_nsec_from_epoch,
         :received_time => packet.received_time.to_nsec_from_epoch,
@@ -35,7 +36,11 @@ module OpenC3
         :buffer => packet.buffer(false)
       }
       msg_hash[:extra] = JSON.generate(packet.extra.as_json, allow_nan: true) if packet.extra
-      Topic.write_topic("#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}", msg_hash)
+      if queued
+        EphemeralStoreQueued.write_topic("#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}", msg_hash)
+      else
+        Topic.write_topic("#{scope}__TELEMETRY__{#{packet.target_name}}__#{packet.packet_name}", msg_hash)
+      end
     end
   end
 end

data/lib/openc3/utilities/aws_bucket.rb

@@ -26,7 +26,9 @@ module OpenC3
     CREATE_CHECK_COUNT = 100 # 10 seconds
 
     def initialize
+      super()
       @client = Aws::S3::Client.new
+      @aws_arn = ENV['OPENC3_AWS_ARN_PREFIX'] || 'arn:aws'
     end
 
     def create(bucket)
@@ -59,7 +61,7 @@ module OpenC3
                 ]
               },
               "Resource": [
-                "arn:aws:s3:::#{bucket}"
+                "#{@aws_arn}:s3:::#{bucket}"
               ],
               "Sid": ""
             },
@@ -74,7 +76,7 @@ module OpenC3
                 ]
               },
               "Resource": [
-                "arn:aws:s3:::#{bucket}/*"
+                "#{@aws_arn}:s3:::#{bucket}/*"
               ],
               "Sid": ""
             }
@@ -198,19 +200,23 @@ module OpenC3
     end
 
     # @returns [Boolean] Whether the file exists
-    def check_object(bucket:, key:)
-      @client.wait_until(:object_exists,
-        {
-          bucket: bucket,
-          key: key
-        },
-        {
-          max_attempts: 30,
-          delay: 0.1, # seconds
-        }
-      )
-      true
-    rescue Aws::Waiters::Errors::TooManyAttemptsError
+    def check_object(bucket:, key:, retries: true)
+      if retries
+        @client.wait_until(:object_exists,
+          {
+            bucket: bucket,
+            key: key
+          },
+          {
+            max_attempts: 30,
+            delay: 0.1, # seconds
+          }
+        )
+        true
+      else
+        head_object(bucket: bucket, key: key)
+      end
+    rescue NotFound, Aws::Waiters::Errors::TooManyAttemptsError
       false
     end
 

data/lib/openc3/utilities/bucket.rb

@@ -69,7 +69,7 @@ module OpenC3
       raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
     end
 
-    def check_object(bucket:, key:)
+    def check_object(bucket:, key:, retries: true)
       raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
     end
 

data/lib/openc3/utilities/logger.rb

@@ -22,7 +22,7 @@
 
 require 'openc3/core_ext/class'
 require 'openc3/core_ext/time'
-require 'openc3/topics/topic'
+require 'openc3/utilities/store_queued'
 require 'socket'
 require 'logger'
 require 'time'
@@ -206,9 +206,9 @@ module OpenC3
         end
         unless @no_store
           if scope
-            Topic.write_topic("#{scope}__openc3_log_messages", data)
+            EphemeralStoreQueued.write_topic("#{scope}__openc3_log_messages", data)
           else
-            Topic.write_topic("NOSCOPE__openc3_log_messages", data)
+            EphemeralStoreQueued.write_topic("NOSCOPE__openc3_log_messages", data)
           end
         end
       end

data/lib/openc3/utilities/process_manager.rb

@@ -68,9 +68,9 @@ module OpenC3
       @monitor_thread = Thread.new do
         begin
           monitor()
-        rescue => err
-          Logger.error("ProcessManager unexpectedly died\n#{err.formatted}", scope: 'DEFAULT')
-          raise "ProcessManager unexpectedly died\n#{err.formatted}"
+        rescue => e
+          Logger.error("ProcessManager unexpectedly died\n#{e.formatted}", scope: 'DEFAULT')
+          raise "ProcessManager unexpectedly died\n#{e.formatted}"
         end
       end
     end
@@ -96,7 +96,9 @@ module OpenC3
             process.status.output = output
             if process.exit_code != 0
               process.status.state = "Crashed"
-            elsif output.include?('"level":"ERROR"') || output.include?('"level":"WARN"')
+            elsif output.include?('"level":"ERROR"')
+              process.status.state = "Error"
+            elsif output.include?('"level":"WARN"')
               process.status.state = "Warning"
             else
               process.status.state = "Complete"
@@ -115,11 +117,15 @@ module OpenC3
           process.status.update
         end
         processes_to_delete.each do |process|
-          if process.status.state == "Complete"
-            Logger.info("Process #{process.status.name}:#{process.process_type}:#{process.detail} completed with state #{process.status.state}", scope: process.scope)
+          message = "Process #{process.status.name}:#{process.process_type}:#{process.detail} completed with state #{process.status.state}\nProcess Output:\n#{process.status.output}"
+          if process.status.state != "Complete"
+            if process.status.state == "Warning"
+              Logger.warn(message, scope: process.scope)
+            else
+              Logger.error(message, scope: process.scope)
+            end
           else
-            Logger.error("Process #{process.status.name}:#{process.process_type}:#{process.detail} completed with state #{process.status.state}", scope: process.scope)
-            Logger.error("Process Output:\n#{process.status.output}", scope: process.scope)
+            Logger.info(message, scope: process.scope)
           end
 
           @processes.delete(process)
@@ -131,7 +137,7 @@ module OpenC3
           scopes = ScopeModel.names
           scopes.each do |scope|
             statuses = ProcessStatusModel.get_all_models(scope: scope)
-            statuses.each do |status_name, status|
+            statuses.each do |_status_name, status|
               if (current_time - Time.from_nsec_from_epoch(status.updated_at)) > CLEANUP_CYCLE_SECONDS
                 status.destroy
               end

data/lib/openc3/utilities/store_autoload.rb

@@ -21,6 +21,7 @@
 # if purchased from OpenC3, Inc.
 
 require 'redis'
+require "hiredis-client"
 require 'json'
 require 'connection_pool'
 
@@ -32,6 +33,32 @@ else
 end
 
 module OpenC3
+  class StoreConnectionPool < ConnectionPool
+    NO_OPTIONS = {}
+
+    def pipelined
+      with(NO_OPTIONS) do |redis|
+        redis.pipelined do |pipeline|
+          Thread.current[:pipeline] = pipeline
+          begin
+            yield
+          ensure
+            Thread.current[:pipeline] = nil
+          end
+        end
+      end
+    end
+
+    def with(options = NO_OPTIONS, &block)
+      pipeline = Thread.current[:pipeline]
+      if pipeline
+        yield pipeline
+      else
+        super(options, &block)
+      end
+    end
+  end
+
   class Store
     # Variable that holds the singleton instance
     @instance = nil
@@ -67,7 +94,7 @@ module OpenC3
       @redis_username = ENV['OPENC3_REDIS_USERNAME']
       @redis_key = ENV['OPENC3_REDIS_PASSWORD']
       @redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME']}:#{ENV['OPENC3_REDIS_PORT']}"
-      @redis_pool = ConnectionPool.new(size: pool_size) { build_redis() }
+      @redis_pool = StoreConnectionPool.new(size: pool_size) { build_redis() }
     end
 
     unless $openc3_redis_cluster
@@ -212,7 +239,7 @@ module OpenC3
     def initialize(pool_size = 10)
       super(pool_size)
       @redis_url = "redis://#{ENV['OPENC3_REDIS_EPHEMERAL_HOSTNAME']}:#{ENV['OPENC3_REDIS_EPHEMERAL_PORT']}"
-      @redis_pool = ConnectionPool.new(size: pool_size) { build_redis() }
+      @redis_pool = StoreConnectionPool.new(size: pool_size) { build_redis() }
     end
   end
 end

data/lib/openc3/utilities/store_queued.rb

@@ -60,19 +60,29 @@ module OpenC3
       end
     end
 
+    def set_update_interval(interval)
+      if interval < @update_interval and interval > 0.0
+        @update_interval = interval
+      end
+    end
+
+    def process_queue
+      unless @store_queue.empty?
+        # Pipeline the requests to redis to improve performance
+        @store.redis_pool.pipelined do
+          while !@store_queue.empty?
+            action = @store_queue.pop()
+            @store.public_send(action.message, *action.args, **action.kwargs, &action.block)
+          end
+        end
+      end
+    end
+
     def store_thread_body
       while true
         start_time = Time.now
 
-        unless @store_queue.empty?
-          # Pipeline the requests to redis to improve performance
-          @store.pipelined do
-            while !@store_queue.empty?
-              action = @store_queue.pop()
-              @store.method_missing(action.message, *action.args, **action.kwargs, &action.block)
-            end
-          end
-        end
+        process_queue()
 
         # Only check whether to update at a set interval
         run_time = Time.now - start_time
@@ -87,25 +97,14 @@ module OpenC3
       OpenC3.kill_thread(self, @update_thread) if @update_thread
       @update_thread = nil
       # Drain the queue before shutdown
-      unless @store_queue.empty?
-        # Pipeline the requests to redis to improve performance
-        @store.pipelined do
-          while !@store_queue.empty?
-            action = @store_queue.pop()
-            @store.method_missing(action.message, *action.args, **action.kwargs, &action.block)
-          end
-        end
-      end
+      process_queue()
     end
 
+    MessageStruct = Struct.new(:message, :args, :kwargs, :block)
+
     # Record the message for pipelining by the thread
     def method_missing(message, *args, **kwargs, &block)
-      o = OpenStruct.new
-      o.message = message
-      o.args = args
-      o.kwargs = kwargs
-      o.block = block
-      @store_queue.push(o)
+      @store_queue.push(MessageStruct.new(message, args, kwargs, block))
     end
 
     # Returns the store we're working with

data/lib/openc3/version.rb

@@ -1,14 +1,14 @@
 # encoding: ascii-8bit
 
-OPENC3_VERSION = '5.14.2'
+OPENC3_VERSION = '5.15.0'
 module OpenC3
   module Version
     MAJOR = '5'
-    MINOR = '14'
-    PATCH = '2'
+    MINOR = '15'
+    PATCH = '0'
     OTHER = ''
-    BUILD = 'e5d9da06d95d4404fba6f9fd5a74a130ef176bd4'
+    BUILD = 'a7dd92ddb0ba4d378da1b611cafc5ad7e89a9223'
   end
-  VERSION = '5.14.2'
-  GEM_VERSION = '5.14.2'
+  VERSION = '5.15.0'
+  GEM_VERSION = '5.15.0'
 end

data/templates/tool_angular/package.json

@@ -1,6 +1,6 @@
 {
   "name": "<%= tool_name %>",
-  "version": "5.14.2",
+  "version": "5.15.0",
   "scripts": {
     "ng": "ng",
     "start": "ng serve",
@@ -12,7 +12,7 @@
   },
   "private": true,
   "dependencies": {
-    "@openc3/tool-common": "5.14.2",
+    "@openc3/tool-common": "5.15.0",
     "@angular/animations": "^17.0.8",
     "@angular/cdk": "^17.0.4",
     "@angular/common": "^17.0.8",

data/templates/tool_svelte/package.json

@@ -11,7 +11,7 @@
     "smui-theme": "smui-theme compile build/smui.css -i src/theme"
   },
   "dependencies": {
-    "@openc3/tool-common": "5.14.2",
+    "@openc3/tool-common": "5.15.0",
     "@astrouxds/astro-web-components": "7.20.0",
     "@smui/button": "^7.0.0-beta.16",
     "@smui/card": "^7.0.0-beta.16",

data/templates/tool_vue/package.json

@@ -1,6 +1,6 @@
 {
   "name": "<%= tool_name %>",
-  "version": "5.14.2",
+  "version": "5.15.0",
   "private": true,
   "scripts": {
     "serve": "vue-cli-service serve",
@@ -11,7 +11,7 @@
     "test:components": "vue-cli-service test:components"
   },
   "dependencies": {
-    "@openc3/tool-common": "5.14.2",
+    "@openc3/tool-common": "5.15.0",
     "@astrouxds/astro-web-components": "7.20.0",
     "axios": "1.6.5",
     "date-fns": "2.30.0",