openc3 5.14.2 → 5.15.1

data/lib/openc3/models/model.rb

@@ -21,6 +21,7 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/utilities/store'
+require 'openc3/utilities/store_queued'
 require 'openc3/config/config_parser'
 
 module OpenC3
@@ -34,6 +35,10 @@ module OpenC3
       Store
     end
 
+    def self.store_queued
+      StoreQueued
+    end
+
     # NOTE: The following three methods must be reimplemented by Model subclasses
     # without primary_key to support other class methods.
 
@@ -75,10 +80,10 @@ module OpenC3
     end
 
     # Sets (updates) the redis hash of this model
-    def self.set(json, scope:)
+    def self.set(json, scope:, queued: false)
       json[:scope] = scope
       json.transform_keys!(&:to_sym)
-      self.new(**json).create(force: true)
+      self.new(**json).create(force: true, queued: queued)
     end
 
     # @return [Model] Model generated from the passed JSON
@@ -134,7 +139,7 @@ module OpenC3
 
     # Update the Redis hash at primary_key and set the field "name"
     # to the JSON generated via calling as_json
-    def create(update: false, force: false)
+    def create(update: false, force: false, queued: false)
       unless force
         existing = self.class.store.hget(@primary_key, @name)
         if existing
@@ -144,12 +149,18 @@ module OpenC3
         end
       end
       @updated_at = Time.now.to_nsec_from_epoch
-      self.class.store.hset(@primary_key, @name, JSON.generate(self.as_json(:allow_nan => true), :allow_nan => true))
+
+      if queued
+        write_store = self.class.store_queued
+      else
+        write_store = self.class.store
+      end
+      write_store.hset(@primary_key, @name, JSON.generate(self.as_json(:allow_nan => true), :allow_nan => true))
     end
 
     # Alias for create(update: true)
-    def update
-      create(update: true)
+    def update(force: false, queued: false)
+      create(update: true, force: force, queued: queued)
     end
 
     # Deploy the model into the OpenC3 system. Subclasses must implement this
@@ -206,5 +217,9 @@ module OpenC3
     def self.store
       EphemeralStore
     end
+
+    def self.store_queued
+      EphemeralStoreQueued
+    end
   end
 end

data/lib/openc3/models/plugin_model.rb

@@ -183,12 +183,18 @@ module OpenC3
         if File.exist?(File.join(gem_path, 'requirements.txt'))
           begin
             pypi_url = get_setting('pypi_url', scope: scope)
+            if pypi_url
+              pypi_url += '/simple'
+            end
           rescue => e
             Logger.error("Failed to retrieve pypi_url: #{e.formatted}")
           ensure
             if pypi_url.nil?
               # If Redis isn't running try the ENV, then simply pypi.org/simple
               pypi_url = ENV['PYPI_URL']
+              if pypi_url
+                pypi_url += '/simple'
+              end
               pypi_url ||= 'https://pypi.org/simple'
             end
           end
@@ -288,9 +294,9 @@ module OpenC3
       @needs_dependencies = ConfigParser.handle_true_false(needs_dependencies)
     end
 
-    def create(update: false, force: false)
+    def create(update: false, force: false, queued: false)
       @name = @name + "__#{Time.now.utc.strftime("%Y%m%d%H%M%S")}" if not update and not @name.index("__")
-      super(update: update, force: force)
+      super(update: update, force: force, queued: queued)
     end
 
     def as_json(*a)

data/lib/openc3/models/python_package_model.rb

@@ -74,6 +74,9 @@ module OpenC3
       rescue
         # If Redis isn't running try the ENV, then simply pypi.org/simple
         pypi_url = ENV['PYPI_URL']
+        if pypi_url
+          pypi_url += '/simple'
+        end
         pypi_url ||= 'https://pypi.org/simple'
       end
       Logger.info "Installing python package: #{name_or_path}"

data/lib/openc3/models/scope_model.rb

@@ -95,11 +95,11 @@ module OpenC3
       @children = []
     end
 
-    def create(update: false, force: false)
+    def create(update: false, force: false, queued: false)
       # Ensure there are no "." in the scope name - prevents gems accidently becoming scope names
       raise "Invalid scope name: #{@name}" if @name !~ /^[a-zA-Z0-9_-]+$/
       @name = @name.upcase
-      super(update: update, force: force)
+      super(update: update, force: force, queued: queued)
     end
 
     def destroy

data/lib/openc3/models/target_model.rb

@@ -97,7 +97,7 @@ module OpenC3
     # All targets with indication of modified targets
     def self.all_modified(scope:)
       targets = self.all(scope: scope)
-      targets.each { |target_name, target| target['modified'] = false }
+      targets.each { |_target_name, target| target['modified'] = false }
 
       if ENV['OPENC3_LOCAL_MODE']
         modified_targets = OpenC3::LocalMode.modified_targets(scope: scope)
@@ -191,13 +191,6 @@ module OpenC3
       return result
     end
 
-    # @return [Array] Array of all the packet names
-    def self.packet_names(target_name, type: :TLM, scope:)
-      raise "Unknown type #{type} for #{target_name}" unless VALID_TYPES.include?(type)
-      # If the key doesn't exist or if there are no packets we return empty array
-      Store.hkeys("#{scope}__openc3#{type.to_s.downcase}__#{target_name}").sort
-    end
-
     # @return [Hash] Packet hash or raises an exception
     def self.packet(target_name, packet_name, type: :TLM, scope:)
       raise "Unknown type #{type} for #{target_name} #{packet_name}" unless VALID_TYPES.include?(type)
@@ -216,7 +209,7 @@ module OpenC3
 
       result = []
       packets = Store.hgetall("#{scope}__openc3#{type.to_s.downcase}__#{target_name}")
-      packets.sort.each do |packet_name, packet_json|
+      packets.sort.each do |_packet_name, packet_json|
         result << JSON.parse(packet_json, :allow_nan => true, :create_additions => true)
       end
       result
@@ -232,9 +225,9 @@ module OpenC3
 
       begin
         Store.hset("#{scope}__openc3#{type.to_s.downcase}__#{target_name}", packet_name, JSON.generate(packet.as_json(:allow_nan => true)))
-      rescue JSON::GeneratorError => err
+      rescue JSON::GeneratorError => e
         Logger.error("Invalid text present in #{target_name} #{packet_name} #{type.to_s.downcase} packet")
-        raise err
+        raise e
       end
     end
 
@@ -403,7 +396,7 @@ module OpenC3
       @children = []
     end
 
-    def as_json(*a)
+    def as_json(*_a)
       {
         'name' => @name,
         'folder_name' => @folder_name,
@@ -586,13 +579,13 @@ module OpenC3
                 end
               end
             end
-          rescue => error
+          rescue => e
             # ERB error parsing a screen is just a logger error because life can go on
             # With cmd/tlm or scripts this is a serious error and we raise
             if (filename.include?('/screens/'))
-              Logger.error("ERB error parsing #{key} due to #{error.message}")
+              Logger.error("ERB error parsing #{key} due to #{e.message}")
             else
-              raise "ERB error parsing #{key} due to #{error.message}"
+              raise "ERB error parsing #{key} due to #{e.message}"
             end
           end
           local_path = File.join(temp_dir, @name, target_folder_path)
@@ -667,8 +660,8 @@ module OpenC3
       @@item_map_cache[@name] = nil
 
       ConfigTopic.write({ kind: 'deleted', type: 'target', name: @name, plugin: @plugin }, scope: @scope)
-    rescue Exception => error
-      Logger.error("Error undeploying target model #{@name} in scope #{@scope} due to #{error}")
+    rescue Exception => e
+      Logger.error("Error undeploying target model #{@name} in scope #{@scope} due to #{e}")
     end
 
     ##################################################
@@ -702,8 +695,8 @@ module OpenC3
             return ERB.new(data.force_encoding("UTF-8").comment_erb(), trim_mode: "-").result(b)
           end
         end
-      rescue => error
-        raise "ERB error parsing: #{path}: #{error.formatted}"
+      rescue => e
+        raise "ERB error parsing: #{path}: #{e.formatted}"
       end
     end
 
@@ -763,9 +756,9 @@ module OpenC3
           Logger.info "Configuring tlm packet: #{target_name} #{packet_name}"
           begin
             Store.hset("#{@scope}__openc3tlm__#{target_name}", packet_name, JSON.generate(packet.as_json(:allow_nan => true)))
-          rescue JSON::GeneratorError => err
+          rescue JSON::GeneratorError => e
             Logger.error("Invalid text present in #{target_name} #{packet_name} tlm packet")
-            raise err
+            raise e
           end
           json_hash = Hash.new
           packet.sorted_items.each do |item|
@@ -783,9 +776,9 @@ module OpenC3
           Logger.info "Configuring cmd packet: #{target_name} #{packet_name}"
           begin
             Store.hset("#{@scope}__openc3cmd__#{target_name}", packet_name, JSON.generate(packet.as_json(:allow_nan => true)))
-          rescue JSON::GeneratorError => err
+          rescue JSON::GeneratorError => e
             Logger.error("Invalid text present in #{target_name} #{packet_name} cmd packet")
-            raise err
+            raise e
           end
         end
       end
@@ -795,9 +788,9 @@ module OpenC3
       system.limits.groups.each do |group, items|
         begin
           Store.hset("#{@scope}__limits_groups", group, JSON.generate(items))
-        rescue JSON::GeneratorError => err
+        rescue JSON::GeneratorError => e
           Logger.error("Invalid text present in #{group} limits group")
-          raise err
+          raise e
         end
       end
     end
@@ -1107,7 +1100,7 @@ module OpenC3
 
           # Figure out if there are individual packets assigned to this microservice
           target_microservices.sort! {|a, b| a.length <=> b.length}
-          target_microservices.each_with_index do |packet_names, index|
+          target_microservices.each_with_index do |packet_names, _index|
             topics = []
             packet_names.each do |packet_name|
               topics << "#{topic_prefix}__#{packet_name}"
@@ -1146,7 +1139,7 @@ module OpenC3
       packet_topic_list = []
       decom_topic_list = []
       begin
-        system.commands.packets(@name).each do |packet_name, packet|
+        system.commands.packets(@name).each do |packet_name, _packet|
           command_topic_list << "#{@scope}__COMMAND__{#{@name}}__#{packet_name}"
           decom_command_topic_list << "#{@scope}__DECOMCMD__{#{@name}}__#{packet_name}"
         end
@@ -1154,7 +1147,7 @@ module OpenC3
         # No command packets for this target
       end
       begin
-        system.telemetry.packets(@name).each do |packet_name, packet|
+        system.telemetry.packets(@name).each do |packet_name, _packet|
           packet_topic_list << "#{@scope}__TELEMETRY__{#{@name}}__#{packet_name}"
           decom_topic_list  << "#{@scope}__DECOM__{#{@name}}__#{packet_name}"
         end

data/lib/openc3/models/tool_model.rb

@@ -157,7 +157,7 @@ module OpenC3
       @disable_erb = disable_erb
     end
 
-    def create(update: false, force: false)
+    def create(update: false, force: false, queued: false)
       tools = self.class.all(scope: @scope)
 
       # Make sure a tool with this folder_name doesn't already exist
@@ -181,7 +181,7 @@ module OpenC3
         end
       end
 
-      super(update: update, force: force)
+      super(update: update, force: force, queued: queued)
     end
 
     def as_json(*a)

data/lib/openc3/packets/json_packet.rb

@@ -67,7 +67,9 @@ module OpenC3
     def read(name, value_type = :CONVERTED, reduced_type = nil)
       value = nil
       array_index = nil
-      if name[-1] == ']'
+      # Check for array index to handle array items but also make sure there
+      # isn't a REAL item that has brackets in the name
+      if name[-1] == ']' and !@json_hash.include?(name)
         open_bracket_index = name.index('[')
         if open_bracket_index
           array_index = name[(open_bracket_index + 1)..-2].to_i
@@ -212,12 +214,12 @@ module OpenC3
         else
           postfix = nil if value_type == :RAW
         end
-        @json_hash.each do |key, value|
+        @json_hash.each do |key, _value|
           key_split = key.split("__")
           result[key_split[0]] = true if key_split[1] == postfix
         end
       else
-        @json_hash.each { |key, value| result[key.split("__")[0]] = true }
+        @json_hash.each { |key, _value| result[key.split("__")[0]] = true }
       end
       return result.keys
     end

data/lib/openc3/script/commands.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2024, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -96,7 +96,7 @@ module OpenC3
 
       # Get the command and validate the parameters
       command = $api_server.get_cmd(target_name, cmd_name, scope: scope)
-      cmd_params.each do |param_name, param_value|
+      cmd_params.each do |param_name, _param_value|
         param = command['items'].find { |item| item['name'] == param_name }
         unless param
           raise "Packet item '#{target_name} #{cmd_name} #{param_name}' does not exist"

data/lib/openc3/script/storage.rb

@@ -43,8 +43,8 @@ module OpenC3
         if response.nil? || response.status != 200
           raise "Failed to delete #{delete_path}"
         end
-      rescue => error
-        raise "Failed deleting #{path} due to #{error.message}"
+      rescue => e
+        raise "Failed deleting #{path} due to #{e.message}"
       end
       nil
     end
@@ -84,8 +84,8 @@ module OpenC3
             end
           end
         end
-      rescue => error
-        raise "Failed to write #{upload_path} due to #{error.message}"
+      rescue => e
+        raise "Failed to write #{upload_path} due to #{e.message}"
       end
       nil
     end
@@ -115,19 +115,33 @@ module OpenC3
           end
 
           return _get_storage_file("#{part}/#{path}", scope: scope)
-        rescue => error
+        rescue => e
           if part == "targets_modified"
             part = "targets"
             redo
           else
-            raise error
+            raise e
           end
         end
         break
       end
     end
 
-    # download_file(path_or_file) is implemented by running_script to download a file
+    def get_download_url(path, scope: $openc3_scope)
+      targets = "targets_modified" # First try targets_modified
+      response = $api_server.request('get', "/openc3-api/storage/exists/#{scope}/#{targets}/#{path}", query: { bucket: 'OPENC3_CONFIG_BUCKET' }, scope: scope)
+      if response.status != 200
+        targets = "targets" # Next try targets
+        response = $api_server.request('get', "/openc3-api/storage/exists/#{scope}/#{targets}/#{path}", query: { bucket: 'OPENC3_CONFIG_BUCKET' }, scope: scope)
+        if response.status != 200
+          raise "File not found: #{path} in scope: #{scope}"
+        end
+      end
+      endpoint = "/openc3-api/storage/download/#{scope}/#{targets}/#{path}"
+      # external must be true because we're using this URL from the frontend
+      result = _get_presigned_request(endpoint, external: true, scope: scope)
+      return result['url']
+    end
 
     # These are helper methods ... should not be used directly
 
@@ -161,7 +175,9 @@ module OpenC3
       if $openc3_in_cluster
         case ENV['OPENC3_CLOUD']
         when 'local'
-          URI.parse("http://openc3-minio:9000" + url)
+          bucket_url = ENV["OPENC3_BUCKET_URL"] || "openc3-minio:9000"
+          # TODO: Bucket schema for http vs https
+          URI.parse("http://#{bucket_url}#{url}")
         when 'aws'
           URI.parse("https://s3.#{ENV['AWS_REGION']}.amazonaws.com" + url)
         when 'gcp'
@@ -175,11 +191,11 @@ module OpenC3
       end
     end
 
-    def _get_presigned_request(endpoint, scope: $openc3_scope)
-      if $openc3_in_cluster
-        response = $api_server.request('get', endpoint, query: { bucket: 'OPENC3_CONFIG_BUCKET', internal: true }, scope: scope)
-      else
+    def _get_presigned_request(endpoint, external: nil, scope: $openc3_scope)
+      if external or !$openc3_in_cluster
         response = $api_server.request('get', endpoint, query: { bucket: 'OPENC3_CONFIG_BUCKET' }, scope: scope)
+      else
+        response = $api_server.request('get', endpoint, query: { bucket: 'OPENC3_CONFIG_BUCKET', internal: true }, scope: scope)
       end
       if response.nil? || response.status != 201
         raise "Failed to get presigned URL for #{endpoint}"

data/lib/openc3/script/suite_results.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2024, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -22,7 +22,7 @@
 
 module OpenC3
   class SuiteResults
-    attr_accessor :metadata
+    attr_accessor :metadata, :context
 
     def initialize
       @report = nil
@@ -41,13 +41,13 @@ module OpenC3
       @report = []
 
       if test_case
-        # Executing a single test case
+        # Executing a script
         @context = "#{test_suite_class.name}:#{test_class.name}:#{test_case} #{test_type}"
       elsif test_class
-        # Executing an entire test
+        # Executing a group
         @context = "#{test_suite_class.name}:#{test_class.name} #{test_type}"
       else
-        # Executing a test suite
+        # Executing a suite
         @context = "#{test_suite_class.name} #{test_type}"
       end
 
@@ -75,7 +75,7 @@ module OpenC3
               line.chomp!
               line = line.inspect.remove_quotes
             end
-            @report << '  ' + line.strip
+            @report << ('  ' + line.strip)
           end
         end
         if result.exceptions
@@ -90,7 +90,7 @@ module OpenC3
                 line.chomp!
                 line = line.inspect.remove_quotes
               end
-              @report << '    ' + line.strip
+              @report << ('    ' + line.strip)
             end
             @report << '' if index != (result.exceptions.length - 1)
           end
@@ -172,11 +172,11 @@ module OpenC3
     end
 
     def write(string)
-      @report << Time.now.sys.formatted + ': ' + string
+      @report << (Time.now.sys.formatted + ': ' + string)
     end
 
     def puts(string)
-      @report << Time.now.sys.formatted + ': ' + string
+      @report << (Time.now.sys.formatted + ': ' + string)
     end
 
     # def collect_metadata(parent = nil)

data/lib/openc3/system/system.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2024, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -31,7 +31,6 @@ require 'openc3/system/target'
 require 'openc3/utilities/bucket'
 require 'openc3/utilities/zip'
 require 'openc3/topics/limits_event_topic'
-require 'thread'
 require 'fileutils'
 
 module OpenC3
@@ -109,7 +108,7 @@ module OpenC3
           # target.txt must be configured to either use all files in cmd_tlm folder (default)
           # or have a predetermined empty file like dynamic_tlm.txt
           bucket_path = "#{scope}/targets_modified/#{target_name}/cmd_tlm"
-          dirs, files = bucket.list_files(bucket: ENV['OPENC3_CONFIG_BUCKET'], path: bucket_path)
+          _, files = bucket.list_files(bucket: ENV['OPENC3_CONFIG_BUCKET'], path: bucket_path)
           files.each do |file|
             bucket_key = File.join(bucket_path, file['name'])
             local_path = "#{base_dir}/targets/#{target_name}/cmd_tlm/#{file['name']}"
@@ -180,8 +179,8 @@ module OpenC3
       errors = [] # Store all errors processing the cmd_tlm files
       target.cmd_tlm_files.each do |cmd_tlm_file|
         @packet_config.process_file(cmd_tlm_file, target.name, target.language)
-      rescue Exception => error
-        errors << "Error processing #{cmd_tlm_file}:\n#{error.message}"
+      rescue Exception => e
+        errors << "Error processing #{cmd_tlm_file}:\n#{e.message}"
       end
       unless errors.empty?
         raise errors.join("\n")