openc3 5.2.0 → 5.4.0

data/lib/openc3/models/secret_model.rb

@@ -0,0 +1,53 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/models/model'
+
+module OpenC3
+  class SecretModel < Model
+    PRIMARY_KEY = 'openc3__secrets'
+
+    # NOTE: The following three class methods are used by the ModelController
+    # and are reimplemented to enable various Model class methods to work
+    def self.get(name:, scope:)
+      super("#{scope}__#{PRIMARY_KEY}", name: name)
+    end
+
+    def self.names(scope:)
+      super("#{scope}__#{PRIMARY_KEY}")
+    end
+
+    def self.all(scope:)
+      super("#{scope}__#{PRIMARY_KEY}")
+    end
+    # END NOTE
+
+    def initialize(name:, value:, scope:)
+      super("#{scope}__#{PRIMARY_KEY}", name: name, scope: scope)
+      @value = value
+    end
+
+    # @return [Hash] JSON encoding of this model
+    def as_json(*a)
+      {
+        'name' => @name,
+        'value' => @value.as_json(*a),
+      }
+    end
+  end
+end

data/lib/openc3/models/target_model.rb

@@ -101,7 +101,7 @@ module OpenC3
           targets[target_name]['modified'] = true if targets[target_name]
         end
       else
-        modified_targets = Bucket.getClient().list_directories(bucket: ENV['OPENC3_CONFIG_BUCKET'], path: "DEFAULT/targets_modified/")
+        modified_targets = Bucket.getClient().list_files(bucket: ENV['OPENC3_CONFIG_BUCKET'], path: "DEFAULT/targets_modified/", only_directories: true)
         modified_targets.each do |target_name|
           # A target could have been deleted without removing the modified files
           # Thus we have to check for the existance of the target_name key
@@ -313,7 +313,7 @@ module OpenC3
       reduced_day_log_retain_time: nil,
       cleanup_poll_time: 900,
       needs_dependencies: false,
-      target_microservices: {},
+      target_microservices: {'REDUCER' => [[]]},
       scope:
     )
       super("#{scope}__#{PRIMARY_KEY}", name: name, plugin: plugin, updated_at: updated_at,

data/lib/openc3/models/tool_model.rb

@@ -93,8 +93,15 @@ module OpenC3
     # The ToolsTab.vue calls the ToolsController which uses this method to reorder the tools
     # Position is index in the list starting with 0 = first
     def self.set_position(name:, position:, scope:)
-      position = Integer(position)
-      next_position = position + 1
+      moving = from_json(get(name: name, scope: scope), scope: scope)
+      old_pos = moving.position
+      new_pos = Integer(position)
+      direction = :down
+      if (old_pos == new_pos)
+        return # we're not doing anything
+      elsif (new_pos > old_pos)
+        direction = :up
+      end
 
       # Go through all the tools and reorder
       all(scope: scope).each do |_tool_name, tool|
@@ -102,12 +109,14 @@ module OpenC3
         # Update the requested model to the new position
         if tool_model.name == name
           tool_model.position = position
-        # Move existing tools down in the order
-        elsif position > 0 && position >= tool_model.position
-          tool_model.position -= 1
-        else # Move existing tools up in the order
-          tool_model.position = next_position
-          next_position += 1
+        elsif direction == :down
+          if tool_model.position >= new_pos && tool_model.position < old_pos
+            tool_model.position += 1
+          end
+        else # up
+          if tool_model.position > old_pos && tool_model.position <= new_pos
+            tool_model.position -= 1
+          end
         end
         tool_model.update
       end

data/lib/openc3/operators/microservice_operator.rb

@@ -23,8 +23,10 @@
 require 'openc3'
 require 'openc3/models/microservice_model'
 require 'openc3/operators/operator'
+require 'openc3/utilities/secrets'
 require 'redis'
 require 'open3'
+require 'fileutils'
 
 module OpenC3
   # Creates new OperatorProcess objects based on querying the Redis key value store.
@@ -34,6 +36,7 @@ module OpenC3
       Logger.microservice_name = "MicroserviceOperator"
       super
 
+      @secrets = Secrets.getClient
       @microservices = {}
       @previous_microservices = {}
       @new_microservices = {}
@@ -53,6 +56,28 @@ module OpenC3
       env['OPENC3_MICROSERVICE_NAME'] = microservice_name
       container = microservice_config["container"]
       scope = microservice_name.split("__")[0]
+
+      # Setup secrets for microservice
+      secrets = microservice_config["secrets"]
+      if secrets
+        secrets.each do |type, secret_name, env_name_or_path|
+          secret_value = @secrets.get(secret_name, scope: scope)
+          if secret_value
+            case type
+            when 'ENV'
+              env[env_name_or_path] = secret_value
+            when 'FILE'
+              FileUtils.mkdir_p(File.dirname(env_name_or_path))
+              File.open(env_name_or_path, 'wb') do |file|
+                file.write(secret_value)
+              end
+            end
+          else
+            Logger.error("Microservice #{microservice_name} references unknown secret: #{secret_name}")
+          end
+        end
+      end
+
       return process_definition, work_dir, env, scope, container
     end
 

data/lib/openc3/operators/operator.rb

@@ -160,7 +160,11 @@ module OpenC3
     def soft_stop
       Thread.new do
         Logger.info("Soft shutting down process: #{cmd_line()}", scope: @scope)
-        Process.kill("SIGINT", @process.pid) if @process # Signal the process to stop
+        begin
+          Process.kill("SIGINT", @process.pid) if @process # Signal the process to stop
+        rescue Errno::ESRCH
+          # Process already gone
+        end
       end
     end
 

data/lib/openc3/packets/packet.rb

@@ -33,7 +33,7 @@ module OpenC3
   # as managing PacketItem's limit states.
   class Packet < Structure
     RESERVED_ITEM_NAMES = ['PACKET_TIMESECONDS'.freeze, 'PACKET_TIMEFORMATTED'.freeze, 'RECEIVED_TIMESECONDS'.freeze, 'RECEIVED_TIMEFORMATTED'.freeze, 'RECEIVED_COUNT'.freeze]
-    CATCH_ALL_STATE = 'ANY'
+    ANY_STATE = 'ANY'
 
     # @return [String] Name of the target this packet is associated with
     attr_reader :target_name
@@ -627,8 +627,8 @@ module OpenC3
             value = value.map do |val, index|
               if item.states.key(val)
                 item.states.key(val)
-              elsif item.states.values.include?(CATCH_ALL_STATE)
-                item.states.key(CATCH_ALL_STATE)
+              elsif item.states.values.include?(ANY_STATE)
+                item.states.key(ANY_STATE)
               else
                 apply_format_string_and_units(item, val, value_type)
               end
@@ -637,8 +637,8 @@ module OpenC3
             state_value = item.states.key(value)
             if state_value
               value = state_value
-            elsif item.states.values.include?(CATCH_ALL_STATE)
-              value = item.states.key(CATCH_ALL_STATE)
+            elsif item.states.values.include?(ANY_STATE)
+              value = item.states.key(ANY_STATE)
             else
               value = apply_format_string_and_units(item, value, value_type)
             end
@@ -653,7 +653,14 @@ module OpenC3
           end
         end
       else
-        raise ArgumentError, "Unknown value type on read: #{value_type}"
+        # Trim a potentially long string (like if they accidentally pass buffer as value_type)
+        if value_type.to_s.length > 10
+          value_type = value_type.to_s[0...10]
+          # Ensure we're not trying to output binary
+          value_type = value_type.simple_formatted unless value_type.is_printable?
+          value_type += '...'
+        end
+        raise ArgumentError, "Unknown value type '#{value_type}', must be :RAW, :CONVERTED, :FORMATTED, or :WITH_UNITS"
       end
       return value
     end
@@ -716,7 +723,14 @@ module OpenC3
       when :FORMATTED, :WITH_UNITS
         raise ArgumentError, "Invalid value type on write: #{value_type}"
       else
-        raise ArgumentError, "Unknown value type on write: #{value_type}"
+        # Trim potentially long string (like if they accidentally pass buffer as value_type)
+        if value_type.to_s.length > 10
+          value_type = value_type.to_s[0...10]
+          # Ensure we're not trying to output binary
+          value_type = value_type.simple_formatted unless value_type.is_printable?
+          value_type += '...'
+        end
+        raise ArgumentError, "Unknown value type '#{value_type}', must be :RAW, :CONVERTED, :FORMATTED, or :WITH_UNITS"
       end
       if @read_conversion_cache
         synchronize() do

data/lib/openc3/packets/packet_item.rb

@@ -493,7 +493,7 @@ module OpenC3
         if self.limits.values
           config['limits'] ||= {}
           config['limits']['persistence_setting'] = self.limits.persistence_setting
-          config['limits']['enabled'] = true if self.limits.enabled
+          config['limits']['response'] = self.limits.response.to_s if self.limits.response
           self.limits.values.each do |limits_set, limits_values|
             limits = {}
             limits['red_low'] =  limits_values[0]
@@ -557,9 +557,10 @@ module OpenC3
 
       item.limits = PacketItemLimits.new
       if hash['limits']
-        # Delete these keys so the only ones left are limits sets
+        # Delete the keys so the only ones left are limits sets
         persistence_setting = hash['limits'].delete('persistence_setting')
         item.limits.persistence_setting = persistence_setting if persistence_setting
+        hash['limits'].delete('response') # Can't round trip response
         item.limits.enabled = true if hash['limits'].delete('enabled')
         values = {}
         hash['limits'].each do |set, items|

data/lib/openc3/script/api_shared.rb

@@ -17,7 +17,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'openc3/script/extract'
@@ -500,7 +500,20 @@ module OpenC3
     def load_utility(procedure_name)
       return start(procedure_name)
     end
-    alias require_utility load_utility
+    def require_utility(procedure_name)
+      @require_utility_cache ||= {}
+      if @require_utility_cache[procedure_name]
+        return false
+      else
+        @require_utility_cache[procedure_name] = true
+        begin
+          return start(procedure_name)
+        rescue LoadError
+          @require_utility_cache[procedure_name] = false
+          raise # reraise the error
+        end
+      end
+    end
 
     ###########################################################################
     # Private implementation details
@@ -714,6 +727,7 @@ module OpenC3
 
     def _openc3_script_wait_implementation(target_name, packet_name, item_name, value_type, timeout, polling_rate, exp_to_eval, scope: $openc3_scope, token: $openc3_token, &block)
       end_time = Time.now.sys + timeout
+      raise "Invalid comparison to non-ascii value" unless exp_to_eval.is_printable?
 
       while true
         work_start = Time.now.sys
@@ -792,6 +806,7 @@ module OpenC3
     # Wait on an expression to be true.
     def openc3_script_wait_implementation_expression(exp_to_eval, timeout, polling_rate, context, scope: $openc3_scope, token: $openc3_token)
       end_time = Time.now.sys + timeout
+      raise "Invalid comparison to non-ascii value" unless exp_to_eval.is_printable?
 
       while true
         work_start = Time.now.sys
@@ -828,6 +843,7 @@ module OpenC3
     end
 
     def check_eval(target_name, packet_name, item_name, comparison_to_eval, value, scope: $openc3_scope, token: $openc3_token)
+      raise "Invalid comparison to non-ascii value" unless comparison_to_eval.is_printable?
       string = "value " + comparison_to_eval
       check_str = "CHECK: #{_upcase(target_name, packet_name, item_name)} #{comparison_to_eval}"
       # Show user the check against a quoted string

data/lib/openc3/script/script.rb

@@ -178,6 +178,14 @@ module OpenC3
       print "Details: #{details}\n" if details
       gets.chomp
     end
+
+    def step_mode
+      # NOOP
+    end
+
+    def run_mode
+      # NOOP
+    end
   end
 
   # Provides a proxy to the JsonDRbObject which communicates with the API server

data/lib/openc3/script/script_runner.rb

@@ -13,7 +13,7 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 # GNU Affero General Public License for more details.
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 module OpenC3
@@ -236,6 +236,5 @@ module OpenC3
         return JSON.parse(response.body, :allow_nan => true, :create_additions => true)
       end
     end
-
   end
 end

data/lib/openc3/script/storage.rb

@@ -157,7 +157,8 @@ module OpenC3
         case ENV['OPENC3_CLOUD']
         when 'local'
           uri = URI.parse("http://openc3-minio:9000" + url)
-        # when 'aws'
+        when 'aws'
+          uri = URI.parse("https://s3.#{ENV['AWS_REGION']}.amazonaws.com" + url)
         when 'gcp'
           uri = URI.parse("https://storage.googleapis.com" + url)
         # when 'azure'

data/lib/openc3/script/suite.rb

@@ -17,7 +17,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'openc3/script/exceptions'
@@ -37,14 +37,12 @@ module OpenC3
     # START PUBLIC API
     ###########################################################################
 
-    # Create a new Suite
-    def initialize
-      @scripts = {}
-      @plans = []
-    end
+    # Explictly avoid creating an initialize method which forces end users to call super()
 
     # Add a group to the suite
     def add_group(group_class)
+      @scripts ||= {}
+      @plans ||= []
       group_class = Object.const_get(group_class.to_s.intern) unless group_class.class == Class
       @scripts[group_class] = group_class.new unless @scripts[group_class]
       @plans << [:GROUP, group_class, nil]
@@ -52,6 +50,8 @@ module OpenC3
 
     # Add a script to the suite
     def add_script(group_class, script)
+      @scripts ||= {}
+      @plans ||= []
       group_class = Object.const_get(group_class.to_s.intern) unless group_class.class == Class
       @scripts[group_class] = group_class.new unless @scripts[group_class]
       @plans << [:SCRIPT, group_class, script]
@@ -59,6 +59,8 @@ module OpenC3
 
     # Add a group setup to the suite
     def add_group_setup(group_class)
+      @scripts ||= {}
+      @plans ||= []
       group_class = Object.const_get(group_class.to_s.intern) unless group_class.class == Class
       @scripts[group_class] = group_class.new unless @scripts[group_class]
       @plans << [:GROUP_SETUP, group_class, nil]
@@ -66,6 +68,8 @@ module OpenC3
 
     # Add a group teardown to the suite
     def add_group_teardown(group_class)
+      @scripts ||= {}
+      @plans ||= []
       group_class = Object.const_get(group_class.to_s.intern) unless group_class.class == Class
       @scripts[group_class] = group_class.new unless @scripts[group_class]
       @plans << [:GROUP_TEARDOWN, group_class, nil]
@@ -269,11 +273,7 @@ module OpenC3
     @@abort_on_exception = false
     @@current_result = nil
 
-    def initialize
-      @output_io = StringIO.new('', 'r+')
-      $stdout = Stdout.instance
-      $stderr = Stderr.instance
-    end
+    # Explictly avoid creating an initialize method which forces end users to call super()
 
     def self.abort_on_exception
       @@abort_on_exception
@@ -350,7 +350,11 @@ module OpenC3
 
       # Verify script method exists
       if object.class.method_defined?(method_name)
+        @output_io ||= StringIO.new('', 'r+')
         # Capture STDOUT and STDERR
+        # $stdout & $stderr must be set to change output
+        $stdout = Stdout.instance
+        $stderr = Stderr.instance
         $stdout.add_stream(@output_io)
         $stderr.add_stream(@output_io)
 

data/lib/openc3/system/system.rb

@@ -131,11 +131,14 @@ module OpenC3
 
       target = Target.new(target_name, target_config_dir)
       @targets[target.name] = target
+      errors = [] # Store all errors processing the cmd_tlm files
       target.cmd_tlm_files.each do |cmd_tlm_file|
         @packet_config.process_file(cmd_tlm_file, target.name)
-      rescue Exception => err
-        Logger.error "Problem processing #{cmd_tlm_file}: #{err}."
-        raise err
+      rescue Exception => error
+        errors << "Error processing #{cmd_tlm_file}:\n#{error.message}"
+      end
+      unless errors.empty?
+        raise errors.join("\n")
       end
     end
   end

data/lib/openc3/topics/interface_topic.rb

@@ -38,7 +38,7 @@ module OpenC3
     def self.receive_commands(interface, scope:)
       while true
         Topic.read_topics(InterfaceTopic.topics(interface, scope: scope)) do |topic, msg_id, msg_hash, redis|
-          result = yield topic, msg_hash
+          result = yield topic, msg_id, msg_hash, redis
           ack_topic = topic.split("__")
           ack_topic[1] = 'ACK' + ack_topic[1]
           ack_topic = ack_topic.join("__")
@@ -76,5 +76,21 @@ module OpenC3
       sleep 1 # Give some time for the interface to shutdown
       InterfaceTopic.clear_topics(InterfaceTopic.topics(interface, scope: scope))
     end
+
+    def interface_cmd(interface_name, cmd_name, *cmd_params, scope:)
+      data = {}
+      data['cmd_name'] = cmd_name
+      data['cmd_params'] = cmd_params
+      Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'interface_cmd' => JSON.generate(data, allow_nan: true) }, '*', 100)
+    end
+
+    def protocol_cmd(interface_name, cmd_name, *cmd_params, read_write: :READ_WRITE, index: -1, scope:)
+      data = {}
+      data['cmd_name'] = cmd_name
+      data['cmd_params'] = cmd_params
+      data['read_write'] = read_write.to_s.upcase
+      data['index'] = index
+      Topic.write_topic("{#{scope}__CMD}INTERFACE__#{interface_name}", { 'protocol_cmd' => JSON.generate(data, allow_nan: true) }, '*', 100)
+    end
   end
 end

data/lib/openc3/topics/router_topic.rb

@@ -40,7 +40,7 @@ module OpenC3
     def self.receive_telemetry(router, scope:)
       while true
         Topic.read_topics(RouterTopic.topics(router, scope: scope)) do |topic, msg_id, msg_hash, redis|
-          result = yield topic, msg_hash
+          result = yield topic, msg_id, msg_hash, redis
           if /CMD}ROUTER/.match?(topic)
             ack_topic = topic.split("__")
             ack_topic[1] = 'ACK' + ack_topic[1]
@@ -88,5 +88,21 @@ module OpenC3
       sleep 1 # Give some time for the interface to shutdown
       RouterTopic.clear_topics(RouterTopic.topics(router, scope: scope))
     end
+
+    def router_cmd(router_name, cmd_name, *cmd_params, scope:)
+      data = {}
+      data['cmd_name'] = cmd_name
+      data['cmd_params'] = cmd_params
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'router_cmd' => JSON.generate(data, allow_nan: true) }, '*', 100)
+    end
+
+    def protocol_cmd(router_name, cmd_name, *cmd_params, read_write: :READ_WRITE, index: -1, scope:)
+      data = {}
+      data['cmd_name'] = cmd_name
+      data['cmd_params'] = cmd_params
+      data['read_write'] = read_write.to_s.upcase
+      data['index'] = index
+      Topic.write_topic("{#{scope}__CMD}ROUTER__#{router_name}", { 'protocol_cmd' => JSON.generate(data, allow_nan: true) }, '*', 100)
+    end
   end
 end

data/lib/openc3/utilities/aws_bucket.rb

@@ -121,15 +121,20 @@ module OpenC3
       result
     end
 
-    # Lists the directories under a specified path
-    def list_directories(bucket:, path:)
+    # Lists the files under a specified path
+    def list_files(bucket:, path:, only_directories: false)
       # Trailing slash is important in AWS S3 when listing files
       # See https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/S3/Types/ListObjectsV2Output.html#common_prefixes-instance_method
       if path[-1] != '/'
         path += '/'
       end
+      # If we're searching for the root then kill the path or AWS will return nothing
+      path = nil if path == '/'
+
       token = nil
       result = []
+      dirs = []
+      files = []
       while true
         resp = @client.list_objects_v2({
           bucket: bucket,
@@ -141,7 +146,19 @@ module OpenC3
         resp.common_prefixes.each do |item|
           # If path was DEFAULT/targets_modified/ then the
           # results look like DEFAULT/targets_modified/INST/
-          result << item.prefix.split('/')[-1]
+          dirs << item.prefix.split('/')[-1]
+        end
+        if only_directories
+          result = dirs
+        else
+          resp.contents.each do |aws_item|
+            item = {}
+            item['name'] = aws_item.key.split('/')[-1]
+            item['modified'] = aws_item.last_modified
+            item['size'] = aws_item.size
+            files << item
+          end
+          result = [dirs, files]
         end
         break unless resp.is_truncated
         token = resp.next_continuation_token

data/lib/openc3/utilities/bucket.rb

@@ -57,7 +57,7 @@ module OpenC3
       raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
     end
 
-    def list_directories(bucket:, path:)
+    def list_files(bucket:, path:, only_directories: false)
       raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
     end
 

data/lib/openc3/utilities/bucket_file_cache.rb

@@ -73,7 +73,7 @@ class BucketFile
       local_path = "#{BucketFileCache.instance.cache_dir}/#{File.basename(@bucket_path)}"
       unless File.exist?(local_path)
         OpenC3::Logger.debug "Retrieving #{@bucket_path} from logs bucket"
-        client.get_object(bucket: "logs", key: @bucket_path, path: local_path)
+        client.get_object(bucket: ENV['OPENC3_LOGS_BUCKET'], key: @bucket_path, path: local_path)
         if File.exist?(local_path)
           basename = File.basename(local_path)
           if uncompress and File.extname(basename) == ".gz"

data/lib/openc3/utilities/bucket_utilities.rb

@@ -46,7 +46,7 @@ module OpenC3
         return oldest_list
       end
 
-      directories = client.list_directories(bucket: bucket, path: prefix)
+      directories = client.list_files(bucket: bucket, path: prefix, only_directories: true)
       filtered_directories = filter_directories_to_time_range(directories, start_time, end_time)
       filtered_directories.each do |directory|
         directory_files = client.list_objects(bucket: bucket, prefix: "#{prefix}/#{directory}", max_request: max_request, max_total: max_total)

data/lib/openc3/utilities/local_mode.rb

@@ -30,6 +30,7 @@ module OpenC3
     DEFAULT_PLUGINS = [
       'openc3-cosmos-tool-admin',
       'openc3-cosmos-tool-autonomic',
+      'openc3-cosmos-tool-bucketexplorer',
       'openc3-cosmos-tool-calendar',
       'openc3-cosmos-tool-cmdsender',
       'openc3-cosmos-tool-cmdtlmserver',