openc3 7.3.0 → 7.4.0

data/lib/openc3/models/trigger_model.rb

@@ -50,6 +50,7 @@ module OpenC3
     STRING_TYPE = 'string'.freeze
     REGEX_TYPE = 'regex'.freeze
     TRIGGER_TYPE = 'trigger'.freeze
+    ITEM_VALUE_TYPES = ['RAW', 'CONVERTED', 'FORMATTED', 'WITH_UNITS'].freeze
 
     def self.create_unique_name(group:, scope:)
       trigger_names = self.names(group: group, scope: scope)
@@ -180,6 +181,24 @@ module OpenC3
       end
     end
 
+    # Validate the items hold a legal valueType and reference telemetry items
+    # which actually exist. Only called on create and update, never from initialize.
+    # Anything which can reject an already persisted trigger must live here rather than
+    # in validate_operand, because validate_operand runs on every from_json
+    def validate_trigger_items
+      {'left' => @left, 'right' => @right}.each do |side, operand|
+        next unless operand.is_a?(Hash) and operand['type'] == ITEM_TYPE
+        unless ITEM_VALUE_TYPES.include?(operand['valueType'])
+          raise TriggerInputError.new "invalid #{side} operand, valueType '#{operand['valueType']}' must be one of #{ITEM_VALUE_TYPES}"
+        end
+        begin
+          TargetModel.packet_item(operand['target'], operand['packet'], operand['item'], scope: @scope)
+        rescue StandardError => e
+          raise TriggerInputError.new "invalid #{side} trigger: #{e.message}"
+        end
+      end
+    end
+
     # Validate that all root triggers exist, but do not persist dependent changes yet.
     # Returns the list of root trigger models that need updating.
     def validate_roots
@@ -207,6 +226,7 @@ module OpenC3
       unless Store.hget(@primary_key, @name).nil?
         raise TriggerInputError.new "existing trigger found: '#{@name}'"
       end
+      validate_trigger_items()
       models = validate_roots()
       @updated_at = Time.now.to_nsec_from_epoch
       Store.hset(@primary_key, @name, JSON.generate(as_json, allow_nan: true))

data/lib/openc3/operators/microservice_operator.rb

@@ -18,6 +18,7 @@
 require 'openc3'
 require 'openc3/models/microservice_model'
 require 'openc3/operators/operator'
+require 'openc3/utilities/python_venv'
 require 'openc3/utilities/secrets'
 require 'redis'
 require 'open3'
@@ -56,28 +57,13 @@ module OpenC3
         plugin_venv_dir = nil
         if plugin_name
           scope = microservice_name.split("__")[0]
-          sanitized_name = "#{scope}__#{plugin_name}".tr('^a-zA-Z0-9_-', '_')
-          candidate = "/gems/plugin_venvs/#{sanitized_name}/.venv"
-          plugin_venv_dir = candidate if File.directory?(candidate)
+          plugin_venv_dir = PythonVenv.plugin_venv_path(scope: scope, plugin_name: plugin_name)
         end
 
         if plugin_venv_dir
-          env['VIRTUAL_ENV'] = plugin_venv_dir
-          env['PATH'] = "#{plugin_venv_dir}/bin:#{ENV.fetch('PATH', '')}"
-          env['PYTHONUSERBASE'] = plugin_venv_dir
-          # Add the plugin venv's site-packages to PYTHONPATH so the base venv's
-          # Python binary can find plugin-specific packages. We keep the base binary
-          # because it has openc3 core installed; PYTHONPATH is always respected by
-          # CPython regardless of venv activation state.
-          site_packages = Dir.glob("#{plugin_venv_dir}/lib/python*/site-packages").first
-          existing_pythonpath = ENV.fetch('PYTHONPATH', '')
-          if site_packages
-            env['PYTHONPATH'] = existing_pythonpath.empty? ? site_packages : "#{site_packages}:#{existing_pythonpath}"
-          else
-            env['PYTHONPATH'] = existing_pythonpath.empty? ? nil : existing_pythonpath
-          end
+          PythonVenv.configure_environment(env, plugin_venv_dir)
         else
-          env['PYTHONUSERBASE'] = '/gems/python_packages'
+          env['PYTHONUSERBASE'] = PythonVenv::DEFAULT_PYTHONUSERBASE
           env['PYTHONPATH'] = ENV.fetch('PYTHONPATH', nil)
         end
       else
@@ -98,10 +84,7 @@ module OpenC3
             if type == 'ENV'
               env[env_name_or_path] = secret_value
             elsif type == 'FILE'
-              FileUtils.mkdir_p(File.dirname(env_name_or_path))
-              File.open(env_name_or_path, 'wb') do |file|
-                file.write(secret_value)
-              end
+              write_secret_file(microservice_name, secret_name, env_name_or_path, secret_value)
             end
           else
             Logger.error("Microservice #{microservice_name} references unknown secret: #{secret_name}")
@@ -112,6 +95,65 @@ module OpenC3
       return process_definition, work_dir, env, scope, container
     end
 
+    # Writes a FILE type secret to disk. The path is first validated lexically
+    # (see Secrets.validate_file_path) and then checked against the filesystem to
+    # make sure no symlink redirects the write outside of Secrets.secret_file_dir.
+    # Both checks happen before any directory is created so that mkdir_p can
+    # never create directories outside of the base.
+    def write_secret_file(microservice_name, secret_name, env_name_or_path, secret_value)
+      begin
+        path = Secrets.validate_file_path(env_name_or_path)
+      rescue ArgumentError => error
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} has an invalid file path: #{error.message}")
+        return
+      end
+
+      # validate_file_path is purely lexical, so a symlink under the secret file
+      # directory could still redirect the write outside of it. Resolve symlinks
+      # here, where the filesystem being written to is the one being checked.
+      begin
+        real_base = File.realpath(Secrets.secret_file_dir)
+      rescue SystemCallError => error
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} secret file directory is unusable: #{error.message}")
+        return
+      end
+
+      # Check the deepest directory that already exists before creating anything.
+      # If an existing component is a symlink out of the base, mkdir_p would
+      # otherwise create the remaining directories outside of it.
+      dir = File.dirname(path)
+      existing = dir
+      existing = File.dirname(existing) until File.directory?(existing) or existing == File.dirname(existing)
+      begin
+        real_existing = File.realpath(existing)
+      rescue SystemCallError => error
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} file path is unusable: #{error.message}")
+        return
+      end
+      unless Secrets.path_contained?(real_base, real_existing)
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} file path contains a symlinked directory which resolves outside of #{real_base}: #{path}")
+        return
+      end
+
+      FileUtils.mkdir_p(dir)
+      # Re-check after creating the directories in case one was created through a
+      # symlink or the tree changed underneath us.
+      real_dir = File.realpath(dir)
+      unless Secrets.path_contained?(real_base, real_dir)
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} file path resolves outside of #{real_base}: #{path}")
+        return
+      end
+      # File.symlink? uses lstat and so does not follow the link, unlike the
+      # File.open below which would write through it.
+      if File.symlink?(path)
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} file path is a symlink which is not allowed: #{path}")
+        return
+      end
+      File.open(path, 'wb') do |file|
+        file.write(secret_value)
+      end
+    end
+
     # Handle the detection of a new microservice model
     def handle_new_microservice(microservice_name, microservice_config)
       parent = microservice_config['parent']

data/lib/openc3/operators/operator.rb

@@ -233,18 +233,18 @@ module OpenC3
     PROCESS_SHUTDOWN_SECONDS = 5.0
 
     def initialize
-      Logger.level = Logger::INFO
+      Logger.level = Logger.default_level
       Logger.microservice_name = 'MicroserviceOperator'
 
       OperatorProcess.setup()
-      @cycle_time = (ENV['OPERATOR_CYCLE_TIME'] and ENV['OPERATOR_CYCLE_TIME'].to_f) || CYCLE_TIME # time in seconds
+      @cycle_time = ENV.fetch('OPERATOR_CYCLE_TIME', CYCLE_TIME).to_f # time in seconds
       # Maximum number of new microservices to start per cycle. This spreads a
       # large startup burst (e.g. installing a plugin with many targets) across
       # multiple cycles so the shared services (object store, redis) aren't
       # stampeded by every microservice connecting at once. 0 = no limit.
-      @max_start_per_cycle = (ENV['OPENC3_OPERATOR_MAX_START_PER_CYCLE'] || 5).to_i
+      @max_start_per_cycle = ENV.fetch('OPENC3_OPERATOR_MAX_START_PER_CYCLE', 5).to_i
 
-      @ruby_process_name = ENV['OPENC3_RUBY']
+      @ruby_process_name = ENV.fetch('OPENC3_RUBY', nil)
       if RUBY_ENGINE != 'ruby'
         @ruby_process_name ||= 'jruby'
       else

data/lib/openc3/packets/packet.rb

@@ -637,6 +637,10 @@ module OpenC3
       else
         value = super(item, :RAW, buffer)
       end
+      # An item beyond the end of a short buffer reads as nil (ALLOW_SHORT).
+      # Conversions, states and format strings can't be applied to a nil value.
+      return nil if value.nil? && @short_buffer_allowed && item.data_type != :DERIVED
+
       derived_raw = false
       if item.data_type == :DERIVED && value_type == :RAW
         value_type = :CONVERTED

data/lib/openc3/packets/structure.rb

@@ -617,7 +617,13 @@ module OpenC3
               return 62
             end
           else
-            return (length_value * item.variable_bit_size['length_bits_per_count']) + item.variable_bit_size['length_value_bit_offset']
+            total_bit_size = (length_value * item.variable_bit_size['length_bits_per_count']) + item.variable_bit_size['length_value_bit_offset']
+            available_bit_size = (@buffer.length * 8) - item.bit_offset
+            if total_bit_size < 0 ||
+               (!@short_buffer_allowed && (available_bit_size < 0 || total_bit_size > available_bit_size))
+              raise ArgumentError, "Variable bit size #{total_bit_size} for item #{item.name} exceeds the #{[available_bit_size, 0].max} bits available in the buffer"
+            end
+            return total_bit_size
           end
         else
           raise "Length value #{item.variable_bit_size['length_item_name']} for item #{item.name} is nil"

data/lib/openc3/processors/processor.rb

@@ -15,6 +15,8 @@
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
+require 'openc3/packets/packet'
+
 module OpenC3
   class Processor
     # @return [Symbol] The value type for the processor

data/lib/openc3/script/autonomic.rb

@@ -67,7 +67,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_group_destroy error: #{parsed['error']}"
+            raise "autonomic_group_destroy error: #{parsed['message']}"
           else
             raise "autonomic_group_destroy failed"
           end
@@ -105,7 +105,7 @@ module OpenC3
         if response.nil? || response.status != 201
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_trigger_create error: #{parsed['error']}"
+            raise "autonomic_trigger_create error: #{parsed['message']}"
           else
             raise "autonomic_trigger_create failed"
           end
@@ -136,7 +136,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_trigger_enable error: #{parsed['error']}"
+            raise "autonomic_trigger_enable error: #{parsed['message']}"
           else
             raise "autonomic_trigger_enable failed"
           end
@@ -154,7 +154,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_trigger_disable error: #{parsed['error']}"
+            raise "autonomic_trigger_disable error: #{parsed['message']}"
           else
             raise "autonomic_trigger_disable failed"
           end
@@ -176,7 +176,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_trigger_update error: #{parsed['error']}"
+            raise "autonomic_trigger_update error: #{parsed['message']}"
           else
             raise "autonomic_trigger_update failed"
           end
@@ -194,7 +194,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_trigger_destroy error: #{parsed['error']}"
+            raise "autonomic_trigger_destroy error: #{parsed['message']}"
           else
             raise "autonomic_trigger_destroy failed"
           end
@@ -232,7 +232,7 @@ module OpenC3
         if response.nil? || response.status != 201
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_reaction_create error: #{parsed['error']}"
+            raise "autonomic_reaction_create error: #{parsed['message']}"
           else
             raise "autonomic_reaction_create failed"
           end
@@ -263,7 +263,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_reaction_enable error: #{parsed['error']}"
+            raise "autonomic_reaction_enable error: #{parsed['message']}"
           else
             raise "autonomic_reaction_enable failed"
           end
@@ -281,7 +281,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_reaction_disable error: #{parsed['error']}"
+            raise "autonomic_reaction_disable error: #{parsed['message']}"
           else
             raise "autonomic_reaction_disable failed"
           end
@@ -299,7 +299,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_reaction_execute error: #{parsed['error']}"
+            raise "autonomic_reaction_execute error: #{parsed['message']}"
           else
             raise "autonomic_reaction_execute failed"
           end
@@ -322,7 +322,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_reaction_update error: #{parsed['error']}"
+            raise "autonomic_reaction_update error: #{parsed['message']}"
           else
             raise "autonomic_reaction_update failed"
           end
@@ -340,7 +340,7 @@ module OpenC3
         if response.nil? || response.status != 200
           if response
             parsed = JSON.parse(response.body)
-            raise "autonomic_reaction_destroy error: #{parsed['error']}"
+            raise "autonomic_reaction_destroy error: #{parsed['message']}"
           else
             raise "autonomic_reaction_destroy failed"
           end

data/lib/openc3/script/script.rb

@@ -241,7 +241,7 @@ module OpenC3
     # script_run("INST/procedures/collect.rb")
     #
     def initialize_offline_access
-      keycloak_url = ENV['OPENC3_KEYCLOAK_URL']
+      keycloak_url = ENV.fetch('OPENC3_KEYCLOAK_URL', nil)
       if keycloak_url.nil? or keycloak_url.empty?
         raise "initialize_offline_access only valid in COSMOS Enterprise. OPENC3_KEYCLOAK_URL environment variable must be set."
       end
@@ -383,4 +383,4 @@ module OpenC3
   end
 end
 
-$VERBOSE = saved_verbose
+$VERBOSE = saved_verbose

data/lib/openc3/script/storage.rb

@@ -169,17 +169,17 @@ module OpenC3
 
     def _get_uri(url)
       if $openc3_in_cluster
-        case ENV['OPENC3_CLOUD']
+        case ENV.fetch('OPENC3_CLOUD', nil)
         when 'local'
-          bucket_url = ENV["OPENC3_BUCKET_URL"] || "http://openc3-buckets:9000"
+          bucket_url = ENV.fetch("OPENC3_BUCKET_URL", "http://openc3-buckets:9000")
           URI.parse("#{bucket_url}#{url}")
         when 'aws'
-          URI.parse("https://s3.#{ENV['AWS_REGION']}.amazonaws.com" + url)
+          URI.parse("https://s3.#{ENV.fetch('AWS_REGION')}.amazonaws.com" + url)
         when 'gcp'
           URI.parse("https://storage.googleapis.com" + url)
         # when 'azure'
         else
-          raise "Unknown cloud #{ENV['OPENC3_CLOUD']}"
+          raise "Unknown cloud #{ENV.fetch('OPENC3_CLOUD', nil)}"
         end
       else
         URI.parse($api_server.generate_url + url)

data/lib/openc3/script/suite_runner.rb

@@ -30,6 +30,12 @@ module OpenC3
     # the values stay consistent in one place.
     DEFAULT_METHOD = 'start'
     DEFAULT_OPTIONS = ['continueAfterError'].freeze
+    # The only SuiteRunner entry points a client may request
+    ALLOWED_METHODS = ['start', 'setup', 'teardown'].freeze
+    # Suite and Group are Ruby constant (class) names, optionally namespaced
+    CLASS_NAME_REGEX = /\A[A-Z]\w*(::[A-Z]\w*)*\z/
+    # Script is a Ruby method name on the Group
+    METHOD_NAME_REGEX = /\A[a-z_]\w*[?!]?\z/
 
     @@suites = []
     @@settings = {}
@@ -55,12 +61,32 @@ module OpenC3
       raise ArgumentError, "Script #{script} requires a Group" if script and not group
     end
 
+    # Validate that the raw (client supplied) suite_runner values are plain
+    # identifiers. These values are interpolated into the Ruby snippet the
+    # running script evaluates, so anything other than a bare class / method
+    # name must be rejected before it reaches the interpreter.
+    def self.validate_identifiers(suite:, group: nil, script: nil, method: nil)
+      unless suite.is_a?(String) and CLASS_NAME_REGEX.match?(suite)
+        raise ArgumentError, "Invalid Suite name: #{suite.inspect}"
+      end
+      if group && !(group.is_a?(String) and CLASS_NAME_REGEX.match?(group))
+        raise ArgumentError, "Invalid Group name: #{group.inspect}"
+      end
+      if script && !(script.is_a?(String) and METHOD_NAME_REGEX.match?(script))
+        raise ArgumentError, "Invalid Script name: #{script.inspect}"
+      end
+      if method && !ALLOWED_METHODS.include?(method.to_s)
+        raise ArgumentError, "Invalid method: #{method.inspect}"
+      end
+    end
+
     # Build the canonical, validated suite_runner hash from raw values,
     # applying defaults. Shared by openc3cli (constructing from CLI flags) and
     # the running script (normalizing a received hash before dispatch) so the
     # hash shape, defaults, and validation live in one place.
     def self.build_options(suite:, group: nil, script: nil, method: nil, options: nil)
       validate_options(group: group, script: script)
+      validate_identifiers(suite: suite, group: group, script: script, method: method)
       suite_runner = { 'suite' => suite }
       if group
         suite_runner['group'] = group

data/lib/openc3/script/telemetry.rb

@@ -40,9 +40,15 @@ module OpenC3
     # inject_tlm, set_tlm, override_tlm, and normalize_tlm are implemented here simply to add a puts
     # these methods modify the telemetry so the user should be notified in the Script Runner log messages
 
-    def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, scope: $openc3_scope, token: $openc3_token)
-      puts "inject_tlm(\"#{target_name}\", \"#{packet_name}\", #{item_hash}, type: #{type}, stored: #{stored})"
-      $api_server.method_missing(:inject_tlm, target_name, packet_name, item_hash, type: type, stored: stored, scope: scope, token: token)
+    def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, stored: false, received_time: nil, scope: $openc3_scope, token: $openc3_token)
+      message = "inject_tlm(\"#{target_name}\", \"#{packet_name}\", #{item_hash}, type: #{type}, stored: #{stored}"
+      kwargs = { type: type, stored: stored, scope: scope, token: token }
+      unless received_time.nil?
+        message << ", received_time: #{received_time}"
+        kwargs[:received_time] = received_time
+      end
+      puts "#{message})"
+      $api_server.method_missing(:inject_tlm, target_name, packet_name, item_hash, **kwargs)
     end
 
     def set_tlm(*args, type: :ALL, scope: $openc3_scope, token: $openc3_token)