openc3 7.2.1 → 7.4.0

data/lib/openc3/operators/microservice_operator.rb

@@ -18,6 +18,7 @@
 require 'openc3'
 require 'openc3/models/microservice_model'
 require 'openc3/operators/operator'
+require 'openc3/utilities/python_venv'
 require 'openc3/utilities/secrets'
 require 'redis'
 require 'open3'
@@ -47,10 +48,24 @@ module OpenC3
       env = microservice_config["env"].dup
       if microservice_config["needs_dependencies"]
         env['GEM_HOME'] = '/gems'
-        env['PYTHONUSERBASE'] = '/gems/python_packages'
-        # Ensure PYTHONPATH includes both the UV venv (for base openc3 module) and user packages
-        # This is critical for UV-based Python installations where openc3 is installed as editable
-        env['PYTHONPATH'] = "#{ENV['PYTHONPATH']}"
+
+        # Resolve the Python virtual environment for this microservice.
+        # If the microservice belongs to a plugin that has a per-plugin UV venv
+        # (created by uvinstall during plugin install), use that isolated venv.
+        # Otherwise fall back to the shared PYTHONUSERBASE for legacy plugins.
+        plugin_name = microservice_config["plugin"]
+        plugin_venv_dir = nil
+        if plugin_name
+          scope = microservice_name.split("__")[0]
+          plugin_venv_dir = PythonVenv.plugin_venv_path(scope: scope, plugin_name: plugin_name)
+        end
+
+        if plugin_venv_dir
+          PythonVenv.configure_environment(env, plugin_venv_dir)
+        else
+          env['PYTHONUSERBASE'] = PythonVenv::DEFAULT_PYTHONUSERBASE
+          env['PYTHONPATH'] = ENV.fetch('PYTHONPATH', nil)
+        end
       else
         env['GEM_HOME'] = nil
         env['PYTHONUSERBASE'] = nil
@@ -69,10 +84,7 @@ module OpenC3
             if type == 'ENV'
               env[env_name_or_path] = secret_value
             elsif type == 'FILE'
-              FileUtils.mkdir_p(File.dirname(env_name_or_path))
-              File.open(env_name_or_path, 'wb') do |file|
-                file.write(secret_value)
-              end
+              write_secret_file(microservice_name, secret_name, env_name_or_path, secret_value)
             end
           else
             Logger.error("Microservice #{microservice_name} references unknown secret: #{secret_name}")
@@ -83,6 +95,65 @@ module OpenC3
       return process_definition, work_dir, env, scope, container
     end
 
+    # Writes a FILE type secret to disk. The path is first validated lexically
+    # (see Secrets.validate_file_path) and then checked against the filesystem to
+    # make sure no symlink redirects the write outside of Secrets.secret_file_dir.
+    # Both checks happen before any directory is created so that mkdir_p can
+    # never create directories outside of the base.
+    def write_secret_file(microservice_name, secret_name, env_name_or_path, secret_value)
+      begin
+        path = Secrets.validate_file_path(env_name_or_path)
+      rescue ArgumentError => error
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} has an invalid file path: #{error.message}")
+        return
+      end
+
+      # validate_file_path is purely lexical, so a symlink under the secret file
+      # directory could still redirect the write outside of it. Resolve symlinks
+      # here, where the filesystem being written to is the one being checked.
+      begin
+        real_base = File.realpath(Secrets.secret_file_dir)
+      rescue SystemCallError => error
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} secret file directory is unusable: #{error.message}")
+        return
+      end
+
+      # Check the deepest directory that already exists before creating anything.
+      # If an existing component is a symlink out of the base, mkdir_p would
+      # otherwise create the remaining directories outside of it.
+      dir = File.dirname(path)
+      existing = dir
+      existing = File.dirname(existing) until File.directory?(existing) or existing == File.dirname(existing)
+      begin
+        real_existing = File.realpath(existing)
+      rescue SystemCallError => error
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} file path is unusable: #{error.message}")
+        return
+      end
+      unless Secrets.path_contained?(real_base, real_existing)
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} file path contains a symlinked directory which resolves outside of #{real_base}: #{path}")
+        return
+      end
+
+      FileUtils.mkdir_p(dir)
+      # Re-check after creating the directories in case one was created through a
+      # symlink or the tree changed underneath us.
+      real_dir = File.realpath(dir)
+      unless Secrets.path_contained?(real_base, real_dir)
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} file path resolves outside of #{real_base}: #{path}")
+        return
+      end
+      # File.symlink? uses lstat and so does not follow the link, unlike the
+      # File.open below which would write through it.
+      if File.symlink?(path)
+        Logger.error("Microservice #{microservice_name} secret #{secret_name} file path is a symlink which is not allowed: #{path}")
+        return
+      end
+      File.open(path, 'wb') do |file|
+        file.write(secret_value)
+      end
+    end
+
     # Handle the detection of a new microservice model
     def handle_new_microservice(microservice_name, microservice_config)
       parent = microservice_config['parent']

data/lib/openc3/operators/operator.rb

@@ -233,18 +233,18 @@ module OpenC3
     PROCESS_SHUTDOWN_SECONDS = 5.0
 
     def initialize
-      Logger.level = Logger::INFO
+      Logger.level = Logger.default_level
       Logger.microservice_name = 'MicroserviceOperator'
 
       OperatorProcess.setup()
-      @cycle_time = (ENV['OPERATOR_CYCLE_TIME'] and ENV['OPERATOR_CYCLE_TIME'].to_f) || CYCLE_TIME # time in seconds
+      @cycle_time = ENV.fetch('OPERATOR_CYCLE_TIME', CYCLE_TIME).to_f # time in seconds
       # Maximum number of new microservices to start per cycle. This spreads a
       # large startup burst (e.g. installing a plugin with many targets) across
       # multiple cycles so the shared services (object store, redis) aren't
       # stampeded by every microservice connecting at once. 0 = no limit.
-      @max_start_per_cycle = (ENV['OPENC3_OPERATOR_MAX_START_PER_CYCLE'] || 5).to_i
+      @max_start_per_cycle = ENV.fetch('OPENC3_OPERATOR_MAX_START_PER_CYCLE', 5).to_i
 
-      @ruby_process_name = ENV['OPENC3_RUBY']
+      @ruby_process_name = ENV.fetch('OPENC3_RUBY', nil)
       if RUBY_ENGINE != 'ruby'
         @ruby_process_name ||= 'jruby'
       else

data/lib/openc3/packets/commands.rb

@@ -61,7 +61,7 @@ module OpenC3
     #   target name keyed by the packet name
     def packets(target_name)
       target_packets = @config.commands[target_name.to_s.upcase]
-      raise "Command target '#{target_name.to_s.upcase}' does not exist" unless target_packets
+      raise "Command target '#{target_name.to_s.upcase}' does not exist (packets lookup)" unless target_packets
 
       target_packets
     end
@@ -73,7 +73,7 @@ module OpenC3
     def packet(target_name, packet_name)
       target_packets = packets(target_name)
       packet = target_packets[packet_name.to_s.upcase]
-      raise "Command packet '#{target_name.to_s.upcase} #{packet_name.to_s.upcase}' does not exist" unless packet
+      raise "Command packet '#{target_name.to_s.upcase} #{packet_name.to_s.upcase}' does not exist (packet lookup)" unless packet
 
       packet
     end
@@ -187,9 +187,9 @@ module OpenC3
 
       # Lookup the command directly - avoid redundant upcase in packet()/packets()
       target_packets = @config.commands[target_upcase]
-      raise "Command target '#{target_upcase}' does not exist" unless target_packets
+      raise "Command target '#{target_upcase}' does not exist (build_cmd)" unless target_packets
       pkt = target_packets[packet_upcase]
-      raise "Command packet '#{target_upcase} #{packet_upcase}' does not exist" unless pkt
+      raise "Command packet '#{target_upcase} #{packet_upcase}' does not exist (build_cmd)" unless pkt
       # Use deep_copy to avoid shared item modifications affecting the template
       # This is critical for variable_bit_size items where handle_write_variable_bit_size
       # modifies item.bit_offset and item.array_size during writes

data/lib/openc3/packets/limits.rb

@@ -159,16 +159,24 @@ module OpenC3
     # @param packet_name [String] Packet Name
     # @param item_name [String] Item Name
     # @param state_name [String] State Name (e.g. 'CONNECTED')
-    # @param color [String or Symbol] New color: GREEN, YELLOW, or RED
-    # @return [Symbol] The color that was set
+    # @param color [String or Symbol, nil] New color: GREEN, YELLOW, or RED.
+    #   Pass nil to clear (remove) the state color.
+    # @return [Symbol, nil] The color that was set, or nil if the color was cleared
     def set_state_color(target_name, packet_name, item_name, state_name, color)
       packet = _get_packet(target_name, packet_name)
       item = packet.get_item(item_name)
-      raise "Item #{target_name} #{packet_name} #{item_name} does not have any states" unless item.states
+      raise "Item '#{target_name} #{packet_name} #{item_name}' does not have any states" unless item.states
       state_name = state_name.to_s.upcase
-      raise "State #{state_name} does not exist for #{target_name} #{packet_name} #{item_name}" unless item.states.key?(state_name)
+      raise "State '#{state_name}' does not exist for item '#{target_name} #{packet_name} #{item_name}'" unless item.states.key?(state_name)
+      if color.nil?
+        # Clear the state color while keeping state_colors as a Hash so limits
+        # checking (which indexes state_colors by value) continues to work.
+        item.state_colors.delete(state_name) if item.state_colors
+        packet.update_limits_items_cache(item)
+        return nil
+      end
       color = color.to_s.upcase.intern
-      raise "Invalid state color #{color}. Must be one of #{PacketItem::STATE_COLORS.join(' ')}." unless PacketItem::STATE_COLORS.include?(color)
+      raise "Invalid state color '#{color}'. Must be one of #{PacketItem::STATE_COLORS.join(', ')}." unless PacketItem::STATE_COLORS.include?(color)
       item.state_colors ||= {}
       item.state_colors[state_name] = color
       item.limits.enabled = true
@@ -182,10 +190,10 @@ module OpenC3
       raise "LATEST packet not valid" if packet_name.upcase == LATEST_PACKET_NAME
 
       packets = @config.telemetry[target_name.to_s.upcase]
-      raise "Telemetry target '#{target_name.to_s.upcase}' does not exist" unless packets
+      raise "Telemetry target '#{target_name.to_s.upcase}' does not exist (limits lookup)" unless packets
 
       packet = packets[packet_name.to_s.upcase]
-      raise "Telemetry packet '#{target_name.to_s.upcase} #{packet_name.to_s.upcase}' does not exist" unless packet
+      raise "Telemetry packet '#{target_name.to_s.upcase} #{packet_name.to_s.upcase}' does not exist (limits lookup)" unless packet
 
       return packet
     end

data/lib/openc3/packets/packet.rb

@@ -616,7 +616,7 @@ module OpenC3
     def get_item(name)
       return super(name)
     rescue ArgumentError
-      raise "Packet item '#{@target_name} #{@packet_name} #{name.upcase}' does not exist"
+      raise "Item '#{@target_name} #{@packet_name} #{name.upcase}' does not exist (Packet)"
     end
 
     # Read an item in the packet
@@ -637,6 +637,10 @@ module OpenC3
       else
         value = super(item, :RAW, buffer)
       end
+      # An item beyond the end of a short buffer reads as nil (ALLOW_SHORT).
+      # Conversions, states and format strings can't be applied to a nil value.
+      return nil if value.nil? && @short_buffer_allowed && item.data_type != :DERIVED
+
       derived_raw = false
       if item.data_type == :DERIVED && value_type == :RAW
         value_type = :CONVERTED

data/lib/openc3/packets/parsers/state_parser.rb

@@ -124,7 +124,7 @@ module OpenC3
     def get_state_colors(item)
       color = @parser.parameters[2].upcase.to_sym
       unless PacketItem::STATE_COLORS.include? color
-        raise @parser.error("Invalid state color #{color}. Must be one of #{PacketItem::STATE_COLORS.join(' ')}.", @usage)
+        raise @parser.error("Invalid state color '#{color}'. Must be one of #{PacketItem::STATE_COLORS.join(', ')}.", @usage)
       end
 
       item.limits.enabled = true