openc3 7.4.0 → 7.5.0

data/lib/openc3/models/metric_model.rb

@@ -23,6 +23,13 @@ module OpenC3
     include DbShardedModel
 
     PRIMARY_KEY = '__openc3__metric'.freeze
+    # Process level metrics (cpu, memory) reported by the operator on behalf of
+    # the microservices it spawned. These live in their own hash so that the
+    # operator and the microservice are never two writers of one field - each
+    # write replaces the whole values hash, so sharing a field would have them
+    # clobbering each other every cycle. The two are merged back together on
+    # read by get/names/all, so callers see a single row per microservice.
+    PROCESS_PRIMARY_KEY = '__openc3__process_metric'.freeze
     METRIC_EXPIRE_SECONDS = 3600 # Expire metrics after 1 hour
 
     attr_accessor :values
@@ -47,17 +54,48 @@ module OpenC3
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
     def self.get(name:, scope:)
-      _db_sharded_get("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
+      _merge_entries(
+        _db_sharded_get("#{scope}#{PRIMARY_KEY}", name: name, scope: scope),
+        _db_sharded_get("#{scope}#{PROCESS_PRIMARY_KEY}", name: name, scope: scope)
+      )
     end
 
     def self.names(scope:)
-      _db_sharded_names("#{scope}#{PRIMARY_KEY}", scope: scope)
+      names = _db_sharded_names("#{scope}#{PRIMARY_KEY}", scope: scope)
+      names.concat(_db_sharded_names("#{scope}#{PROCESS_PRIMARY_KEY}", scope: scope))
+      names.uniq.sort
     end
 
     def self.all(scope:)
+      result = _db_sharded_all("#{scope}#{PRIMARY_KEY}", scope: scope)
+      _db_sharded_all("#{scope}#{PROCESS_PRIMARY_KEY}", scope: scope).each do |name, process_entry|
+        result[name] = _merge_entries(result[name], process_entry)
+      end
+      result
+    end
+
+    # Only the metrics each microservice reported about itself, without the
+    # operator's process metrics. Their updated_at is when the microservice
+    # last published, which the merged rows from all can't tell you because
+    # the operator keeps refreshing them while the process is alive.
+    def self.all_self_reported(scope:)
       _db_sharded_all("#{scope}#{PRIMARY_KEY}", scope: scope)
     end
 
+    # Combine a microservice's self reported metrics with the process metrics
+    # the operator reported for it. The microservice's own values win on a key
+    # collision - a process that can measure itself knows better than the
+    # operator watching from outside.
+    def self._merge_entries(entry, process_entry)
+      return entry unless process_entry
+      return process_entry unless entry
+
+      entry['values'] = (process_entry['values'] || {}).merge(entry['values'] || {})
+      updated_at = [entry['updated_at'], process_entry['updated_at']].compact.max
+      entry['updated_at'] = updated_at if updated_at
+      entry
+    end
+
     # Sets (updates) the redis hash of this model
     # Queued defaults to true for MetricModel
     def self.set(json, scope:, queued: true)
@@ -66,13 +104,23 @@ module OpenC3
       self.new(**json).create(force: true, queued: queued, expire_seconds: METRIC_EXPIRE_SECONDS)
     end
 
+    # Sets (updates) the process metrics reported by the operator for a
+    # microservice it spawned. See PROCESS_PRIMARY_KEY.
+    def self.set_process(json, scope:, queued: true)
+      json[:scope] = scope
+      json.transform_keys!(&:to_sym)
+      json[:primary_key] = PROCESS_PRIMARY_KEY
+      self.new(**json).create(force: true, queued: queued, expire_seconds: METRIC_EXPIRE_SECONDS)
+    end
+
     def self.destroy(scope:, name:)
       db_shard = _db_shard_for_name(name, scope: scope)
       store.instance(db_shard: db_shard).hdel("#{scope}#{PRIMARY_KEY}", name)
+      store.instance(db_shard: db_shard).hdel("#{scope}#{PROCESS_PRIMARY_KEY}", name)
     end
 
-    def initialize(name:, values: {}, db_shard: 0, scope:)
-      super("#{scope}#{PRIMARY_KEY}", name: name, scope: scope)
+    def initialize(name:, values: {}, db_shard: 0, scope:, primary_key: PRIMARY_KEY)
+      super("#{scope}#{primary_key}", name: name, scope: scope)
       @values = values
       @db_shard = db_shard.to_i
     end

data/lib/openc3/models/microservice_model.rb

@@ -308,9 +308,13 @@ module OpenC3
     end
 
     def cleanup
-      # Cleanup metrics
-      metric_model = MetricModel.new(name: @name, scope: @scope)
-      metric_model.destroy
+      # Cleanup metrics - both the microservice's own and the process metrics
+      # the operator reported for it
+      # The MicroserviceModel has already been deleted during uninstall, so
+      # shard lookup by name can no longer find its configured database.
+      [MetricModel::PRIMARY_KEY, MetricModel::PROCESS_PRIMARY_KEY].each do |key|
+        MetricModel.new(name: @name, scope: @scope, db_shard: @db_shard, primary_key: key).destroy
+      end
     end
   end
 end

data/lib/openc3/models/plugin_model.rb

@@ -278,7 +278,10 @@ module OpenC3
 
         if File.exist?(pyproject_path) || File.exist?(requirements_path)
           pypi_url = resolve_pypi_url(scope: scope)
-          unless validate_only
+          # Skipped for any dry run: diff_only runs inline in an HTTP request, so
+          # a slow or unreachable PyPI would time it out, and it must not mutate
+          # the plugin venv.
+          unless dry_run
             pypi_args = build_pypi_args(pypi_url)
 
             # Install Python dependencies into an isolated per-plugin venv when UV
@@ -428,11 +431,9 @@ module OpenC3
     # Dry run: which modified files would this plugin's install supersede?
     # Returns a list of "TARGET/path" names whose live (modified) content
     # differs from the rendered plugin content. Read-only; no side effects.
+    # Errors propagate so callers can tell a failed check from an empty diff.
     def self.modified_diff(plugin_hash, scope:)
       install_phase2(plugin_hash, scope: scope, diff_only: true)
-    rescue => e
-      Logger.warn("PluginModel.modified_diff failed: #{e.message}")
-      []
     end
 
     def initialize(
@@ -597,25 +598,24 @@ module OpenC3
     def self.resolve_pypi_url(scope:)
       pypi_url = nil
       begin
-        pypi_url = get_setting('pypi_url', scope: scope)
-        pypi_url += '/simple' if pypi_url
+        pypi_url = PypiUrl.index_url(get_setting('pypi_url', scope: scope))
       rescue => e
         Logger.error("Failed to retrieve pypi_url: #{e.formatted}")
       ensure
         if pypi_url.nil?
-          pypi_url = ENV.fetch('PYPI_URL', nil)
-          pypi_url += '/simple' if pypi_url
+          pypi_url = PypiUrl.index_url(ENV.fetch('PYPI_URL', nil))
           pypi_url ||= PypiUrl::DEFAULT
         end
       end
       PypiUrl.validate(pypi_url)
     end
 
-    # Build the argv array for pypi index and trusted-host arguments.
+    # Build the argv array of pypi index arguments. Kept as a delegate so
+    # callers that already reference PluginModel keep working; the logic lives
+    # in PypiUrl because PythonPackageModel needs it too and must not have to
+    # require this file.
     def self.build_pypi_args(pypi_url)
-      args = ["-i", pypi_url]
-      args += ["--trusted-host", URI.parse(pypi_url).host] unless ENV['PIP_ENABLE_TRUSTED_HOST'].nil?
-      args
+      PypiUrl.build_args(pypi_url)
     end
 
     # Build a sanitized venv directory name from scope and plugin name.

data/lib/openc3/models/python_package_model.rb

@@ -235,29 +235,19 @@ module OpenC3
       end
       package_filename = File.basename(package_file_path)
       begin
-        pypi_url = get_setting('pypi_url', scope: scope)
-        if pypi_url
-          pypi_url += '/simple'
-        end
+        pypi_url = PypiUrl.index_url(get_setting('pypi_url', scope: scope))
       rescue => e
         Logger.error("Failed to retrieve pypi_url: #{e.formatted}")
       ensure
         if pypi_url.nil?
           # If Redis isn't running try the ENV, then simply pypi.org/simple
-          pypi_url = ENV.fetch('PYPI_URL', nil)
-          if pypi_url
-            pypi_url += '/simple'
-          end
+          pypi_url = PypiUrl.index_url(ENV.fetch('PYPI_URL', nil))
           pypi_url ||= PypiUrl::DEFAULT
         end
       end
       pypi_url = PypiUrl.validate(pypi_url)
       Logger.info "Installing python package: #{name_or_path}"
-      if ENV['PIP_ENABLE_TRUSTED_HOST'].nil?
-        pip_args = ["-i", pypi_url, package_file_path]
-      else
-        pip_args = ["-i", pypi_url, "--trusted-host", URI.parse(pypi_url).host, package_file_path]
-      end
+      pip_args = PypiUrl.build_args(pypi_url) + [package_file_path]
       spawn_env = {}
       if plugin
         venv_path = "#{PLUGIN_VENVS_DIR}/#{plugin}/.venv"

data/lib/openc3/models/reaction_model.rb

@@ -150,11 +150,14 @@ module OpenC3
         if trigger['name'].nil? || trigger['group'].nil?
           raise ReactionInputError.new "invalid trigger, must contain 'name' and 'group' keys: #{trigger}"
         end
-        trigger_name = trigger['name']
-        unless trigger_hash[trigger_name].nil?
+        # Trigger names are only unique within a group so both parts identify
+        # the trigger. Keying on name alone rejects a reaction which references
+        # the same trigger name in two different groups.
+        trigger_key = "#{trigger['group']}__#{trigger['name']}"
+        unless trigger_hash[trigger_key].nil?
           raise ReactionInputError.new "no duplicate triggers allowed: #{triggers}"
         else
-          trigger_hash[trigger_name] = 1
+          trigger_hash[trigger_key] = 1
         end
       end
       return triggers

data/lib/openc3/models/setting_model.rb

@@ -182,9 +182,11 @@ module OpenC3
     # Seed settings from OPENC3_SETTING_<NAME> environment variables. Called by
     # `openc3cli initsettings` during init container startup.
     #
-    # By default a setting is only written when it does not already exist, so a
-    # value changed in the Admin Console survives a container restart. Set
-    # OPENC3_SETTINGS_OVERWRITE to write on every run instead.
+    # By default a setting is written when it does not exist yet, or when it
+    # still holds the value last seeded (see seeded_value?), so a changed env
+    # var takes effect but a value changed in the Admin Console survives a
+    # container restart. Set OPENC3_SETTINGS_OVERWRITE to write on every run
+    # instead.
     #
     # Nothing here aborts init by default. A bad setting name, a bad value, or a
     # malformed control variable is reported on stdout and that one setting is

data/lib/openc3/models/target_model.rb

@@ -109,10 +109,14 @@ module OpenC3
       modified_targets = []
       if ENV['OPENC3_LOCAL_MODE']
         modified_targets += OpenC3::LocalMode.modified_targets(scope: scope)
+      elsif !OpenC3::LocalMode.local_only_targets.empty?
+        modified_targets += OpenC3::LocalMode.modified_targets(scope: scope).select { |name| OpenC3::LocalMode.local_only_target?(name) }
       end
       # Always list the bucket, even in local mode, since the local directory
-      # only holds what COSMOS itself wrote there
-      modified_targets += Bucket.getClient().list_files(bucket: ENV['OPENC3_CONFIG_BUCKET'], path: "#{scope}/targets_modified/", only_directories: true)
+      # only holds what COSMOS itself wrote there. Local only targets are
+      # never read from the bucket.
+      bucket_targets = Bucket.getClient().list_files(bucket: ENV['OPENC3_CONFIG_BUCKET'], path: "#{scope}/targets_modified/", only_directories: true)
+      modified_targets += bucket_targets.reject { |name| OpenC3::LocalMode.local_only_target?(name) }
       modified_targets.each do |target_name|
         # A target could have been deleted without removing the modified files
         # Thus we have to check for the existence of the target_name key
@@ -186,13 +190,15 @@ module OpenC3
     def self.modified_files(target_name, scope:)
       modified = []
 
-      if ENV['OPENC3_LOCAL_MODE']
+      if ENV['OPENC3_LOCAL_MODE'] or OpenC3::LocalMode.local_only_target?(target_name)
         # LocalMode reports target relative paths, i.e. "procedures/new.rb",
         # so add the target name to match the bucket listing below
         modified += OpenC3::LocalMode.modified_files(target_name, scope: scope).map do |file_path|
           "#{target_name}/#{file_path}"
         end
       end
+      # Local only targets are never read from the bucket
+      return modified.uniq.sort if OpenC3::LocalMode.local_only_target?(target_name)
       # Always list the bucket, even in local mode. The local directory only
       # holds what COSMOS itself wrote there, so a file placed directly in the
       # bucket is modified but has no local copy.
@@ -216,16 +222,28 @@ module OpenC3
     # some modified files while keeping others); when nil, every modified file
     # for the target is removed (the original behavior).
     def self.delete_modified(target_name, scope:, files: nil)
+      # Validate target_name to not allow directory traversal
+      unless OpenC3::LocalMode.safe_key?(target_name) and !target_name.include?('/')
+        raise ArgumentError, "Invalid target_name: #{target_name.inspect}"
+      end
       if files && !files.empty?
+        # Only delete names that are actually modified files of this target.
+        # Anything else (another target, another scope via '..') is ignored.
+        allowed = modified_files(target_name, scope: scope)
         files.each do |name|
+          next unless String === name
+          name = OpenC3::TargetFile.strip_modified(name)
+          next unless allowed.include?(name)
           # TargetFile.destroy handles both local mode and the bucket.
           OpenC3::TargetFile.destroy(scope, name)
         end
         return
       end
-      if ENV['OPENC3_LOCAL_MODE']
+      if ENV['OPENC3_LOCAL_MODE'] or OpenC3::LocalMode.local_only_target?(target_name)
         OpenC3::LocalMode.delete_modified(target_name, scope: scope)
       end
+      # Local only targets are never written to the bucket
+      return if OpenC3::LocalMode.local_only_target?(target_name)
       bucket = Bucket.getClient()
       # Delete the remote files as well
       resp = bucket.list_objects(
@@ -249,7 +267,7 @@ module OpenC3
         Zip.continue_on_exists_proc = true
         zip = Zip::File.open(zip_filename, create: true)
 
-        if ENV['OPENC3_LOCAL_MODE']
+        if ENV['OPENC3_LOCAL_MODE'] or OpenC3::LocalMode.local_only_target?(target_name)
           OpenC3::LocalMode.zip_target(target_name, zip, scope: scope)
         else
           bucket = Bucket.getClient()
@@ -332,6 +350,35 @@ module OpenC3
       result
     end
 
+    # Returns just the packet names without fetching and parsing every packet
+    # definition, which is expensive for targets with thousands of packets.
+    # @return [Array<String>] Sorted packet names under the target_name
+    def self.packet_names(target_name, type: :TLM, scope:)
+      raise "Unknown type #{type} for #{target_name}" unless VALID_TYPES.include?(type)
+      raise "Target '#{target_name}' does not exist for scope: #{scope} (TargetModel)" unless get(name: target_name, scope: scope)
+
+      store_for_target(target_name, scope: scope).hkeys("#{scope}__openc3#{type.to_s.downcase}__#{target_name}").sort
+    end
+
+    # Returns the names of the hidden packets without parsing every packet definition.
+    # The list is stored when the target is installed. Targets installed before the
+    # list existed have it built once from the packet definitions and stored.
+    # @return [Array<String>] Sorted hidden packet names under the target_name
+    def self.hidden_packet_names(target_name, type: :TLM, scope:)
+      raise "Unknown type #{type} for #{target_name}" unless VALID_TYPES.include?(type)
+
+      store = store_for_target(target_name, scope: scope)
+      # Stored as a JSON string rather than a set so an empty list is distinct from a missing one
+      # e.g. DEFAULT__openc3tlm__INST__hidden
+      key = "#{scope}__openc3#{type.to_s.downcase}__#{target_name}__hidden"
+      names_json = store.get(key)
+      return JSON.parse(names_json) if names_json
+
+      names = packets(target_name, type: type, scope: scope).select { |packet| packet['hidden'] }.map { |packet| packet['packet_name'] }
+      store.set(key, JSON.generate(names))
+      names
+    end
+
     # @return [Array<Hash>] All packet hashes under the target_name
     def self.all_packet_name_descriptions(target_name, type: :TLM, scope:)
       self.packets(target_name, type: type, scope: scope).map! { |hash| hash.slice("packet_name", "description") }
@@ -751,11 +798,12 @@ module OpenC3
     # Version History is an Enterprise feature. The plugin deploy runs in the
     # cmd-tlm-api process, which (unlike script-runner-api) doesn't otherwise
     # require the store, so lazily load it on demand. Returns false in Core
-    # builds where the Enterprise gem isn't present.
+    # builds where the Enterprise gem isn't present, and when the store is
+    # disabled (OPENC3_VERSION_HISTORY_DIR unset): its commits no-op there, so
+    # versioning would drop the modified copy without preserving it.
     def version_store_available?
-      return true if defined?(::VersionStore)
-      require 'openc3-enterprise/utilities/version_store'
-      defined?(::VersionStore) ? true : false
+      require 'openc3-enterprise/utilities/version_store' unless defined?(::VersionStore)
+      defined?(::VersionStore) && ::VersionStore.enabled? ? true : false
     rescue LoadError
       false
     end
@@ -765,6 +813,8 @@ module OpenC3
     # copy), commits it, removes it, then commits the incoming plugin content.
     # Best-effort: a versioning failure must not abort the plugin upgrade.
     def apply_upgrade_version(name, new_data, ctx)
+      # Local only targets have no modified copy in the bucket
+      return if OpenC3::LocalMode.local_only_name?(name)
       modified_key = "#{@scope}/targets_modified/#{name}"
       resp = @bucket.get_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: modified_key)
       return unless resp && resp.body
@@ -791,6 +841,7 @@ module OpenC3
     # a modified copy exists and differs (by bytes) from the rendered plugin
     # content. Read-only.
     def collect_modified_diff(name, new_data, collector)
+      return if OpenC3::LocalMode.local_only_name?(name)
       resp = @bucket.get_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: "#{@scope}/targets_modified/#{name}")
       return unless resp && resp.body
       collector << name if resp.body.read.b != new_data.b
@@ -843,6 +894,8 @@ module OpenC3
       db_shard_store = Store.instance(db_shard: @db_shard)
       db_shard_store.del("#{@scope}__openc3tlm__#{@name}")
       db_shard_store.del("#{@scope}__openc3cmd__#{@name}")
+      db_shard_store.del("#{@scope}__openc3tlm__#{@name}__hidden")
+      db_shard_store.del("#{@scope}__openc3cmd__#{@name}__hidden")
       db_shard_store.del("#{@scope}__TELEMETRYCNTS__{#{@name}}")
       db_shard_store.del("#{@scope}__COMMANDCNTS__{#{@name}}")
 
@@ -980,6 +1033,7 @@ module OpenC3
           end
           CvtModel.set(json_hash, target_name: packet.target_name, packet_name: packet.packet_name, scope: @scope)
         end
+        update_store_hidden_packet_names(db_shard_store, target_name, packets, type: :TLM, clear_old: clear_old)
       end
     end
 
@@ -999,7 +1053,27 @@ module OpenC3
             raise e
           end
         end
+        update_store_hidden_packet_names(db_shard_store, target_name, packets, type: :CMD, clear_old: clear_old)
+      end
+    end
+
+    # Store the names of the hidden packets so listing packet names doesn't require
+    # parsing every packet definition. When only some packets are updated
+    # (clear_old false) merge them into the existing list.
+    def update_store_hidden_packet_names(db_shard_store, target_name, packets, type:, clear_old:)
+      # e.g. DEFAULT__openc3tlm__INST__hidden
+      key = "#{@scope}__openc3#{type.to_s.downcase}__#{target_name}__hidden"
+      names = []
+      unless clear_old
+        names_json = db_shard_store.get(key)
+        # Without an existing list hidden_packet_names builds it from all the packets
+        return unless names_json
+        names = JSON.parse(names_json) - packets.keys
+      end
+      packets.each do |packet_name, packet|
+        names << packet_name if packet.hidden
       end
+      db_shard_store.set(key, JSON.generate(names.sort))
     end
 
     def update_store_limits_groups(system)

data/lib/openc3/operators/microservice_operator.rb

@@ -72,6 +72,11 @@ module OpenC3
         env['PYTHONPATH'] = nil
       end
       env['OPENC3_MICROSERVICE_NAME'] = microservice_name
+      # Tell the microservice not to report its own cpu/memory - we do that for
+      # it in publish_process_metrics, which also covers microservices that
+      # exec() into something that never loads the OpenC3 libraries. Only claim
+      # this where we can actually deliver on it.
+      env[Metric::SUPERVISED_ENV_VAR] = '1' if ProcessStats.supported?
       container = microservice_config["container"]
       scope = microservice_name.split("__")[0]
 

data/lib/openc3/operators/operator.rb

@@ -19,6 +19,8 @@ require 'childprocess'
 require 'openc3'
 require 'fileutils'
 require 'tempfile'
+require 'openc3/models/metric_model'
+require 'openc3/utilities/process_stats'
 
 module OpenC3
   # Class to prevent an infinitely growing log file
@@ -156,6 +158,15 @@ module OpenC3
       end
     end
 
+    # The pid of the spawned process, or nil if it isn't running. Note this is
+    # the pid of plugin_microservice.rb, which exec()s the configured cmd, so it
+    # stays the pid of whatever the microservice actually ends up running.
+    def pid
+      @process ? @process.pid : nil
+    rescue Exception
+      nil
+    end
+
     def soft_stop
       Thread.new do
         Logger.info("Soft shutting down process: #{cmd_line()}", scope: @scope)
@@ -255,6 +266,9 @@ module OpenC3
       @new_processes = {}
       @changed_processes = {}
       @removed_processes = {}
+      # microservice name => ProcessStats, so cpu deltas survive across cycles
+      @process_stats = {}
+      @process_stats_db_shards = {}
       @mutex = Mutex.new
       @shutdown = false
       @shutdown_complete = false
@@ -264,6 +278,60 @@ module OpenC3
       raise "Implement in subclass"
     end
 
+    # Report cpu and memory for every microservice this operator is running.
+    #
+    # Microservices that are themselves OpenC3::Microservices could report this
+    # about themselves, but ones that exec() into a Rails app, a Python script
+    # or any other binary never load the OpenC3 libraries and so can't. Doing it
+    # here covers every microservice the same way, including plugins we don't
+    # ship, without asking anything of the plugin author.
+    def publish_process_metrics
+      return unless ProcessStats.supported?
+
+      @mutex.synchronize do
+        @processes.each do |name, p|
+          break if @shutdown
+          # Not started yet - queued behind the per cycle start limit
+          next if @new_processes[name]
+
+          # Only processes this operator spawned locally have a pid we can read
+          # from /proc. The KubernetesOperator's processes are pods elsewhere in
+          # the cluster, so they don't implement pid and keep reporting their
+          # own metrics from inside the pod.
+          next unless p.respond_to?(:pid)
+          pid = p.pid
+          next unless pid
+
+          # Isolate each process so one failure doesn't starve the rest of metrics
+          begin
+            stats = @process_stats[name]
+            # A respawn gets a new pid, and the old cpu totals don't carry over
+            if stats.nil? or stats.pid != pid
+              # Look up the shard before caching stats so a failed lookup is retried next cycle
+              @process_stats_db_shards[name] = MetricModel._lookup_db_shard(name, scope: p.scope)
+              stats = ProcessStats.new(pid)
+              @process_stats[name] = stats
+            end
+            next unless stats.sample
+
+            MetricModel.set_process(
+              { 'name' => name, 'db_shard' => @process_stats_db_shards[name], 'values' => stats.values },
+              scope: p.scope
+            )
+          rescue => e
+            Logger.error("#{self.class} failed to publish process metrics for #{name}: #{e.class} #{e.message}")
+          end
+        end
+
+        # Stop tracking microservices that have gone away. Their metrics rows
+        # expire on their own (METRIC_EXPIRE_SECONDS).
+        (@process_stats.keys - @processes.keys).each do |name|
+          @process_stats.delete(name)
+          @process_stats_db_shards.delete(name)
+        end
+      end
+    end
+
     def start_new
       @mutex.synchronize do
         if @new_processes.length > 0
@@ -393,6 +461,7 @@ module OpenC3
           respawn_changed()
           start_new()
           respawn_dead()
+          publish_process_metrics()
         rescue => e
           Logger.error("#{self.class} cycle error, continuing: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
         end

data/lib/openc3/packets/packet.rb

@@ -879,7 +879,7 @@ module OpenC3
       result = nil
       synchronize_allow_reads(true) do
         result = read_all(value_type, buffer, false).map! do |array|
-          array << @items[array[0]].limits.state
+          array << @items[array[0]].limits_state
         end
       end
       return result
@@ -996,7 +996,7 @@ module OpenC3
     # This is an optimization so we don't have to iterate through all the items when
     # checking for limits.
     def update_limits_items_cache(item)
-      if item.limits.values || item.state_colors
+      if item.limits_values || item.state_colors
         @limits_items ||= []
         @limits_items_hash ||= {}
         unless @limits_items_hash[item]
@@ -1306,7 +1306,7 @@ module OpenC3
             json_hash["#{item.name}__F"] = read_item(item, :FORMATTED, @buffer, given_raw)
           end
           if include_limits_states
-            limits_state = item.limits.state
+            limits_state = item.limits_state
           end
           if limits_state
             json_hash["#{item.name}__L"] = limits_state
@@ -1339,6 +1339,12 @@ module OpenC3
       return unless @buffer
       return unless @obfuscated_items
 
+      # Given values hold the plaintext the user sent, so drop them for obfuscated items
+      if @given_values
+        obfuscated_names = @obfuscated_items.map { |item| item.name.upcase }
+        @given_values.delete_if { |key, _| obfuscated_names.include?(key.to_s.upcase) }
+      end
+
       @obfuscated_items.each do |item|
         next if item.data_type == :DERIVED
 
@@ -1441,7 +1447,7 @@ module OpenC3
 
     def handle_limits_states(item, value)
       # Retrieve limits state for the given value
-      limits_state = item.state_colors[value]
+      limits_state = item.state_colors ? item.state_colors[value] : nil
 
       if item.limits.state != limits_state # PacketItemLimits state has changed
         # Save old limits state

data/lib/openc3/packets/packet_config.rb

@@ -99,7 +99,11 @@ module OpenC3
     # Note: DERIVED is not a valid converted type. Also TIME is currently only a converted type
     CONVERTED_DATA_TYPES = [:INT, :UINT, :FLOAT, :STRING, :BLOCK, :BOOL, :OBJECT, :ARRAY, :ANY, :TIME]
 
-    def initialize
+    # @param descriptions [Boolean] Whether to keep packet and item descriptions.
+    #   Pass false in processes that never display them (e.g. decom and
+    #   interface microservices) to reduce memory on large definitions.
+    def initialize(descriptions: true)
+      @descriptions = descriptions
       @name = nil
       @telemetry = {}
       @commands = {}
@@ -145,7 +149,12 @@ module OpenC3
       # Handle .xtce files
       extension = File.extname(filename).to_s.downcase
       if extension == ".xtce" or extension == ".xml"
-        XtceParser.process(@commands, @telemetry, @warnings, filename, process_target_name)
+        target_name = XtceParser.process(@commands, @telemetry, @warnings, filename, process_target_name)
+        # XtceParser adds packets straight into @commands / @telemetry without
+        # going through finish_packet, so the ID lookup metadata finish_packet
+        # builds has to be built here instead. Without it Telemetry#identify
+        # and Commands#identify find no hash for the target and raise on nil.
+        build_xtce_id_metadata(target_name)
         return
       end
 
@@ -340,6 +349,12 @@ module OpenC3
     def finish_packet
       finish_item()
       if @current_packet
+        unless @descriptions
+          @current_packet.description = nil
+          # Reserved items (PACKET_TIMESECONDS, etc.) get descriptions from
+          # define_reserved_items and never pass through finish_item
+          @current_packet.sorted_items.each { |item| item.description = nil }
+        end
         @warnings += @current_packet.check_bit_offsets
         if !@current_packet.virtual && !@current_packet.disabled && !@current_packet.catchall && @current_packet.target_name != 'UNKNOWN' && @current_packet.id_items.empty?
           type = @current_cmd_or_tlm == COMMAND ? "Command" : "Telemetry"
@@ -430,6 +445,36 @@ module OpenC3
 
     protected
 
+    # Build the ID lookup metadata for every packet an XTCE file just added to
+    # the given target. Packets are walked in the order XtceParser left them in,
+    # which is the order the rest of COSMOS sees them, so duplicate ID values
+    # resolve the same way here as they do everywhere else.
+    #
+    # @param target_name [String] Target the XTCE file defined, or nil if the
+    #   file defined no target
+    def build_xtce_id_metadata(target_name)
+      return unless target_name
+
+      @commands[target_name]&.each_value do |packet|
+        next if packet.virtual
+
+        if packet.subpacket
+          build_id_metadata(packet, @cmd_subpacket_id_value_hash, @cmd_subpacket_id_signature, @cmd_subpacket_unique_id_mode)
+        else
+          build_id_metadata(packet, @cmd_id_value_hash, @cmd_id_signature, @cmd_unique_id_mode)
+        end
+      end
+      @telemetry[target_name]&.each_value do |packet|
+        next if packet.virtual
+
+        if packet.subpacket
+          build_id_metadata(packet, @tlm_subpacket_id_value_hash, @tlm_subpacket_id_signature, @tlm_subpacket_unique_id_mode)
+        else
+          build_id_metadata(packet, @tlm_id_value_hash, @tlm_id_signature, @tlm_unique_id_mode)
+        end
+      end
+    end
+
     def build_id_metadata(packet, id_value_hash, id_signature_hash, unique_id_mode_hash)
       target_id_value_hash = id_value_hash[packet.target_name]
       target_id_value_hash = {} unless target_id_value_hash
@@ -891,6 +936,7 @@ module OpenC3
     # Finish updating packet item
     def finish_item
       if @current_item
+        @current_item.description = nil unless @descriptions
         @current_packet.set_item(@current_item)
         if @current_cmd_or_tlm == TELEMETRY
           target_latest_data = @latest_data[@current_packet.target_name]