openc3 7.4.1 → 7.5.1

data/lib/openc3/models/microservice_model.rb

@@ -308,9 +308,13 @@ module OpenC3
     end
 
     def cleanup
-      # Cleanup metrics
-      metric_model = MetricModel.new(name: @name, scope: @scope)
-      metric_model.destroy
+      # Cleanup metrics - both the microservice's own and the process metrics
+      # the operator reported for it
+      # The MicroserviceModel has already been deleted during uninstall, so
+      # shard lookup by name can no longer find its configured database.
+      [MetricModel::PRIMARY_KEY, MetricModel::PROCESS_PRIMARY_KEY].each do |key|
+        MetricModel.new(name: @name, scope: @scope, db_shard: @db_shard, primary_key: key).destroy
+      end
     end
   end
 end

data/lib/openc3/models/plugin_model.rb

@@ -278,7 +278,10 @@ module OpenC3
 
         if File.exist?(pyproject_path) || File.exist?(requirements_path)
           pypi_url = resolve_pypi_url(scope: scope)
-          unless validate_only
+          # Skipped for any dry run: diff_only runs inline in an HTTP request, so
+          # a slow or unreachable PyPI would time it out, and it must not mutate
+          # the plugin venv.
+          unless dry_run
             pypi_args = build_pypi_args(pypi_url)
 
             # Install Python dependencies into an isolated per-plugin venv when UV
@@ -428,11 +431,9 @@ module OpenC3
     # Dry run: which modified files would this plugin's install supersede?
     # Returns a list of "TARGET/path" names whose live (modified) content
     # differs from the rendered plugin content. Read-only; no side effects.
+    # Errors propagate so callers can tell a failed check from an empty diff.
     def self.modified_diff(plugin_hash, scope:)
       install_phase2(plugin_hash, scope: scope, diff_only: true)
-    rescue => e
-      Logger.warn("PluginModel.modified_diff failed: #{e.message}")
-      []
     end
 
     def initialize(
@@ -597,25 +598,24 @@ module OpenC3
     def self.resolve_pypi_url(scope:)
       pypi_url = nil
       begin
-        pypi_url = get_setting('pypi_url', scope: scope)
-        pypi_url += '/simple' if pypi_url
+        pypi_url = PypiUrl.index_url(get_setting('pypi_url', scope: scope))
       rescue => e
         Logger.error("Failed to retrieve pypi_url: #{e.formatted}")
       ensure
         if pypi_url.nil?
-          pypi_url = ENV.fetch('PYPI_URL', nil)
-          pypi_url += '/simple' if pypi_url
+          pypi_url = PypiUrl.index_url(ENV.fetch('PYPI_URL', nil))
           pypi_url ||= PypiUrl::DEFAULT
         end
       end
       PypiUrl.validate(pypi_url)
     end
 
-    # Build the argv array for pypi index and trusted-host arguments.
+    # Build the argv array of pypi index arguments. Kept as a delegate so
+    # callers that already reference PluginModel keep working; the logic lives
+    # in PypiUrl because PythonPackageModel needs it too and must not have to
+    # require this file.
     def self.build_pypi_args(pypi_url)
-      args = ["-i", pypi_url]
-      args += ["--trusted-host", URI.parse(pypi_url).host] unless ENV['PIP_ENABLE_TRUSTED_HOST'].nil?
-      args
+      PypiUrl.build_args(pypi_url)
     end
 
     # Build a sanitized venv directory name from scope and plugin name.

data/lib/openc3/models/python_package_model.rb

@@ -235,29 +235,19 @@ module OpenC3
       end
       package_filename = File.basename(package_file_path)
       begin
-        pypi_url = get_setting('pypi_url', scope: scope)
-        if pypi_url
-          pypi_url += '/simple'
-        end
+        pypi_url = PypiUrl.index_url(get_setting('pypi_url', scope: scope))
       rescue => e
         Logger.error("Failed to retrieve pypi_url: #{e.formatted}")
       ensure
         if pypi_url.nil?
           # If Redis isn't running try the ENV, then simply pypi.org/simple
-          pypi_url = ENV.fetch('PYPI_URL', nil)
-          if pypi_url
-            pypi_url += '/simple'
-          end
+          pypi_url = PypiUrl.index_url(ENV.fetch('PYPI_URL', nil))
           pypi_url ||= PypiUrl::DEFAULT
         end
       end
       pypi_url = PypiUrl.validate(pypi_url)
       Logger.info "Installing python package: #{name_or_path}"
-      if ENV['PIP_ENABLE_TRUSTED_HOST'].nil?
-        pip_args = ["-i", pypi_url, package_file_path]
-      else
-        pip_args = ["-i", pypi_url, "--trusted-host", URI.parse(pypi_url).host, package_file_path]
-      end
+      pip_args = PypiUrl.build_args(pypi_url) + [package_file_path]
       spawn_env = {}
       if plugin
         venv_path = "#{PLUGIN_VENVS_DIR}/#{plugin}/.venv"

data/lib/openc3/models/target_model.rb

@@ -109,10 +109,14 @@ module OpenC3
       modified_targets = []
       if ENV['OPENC3_LOCAL_MODE']
         modified_targets += OpenC3::LocalMode.modified_targets(scope: scope)
+      elsif !OpenC3::LocalMode.local_only_targets.empty?
+        modified_targets += OpenC3::LocalMode.modified_targets(scope: scope).select { |name| OpenC3::LocalMode.local_only_target?(name) }
       end
       # Always list the bucket, even in local mode, since the local directory
-      # only holds what COSMOS itself wrote there
-      modified_targets += Bucket.getClient().list_files(bucket: ENV['OPENC3_CONFIG_BUCKET'], path: "#{scope}/targets_modified/", only_directories: true)
+      # only holds what COSMOS itself wrote there. Local only targets are
+      # never read from the bucket.
+      bucket_targets = Bucket.getClient().list_files(bucket: ENV['OPENC3_CONFIG_BUCKET'], path: "#{scope}/targets_modified/", only_directories: true)
+      modified_targets += bucket_targets.reject { |name| OpenC3::LocalMode.local_only_target?(name) }
       modified_targets.each do |target_name|
         # A target could have been deleted without removing the modified files
         # Thus we have to check for the existence of the target_name key
@@ -186,13 +190,15 @@ module OpenC3
     def self.modified_files(target_name, scope:)
       modified = []
 
-      if ENV['OPENC3_LOCAL_MODE']
+      if ENV['OPENC3_LOCAL_MODE'] or OpenC3::LocalMode.local_only_target?(target_name)
         # LocalMode reports target relative paths, i.e. "procedures/new.rb",
         # so add the target name to match the bucket listing below
         modified += OpenC3::LocalMode.modified_files(target_name, scope: scope).map do |file_path|
           "#{target_name}/#{file_path}"
         end
       end
+      # Local only targets are never read from the bucket
+      return modified.uniq.sort if OpenC3::LocalMode.local_only_target?(target_name)
       # Always list the bucket, even in local mode. The local directory only
       # holds what COSMOS itself wrote there, so a file placed directly in the
       # bucket is modified but has no local copy.
@@ -216,16 +222,28 @@ module OpenC3
     # some modified files while keeping others); when nil, every modified file
     # for the target is removed (the original behavior).
     def self.delete_modified(target_name, scope:, files: nil)
+      # Validate target_name to not allow directory traversal
+      unless OpenC3::LocalMode.safe_key?(target_name) and !target_name.include?('/')
+        raise ArgumentError, "Invalid target_name: #{target_name.inspect}"
+      end
       if files && !files.empty?
+        # Only delete names that are actually modified files of this target.
+        # Anything else (another target, another scope via '..') is ignored.
+        allowed = modified_files(target_name, scope: scope)
         files.each do |name|
+          next unless String === name
+          name = OpenC3::TargetFile.strip_modified(name)
+          next unless allowed.include?(name)
           # TargetFile.destroy handles both local mode and the bucket.
           OpenC3::TargetFile.destroy(scope, name)
         end
         return
       end
-      if ENV['OPENC3_LOCAL_MODE']
+      if ENV['OPENC3_LOCAL_MODE'] or OpenC3::LocalMode.local_only_target?(target_name)
         OpenC3::LocalMode.delete_modified(target_name, scope: scope)
       end
+      # Local only targets are never written to the bucket
+      return if OpenC3::LocalMode.local_only_target?(target_name)
       bucket = Bucket.getClient()
       # Delete the remote files as well
       resp = bucket.list_objects(
@@ -249,7 +267,7 @@ module OpenC3
         Zip.continue_on_exists_proc = true
         zip = Zip::File.open(zip_filename, create: true)
 
-        if ENV['OPENC3_LOCAL_MODE']
+        if ENV['OPENC3_LOCAL_MODE'] or OpenC3::LocalMode.local_only_target?(target_name)
           OpenC3::LocalMode.zip_target(target_name, zip, scope: scope)
         else
           bucket = Bucket.getClient()
@@ -332,6 +350,35 @@ module OpenC3
       result
     end
 
+    # Returns just the packet names without fetching and parsing every packet
+    # definition, which is expensive for targets with thousands of packets.
+    # @return [Array<String>] Sorted packet names under the target_name
+    def self.packet_names(target_name, type: :TLM, scope:)
+      raise "Unknown type #{type} for #{target_name}" unless VALID_TYPES.include?(type)
+      raise "Target '#{target_name}' does not exist for scope: #{scope} (TargetModel)" unless get(name: target_name, scope: scope)
+
+      store_for_target(target_name, scope: scope).hkeys("#{scope}__openc3#{type.to_s.downcase}__#{target_name}").sort
+    end
+
+    # Returns the names of the hidden packets without parsing every packet definition.
+    # The list is stored when the target is installed. Targets installed before the
+    # list existed have it built once from the packet definitions and stored.
+    # @return [Array<String>] Sorted hidden packet names under the target_name
+    def self.hidden_packet_names(target_name, type: :TLM, scope:)
+      raise "Unknown type #{type} for #{target_name}" unless VALID_TYPES.include?(type)
+
+      store = store_for_target(target_name, scope: scope)
+      # Stored as a JSON string rather than a set so an empty list is distinct from a missing one
+      # e.g. DEFAULT__openc3tlm__INST__hidden
+      key = "#{scope}__openc3#{type.to_s.downcase}__#{target_name}__hidden"
+      names_json = store.get(key)
+      return JSON.parse(names_json) if names_json
+
+      names = packets(target_name, type: type, scope: scope).select { |packet| packet['hidden'] }.map { |packet| packet['packet_name'] }
+      store.set(key, JSON.generate(names))
+      names
+    end
+
     # @return [Array<Hash>] All packet hashes under the target_name
     def self.all_packet_name_descriptions(target_name, type: :TLM, scope:)
       self.packets(target_name, type: type, scope: scope).map! { |hash| hash.slice("packet_name", "description") }
@@ -411,10 +458,13 @@ module OpenC3
     end
 
     def self.get_item_to_packet_map(target_name, scope:)
-      cache_time, item_map = @@item_map_cache[target_name]
-      return item_map if item_map and (Time.now - cache_time) < ITEM_MAP_CACHE_TIMEOUT
-      item_map_key = "#{scope}__#{target_name}__item_to_packet_map"
+      # Normalize before building the key so lowercase lookups share the entry
+      # that update_store_item_map and undeploy refresh and invalidate
       target_name = target_name.upcase
+      # The cache is keyed by item_map_key since the same target can exist in multiple scopes
+      item_map_key = "#{scope}__#{target_name}__item_to_packet_map"
+      cache_time, item_map = @@item_map_cache[item_map_key]
+      return item_map if item_map and (Time.now - cache_time) < ITEM_MAP_CACHE_TIMEOUT
       store = store_for_target(target_name, scope: scope)
       json_data = store.get(item_map_key)
       if json_data
@@ -423,7 +473,7 @@ module OpenC3
         item_map = build_item_to_packet_map(target_name, scope: scope)
         store.set(item_map_key, JSON.generate(item_map, allow_nan: true))
       end
-      @@item_map_cache[target_name] = [Time.now, item_map]
+      @@item_map_cache[item_map_key] = [Time.now, item_map]
       return item_map
     end
 
@@ -751,11 +801,12 @@ module OpenC3
     # Version History is an Enterprise feature. The plugin deploy runs in the
     # cmd-tlm-api process, which (unlike script-runner-api) doesn't otherwise
     # require the store, so lazily load it on demand. Returns false in Core
-    # builds where the Enterprise gem isn't present.
+    # builds where the Enterprise gem isn't present, and when the store is
+    # disabled (OPENC3_VERSION_HISTORY_DIR unset): its commits no-op there, so
+    # versioning would drop the modified copy without preserving it.
     def version_store_available?
-      return true if defined?(::VersionStore)
-      require 'openc3-enterprise/utilities/version_store'
-      defined?(::VersionStore) ? true : false
+      require 'openc3-enterprise/utilities/version_store' unless defined?(::VersionStore)
+      defined?(::VersionStore) && ::VersionStore.enabled? ? true : false
     rescue LoadError
       false
     end
@@ -765,6 +816,8 @@ module OpenC3
     # copy), commits it, removes it, then commits the incoming plugin content.
     # Best-effort: a versioning failure must not abort the plugin upgrade.
     def apply_upgrade_version(name, new_data, ctx)
+      # Local only targets have no modified copy in the bucket
+      return if OpenC3::LocalMode.local_only_name?(name)
       modified_key = "#{@scope}/targets_modified/#{name}"
       resp = @bucket.get_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: modified_key)
       return unless resp && resp.body
@@ -791,6 +844,7 @@ module OpenC3
     # a modified copy exists and differs (by bytes) from the rendered plugin
     # content. Read-only.
     def collect_modified_diff(name, new_data, collector)
+      return if OpenC3::LocalMode.local_only_name?(name)
       resp = @bucket.get_object(bucket: ENV['OPENC3_CONFIG_BUCKET'], key: "#{@scope}/targets_modified/#{name}")
       return unless resp && resp.body
       collector << name if resp.body.read.b != new_data.b
@@ -843,6 +897,8 @@ module OpenC3
       db_shard_store = Store.instance(db_shard: @db_shard)
       db_shard_store.del("#{@scope}__openc3tlm__#{@name}")
       db_shard_store.del("#{@scope}__openc3cmd__#{@name}")
+      db_shard_store.del("#{@scope}__openc3tlm__#{@name}__hidden")
+      db_shard_store.del("#{@scope}__openc3cmd__#{@name}__hidden")
       db_shard_store.del("#{@scope}__TELEMETRYCNTS__{#{@name}}")
       db_shard_store.del("#{@scope}__COMMANDCNTS__{#{@name}}")
 
@@ -864,7 +920,7 @@ module OpenC3
       # Delete item_map
       item_map_key = "#{@scope}__#{@name}__item_to_packet_map"
       db_shard_store.del(item_map_key)
-      @@item_map_cache[@name] = nil
+      @@item_map_cache[item_map_key] = nil
 
       topic = { kind: 'deleted', type: 'target', name: @name }
       # The UNKNOWN target doesn't have an associated plugin
@@ -980,6 +1036,7 @@ module OpenC3
           end
           CvtModel.set(json_hash, target_name: packet.target_name, packet_name: packet.packet_name, scope: @scope)
         end
+        update_store_hidden_packet_names(db_shard_store, target_name, packets, type: :TLM, clear_old: clear_old)
       end
     end
 
@@ -999,7 +1056,27 @@ module OpenC3
             raise e
           end
         end
+        update_store_hidden_packet_names(db_shard_store, target_name, packets, type: :CMD, clear_old: clear_old)
+      end
+    end
+
+    # Store the names of the hidden packets so listing packet names doesn't require
+    # parsing every packet definition. When only some packets are updated
+    # (clear_old false) merge them into the existing list.
+    def update_store_hidden_packet_names(db_shard_store, target_name, packets, type:, clear_old:)
+      # e.g. DEFAULT__openc3tlm__INST__hidden
+      key = "#{@scope}__openc3#{type.to_s.downcase}__#{target_name}__hidden"
+      names = []
+      unless clear_old
+        names_json = db_shard_store.get(key)
+        # Without an existing list hidden_packet_names builds it from all the packets
+        return unless names_json
+        names = JSON.parse(names_json) - packets.keys
+      end
+      packets.each do |packet_name, packet|
+        names << packet_name if packet.hidden
       end
+      db_shard_store.set(key, JSON.generate(names.sort))
     end
 
     def update_store_limits_groups(system)
@@ -1027,7 +1104,7 @@ module OpenC3
       item_map_key = "#{@scope}__#{@name}__item_to_packet_map"
       item_map = self.class.build_item_to_packet_map(@name, scope: @scope)
       Store.instance(db_shard: @db_shard).set(item_map_key, JSON.generate(item_map, allow_nan: true))
-      @@item_map_cache[@name] = [Time.now, item_map]
+      @@item_map_cache[item_map_key] = [Time.now, item_map]
     end
 
     def update_store(system, clear_old: true)

data/lib/openc3/operators/microservice_operator.rb

@@ -72,6 +72,11 @@ module OpenC3
         env['PYTHONPATH'] = nil
       end
       env['OPENC3_MICROSERVICE_NAME'] = microservice_name
+      # Tell the microservice not to report its own cpu/memory - we do that for
+      # it in publish_process_metrics, which also covers microservices that
+      # exec() into something that never loads the OpenC3 libraries. Only claim
+      # this where we can actually deliver on it.
+      env[Metric::SUPERVISED_ENV_VAR] = '1' if ProcessStats.supported?
       container = microservice_config["container"]
       scope = microservice_name.split("__")[0]
 

data/lib/openc3/operators/operator.rb

@@ -19,6 +19,8 @@ require 'childprocess'
 require 'openc3'
 require 'fileutils'
 require 'tempfile'
+require 'openc3/models/metric_model'
+require 'openc3/utilities/process_stats'
 
 module OpenC3
   # Class to prevent an infinitely growing log file
@@ -156,6 +158,15 @@ module OpenC3
       end
     end
 
+    # The pid of the spawned process, or nil if it isn't running. Note this is
+    # the pid of plugin_microservice.rb, which exec()s the configured cmd, so it
+    # stays the pid of whatever the microservice actually ends up running.
+    def pid
+      @process ? @process.pid : nil
+    rescue Exception
+      nil
+    end
+
     def soft_stop
       Thread.new do
         Logger.info("Soft shutting down process: #{cmd_line()}", scope: @scope)
@@ -255,6 +266,9 @@ module OpenC3
       @new_processes = {}
       @changed_processes = {}
       @removed_processes = {}
+      # microservice name => ProcessStats, so cpu deltas survive across cycles
+      @process_stats = {}
+      @process_stats_db_shards = {}
       @mutex = Mutex.new
       @shutdown = false
       @shutdown_complete = false
@@ -264,6 +278,60 @@ module OpenC3
       raise "Implement in subclass"
     end
 
+    # Report cpu and memory for every microservice this operator is running.
+    #
+    # Microservices that are themselves OpenC3::Microservices could report this
+    # about themselves, but ones that exec() into a Rails app, a Python script
+    # or any other binary never load the OpenC3 libraries and so can't. Doing it
+    # here covers every microservice the same way, including plugins we don't
+    # ship, without asking anything of the plugin author.
+    def publish_process_metrics
+      return unless ProcessStats.supported?
+
+      @mutex.synchronize do
+        @processes.each do |name, p|
+          break if @shutdown
+          # Not started yet - queued behind the per cycle start limit
+          next if @new_processes[name]
+
+          # Only processes this operator spawned locally have a pid we can read
+          # from /proc. The KubernetesOperator's processes are pods elsewhere in
+          # the cluster, so they don't implement pid and keep reporting their
+          # own metrics from inside the pod.
+          next unless p.respond_to?(:pid)
+          pid = p.pid
+          next unless pid
+
+          # Isolate each process so one failure doesn't starve the rest of metrics
+          begin
+            stats = @process_stats[name]
+            # A respawn gets a new pid, and the old cpu totals don't carry over
+            if stats.nil? or stats.pid != pid
+              # Look up the shard before caching stats so a failed lookup is retried next cycle
+              @process_stats_db_shards[name] = MetricModel._lookup_db_shard(name, scope: p.scope)
+              stats = ProcessStats.new(pid)
+              @process_stats[name] = stats
+            end
+            next unless stats.sample
+
+            MetricModel.set_process(
+              { 'name' => name, 'db_shard' => @process_stats_db_shards[name], 'values' => stats.values },
+              scope: p.scope
+            )
+          rescue => e
+            Logger.error("#{self.class} failed to publish process metrics for #{name}: #{e.class} #{e.message}")
+          end
+        end
+
+        # Stop tracking microservices that have gone away. Their metrics rows
+        # expire on their own (METRIC_EXPIRE_SECONDS).
+        (@process_stats.keys - @processes.keys).each do |name|
+          @process_stats.delete(name)
+          @process_stats_db_shards.delete(name)
+        end
+      end
+    end
+
     def start_new
       @mutex.synchronize do
         if @new_processes.length > 0
@@ -393,6 +461,7 @@ module OpenC3
           respawn_changed()
           start_new()
           respawn_dead()
+          publish_process_metrics()
         rescue => e
           Logger.error("#{self.class} cycle error, continuing: #{e.class} #{e.message}\n#{e.backtrace.join("\n")}")
         end

data/lib/openc3/packets/packet.rb

@@ -879,7 +879,7 @@ module OpenC3
       result = nil
       synchronize_allow_reads(true) do
         result = read_all(value_type, buffer, false).map! do |array|
-          array << @items[array[0]].limits.state
+          array << @items[array[0]].limits_state
         end
       end
       return result
@@ -996,7 +996,7 @@ module OpenC3
     # This is an optimization so we don't have to iterate through all the items when
     # checking for limits.
     def update_limits_items_cache(item)
-      if item.limits.values || item.state_colors
+      if item.limits_values || item.state_colors
         @limits_items ||= []
         @limits_items_hash ||= {}
         unless @limits_items_hash[item]
@@ -1306,7 +1306,7 @@ module OpenC3
             json_hash["#{item.name}__F"] = read_item(item, :FORMATTED, @buffer, given_raw)
           end
           if include_limits_states
-            limits_state = item.limits.state
+            limits_state = item.limits_state
           end
           if limits_state
             json_hash["#{item.name}__L"] = limits_state
@@ -1339,6 +1339,12 @@ module OpenC3
       return unless @buffer
       return unless @obfuscated_items
 
+      # Given values hold the plaintext the user sent, so drop them for obfuscated items
+      if @given_values
+        obfuscated_names = @obfuscated_items.map { |item| item.name.upcase }
+        @given_values.delete_if { |key, _| obfuscated_names.include?(key.to_s.upcase) }
+      end
+
       @obfuscated_items.each do |item|
         next if item.data_type == :DERIVED
 
@@ -1441,7 +1447,7 @@ module OpenC3
 
     def handle_limits_states(item, value)
       # Retrieve limits state for the given value
-      limits_state = item.state_colors[value]
+      limits_state = item.state_colors ? item.state_colors[value] : nil
 
       if item.limits.state != limits_state # PacketItemLimits state has changed
         # Save old limits state

data/lib/openc3/packets/packet_config.rb

@@ -99,7 +99,11 @@ module OpenC3
     # Note: DERIVED is not a valid converted type. Also TIME is currently only a converted type
     CONVERTED_DATA_TYPES = [:INT, :UINT, :FLOAT, :STRING, :BLOCK, :BOOL, :OBJECT, :ARRAY, :ANY, :TIME]
 
-    def initialize
+    # @param descriptions [Boolean] Whether to keep packet and item descriptions.
+    #   Pass false in processes that never display them (e.g. decom and
+    #   interface microservices) to reduce memory on large definitions.
+    def initialize(descriptions: true)
+      @descriptions = descriptions
       @name = nil
       @telemetry = {}
       @commands = {}
@@ -345,6 +349,12 @@ module OpenC3
     def finish_packet
       finish_item()
       if @current_packet
+        unless @descriptions
+          @current_packet.description = nil
+          # Reserved items (PACKET_TIMESECONDS, etc.) get descriptions from
+          # define_reserved_items and never pass through finish_item
+          @current_packet.sorted_items.each { |item| item.description = nil }
+        end
         @warnings += @current_packet.check_bit_offsets
         if !@current_packet.virtual && !@current_packet.disabled && !@current_packet.catchall && @current_packet.target_name != 'UNKNOWN' && @current_packet.id_items.empty?
           type = @current_cmd_or_tlm == COMMAND ? "Command" : "Telemetry"
@@ -926,6 +936,7 @@ module OpenC3
     # Finish updating packet item
     def finish_item
       if @current_item
+        @current_item.description = nil unless @descriptions
         @current_packet.set_item(@current_item)
         if @current_cmd_or_tlm == TELEMETRY
           target_latest_data = @latest_data[@current_packet.target_name]

data/lib/openc3/packets/packet_item.rb

@@ -85,8 +85,22 @@ module OpenC3
     # The allowable state colors
     STATE_COLORS = [:GREEN, :YELLOW, :RED]
 
+    # PacketItemLimits is created on first access since most items have no
+    # limits and large definitions have hundreds of thousands of items.
     # @return [PacketItemLimits] All information regarding limits for this PacketItem
-    attr_reader :limits
+    def limits
+      @limits ||= PacketItemLimits.new
+    end
+
+    # @return [Symbol, nil] The limits state without creating PacketItemLimits
+    def limits_state
+      @limits ? @limits.state : nil
+    end
+
+    # @return [Hash, nil] The limits values without creating PacketItemLimits
+    def limits_values
+      @limits ? @limits.values : nil
+    end
 
     # @return [Boolean] Whether the parameter must be obfuscated from logs or not
     attr_accessor :obfuscate
@@ -110,7 +124,7 @@ module OpenC3
       @obfuscate = false
       @messages_disabled = nil
       @state_colors = nil
-      @limits = PacketItemLimits.new
+      @limits = nil
       @meta = nil
     end
 
@@ -165,7 +179,6 @@ module OpenC3
         end
 
         @states = upcase_states
-        @state_colors ||= {}
       else
         @states = nil
       end
@@ -317,7 +330,7 @@ module OpenC3
       item.hazardous = self.hazardous.clone if self.hazardous
       item.messages_disabled = self.messages_disabled.clone if self.messages_disabled
       item.state_colors = self.state_colors.clone if self.state_colors
-      item.limits = self.limits.clone if self.limits
+      item.limits = @limits.clone if @limits
       item.meta = self.meta.clone if @meta
       item.obfuscate = self.obfuscate.clone if @obfuscate
       item
@@ -385,14 +398,19 @@ module OpenC3
       if @states
         @states.each do |state_name, state_value|
           config << "    STATE #{state_name.to_s.quote_if_necessary} #{state_value.to_s.quote_if_necessary}"
-          if @hazardous and @hazardous[state_name]
-            config << " HAZARDOUS #{@hazardous[state_name].to_s.quote_if_necessary}"
-          end
-          if @messages_disabled and @messages_disabled[state_name]
-            config << " DISABLE_MESSAGES"
-          end
-          if @state_colors and @state_colors[state_name]
-            config << " #{@state_colors[state_name]}"
+          # Only write the attributes STATE accepts for this packet type so the config parses back
+          if cmd_or_tlm == :TELEMETRY
+            if @state_colors and @state_colors[state_name]
+              config << " #{@state_colors[state_name]}"
+            end
+          else
+            if @hazardous and @hazardous[state_name]
+              config << " HAZARDOUS"
+              config << " #{@hazardous[state_name].to_s.quote_if_necessary}" unless @hazardous[state_name].to_s.empty?
+            end
+            if @messages_disabled and @messages_disabled[state_name]
+              config << " DISABLE_MESSAGES"
+            end
           end
           config << "\n"
         end
@@ -401,16 +419,16 @@ module OpenC3
       config << self.read_conversion.to_config(:READ) if self.read_conversion
       config << self.write_conversion.to_config(:WRITE) if self.write_conversion
 
-      if self.limits.values
-        self.limits.values.each do |limits_set, limits_values|
-          config << "    LIMITS #{limits_set} #{self.limits.persistence_setting} #{self.limits.enabled ? 'ENABLED' : 'DISABLED'} #{limits_values[0]} #{limits_values[1]} #{limits_values[2]} #{limits_values[3]}"
+      if limits_values
+        @limits.values.each do |limits_set, limits_values|
+          config << "    LIMITS #{limits_set} #{@limits.persistence_setting} #{@limits.enabled ? 'ENABLED' : 'DISABLED'} #{limits_values[0]} #{limits_values[1]} #{limits_values[2]} #{limits_values[3]}"
           if limits_values[4] && limits_values[5]
             config << " #{limits_values[4]} #{limits_values[5]}\n"
           else
             config << "\n"
           end
         end
-        config << self.limits.response.to_config if self.limits.response
+        config << @limits.response.to_config if @limits.response
       end
 
       if @meta
@@ -456,19 +474,19 @@ module OpenC3
       config['write_conversion'] = self.write_conversion.as_json(*a) if self.write_conversion
 
       config['limits'] ||= {}
-      if self.limits.enabled
+      if @limits && @limits.enabled
         config['limits']['enabled'] = true
-      elsif self.limits.values || (self.state_colors && self.state_colors.length > 0)
+      elsif limits_values || (self.state_colors && self.state_colors.length > 0)
         # Only set to false if there are limits or state colors
         # to avoid items without limits acting like they can be enabled
         config['limits']['enabled'] = false
       end
-      if self.limits.values
+      if limits_values
         # Only set these if there are limits.values because persistence_setting has a default
         # and we don't want keys on the 'limits' hash if there aren't any limits
-        config['limits']['persistence_setting'] = self.limits.persistence_setting if self.limits.persistence_setting
-        config['limits']['response'] = self.limits.response.to_s if self.limits.response
-        self.limits.values.each do |limits_set, limits_values|
+        config['limits']['persistence_setting'] = @limits.persistence_setting if @limits.persistence_setting
+        config['limits']['response'] = @limits.response.to_s if @limits.response
+        @limits.values.each do |limits_set, limits_values|
           limits = {}
           limits['red_low'] =  limits_values[0]
           limits['yellow_low'] = limits_values[1]