openc3 5.17.0 → 5.18.0

data/lib/openc3/models/scope_model.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2024, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -26,6 +26,18 @@ require 'openc3/models/plugin_model'
 require 'openc3/models/microservice_model'
 require 'openc3/models/setting_model'
 require 'openc3/models/trigger_group_model'
+require 'openc3/topics/system_events_topic'
+
+begin
+  require 'openc3-enterprise/models/cmd_authority_model'
+rescue LoadError
+  # Stub out the Enterprise CmdAuthorityModel to do nothing
+  class CmdAuthorityModel
+    def self.names(scope:)
+      []
+    end
+  end
+end
 
 module OpenC3
   class ScopeModel < Model
@@ -37,9 +49,13 @@ module OpenC3
     attr_accessor :text_log_retain_time
     attr_accessor :tool_log_retain_time
     attr_accessor :cleanup_poll_time
+    attr_accessor :command_authority
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
+    #
+    # The scope keyword is given to support the ModelController method signature
+    # even though it is not used
     def self.get(name:, scope: nil)
       super(PRIMARY_KEY, name: name)
     end
@@ -52,13 +68,13 @@ module OpenC3
       super(PRIMARY_KEY)
     end
 
-    def self.from_json(json, scope: nil)
+    def self.from_json(json)
       json = JSON.parse(json, :allow_nan => true, :create_additions => true) if String === json
       raise "json data is nil" if json.nil?
-      self.new(**json.transform_keys(&:to_sym), scope: scope)
+      self.new(**json.transform_keys(&:to_sym))
     end
 
-    def self.get_model(name:, scope: nil)
+    def self.get_model(name:)
       json = get(name: name)
       if json
         return from_json(json)
@@ -73,18 +89,15 @@ module OpenC3
       text_log_retain_time: nil,
       tool_log_retain_time: nil,
       cleanup_poll_time: 900,
-      updated_at: nil,
-      scope: nil
+      command_authority: false,
+      updated_at: nil
     )
       super(
         PRIMARY_KEY,
         name: name,
-        text_log_cycle_time: text_log_cycle_time,
-        text_log_cycle_size: text_log_cycle_size,
-        text_log_retain_time: text_log_retain_time,
-        tool_log_retain_time: tool_log_retain_time,
-        cleanup_poll_time: cleanup_poll_time,
         updated_at: updated_at,
+        # This sets the @scope variable which is sort of redundant for the ScopeModel
+        # (since its the same as @name) but every model has a @scope
         scope: name
       )
       @text_log_cycle_time = text_log_cycle_time
@@ -92,6 +105,7 @@ module OpenC3
       @text_log_retain_time = text_log_retain_time
       @tool_log_retain_time = tool_log_retain_time
       @cleanup_poll_time = cleanup_poll_time
+      @command_authority = command_authority
       @children = []
     end
 
@@ -99,14 +113,26 @@ module OpenC3
       # Ensure there are no "." in the scope name - prevents gems accidently becoming scope names
       raise "Invalid scope name: #{@name}" if @name !~ /^[a-zA-Z0-9_-]+$/
       @name = @name.upcase
+      @scope = @name # Ensure @scope matches @name
       super(update: update, force: force, queued: queued)
+
+      # If we're updating the scope and disabling command_authority
+      # then we clear out all the existing values so it comes up fresh
+      if update and @command_authority == false
+        CmdAuthorityModel.names(scope: @name).each do |auth_name|
+          model = CmdAuthorityModel.get_model(name: auth_name, scope: @name)
+          model.destroy if model
+        end
+      end
+
+      SystemEventsTopic.write(:scope, as_json())
     end
 
     def destroy
       if @name != 'DEFAULT'
         # Remove all the plugins for this scope
         plugins = PluginModel.get_all_models(scope: @name)
-        plugins.each do |plugin_name, plugin|
+        plugins.each do |_plugin_name, plugin|
           plugin.destroy
         end
         super()
@@ -115,7 +141,7 @@ module OpenC3
       end
     end
 
-    def as_json(*a)
+    def as_json(*_a)
       { 'name' => @name,
         'updated_at' => @updated_at,
         'text_log_cycle_time' => @text_log_cycle_time,
@@ -123,6 +149,7 @@ module OpenC3
         'text_log_retain_time' => @text_log_retain_time,
         'tool_log_retain_time' => @tool_log_retain_time,
         'cleanup_poll_time' => @cleanup_poll_time,
+        'command_authority' => @command_authority,
        }
     end
 
@@ -276,6 +303,10 @@ module OpenC3
     end
 
     def undeploy
+      # Delete UNKNOWN target
+      target = TargetModel.get_model(name: "UNKNOWN", scope: @scope)
+      target.destroy
+
       model = MicroserviceModel.get_model(name: "#{@scope}__SCOPEMULTI__#{@scope}", scope: @scope)
       model.destroy if model
       model = MicroserviceModel.get_model(name: "#{@scope}__SCOPECLEANUP__#{@scope}", scope: @scope)

data/lib/openc3/models/sorted_model.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2023, OpenC3, Inc.
+# All changes Copyright 2024, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license

data/lib/openc3/models/target_model.rb

@@ -660,7 +660,10 @@ module OpenC3
       Store.del(item_map_key)
       @@item_map_cache[@name] = nil
 
-      ConfigTopic.write({ kind: 'deleted', type: 'target', name: @name, plugin: @plugin }, scope: @scope)
+      topic = { kind: 'deleted', type: 'target', name: @name }
+      # The UNKNOWN target doesn't have an associated plugin
+      topic[:plugin] = @plugin if @plugin
+      ConfigTopic.write(topic, scope: @scope)
     rescue Exception => e
       Logger.error("Error undeploying target model #{@name} in scope #{@scope} due to #{e}")
     end

data/lib/openc3/operators/microservice_operator.rb

@@ -14,7 +14,7 @@
 # GNU Affero General Public License for more details.
 
 # Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
+# All changes Copyright 2024, OpenC3, Inc.
 # All Rights Reserved
 #
 # This file may also be used under the terms of a commercial license
@@ -186,7 +186,7 @@ module OpenC3
           end
         end
 
-        @removed_microservices.each do |microservice_name, microservice_config|
+        @removed_microservices.each do |microservice_name, _microservice_config|
           process = @processes[microservice_name]
           @processes.delete(microservice_name)
           @removed_processes[microservice_name] = process

data/lib/openc3/operators/operator.rb

@@ -127,7 +127,7 @@ module OpenC3
       # @process.io.inherit!
       @process.cwd = @work_dir
       # Spawned process should not be controlled by same Bundler constraints as spawning process
-      ENV.each do |key, value|
+      ENV.each do |key, _value|
         if key =~ /^BUNDLER/
           @process.environment[key] = nil
         end
@@ -265,7 +265,7 @@ module OpenC3
         if @new_processes.length > 0
           # Start all the processes
           Logger.info("#{self.class} starting each new process...")
-          @new_processes.each { |name, p| p.start }
+          @new_processes.each { |_name, p| p.start }
           @new_processes = {}
         end
       end
@@ -278,7 +278,7 @@ module OpenC3
           shutdown_processes(@changed_processes)
           break if @shutdown
 
-          @changed_processes.each { |name, p| p.start }
+          @changed_processes.each { |_name, p| p.start }
           @changed_processes = {}
         end
       end
@@ -296,7 +296,7 @@ module OpenC3
 
     def respawn_dead
       @mutex.synchronize do
-        @processes.each do |name, p|
+        @processes.each do |_name, p|
           break if @shutdown
           p.output_increment
           unless p.alive?
@@ -314,7 +314,7 @@ module OpenC3
       processes = processes.dup
 
       Logger.info("Commanding soft stops...")
-      processes.each { |name, p| p.soft_stop }
+      processes.each { |_name, p| p.soft_stop }
       start_time = Time.now
       # Allow sufficient time for processes to shutdown cleanly
       while (Time.now - start_time) < PROCESS_SHUTDOWN_SECONDS
@@ -322,21 +322,21 @@ module OpenC3
         processes.each do |name, p|
           unless p.alive?
             processes_to_remove << name
-            Logger.info("Soft stop process successful: #{p.cmd_line}", scope: p.scope)
+            Logger.debug("Soft stop process successful: #{p.cmd_line}", scope: p.scope)
           end
         end
         processes_to_remove.each do |name|
           processes.delete(name)
         end
         if processes.length <= 0
-          Logger.info("Soft stop all successful")
+          Logger.debug("Soft stop all successful")
           break
         end
         sleep(0.1)
       end
       if processes.length > 0
-        Logger.info("Commanding hard stops...")
-        processes.each { |name, p| p.hard_stop }
+        Logger.debug("Commanding hard stops...")
+        processes.each { |_name, p| p.hard_stop }
       end
     end
 

data/lib/openc3/packets/packet.rb

@@ -104,6 +104,9 @@ module OpenC3
     # @return [Boolean] Whether to ignore overlapping items
     attr_accessor :ignore_overlap
 
+    # @return [Boolean] If this packet should be used for identification
+    attr_reader :virtual
+
     # Valid format types
     VALUE_TYPES = [:RAW, :CONVERTED, :FORMATTED, :WITH_UNITS]
 
@@ -144,6 +147,7 @@ module OpenC3
         @template = nil
         @packet_time = nil
         @ignore_overlap = false
+        @virtual = false
       end
 
       # Sets the target name this packet is associated with. Unidentified packets
@@ -233,6 +237,7 @@ module OpenC3
     # @return [Boolean] Whether or not the buffer of data is this packet
     def identify?(buffer)
       return false unless buffer
+      return false if @virtual
       return true unless @id_items
 
       @id_items.each do |item|
@@ -286,6 +291,14 @@ module OpenC3
       @packet_time = time
     end
 
+    def virtual=(v)
+      @virtual = v
+      if @virtual
+        @hidden = true
+        @disabled = true
+      end
+    end
+
     # Calculates a unique hashing sum that changes if the parts of the packet configuration change that could affect
     # the "shape" of the packet.  This value is cached and that packet should not be changed if this method is being used
     def config_name
@@ -1058,7 +1071,9 @@ module OpenC3
       config << "  ALLOW_SHORT\n" if @short_buffer_allowed
       config << "  HAZARDOUS #{@hazardous_description.to_s.quote_if_necessary}\n" if @hazardous
       config << "  DISABLE_MESSAGES\n" if @messages_disabled
-      if @disabled
+      if @virtual
+        config << "  VIRTUAL\n"
+      elsif @disabled
         config << "  DISABLED\n"
       elsif @hidden
         config << "  HIDDEN\n"
@@ -1122,6 +1137,7 @@ module OpenC3
       config['messages_disabled'] = true if @messages_disabled
       config['disabled'] = true if @disabled
       config['hidden'] = true if @hidden
+      config['virtual'] = true if @virtual
       config['accessor'] = @accessor.class.to_s
       config['accessor_args'] = @accessor.args
       config['template'] = Base64.encode64(@template) if @template
@@ -1176,6 +1192,7 @@ module OpenC3
       packet.messages_disabled = hash['messages_disabled']
       packet.disabled = hash['disabled']
       packet.hidden = hash['hidden']
+      packet.virtual = hash['virtual']
       if hash['accessor']
         begin
           accessor = OpenC3::const_get(hash['accessor'])

data/lib/openc3/packets/packet_config.rb

@@ -74,13 +74,13 @@ module OpenC3
     attr_reader :latest_data
 
     # @return [Hash<String>=>Hash<Array>=>Packet] Hash keyed by target name
-    # that returns a hash keyed by an array of id values.  The id values resolve to the packet
-    # defined by that identification.  Command version
+    # that returns a hash keyed by an array of id values. The id values resolve to the packet
+    # defined by that identification. Command version
     attr_reader :cmd_id_value_hash
 
     # @return [Hash<String>=>Hash<Array>=>Packet] Hash keyed by target name
-    # that returns a hash keyed by an array of id values.  The id values resolve to the packet
-    # defined by that identification.  Telemetry version
+    # that returns a hash keyed by an array of id values. The id values resolve to the packet
+    # defined by that identification. Telemetry version
     attr_reader :tlm_id_value_hash
 
     # @return [String] Language of current target (ruby or python)
@@ -219,7 +219,7 @@ module OpenC3
               'PARAMETER', 'ID_ITEM', 'ID_PARAMETER', 'ARRAY_ITEM', 'ARRAY_PARAMETER', 'APPEND_ITEM',\
               'APPEND_PARAMETER', 'APPEND_ID_ITEM', 'APPEND_ID_PARAMETER', 'APPEND_ARRAY_ITEM',\
               'APPEND_ARRAY_PARAMETER', 'ALLOW_SHORT', 'HAZARDOUS', 'PROCESSOR', 'META',\
-              'DISABLE_MESSAGES', 'HIDDEN', 'DISABLED', 'ACCESSOR', 'TEMPLATE', 'TEMPLATE_FILE',\
+              'DISABLE_MESSAGES', 'HIDDEN', 'DISABLED', 'VIRTUAL', 'ACCESSOR', 'TEMPLATE', 'TEMPLATE_FILE',\
               'RESPONSE', 'ERROR_RESPONSE', 'SCREEN', 'RELATED_ITEM', 'IGNORE_OVERLAP'
             raise parser.error("No current packet for #{keyword}") unless @current_packet
 
@@ -232,7 +232,7 @@ module OpenC3
               'POLY_WRITE_CONVERSION', 'SEG_POLY_READ_CONVERSION', 'SEG_POLY_WRITE_CONVERSION',\
               'GENERIC_READ_CONVERSION_START', 'GENERIC_WRITE_CONVERSION_START', 'REQUIRED',\
               'LIMITS', 'LIMITS_RESPONSE', 'UNITS', 'FORMAT_STRING', 'DESCRIPTION',\
-              'MINIMUM_VALUE', 'MAXIMUM_VALUE', 'DEFAULT_VALUE', 'OVERFLOW', 'OVERLAP', 'KEY'
+              'MINIMUM_VALUE', 'MAXIMUM_VALUE', 'DEFAULT_VALUE', 'OVERFLOW', 'OVERLAP', 'KEY', 'VARIABLE_BIT_SIZE'
             raise parser.error("No current item for #{keyword}") unless @current_item
 
             process_current_item(parser, keyword, params)
@@ -317,16 +317,20 @@ module OpenC3
         if @current_cmd_or_tlm == COMMAND
           PacketParser.check_item_data_types(@current_packet)
           @commands[@current_packet.target_name][@current_packet.packet_name] = @current_packet
-          hash = @cmd_id_value_hash[@current_packet.target_name]
-          hash = {} unless hash
-          @cmd_id_value_hash[@current_packet.target_name] = hash
-          update_id_value_hash(@current_packet, hash)
+          unless @current_packet.virtual
+            hash = @cmd_id_value_hash[@current_packet.target_name]
+            hash = {} unless hash
+            @cmd_id_value_hash[@current_packet.target_name] = hash
+            update_id_value_hash(@current_packet, hash)
+          end
         else
           @telemetry[@current_packet.target_name][@current_packet.packet_name] = @current_packet
-          hash = @tlm_id_value_hash[@current_packet.target_name]
-          hash = {} unless hash
-          @tlm_id_value_hash[@current_packet.target_name] = hash
-          update_id_value_hash(@current_packet, hash)
+          unless @current_packet.virtual
+            hash = @tlm_id_value_hash[@current_packet.target_name]
+            hash = {} unless hash
+            @tlm_id_value_hash[@current_packet.target_name] = hash
+            update_id_value_hash(@current_packet, hash)
+          end
         end
         @current_packet = nil
         @current_item = nil
@@ -337,7 +341,7 @@ module OpenC3
       if cmd_or_tlm == :COMMAND
         @commands[packet.target_name][packet.packet_name] = packet
 
-        if affect_ids
+        if affect_ids and not packet.virtual
           hash = @cmd_id_value_hash[packet.target_name]
           hash = {} unless hash
           @cmd_id_value_hash[packet.target_name] = hash
@@ -354,7 +358,7 @@ module OpenC3
           latest_data_packets << packet unless latest_data_packets.include?(packet)
         end
 
-        if affect_ids
+        if affect_ids and not packet.virtual
           hash = @tlm_id_value_hash[packet.target_name]
           hash = {} unless hash
           @tlm_id_value_hash[packet.target_name] = hash
@@ -469,6 +473,13 @@ module OpenC3
         @current_packet.hidden = true
         @current_packet.disabled = true
 
+      when 'VIRTUAL'
+        usage = "#{keyword}"
+        parser.verify_num_parameters(0, 0, usage)
+        @current_packet.hidden = true
+        @current_packet.disabled = true
+        @current_packet.virtual = true
+
       when 'ACCESSOR'
         usage = "#{keyword} <Accessor class name>"
         parser.verify_num_parameters(1, nil, usage)
@@ -720,6 +731,16 @@ module OpenC3
       when 'KEY'
         parser.verify_num_parameters(1, 1, 'KEY <key or path into data>')
         @current_item.key = params[0]
+
+      when 'VARIABLE_BIT_SIZE'
+        parser.verify_num_parameters(1, 3, 'VARIABLE_BIT_SIZE <length_item_name> <length_bits_per_count = 8> <length_value_bit_offset = 0>')
+
+        variable_bit_size = {'length_bits_per_count' => 8, 'length_value_bit_offset' => 0}
+        variable_bit_size['length_item_name'] = params[0].upcase
+        variable_bit_size['length_bits_per_count'] = Integer(params[1]) if params[1]
+        variable_bit_size['length_value_bit_offset'] = Integer(params[2]) if params[2]
+
+        @current_item.variable_bit_size = variable_bit_size
       end
     end
 

data/lib/openc3/packets/packet_item.rb

@@ -401,6 +401,7 @@ module OpenC3
       config << " #{self.endianness}" if self.endianness != default_endianness && self.data_type != :STRING && self.data_type != :BLOCK
       config << "\n"
 
+      config << "    VARIABLE_BIT_SIZE '#{self.variable_bit_size['length_item_name']}' #{self.variable_bit_size['length_bits_per_count']} #{self.variable_bit_size['length_value_bit_offset']}\n" if self.variable_bit_size
       config << "    REQUIRED\n" if self.required
       config << "    FORMAT_STRING #{self.format_string.to_s.quote_if_necessary}\n" if self.format_string
       config << "    UNITS #{self.units_full.to_s.quote_if_necessary} #{self.units.to_s.quote_if_necessary}\n" if self.units
@@ -510,6 +511,9 @@ module OpenC3
       end
 
       config['meta'] = @meta if @meta
+      if @variable_bit_size
+        config['variable_bit_size'] = @variable_bit_size
+      end
       config
     end
 
@@ -571,6 +575,7 @@ module OpenC3
         item.limits.values = values if values.length > 0
       end
       item.meta = hash['meta']
+      item.variable_bit_size = hash['variable_bit_size']
       item
     end
 

data/lib/openc3/packets/structure.rb

@@ -299,7 +299,6 @@ module OpenC3
     # @param overflow (see #define_item)
     # @return (see #define_item)
     def append_item(name, bit_size, data_type, array_size = nil, endianness = @default_endianness, overflow = :ERROR)
-      raise ArgumentError, "Can't append an item after a variably sized item" if !@fixed_size
       if data_type == :DERIVED
         return define_item(name, 0, bit_size, data_type, array_size, endianness, overflow)
       else
@@ -313,12 +312,15 @@ module OpenC3
     # @param item (see #define)
     # @return (see #define)
     def append(item)
-      raise ArgumentError, "Can't append an item after a variably sized item" if !@fixed_size
-
       if item.data_type == :DERIVED
         item.bit_offset = 0
       else
+        # We're appending a new item so set the bit_offset
         item.bit_offset = @defined_length_bits
+        # Also set original_bit_offset because it's currently 0
+        # due to PacketItemParser::create_packet_item
+        # get_bit_offset() returning 0 if append
+        item.original_bit_offset = @defined_length_bits
       end
       return define(item)
     end
@@ -337,6 +339,20 @@ module OpenC3
     def set_item(item)
       if @items[item.name]
         @items[item.name] = item
+        # Need to allocate space for the variable length item if its minimum size is greater than zero
+        if item.variable_bit_size
+          minimum_data_bits = 0
+          if (item.data_type == :INT or item.data_type == :UINT) and not item.original_array_size
+            # Minimum QUIC encoded integer, see https://datatracker.ietf.org/doc/html/rfc9000#name-variable-length-integer-enc
+            minimum_data_bits = 6
+          # :STRING, :BLOCK, or array item
+          elsif item.variable_bit_size['length_value_bit_offset'] > 0
+            minimum_data_bits = item.variable_bit_size['length_value_bit_offset'] * item.variable_bit_size['length_bits_per_count']
+          end
+          if minimum_data_bits > 0 and item.bit_offset >= 0 and @defined_length_bits == item.bit_offset
+            @defined_length_bits += minimum_data_bits
+          end
+        end
       else
         raise ArgumentError, "Unknown item: #{item.name} - Ensure item name is uppercase"
       end
@@ -566,6 +582,51 @@ module OpenC3
       end
     end
 
+    def calculate_total_bit_size(item)
+      if item.variable_bit_size
+        # Bit size is determined by length field
+        length_value = self.read(item.variable_bit_size['length_item_name'], :CONVERTED)
+        if item.data_type == :INT or item.data_type == :UINT and not item.original_array_size
+          case length_value
+          when 0
+            return 6
+          when 1
+            return 14
+          when 2
+            return 30
+          else
+            return 62
+          end
+        else
+          return (length_value * item.variable_bit_size['length_bits_per_count']) + item.variable_bit_size['length_value_bit_offset']
+        end
+      elsif item.original_bit_size <= 0
+        # Bit size is full packet length - bits before item + negative bits saved at end
+        return (@buffer.length * 8) - item.bit_offset + item.original_bit_size
+      elsif item.original_array_size and item.original_array_size <= 0
+        # Bit size is full packet length - bits before item + negative bits saved at end
+        return (@buffer.length * 8) - item.bit_offset + item.original_array_size
+      else
+        raise "Unexpected use of calculate_total_bit_size for non-variable-sized item"
+      end
+    end
+
+    def recalculate_bit_offsets
+      adjustment = 0
+      @sorted_items.each do |item|
+        # Anything with a negative bit offset should be left alone
+        if item.original_bit_offset >= 0
+          item.bit_offset = item.original_bit_offset + adjustment
+          if item.data_type != :DERIVED and (item.variable_bit_size or item.original_bit_size <= 0 or (item.original_array_size and item.original_array_size <= 0))
+            new_bit_size = calculate_total_bit_size(item)
+            if item.original_bit_size != new_bit_size
+              adjustment += (new_bit_size - item.original_bit_size)
+            end
+          end
+        end
+      end
+    end
+
     def internal_buffer_equals(buffer)
       raise ArgumentError, "Buffer class is #{buffer.class} but must be String" unless String === buffer
 
@@ -573,6 +634,9 @@ module OpenC3
       if @accessor.enforce_encoding
         @buffer.force_encoding(@accessor.enforce_encoding)
       end
+      if not @fixed_size
+        recalculate_bit_offsets()
+      end
       if @accessor.enforce_length
         if @buffer.length != @defined_length
           if @buffer.length < @defined_length