openc3 5.0.6 → 5.0.9

data/lib/openc3/models/scope_model.rb

@@ -40,10 +40,47 @@ module OpenC3
       super(PRIMARY_KEY)
     end
 
+    def self.from_json(json, scope: nil)
+      json = JSON.parse(json, :allow_nan => true, :create_additions => true) if String === json
+      raise "json data is nil" if json.nil?
+
+      json.transform_keys!(&:to_sym)
+      self.new(**json, scope: scope)
+    end
+
+    def self.get_model(name:, scope: nil)
+      json = get(name: name)
+      if json
+        return from_json(json)
+      else
+        return nil
+      end
+    end
+
     def initialize(name:, updated_at: nil, scope: nil)
       super(PRIMARY_KEY, name: name, scope: name, updated_at: updated_at)
     end
 
+    def create(update: false, force: false)
+      # Ensure there are no "." in the scope name - prevents gems accidently becoming scope names
+      raise "Invalid scope name: #{@name}" if @name !~ /^[a-zA-Z0-9_-]+$/
+      @name = @name.upcase
+      super(update: update, force: force)
+    end
+
+    def destroy
+      if @name != 'DEFAULT'
+        # Remove all the plugins for this scope
+        plugins = PluginModel.get_all_models(scope: @name)
+        plugins.each do |plugin_name, plugin|
+          plugin.destroy
+        end
+        super()
+      else
+        raise "DEFAULT scope cannot be destroyed"
+      end
+    end
+
     def as_json(*a)
       { 'name' => @name,
         'updated_at' => @updated_at }
@@ -51,9 +88,12 @@ module OpenC3
 
     def deploy(gem_path, variables)
       seed_database()
-
       ConfigTopic.initialize_stream(@scope)
 
+      # Create UNKNOWN target for display of unknown data
+      model = TargetModel.new(name: "UNKNOWN", scope: @scope)
+      model.create
+
       # OpenC3 Log Microservice
       microservice_name = "#{@scope}__OPENC3__LOG"
       microservice = MicroserviceModel.new(

data/lib/openc3/models/target_model.rb

@@ -24,6 +24,7 @@ require 'openc3/models/microservice_model'
 require 'openc3/topics/limits_event_topic'
 require 'openc3/topics/config_topic'
 require 'openc3/system'
+require 'openc3/utilities/local_mode'
 require 'openc3/utilities/s3'
 require 'openc3/utilities/zip'
 require 'fileutils'
@@ -82,13 +83,149 @@ module OpenC3
       super("#{scope}__#{PRIMARY_KEY}")
     end
 
+    # All targets with indication of modified targets
+    def self.all_modified(scope:)
+      targets = self.all(scope: scope)
+      targets.each { |target_name, target| target['modified'] = false }
+
+      if ENV['OPENC3_LOCAL_MODE']
+        modified_targets = OpenC3::LocalMode.modified_targets(scope: scope)
+        modified_targets.each do |target_name|
+          targets[target_name]['modified'] = true if targets[target_name]
+        end
+      else
+        rubys3_client = Aws::S3::Client.new
+        token = nil
+        while true
+          resp = rubys3_client.list_objects_v2({
+            bucket: 'config',
+            max_keys: 1000,
+            # The trailing slash is important!
+            prefix: "#{scope}/targets_modified/",
+            delimiter: '/',
+            continuation_token: token
+          })
+          resp.common_prefixes.each do |item|
+            # Results look like DEFAULT/targets_modified/INST/
+            # so split on '/' and pull out the last value
+            target_name = item.prefix.split('/')[-1]
+            # A target could have been deleted without removing the modified files
+            # Thus we have to check for the existance of the target_name key
+            if targets.has_key?(target_name)
+              targets[target_name]['modified'] = true
+            end
+          end
+          break unless resp.is_truncated
+          token = resp.next_continuation_token
+        end
+      end
+      # Sort (which turns hash to array) and return hash
+      # This enables a consistent listing of the targets
+      targets.sort.to_h
+    end
+
+    # Given target's modified file list
+    def self.modified_files(target_name, scope:)
+      modified = []
+
+      if ENV['OPENC3_LOCAL_MODE']
+        modified = OpenC3::LocalMode.modified_files(target_name, scope: scope)
+      else
+        rubys3_client = Aws::S3::Client.new
+        token = nil
+        while true
+          resp = rubys3_client.list_objects_v2({
+            bucket: 'config',
+            max_keys: 1000,
+            # The trailing slash is important!
+            prefix: "#{scope}/targets_modified/#{target_name}/",
+            continuation_token: token
+          })
+          resp.contents.each do |item|
+            # Results look like DEFAULT/targets_modified/INST/procedures/new.rb
+            # so split on '/' and ignore the first two values
+            modified << item.key.split('/')[2..-1].join('/')
+          end
+          break unless resp.is_truncated
+          token = resp.next_continuation_token
+        end
+      end
+      # Sort to enable a consistent listing of the modified files
+      modified.sort
+    end
+
+    def self.delete_modified(target_name, scope:)
+      if ENV['OPENC3_LOCAL_MODE']
+        OpenC3::LocalMode.delete_modified(target_name, scope: scope)
+      end
+
+      # Delete the remote files as well
+      rubys3_client = Aws::S3::Client.new
+      token = nil
+      while true
+        resp = rubys3_client.list_objects_v2({
+          bucket: 'config',
+          max_keys: 1000,
+          # The trailing slash is important!
+          prefix: "#{scope}/targets_modified/#{target_name}/",
+          continuation_token: token
+        })
+        resp.contents.each do |item|
+          rubys3_client.delete_object(bucket: 'config', key: item.key)
+        end
+        break unless resp.is_truncated
+        token = resp.next_continuation_token
+      end
+    end
+
+    def self.download(target_name, scope:)
+      tmp_dir = Dir.mktmpdir
+      zip_filename = File.join(tmp_dir, "#{target_name}.zip")
+      Zip.continue_on_exists_proc = true
+      zip = Zip::File.open(zip_filename, Zip::File::CREATE)
+
+      if ENV['OPENC3_LOCAL_MODE']
+        OpenC3::LocalMode.zip_target(target_name, zip, scope: scope)
+      else
+        rubys3_client = Aws::S3::Client.new
+        token = nil
+        # The trailing slash is important!
+        prefix = "#{scope}/targets_modified/#{target_name}/"
+        while true
+          resp = rubys3_client.list_objects_v2({
+            bucket: 'config',
+            max_keys: 1000,
+            prefix: prefix,
+            continuation_token: token
+          })
+          resp.contents.each do |item|
+            # item.key looks like DEFAULT/targets_modified/INST/screens/blah.txt
+            base_path = item.key.sub(prefix, '') # remove prefix
+            local_path = File.join(tmp_dir, base_path)
+            # Ensure dir structure exists, get_object fails if not
+            FileUtils.mkdir_p(File.dirname(local_path))
+            rubys3_client.get_object(bucket: 'config', key: item.key, response_target: local_path)
+            zip.add(base_path, local_path)
+          end
+          break unless resp.is_truncated
+          token = resp.next_continuation_token
+        end
+      end
+      zip.close
+
+      result = OpenStruct.new
+      result.filename = File.basename(zip_filename)
+      result.contents = File.read(zip_filename, mode: 'rb')
+      return result
+    end
+
     # @return [Array] Array of all the packet names
     def self.packet_names(target_name, type: :TLM, scope:)
       raise "Unknown type #{type} for #{target_name}" unless VALID_TYPES.include?(type)
       # If the key doesn't exist or if there are no packets we return empty array
       Store.hkeys("#{scope}__openc3#{type.to_s.downcase}__#{target_name}").sort
     end
-  
+
     # @return [Hash] Packet hash or raises an exception
     def self.packet(target_name, packet_name, type: :TLM, scope:)
       raise "Unknown type #{type} for #{target_name} #{packet_name}" unless VALID_TYPES.include?(type)
@@ -420,9 +557,6 @@ module OpenC3
     end
 
     def undeploy
-      # Note: The plugin_model undeploy method removes all the microservices first
-      # so we don't need to destroy them here
-
       rubys3_client = Aws::S3::Client.new
       prefix = "#{@scope}/targets/#{@name}/"
       rubys3_client.list_objects(bucket: 'config', prefix: prefix).contents.each do |object|

data/lib/openc3/models/tool_model.rb

@@ -203,6 +203,9 @@ module OpenC3
       when 'SHOWN'
         parser.verify_num_parameters(1, 1, "SHOWN <true/false>")
         @shown = ConfigParser.handle_true_false(parameters[0])
+      when 'POSITION'
+        parser.verify_num_parameters(1, 1, "POSITION <value>")
+        @position = parameters[0].to_i
       else
         raise ConfigParser::Error.new(parser, "Unknown keyword and parameters for Tool: #{keyword} #{parameters.join(" ")}")
       end

data/lib/openc3/operators/microservice_operator.rb

@@ -90,7 +90,7 @@ module OpenC3
         @new_microservices.each do |microservice_name, microservice_config|
           cmd_array, work_dir, env, scope, container = convert_microservice_to_process_definition(microservice_name, microservice_config)
           if cmd_array
-            process = OperatorProcess.new(cmd_array, work_dir: work_dir, env: env, scope: scope, container: container)
+            process = OperatorProcess.new(cmd_array, work_dir: work_dir, env: env, scope: scope, container: container, config: microservice_config)
             @new_processes[microservice_name] = process
             @processes[microservice_name] = process
           end
@@ -108,7 +108,7 @@ module OpenC3
               @changed_processes[microservice_name] = process
             else # TODO: How is this even possible?
               Logger.error("Changed microservice #{microservice_name} does not exist. Creating new...", scope: scope)
-              process = OperatorProcess.new(cmd_array, work_dir: work_dir, env: env, scope: scope, container: container)
+              process = OperatorProcess.new(cmd_array, work_dir: work_dir, env: env, scope: scope, container: container, config: microservice_config)
               @new_processes[microservice_name] = process
               @processes[microservice_name] = process
             end

data/lib/openc3/operators/operator.rb

@@ -35,7 +35,8 @@ module OpenC3
       # Perform any setup steps necessary
     end
 
-    def initialize(process_definition, work_dir: '/openc3/lib/openc3/microservices', temp_dir: nil, env: {}, scope:, container: nil) # container is not used, it's just here for Enterprise
+    # container is not used, it's just here for Enterprise
+    def initialize(process_definition, work_dir: '/openc3/lib/openc3/microservices', temp_dir: nil, env: {}, scope:, container: nil, config: nil)
       @process = nil
       @process_definition = process_definition
       @work_dir = work_dir
@@ -43,12 +44,26 @@ module OpenC3
       @new_temp_dir = temp_dir
       @env = env
       @scope = scope
+      # @config only used in start to help print a better Logger message
+      @config = config
     end
 
     def start
       @temp_dir = @new_temp_dir
       @new_temp_dir = nil
-      Logger.info("Starting: #{@process_definition.join(' ')}", scope: @scope)
+
+      # In ProcessManager processes, the process_definition is the actual thing run
+      # e.g. OpenC3::ProcessManager.instance.spawn(["ruby", "/openc3/bin/openc3cli", "load", ...])
+      # However, if the MicroserviceOperator is spawning the proceses it sets
+      # process_definition = ["ruby", "plugin_microservice.rb"]
+      # which then calls exec(*@config["cmd"]) to actually run
+      # So check if the @config['cmd'] is defined to give the user more info in the log
+      cmd = @process_definition.join(' ')
+      if @config && @config['cmd']
+        cmd = @config['cmd'].join(' ')
+      end
+      Logger.info("Starting: #{cmd}", scope: @scope)
+
       @process = ChildProcess.build(*@process_definition)
       # This lets the ChildProcess use the parent IO ... but it breaks unit tests
       # @process.io.inherit!

data/lib/openc3/packets/parsers/xtce_parser.rb

@@ -178,14 +178,40 @@ module OpenC3
       when 'ParameterTypeSet', 'EnumerationList', 'ParameterSet', 'ContainerSet',
         'EntryList', 'DefaultCalibrator', 'DefaultAlarm', 'RestrictionCriteria',
         'ComparisonList', 'MetaCommandSet', 'ArgumentTypeSet', 'ArgumentList',
-        'ArgumentAssignmentList', 'LocationInContainerInBits'
+        'ArgumentAssignmentList', 'LocationInContainerInBits', 'ReferenceTime'
         # Do Nothing
 
+      when 'ErrorDetectCorrect'
+        # TODO: Setup an algorithm to calculate the CRC
+        exponents = []
+        xtce_recurse_element(element) do |crc_element|
+          if crc_element.name == 'CRC'
+            xtce_recurse_element(crc_element) do |poly_element|
+              if poly_element['Polynomial']
+                xtce_recurse_element(poly_element) do |term_element|
+                  if term_element['Term']
+                    exponents << term_element['exponent'].to_i
+                  end
+                  true
+                end
+              end
+              true
+            end
+          end
+          true
+        end
+        @current_type.xtce_encoding = 'IntegerDataEncoding'
+        @current_type.signed = 'false'
+        return false # Already recursed
+
       when 'EnumeratedParameterType', 'EnumeratedArgumentType',
         'IntegerParameterType', 'IntegerArgumentType',
         'FloatParameterType', 'FloatArgumentType',
         'StringParameterType', 'StringArgumentType',
-        'BinaryParameterType', 'BinaryArgumentType'
+        'BinaryParameterType', 'BinaryArgumentType',
+        'BooleanParameterType', 'BooleanArgumentType',
+        'AbsoluteTimeParameterType', 'AbsoluteTimeArgumentType',
+        'RelativeTimeParameterType', 'RelativeTimeArgumentType'
         @current_type = create_new_type(element)
         @current_type.endianness = :BIG_ENDIAN
 
@@ -204,6 +230,20 @@ module OpenC3
         when 'BinaryParameterType', 'BinaryArgumentType'
           @current_type.xtce_encoding = 'BinaryDataEncoding'
           @current_type.sizeInBits = 8 # This is undocumented but appears to be the design
+        when 'BooleanParameterType', 'BooleanArgumentType'
+          @current_type.xtce_encoding = 'StringDataEncoding'
+          @current_type.sizeInBits = 8 # This is undocumented but appears to be the design
+          @current_type.states ||= {}
+          if element.attributes['zeroStringValue'] && element.attributes['oneStringValue']
+            @current_type.states[element.attributes['zeroStringValue'].value] = 0
+            @current_type.states[element.attributes['oneStringValue'].value] = 1
+          else
+            @current_type.states['FALSE'] = 0
+            @current_type.states['TRUE'] = 1
+          end
+        # No defaults for the time types
+        # when 'AbsoluteTimeParameterType', 'AbsoluteTimeArgumentType',
+        # when'RelativeTimeParameterType', 'RelativeTimeArgumentType'
         end
 
       when 'ArrayParameterType', 'ArrayArgumentType'
@@ -251,7 +291,7 @@ module OpenC3
 
         return false # Already recursed
 
-      when "SizeInBits"
+      when 'SizeInBits'
         xtce_recurse_element(element) do |block_element|
           if block_element.name == 'FixedValue'
             @current_type.sizeInBits = Integer(block_element.text)
@@ -326,6 +366,26 @@ module OpenC3
         @current_type.states ||= {}
         @current_type.states[element['label']] = Integer(element['value'])
 
+      when 'Encoding'
+        if element.attributes['units']
+          @current_type.units_full = element.attributes['units'].value
+          # Could do a better job mapping units to abbreviations
+          @current_type.units = element.attributes['units'].value[0]
+        end
+        # TODO: Not sure if this is correct
+        # if @current_type.attributes['scale'] || @current_type.attributes['offset']
+        #   @current_type.conversion ||= PolynomialConversion.new()
+        #   if @current_type.attributes['offset']
+        #     @current_type.conversion.coeffs[0] = @current_type.attributes['offset']
+        #   end
+        #   if @current_type.attributes['scale']
+        #     @current_type.conversion.coeffs[1] = @current_type.attributes['scale']
+        #   end
+        # end
+
+      when 'Epoch'
+        # TODO: How to handle this ... it affects the conversion applied
+
       when 'IntegerDataEncoding', 'FloatDataEncoding', 'StringDataEncoding', 'BinaryDataEncoding'
         @current_type.xtce_encoding = element.name
         element.attributes.each do |att_name, att|
@@ -340,6 +400,11 @@ module OpenC3
           @current_type.endianness = :LITTLE_ENDIAN
         end
 
+        # Ensure if the encoding is a string we convert state values to strings
+        if @current_type.xtce_encoding == 'StringDataEncoding' && @current_type.states
+          @current_type.states.transform_values!(&:to_s)
+        end
+
       when 'Parameter'
         @current_parameter = OpenStruct.new
         element.attributes.each do |att_name, att|

data/lib/openc3/script/storage.rb

@@ -34,7 +34,7 @@ module OpenC3
         OpenC3::Logger.info "Deleting #{delete_path}"
         response = $api_server.request('delete', endpoint, query: {bucket: 'config'}, scope: scope)
         if response.nil? || response.code != 200
-          raise "Failed to delete #{delete_path}. Note: #{scope}/targets is read-only."
+          raise "Failed to delete #{delete_path}"
         end
       rescue => error
         raise "Failed deleting #{path} due to #{error.message}"
@@ -48,7 +48,13 @@ module OpenC3
     # @param io_or_string [Io or String] IO object
     def put_target_file(path, io_or_string, scope: $openc3_scope)
       raise "Disallowed path modifier '..' found in #{path}" if path.include?('..')
+
       upload_path = "#{scope}/targets_modified/#{path}"
+
+      if ENV['OPENC3_LOCAL_MODE'] and $openc3_in_cluster
+        OpenC3::LocalMode.put_target_file(upload_path, io_or_string, scope: scope)
+      end
+
       endpoint = "/openc3-api/storage/upload/#{upload_path}"
       OpenC3::Logger.info "Writing #{upload_path}"
       result = _get_presigned_request(endpoint, scope: scope)
@@ -86,6 +92,17 @@ module OpenC3
       # Loop to allow redo when switching from modified to original
       loop do
         begin
+          if part == "targets_modified" and ENV['OPENC3_LOCAL_MODE']
+            local_file = OpenC3::LocalMode.open_local_file(path, scope: scope)
+            if local_file
+              file = Tempfile.new('target', binmode: true)
+              file.write(local_file.read)
+              local_file.close
+              file.rewind
+              return file if local_file
+            end
+          end
+
           return _get_storage_file("#{part}/#{path}", scope: scope)
         rescue => error
           if part == "targets_modified"