openc3 5.0.8 → 5.0.10

data/lib/openc3/accessors/cbor_accessor.rb

@@ -0,0 +1,83 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+require 'cbor'
+require 'openc3/accessors/json_accessor'
+
+module OpenC3
+  class CborAccessor < JsonAccessor
+    def self.read_item(item, buffer)
+      return nil if item.data_type == :DERIVED
+      if String === buffer
+        parsed = CBOR.decode(buffer)
+      else
+        parsed = buffer
+      end
+      return super(item, parsed)
+    end
+
+    def self.write_item(item, value, buffer)
+      return nil if item.data_type == :DERIVED
+
+      # Convert to ruby objects
+      if String === buffer
+        decoded = CBOR.decode(buffer)
+      else
+        decoded = buffer
+      end
+
+      # Write the value
+      write_item_internal(item, value, decoded)
+
+      # Update buffer
+      if String === buffer
+        buffer.replace(decoded.to_cbor)
+      end
+
+      return buffer
+    end
+
+    def self.read_items(items, buffer)
+      # Prevent JsonPath from decoding every call
+      if String === buffer
+        decoded = CBOR.decode(buffer)
+      else
+        decoded = buffer
+      end
+      super(items, decoded)
+    end
+
+    def self.write_items(items, values, buffer)
+      # Convert to ruby objects
+      if String === buffer
+        decoded = CBOR.decode(buffer)
+      else
+        decoded = buffer
+      end
+
+      items.each_with_index do |item, index|
+        write_item_internal(item, values[index], decoded)
+      end
+
+      # Update buffer
+      if String === buffer
+        buffer.replace(decoded.to_cbor)
+      end
+
+      return buffer
+    end
+
+  end
+end

data/lib/openc3/accessors/html_accessor.rb

@@ -0,0 +1,28 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+require 'openc3/accessors/xml_accessor'
+
+module OpenC3
+  class HtmlAccessor < XmlAccessor
+    def self.buffer_to_doc(buffer)
+      Nokogiri.HTML(buffer)
+    end
+
+    def self.doc_to_buffer(doc)
+      doc.to_html
+    end
+  end
+end

data/lib/openc3/accessors/json_accessor.rb

@@ -0,0 +1,131 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+require 'json'
+require 'jsonpath'
+require 'openc3/accessors/accessor'
+
+module OpenC3
+  class JsonAccessor < Accessor
+    def self.read_item(item, buffer)
+      return nil if item.data_type == :DERIVED
+      return JsonPath.on(buffer, item.key).first
+    end
+
+    def self.write_item(item, value, buffer)
+      return nil if item.data_type == :DERIVED
+
+      # Convert to ruby objects
+      if String === buffer
+        decoded = JSON.parse(buffer, :allow_nan => true)
+      else
+        decoded = buffer
+      end
+
+      # Write the value
+      write_item_internal(item, value, decoded)
+
+      # Update buffer
+      if String === buffer
+        buffer.replace(JSON.generate(decoded, :allow_nan => true))
+      end
+
+      return buffer
+    end
+
+    def self.read_items(items, buffer)
+      # Prevent JsonPath from decoding every call
+      if String === buffer
+        decoded = JSON.parse(buffer)
+      else
+        decoded = buffer
+      end
+      super(items, decoded)
+    end
+
+    def self.write_items(items, values, buffer)
+      # Start with an empty object if no buffer
+      buffer.replace("{}") if buffer.length == 0 or buffer[0] == "\x00"
+
+      # Convert to ruby objects
+      if String === buffer
+        decoded = JSON.parse(buffer, :allow_nan => true)
+      else
+        decoded = buffer
+      end
+
+      items.each_with_index do |item, index|
+        write_item_internal(item, values[index], decoded)
+      end
+
+      # Update buffer
+      if String === buffer
+        buffer.replace(JSON.generate(decoded, :allow_nan => true))
+      end
+
+      return buffer
+    end
+
+    def self.write_item_internal(item, value, decoded)
+      return nil if item.data_type == :DERIVED
+
+      # Save traversal state
+      parent_node = nil
+      parent_key = nil
+      node = decoded
+
+      # Parse the JsonPath
+      json_path = JsonPath.new(item.key)
+
+      # Handle each token
+      json_path.path.each do |token|
+        case token
+        when '$'
+          # Ignore start - it is implied
+          next
+        when /\[.*\]/
+          # Array or Hash Index
+          if token.index("'") # Hash index
+            key = token[2..-3]
+            if not (Hash === node)
+              node = {}
+              parent_node[parent_key] = node
+            end
+            parent_node = node
+            parent_key = key
+            node = node[key]
+          else # Array index
+            key = token[1..-2].to_i
+            if not (Array === node)
+              node = []
+              parent_node[parent_key] = node
+            end
+            parent_node = node
+            parent_key = key
+            node = node[key]
+          end
+        else
+          raise "Unsupported key/token: #{item.key} - #{token}"
+        end
+      end
+      if parent_node
+        parent_node[parent_key] = value
+      else
+        decoded.replace(value)
+      end
+      return decoded
+    end
+  end
+end

data/lib/openc3/accessors/xml_accessor.rb

@@ -0,0 +1,67 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+require 'nokogiri'
+require 'openc3/accessors/accessor'
+
+module OpenC3
+  class XmlAccessor < Accessor
+    def self.read_item(item, buffer)
+      return nil if item.data_type == :DERIVED
+      doc = buffer_to_doc(buffer)
+      return convert_to_type(doc.xpath(item.key).first.to_s, item)
+    end
+
+    def self.write_item(item, value, buffer)
+      return nil if item.data_type == :DERIVED
+      doc = buffer_to_doc(buffer)
+      node = doc.xpath(item.key).first
+      node.content = value.to_s
+      buffer.replace(doc_to_buffer(doc))
+    end
+
+    def self.read_items(items, buffer)
+      doc = buffer_to_doc(buffer)
+      result = {}
+      items.each do |item|
+        if item.data_type == :DERIVED
+          result[item.name] = nil
+        else
+          result[item.name] = convert_to_type(doc.xpath(item.key).first.to_s, item)
+        end
+      end
+      return result
+    end
+
+    def self.write_items(items, values, buffer)
+      doc = buffer_to_doc(buffer)
+      items.each_with_index do |item, index|
+        next if item.data_type == :DERIVED
+        node = doc.xpath(item.key).first
+        node.content = values[index].to_s
+      end
+      buffer.replace(doc_to_buffer(doc))
+    end
+
+    def self.buffer_to_doc(buffer)
+      Nokogiri.XML(buffer)
+    end
+
+    def self.doc_to_buffer(doc)
+      doc.to_xml
+    end
+
+  end
+end

data/lib/openc3/accessors.rb

@@ -0,0 +1,23 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+module OpenC3
+  autoload(:Accessor, 'openc3/accessors/accessor.rb')
+  autoload(:BinaryAccessor, 'openc3/accessors/binary_accessor.rb')
+  autoload(:CborAccessor, 'openc3/accessors/cbor_accessor.rb')
+  autoload(:HtmlAccessor, 'openc3/accessors/html_accessor.rb')
+  autoload(:JsonAccessor, 'openc3/accessors/json_accessor.rb')
+  autoload(:XmlAccessor, 'openc3/accessors/xml_accessor.rb')
+end

data/lib/openc3/api/cmd_api.rb

@@ -281,9 +281,12 @@ module OpenC3
 
         # States are an array of the name followed by a hash of 'value' and sometimes 'hazardous'
         item['states'].each do |name, hash|
+          param_name = params[item['name']]
+          # Remove quotes from string parameters
+          param_name = param_name.gsub('"', '').gsub("'", '') if param_name.is_a?(String)
           # To be hazardous the state must be marked hazardous
           # Check if either the state name or value matches the param passed
-          if hash['hazardous'] && (name == params[item['name']] || hash['value'].to_f == params[item['name']].to_f)
+          if hash['hazardous'] && (name == param_name || hash['value'] == param_name)
             return true
           end
         end
@@ -396,7 +399,7 @@ module OpenC3
         'hazardous_check' => hazardous_check,
         'raw' => raw
       }
-      Logger.info build_cmd_output_string(target_name, cmd_name, cmd_params, packet, raw) if !packet["messages_disabled"]
+      Logger.info(build_cmd_output_string(target_name, cmd_name, cmd_params, packet, raw), scope: scope) if !packet["messages_disabled"]
       CommandTopic.send_command(command, scope: scope)
     end
 

data/lib/openc3/api/interface_api.rb

@@ -32,6 +32,7 @@ module OpenC3
                        'start_raw_logging_interface',
                        'stop_raw_logging_interface',
                        'get_all_interface_info',
+                       'map_target_to_interface',
                      ])
 
     # Get information about an interface
@@ -113,5 +114,19 @@ module OpenC3
       info.sort! { |a, b| a[0] <=> b[0] }
       info
     end
+
+    # Associates a target and all its commands and telemetry with a particular
+    # interface. All the commands will go out over and telemetry be received
+    # from that interface.
+    #
+    # @param target_name [String] The name of the target
+    # @param interface_name (see #connect_interface)
+    def map_target_to_interface(target_name, interface_name, scope: $openc3_scope, token: $openc3_token)
+      authorize(permission: 'system_set', interface_name: interface_name, scope: scope, token: token)
+      new_interface = InterfaceModel.get_model(name: interface_name, scope: scope)
+      new_interface.map_target(target_name)
+      Logger.info("Target #{target_name} mapped to Interface #{interface_name}", scope: scope)
+      nil
+    end
   end
 end

data/lib/openc3/api/limits_api.rb

@@ -260,7 +260,7 @@ module OpenC3
       message = "Setting '#{target_name} #{packet_name} #{item_name}' limits to #{red_low} #{yellow_low} #{yellow_high} #{red_high}"
       message << " #{green_low} #{green_high}" if green_low && green_high
       message << " in set #{limits_set} with persistence #{persistence} as enabled #{enabled}"
-      Logger.info(message)
+      Logger.info(message, scope: scope)
 
       TargetModel.set_packet(target_name, packet_name, packet, scope: scope)
 
@@ -307,7 +307,7 @@ module OpenC3
     def set_limits_set(limits_set, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'tlm_set', scope: scope, token: token)
       message = "Setting Limits Set: #{limits_set}"
-      Logger.info(message)
+      Logger.info(message, scope: scope)
       LimitsEventTopic.write({ type: :LIMITS_SET, set: limits_set.to_s,
         time_nsec: Time.now.to_nsec_from_epoch, message: message }, scope: scope)
     end
@@ -343,7 +343,7 @@ module OpenC3
       group = get_limits_groups()[group_name]
       raise "LIMITS_GROUP #{group_name} undefined. Ensure your telemetry definition contains the line: LIMITS_GROUP #{group_name}" unless group
 
-      Logger.info("Disabling Limits Group: #{group_name}")
+      Logger.info("Disabling Limits Group: #{group_name}", scope: scope)
       last_target_name = nil
       last_packet_name = nil
       packet = nil

data/lib/openc3/config/config_parser.rb

@@ -161,14 +161,20 @@ module OpenC3
       if options[:locals]
         options[:locals].each { |key, value| b.local_variable_set(key, value) }
       end
+
+      return ERB.new(read_file(template_name), trim_mode: "-").result(b)
+    end
+
+    # Can be called during parsing to read a referenced file
+    def read_file(filename)
       # Assume the file is there. If not we raise a pretty obvious error
-      if File.expand_path(template_name) == template_name # absolute path
-        path = template_name
+      if File.expand_path(filename) == filename # absolute path
+        path = filename
       else # relative to the current @filename
-        path = File.join(File.dirname(@filename), template_name)
+        path = File.join(File.dirname(@filename), filename)
       end
       OpenC3.set_working_dir(File.dirname(path)) do
-        return ERB.new(File.read(path), trim_mode: "-").result(b)
+        return File.read(path)
       end
     end
 

data/lib/openc3/core_ext/file.rb

@@ -80,4 +80,9 @@ class File
     end
     return nil
   end
+
+  def delete
+    self.close
+    File.delete(self.path)
+  end
 end

data/lib/openc3/core_ext/tempfile.rb

@@ -0,0 +1,20 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+
+require 'tempfile'
+
+class Tempfile
+  attr_accessor :filename
+end

data/lib/openc3/io/json_rpc.rb

@@ -80,9 +80,9 @@ class Float
   end
 
   def as_json(options = nil)
-    return { "json_class" => Float, "raw" => "Infinity" }  if self.infinite? ==  1
-    return { "json_class" => Float, "raw" => "-Infinity" } if self.infinite? == -1
-    return { "json_class" => Float, "raw" => "NaN" }       if self.nan?
+    return { "json_class" => "Float", "raw" => "Infinity" }  if self.infinite? ==  1
+    return { "json_class" => "Float", "raw" => "-Infinity" } if self.infinite? == -1
+    return { "json_class" => "Float", "raw" => "NaN" }       if self.nan?
 
     return self
   end

data/lib/openc3/microservices/cleanup_microservice.rb

@@ -35,9 +35,9 @@ module OpenC3
         @state = 'GETTING_OBJECTS'
         start_time = Time.now
         [
-         ["#{@scope}/raw_logs/cmd/#{target_name}/", target.cmd_log_retain_time], 
-         ["#{@scope}/decom_logs/cmd/#{target_name}/", target.cmd_decom_log_retain_time], 
-         ["#{@scope}/raw_logs/tlm/#{target_name}/", target.tlm_log_retain_time], 
+         ["#{@scope}/raw_logs/cmd/#{target_name}/", target.cmd_log_retain_time],
+         ["#{@scope}/decom_logs/cmd/#{target_name}/", target.cmd_decom_log_retain_time],
+         ["#{@scope}/raw_logs/tlm/#{target_name}/", target.tlm_log_retain_time],
          ["#{@scope}/decom_logs/tlm/#{target_name}/", target.tlm_decom_log_retain_time],
          ["#{@scope}/reduced_minute_logs/tlm/#{target_name}/", target.reduced_minute_log_retain_time],
          ["#{@scope}/reduced_hour_logs/tlm/#{target_name}/", target.reduced_hour_log_retain_time],
@@ -52,8 +52,10 @@ module OpenC3
           end
           if delete_items.length > 0
             @state = 'DELETING_OBJECTS'
-            rubys3_client.delete_objects({ bucket: 'logs', delete: { objects: delete_items } })
-            Logger.info("Deleted #{delete_items.length} #{target_name} log files")
+            delete_items.each_slice(1000) do |delete_slice|
+              rubys3_client.delete_objects({ bucket: 'logs', delete: { objects: delete_slice } })
+              Logger.info("Deleted #{delete_slice.length} #{target_name} log files")
+            end
           end
         end
 

data/lib/openc3/models/cvt_model.rb

@@ -26,16 +26,7 @@ module OpenC3
     @overrides = {}
 
     def self.build_json_from_packet(packet)
-      json_hash = {}
-      packet.sorted_items.each do |item|
-        json_hash[item.name] = packet.read_item(item, :RAW)
-        json_hash["#{item.name}__C"] = packet.read_item(item, :CONVERTED) if item.read_conversion or item.states
-        json_hash["#{item.name}__F"] = packet.read_item(item, :FORMATTED) if item.format_string
-        json_hash["#{item.name}__U"] = packet.read_item(item, :WITH_UNITS) if item.units
-        limits_state = item.limits.state
-        json_hash["#{item.name}__L"] = limits_state if limits_state
-      end
-      json_hash
+      packet.decom
     end
 
     # Delete the current value table for a target

data/lib/openc3/models/interface_model.rb

@@ -277,5 +277,46 @@ module OpenC3
       end
       status_model.destroy if status_model
     end
+
+    def unmap_target(target_name)
+      target_name = target_name.to_s.upcase
+
+      # Remove from this interface
+      @target_names.delete(target_name)
+      update()
+
+      # Respawn the microservice
+      type = self.class._get_type
+      microservice_name = "#{@scope}__#{type}__#{@name}"
+      microservice = MicroserviceModel.get_model(name: microservice_name, scope: scope)
+      microservice.target_names.delete(target_name)
+      microservice.update
+    end
+
+    def map_target(target_name)
+      target_name = target_name.to_s.upcase
+
+      # Remove from old interface
+      all_interfaces = InterfaceModel.all(scope: scope)
+      old_interface = nil
+      all_interfaces.each do |old_interface_name, old_interface_details|
+        if old_interface_details['target_names'].include?(target_name)
+          old_interface = InterfaceModel.from_json(old_interface_details, scope: scope)
+          break
+        end
+      end
+      old_interface.unmap_target(target_name) if old_interface
+
+      # Add to this interface
+      @target_names << target_name unless @target_names.include?(target_name)
+      update()
+
+      # Respawn the microservice
+      type = self.class._get_type
+      microservice_name = "#{@scope}__#{type}__#{@name}"
+      microservice = MicroserviceModel.get_model(name: microservice_name, scope: scope)
+      microservice.target_names << target_name unless microservice.target_names.include?(target_name)
+      microservice.update
+    end
   end
 end

data/lib/openc3/models/microservice_model.rb

@@ -26,8 +26,15 @@ module OpenC3
     PRIMARY_KEY = 'openc3_microservices'
 
     attr_accessor :cmd
-    attr_accessor :options
+    attr_accessor :container
+    attr_accessor :env
+    attr_accessor :folder_name
     attr_accessor :needs_dependencies
+    attr_accessor :options
+    attr_accessor :target_names
+    attr_accessor :topics
+    attr_accessor :work_dir
+    attr_accessor :ports
 
     # NOTE: The following three class methods are used by the ModelController
     # and are reimplemented to enable various Model class methods to work
@@ -75,6 +82,7 @@ module OpenC3
       folder_name: nil,
       cmd: [],
       work_dir: '.',
+      ports: [],
       env: {},
       topics: [],
       target_names: [],
@@ -97,6 +105,7 @@ module OpenC3
       @folder_name = folder_name
       @cmd = cmd
       @work_dir = work_dir
+      @ports = ports
       @env = env
       @topics = topics
       @target_names = target_names
@@ -111,6 +120,7 @@ module OpenC3
         'folder_name' => @folder_name,
         'cmd' => @cmd,
         'work_dir' => @work_dir,
+        'ports' => @ports,
         'env' => @env,
         'topics' => @topics,
         'target_names' => @target_names,
@@ -126,6 +136,9 @@ module OpenC3
       result = "MICROSERVICE #{@folder_name ? @folder_name : 'nil'} #{@name.split("__")[-1]}\n"
       result << "  CMD #{@cmd.join(' ')}\n"
       result << "  WORK_DIR \"#{@work_dir}\"\n"
+      @ports.each do |port|
+        result << "  PORT #{port}\n"
+      end
       @topics.each do |topic_name|
         result << "  TOPIC #{topic_name}\n"
       end
@@ -150,6 +163,25 @@ module OpenC3
       when 'WORK_DIR'
         parser.verify_num_parameters(1, 1, "#{keyword} <Dir>")
         @work_dir = parameters[0]
+      when 'PORT'
+        usage = "PORT <Number> <Protocol (Optional)"
+        parser.verify_num_parameters(1, 2, usage)
+        begin
+          @ports << [Integer(parameters[0])]
+        rescue => err # In case Integer fails
+          raise ConfigParser::Error.new(parser, "Port must be an integer: #{parameters[0]}", usage)
+        end
+        protocol = ConfigParser.handle_nil(parameters[1])
+        if protocol
+          # Per https://kubernetes.io/docs/concepts/services-networking/service/#protocol-support
+          if %w(TCP UDP SCTP HTTP SCTP).include?(protocol.upcase)
+            @ports[-1] << protocol.upcase
+          else
+            raise ConfigParser::Error.new(parser, "Unknown port protocol: #{parameters[1]}", usage)
+          end
+        else
+          @ports[-1] << 'TCP'
+        end
       when 'TOPIC'
         parser.verify_num_parameters(1, 1, "#{keyword} <Topic Name>")
         @topics << parameters[0]

data/lib/openc3/models/plugin_model.rb

@@ -251,7 +251,7 @@ module OpenC3
     end
 
     def create(update: false, force: false)
-      @name = @name + "__#{Time.now.utc.strftime("%Y%m%d%H%M%S")}" unless update
+      @name = @name + "__#{Time.now.utc.strftime("%Y%m%d%H%M%S")}" if not update and not @name.index("__")
       super(update: update, force: force)
     end