openc3 5.2.0 → 5.4.0

data/lib/openc3/api/target_api.rb

@@ -28,7 +28,8 @@ module OpenC3
     WHITELIST.concat([
                        'get_target_list',
                        'get_target',
-                       'get_all_target_info',
+                       'get_target_interfaces',
+                       'get_all_target_info', # DEPRECATED
                      ])
 
     # Returns the list of all target names
@@ -49,9 +50,29 @@ module OpenC3
       TargetModel.get(name: target_name, scope: scope)
     end
 
-    # Get information about all targets
+    # Get all targets and their interfaces
     #
-    # @return [Array<Array<String, Numeric, Numeric>] Array of Arrays \[name, interface, cmd_cnt, tlm_cnt]
+    # @return [Array<Array<String, String] Array of Arrays \[name, interfaces]
+    def get_target_interfaces(scope: $openc3_scope, token: $openc3_token)
+      authorize(permission: 'system', scope: scope, token: token)
+      info = []
+      interfaces = InterfaceModel.all(scope: scope)
+      get_target_list(scope: scope, token: token).each do |target_name|
+        interface_names = []
+        interfaces.each do |name, interface|
+          if interface['target_names'].include? target_name
+            interface_names << interface['name']
+          end
+        end
+        info << [target_name, interface_names.join(",")]
+      end
+      info
+    end
+
+    # DEPRECATED: Get information about all targets
+    # Warning this call can take a long time with many defined packets
+    #
+    # @return [Array<Array<String, String, Numeric, Numeric>] Array of Arrays \[name, interface, cmd_cnt, tlm_cnt]
     def get_all_target_info(scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'system', scope: scope, token: token)
       info = []

data/lib/openc3/api/tlm_api.rb

@@ -118,7 +118,8 @@ module OpenC3
     # @param type [Symbol] Telemetry type, :RAW, :CONVERTED (default), :FORMATTED, or :WITH_UNITS
     def inject_tlm(target_name, packet_name, item_hash = nil, type: :CONVERTED, scope: $openc3_scope, token: $openc3_token)
       authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, scope: scope, token: token)
-      unless CvtModel::VALUE_TYPES.include?(type.intern)
+      type = type.to_s.intern
+      unless CvtModel::VALUE_TYPES.include?(type)
         raise "Unknown type '#{type}' for #{target_name} #{packet_name}"
       end
 
@@ -161,8 +162,8 @@ module OpenC3
     # @param args The args must either be a string followed by a value or
     #   three strings followed by a value (see the calling style in the
     #   description).
-    # @param type [Symbol] Telemetry type, :RAW, :CONVERTED (default), :FORMATTED, or :WITH_UNITS
-    def override_tlm(*args, type: :CONVERTED, scope: $openc3_scope, token: $openc3_token)
+    # @param type [Symbol] Telemetry type, :ALL (default), :RAW, :CONVERTED, :FORMATTED, :WITH_UNITS
+    def override_tlm(*args, type: :ALL, scope: $openc3_scope, token: $openc3_token)
       target_name, packet_name, item_name, value = set_tlm_process_args(args, __method__, scope: scope)
       authorize(permission: 'tlm_set', target_name: target_name, packet_name: packet_name, scope: scope, token: token)
       CvtModel.override(target_name, packet_name, item_name, value, type: type.intern, scope: scope)
@@ -179,7 +180,7 @@ module OpenC3
     #
     # @param args The args must either be a string or three strings
     #   (see the calling style in the description).
-    # @param type [Symbol] Telemetry type, :RAW, :CONVERTED (default), :FORMATTED, or :WITH_UNITS
+    # @param type [Symbol] Telemetry type, :ALL (default), :RAW, :CONVERTED, :FORMATTED, :WITH_UNITS
     #   Also takes :ALL which means to normalize all telemetry types
     def normalize_tlm(*args, type: :ALL, scope: $openc3_scope, token: $openc3_token)
       target_name, packet_name, item_name = tlm_process_args(args, __method__, scope: scope)

data/lib/openc3/config/config_parser.rb

@@ -219,8 +219,6 @@ module OpenC3
                    size,
                    PARSING_REGEX,
                    &block)
-      rescue Exception => e # Catch EVERYTHING so we can re-raise with additional info
-        raise e, "#{e}\n\nParsed output in #{file.path}", e.backtrace
       ensure
         file.close unless file.closed?
       end
@@ -446,6 +444,22 @@ module OpenC3
       value
     end
 
+    def parse_errors(errors)
+      return if errors.empty?
+      message = ''
+      errors.each do |error|
+        if error.is_a? OpenC3::ConfigParser::Error
+          message += "\n#{File.basename(error.filename)}:#{error.line_number}: #{error.line}"
+          message += "\nError: #{error.message}"
+          message += "\nUsage: #{error.usage}" unless error.usage.empty?
+        else
+          message += "\n#{error.message}"
+        end
+        message += "\n"
+      end
+      raise message
+    end
+
     if RUBY_ENGINE != 'ruby' or ENV['OPENC3_NO_EXT']
       # Iterates over each line of the io object and yields the keyword and parameters
       def parse_loop(io, yield_non_keyword_lines, remove_quotes, size, rx)
@@ -454,6 +468,7 @@ module OpenC3
         @keyword = nil
         @parameters = []
         @line = ''
+        errors = []
 
         while true
           @line_number += 1
@@ -516,7 +531,11 @@ module OpenC3
           # Ignore lines without keywords: comments and blank lines
           if @keyword.nil?
             if yield_non_keyword_lines
-              yield(@keyword, @parameters)
+              begin
+                yield(@keyword, @parameters)
+              rescue => error
+                errors << error
+              end
             end
             @line = ''
             next
@@ -545,10 +564,16 @@ module OpenC3
             end
           end
 
-          yield(@keyword, @parameters)
+          begin
+            yield(@keyword, @parameters)
+          rescue => error
+            errors << error
+          end
           @line = ''
         end
 
+        parse_errors(errors)
+
         @@progress_callback.call(1.0) if @@progress_callback
 
         return nil

data/lib/openc3/core_ext/time.rb

@@ -17,7 +17,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'date'
@@ -58,6 +58,7 @@ class Time
   MINUTES_PER_HOUR = 60
   HOURS_PER_DAY = 24
   NSEC_PER_SECOND = 1_000_000_000
+  NSEC_PER_MSEC = 1_000_000
   USEC_PER_SECOND = USEC_PER_MSEC * MSEC_PER_SECOND
   MSEC_PER_MINUTE = 60 * MSEC_PER_SECOND
   MSEC_PER_HOUR = 60 * MSEC_PER_MINUTE
@@ -508,4 +509,8 @@ class Time
     nanoseconds = nsec_from_epoch % NSEC_PER_SECOND
     Time.at(seconds, nanoseconds, :nsec)
   end
+
+  def to_msec_from_epoch
+    (self.tv_sec * MSEC_PER_SECOND) + (self.tv_nsec / NSEC_PER_MSEC)
+  end
 end

data/lib/openc3/interfaces/interface.rb

@@ -22,6 +22,7 @@
 
 require 'openc3/api/api'
 require 'openc3/io/raw_logger_pair'
+require 'openc3/utilities/secrets'
 
 module OpenC3
   # Defines all the attributes and methods common to all interface classes
@@ -112,9 +113,6 @@ module OpenC3
     # @return [Array<[Protocol Class, Protocol Args, Protocol kind (:READ, :WRITE, :READ_WRITE)>] Info to recreate protocols
     attr_accessor :protocol_info
 
-    # @return [Hash or nil] Hash of overridden telemetry points
-    attr_accessor :override_tlm
-
     # @return [String] Most recently read raw data
     attr_accessor :read_raw_data
 
@@ -134,6 +132,9 @@ module OpenC3
     #   (when used as a BridgeRouter)
     attr_accessor :interfaces
 
+    # @return [Secrets] Interface secrets manager class
+    attr_accessor :secrets
+
     # Initialize default attribute values
     def initialize
       @name = self.class.to_s.split("::")[-1] # Remove namespacing if present
@@ -166,13 +167,13 @@ module OpenC3
       @read_protocols = []
       @write_protocols = []
       @protocol_info = []
-      @override_tlm = nil
       @read_raw_data = ''
       @written_raw_data = ''
       @read_raw_data_time = nil
       @written_raw_data_time = nil
       @config_params = []
       @interfaces = []
+      @secrets = Secrets.getClient
     end
 
     # Connects the interface to its target(s). Must be implemented by a
@@ -406,13 +407,13 @@ module OpenC3
       # num_clients is per interface so don't copy
       # read_queue_size is the number of packets in the queue so don't copy
       # write_queue_size is the number of packets in the queue so don't copy
-      other_interface.options = self.options.clone
+      self.options.each do |option_name, option_values|
+        other_interface.set_option(option_name, option_values)
+      end
       other_interface.protocol_info = []
       self.protocol_info.each do |protocol_class, protocol_args, read_write|
         other_interface.add_protocol(protocol_class, protocol_args, read_write)
       end
-      other_interface.override_tlm = nil
-      other_interface.override_tlm = self.override_tlm.clone if self.override_tlm
     end
 
     # Set an interface or router specific option
@@ -484,28 +485,28 @@ module OpenC3
       protocol.interface = self
     end
 
-    def _override_tlm(target_name, packet_name, item_name, value)
-      _override(target_name, packet_name, item_name, value, :CONVERTED)
+    def interface_cmd(cmd_name, *cmd_args)
+      # Default do nothing - Implemented by subclasses
+      return false
     end
 
-    def _override_tlm_raw(target_name, packet_name, item_name, value)
-      _override(target_name, packet_name, item_name, value, :RAW)
-    end
-
-    def _normalize_tlm(target_name, packet_name, item_name)
-      @override_tlm ||= {}
-      pkt = @override_tlm[target_name]
-      if pkt
-        items = @override_tlm[target_name][packet_name]
-        items.delete(item_name) if items
+    def protocol_cmd(cmd_name, *cmd_args, read_write: :READ_WRITE, index: -1)
+      read_write = read_write.to_s.upcase.intern
+      protocols = nil
+      case read_write
+      when :READ, :READ_WRITE
+        protocols = @read_protocols
+      when :WRITE
+        protocols = @write_protocols.reverse # Reverse so ordering matches configuration ordering
+      else
+        raise "Unknown protocol descriptor: #{read_write}. Must be :READ, :WRITE, or :READ_WRITE."
       end
-    end
-
-    def _override(target_name, packet_name, item_name, value, type)
-      @override_tlm ||= {}
-      @override_tlm[target_name] ||= {}
-      @override_tlm[target_name][packet_name] ||= {}
-      @override_tlm[target_name][packet_name][item_name] = [value, type]
+      handled = false
+      protocols.each_with_index do |protocol, protocol_index|
+        result = protocol.protocol_cmd(cmd_name, @cmd_args) if index == protocol_index or index == -1
+        handled = true if result
+      end
+      return handled
     end
   end
 end

data/lib/openc3/interfaces/mqtt_interface.rb

@@ -0,0 +1,240 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+# You can quickly setup an unauthenticated MQTT server in Docker with
+# docker run -it -p 1883:1883 eclipse-mosquitto:2.0.15 mosquitto -c /mosquitto-no-auth.conf
+
+require 'openc3/interfaces/interface'
+require 'openc3/config/config_parser'
+require 'mqtt'
+
+# Patches to the Ruby MQTT library so that it will work reliably with COSMOS
+saved_verbose = $VERBOSE
+$VERBOSE = nil
+module MQTT
+  class Client
+    def get(topic = nil, options = {})
+      if block_given?
+        get_packet(topic) do |packet|
+          yield(packet.topic, packet.payload) unless packet.retain && options[:omit_retained]
+        end
+      else
+        loop do
+          # Wait for one packet to be available
+          packet = get_packet(topic)
+          return nil unless packet # Patch for COSMOS
+          return packet.topic, packet.payload unless packet.retain && options[:omit_retained]
+        end
+      end
+    end
+
+    def get_packet(topic = nil)
+      # Subscribe to a topic, if an argument is given
+      subscribe(topic) unless topic.nil?
+
+      if block_given?
+        # Loop forever!
+        loop do
+          packet = @read_queue.pop
+          return nil unless packet # Patch for COSMOS
+          yield(packet)
+          puback_packet(packet) if packet.qos > 0
+        end
+      else
+        # Wait for one packet to be available
+        packet = @read_queue.pop
+        return nil unless packet # Patch for COSMOS
+        puback_packet(packet) if packet.qos > 0
+        return packet
+      end
+    end
+
+    def disconnect(send_msg = true)
+      # Stop reading packets from the socket first
+      @read_thread.kill if @read_thread && @read_thread.alive?
+      @read_thread = nil
+
+      @read_queue << nil # Patch for COSMOS
+
+      # Close the socket if it is open
+      if connected?
+        if send_msg
+          packet = MQTT::Packet::Disconnect.new
+          send_packet(packet)
+        end
+        @socket.close unless @socket.nil?
+        @socket = nil
+      end
+    end
+
+  end
+end
+$VERBOSE = saved_verbose
+
+module OpenC3
+  # Base class for interfaces that send and receive messages over MQTT
+  class MqttInterface < Interface
+    # @param hostname [String] MQTT server to connect to
+    # @param port [Integer] MQTT port
+    # @param ssl [Boolean] Use SSL true/false
+    def initialize(hostname, port = 1883, ssl = false)
+      super()
+      @hostname = hostname
+      @port = Integer(port)
+      @ssl = ConfigParser.handle_true_false(ssl)
+      @username = nil
+      @password = nil
+      @cert = nil
+      @key = nil
+      @ca_file = nil
+
+      @write_topics = []
+      @read_topics = []
+
+      # Build list of packets by topic
+      @read_packets_by_topic = {}
+      System.telemetry.all.each do |target_name, target_packets|
+        target_packets.each do |packet_name, packet|
+          topics = packet.meta['TOPIC']
+          topics = packet.meta['TOPICS'] unless topics
+          if topics
+            topics.each do |topic|
+              @read_packets_by_topic[topic] = packet
+            end
+          end
+        end
+      end
+    end
+
+    # Connects the interface to its target(s)
+    def connect
+      @write_topics = []
+      @read_topics = []
+      @client = MQTT::Client.new
+      @client.host = @hostname
+      @client.port = @port
+      @client.ssl = @ssl
+      @client.username = @username if @username
+      @client.password = @password if @password
+      @client.cert = @cert if @cert
+      @client.key = @key if @key
+      @client.ca_file = @ca_file.path if @ca_file
+      @client.connect
+      @read_packets_by_topic.each do |topic, _|
+        Logger.info "#{@name}: Subscribing to #{topic}"
+        @client.subscribe(topic)
+      end
+      super()
+    end
+
+    # @return [Boolean] Whether the active ports (read and/or write) have
+    #   created sockets. Since UDP is connectionless, creation of the sockets
+    #   is used to determine connection.
+    def connected?
+      if @client
+        return @client.connected?
+      else
+        return false
+      end
+    end
+
+    # Disconnects the interface from its target(s)
+    def disconnect
+      @client.disconnect
+      @client = nil
+      super()
+    end
+
+    def read
+      topic = @read_topics.shift
+      packet = super()
+      return nil unless packet
+      identified_packet = @read_packets_by_topic[topic]
+      if identified_packet
+        identified_packet = identified_packet.dup
+        identified_packet.buffer = packet.buffer
+        packet = identified_packet
+      end
+      packet.received_time = nil
+      return packet
+    end
+
+    def write(packet)
+      topics = packet.meta['TOPIC']
+      topics = packet.meta['TOPICS'] unless topics
+      if topics
+        topics.each do |topic|
+          @write_topics << topic
+          super(packet)
+        end
+      else
+        raise "Command packet #{packet.target_name} #{packet.packet_name} requires a META TOPIC or TOPICS"
+      end
+    end
+
+    # Reads from the socket if the read_port is defined
+    def read_interface
+      topic, data = @client.get
+      if data.nil? or data.length <= 0
+        Logger.info "#{@name}: read returned nil" if data.nil?
+        Logger.info "#{@name}: read returned 0 bytes" if not data.nil? and data.length <= 0
+        return nil
+      end
+      @read_topics << topic
+      read_interface_base(data)
+      return data
+    rescue IOError # Disconnected
+      return nil
+    end
+
+    # Writes to the socket
+    # @param data [String] Raw packet data
+    def write_interface(data)
+      write_interface_base(data)
+      topic = @write_topics.shift
+      @client.publish(topic, data)
+      data
+    end
+
+    # Supported Options
+    # USERNAME - Username for Mqtt Server
+    # PASSWORD - Password for Mqtt Server
+    # CERT - Public Key for Client Cert Auth
+    # KEY - Private Key for Client Cert Auth
+    # CA_FILE - Certificate Authority for Client Cert Auth
+    # (see Interface#set_option)
+    def set_option(option_name, option_values)
+      super(option_name, option_values)
+      case option_name.upcase
+      when 'USERNAME'
+        @username = option_values[0]
+      when 'PASSWORD'
+        @password = option_values[0]
+      when 'CERT'
+        @cert = option_values[0]
+      when 'KEY'
+        @key = option_values[0]
+      when 'CA_FILE'
+        # CA_FILE must be given as a file
+        @ca_file = Tempfile.new('ca_file')
+        @ca_file.write(option_values[0])
+        @ca_file.close
+      end
+    end
+  end
+end

data/lib/openc3/interfaces/protocols/override_protocol.rb

@@ -1,63 +1,4 @@
 # encoding: ascii-8bit
 
-# Copyright 2022 Ball Aerospace & Technologies Corp.
-# All Rights Reserved.
-#
-# This program is free software; you can modify and/or redistribute it
-# under the terms of the GNU Affero General Public License
-# as published by the Free Software Foundation; version 3 with
-# attribution addendums as found in the LICENSE.txt
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-
-# Modified by OpenC3, Inc.
-# All changes Copyright 2022, OpenC3, Inc.
-# All Rights Reserved
-#
-# This file may also be used under the terms of a commercial license
-# if purchased from OpenC3, Inc.
-
-require 'openc3/interfaces/protocols/protocol'
-
-module OpenC3
-  # Protocol which permanently overrides an item value such that reading the
-  # item returns the overriden value. Methods are prefixed with underscores
-  # so the API can include the original name which calls out to these
-  # methods. Clearing the override requires calling normalize_tlm.
-  class OverrideProtocol < Protocol
-    # @param allow_empty_data [true/false/nil] See Protocol#initialize
-    def initialize(allow_empty_data = nil)
-      super(allow_empty_data)
-    end
-
-    # Called to perform modifications on a read packet before it is given to the user
-    #
-    # @param packet [Packet] Original packet
-    # @return [Packet] Potentially modified packet
-    def read_packet(packet)
-      if @interface.override_tlm && !@interface.override_tlm.empty?
-        # Need to make sure packet is identified and defined
-        target_names = nil
-        target_names = @interface.tlm_target_names if @interface
-        identified_packet = System.telemetry.identify_and_define_packet(packet, target_names)
-        if identified_packet
-          packet = identified_packet
-          packets = @interface.override_tlm[packet.target_name]
-          if packets
-            items = packets[packet.packet_name]
-            if items
-              items.each do |item_name, value|
-                # This should be safe because we check at the API level it exists
-                packet.write(item_name, value[0], value[1])
-              end
-            end
-          end
-        end
-      end
-      return packet
-    end
-  end
-end
+# This class is deprecated and this file exists only to satisfy existing code requiring it
+# TODO: Remove this in a future release

data/lib/openc3/interfaces/protocols/protocol.rb

@@ -17,7 +17,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'openc3/config/config_parser'
@@ -81,5 +81,10 @@ module OpenC3
     def post_write_interface(packet, data)
       return packet, data
     end
+
+    def protocol_cmd(cmd_name, *cmd_args)
+      # Default do nothing - Implemented by subclasses
+      return false
+    end
   end
 end

data/lib/openc3/interfaces/simulated_target_interface.rb

@@ -17,11 +17,10 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'openc3/interfaces/interface'
-require 'openc3/interfaces/protocols/override_protocol'
 
 module OpenC3
   # An interface class that provides simulated telemetry and command responses
@@ -38,7 +37,6 @@ module OpenC3
       @sim_target = nil
       @write_raw_allowed = false
       @raw_logger_pair = nil
-      add_protocol(OverrideProtocol, [], :READ)
     end
 
     # Initialize the simulated target object and "connect" to the target

data/lib/openc3/interfaces/tcpip_server_interface.rb

@@ -58,8 +58,6 @@ module OpenC3
     attr_accessor :raw_logger_pair
     # @return [String] The ip address to bind to.  Default to ANY (0.0.0.0)
     attr_accessor :listen_address
-    # @return [boolean] Automatically send SYSTEM META on connect - Default false - Can be CMD/TLM
-    attr_accessor :auto_system_meta
 
     # @param write_port [Integer] The server write port. Clients should connect
     #   and expect to receive data from this port.
@@ -114,7 +112,6 @@ module OpenC3
       @raw_logging_enabled = false
       @connection_mutex = Mutex.new
       @listen_address = "0.0.0.0"
-      @auto_system_meta = false
 
       @read_allowed = false unless ConfigParser.handle_nil(read_port)
       @write_allowed = false unless ConfigParser.handle_nil(write_port)
@@ -282,15 +279,12 @@ module OpenC3
 
     # Supported Options
     # LISTEN_ADDRESS - Ip address of the interface to accept connections on - Default: 0.0.0.0
-    # AUTO_SYSTEM_META - Automatically send SYSTEM META on connect - Default false
     # (see Interface#set_option)
     def set_option(option_name, option_values)
       super(option_name, option_values)
       case option_name.upcase
       when 'LISTEN_ADDRESS'
         @listen_address = option_values[0]
-      when 'AUTO_SYSTEM_META'
-        @auto_system_meta = ConfigParser.handle_true_false(option_values[0])
       end
     end
 
@@ -411,11 +405,6 @@ module OpenC3
       interface.connect
 
       if listen_write
-        if @auto_system_meta
-          meta_packet = System.telemetry.packet('SYSTEM', 'META').clone
-          interface.write(meta_packet)
-        end
-
         @write_connection_callback.call(interface) if @write_connection_callback
         @connection_mutex.synchronize do
           @write_interface_infos << InterfaceInfo.new(interface, hostname, host_ip, port)