openc3 5.12.0 → 5.14.0

data/lib/openc3/interfaces/interface.rb

@@ -198,14 +198,16 @@ module OpenC3
             log_dont_log.upcase!
             period = "#{period.to_f}s"
             @scheduler.every period do
-              begin
-                if log_dont_log == 'DONT_LOG'
-                  cmd(cmd_string, log_message: false)
-                else
-                  cmd(cmd_string)
+              if connected?()
+                begin
+                  if log_dont_log == 'DONT_LOG'
+                    cmd(cmd_string, log_message: false)
+                  else
+                    cmd(cmd_string)
+                  end
+                rescue Exception => err
+                  Logger.error("Error sending periodic cmd(#{cmd_string}):\n#{err.formatted}")
                 end
-              rescue Exception => err
-                Logger.error("Error sending periodic cmd(#{cmd_string}):\n#{err.formatted}")
               end
             end
           end

data/lib/openc3/interfaces/mqtt_interface.rb

@@ -161,8 +161,8 @@ module OpenC3
     end
 
     def read
-      topic = @read_topics.shift
       packet = super()
+      topic = @read_topics.shift
       return nil unless packet
       identified_packet = @read_packets_by_topic[topic]
       if identified_packet
@@ -175,15 +175,17 @@ module OpenC3
     end
 
     def write(packet)
-      topics = packet.meta['TOPIC']
-      topics = packet.meta['TOPICS'] unless topics
-      if topics
-        topics.each do |topic|
-          @write_topics << topic
-          super(packet)
+      @write_mutex.synchronize do
+        topics = packet.meta['TOPIC']
+        topics = packet.meta['TOPICS'] unless topics
+        if topics
+          topics.each do |topic|
+            @write_topics << topic
+            super(packet)
+          end
+        else
+          raise "Command packet #{packet.target_name} #{packet.packet_name} requires a META TOPIC or TOPICS"
         end
-      else
-        raise "Command packet #{packet.target_name} #{packet.packet_name} requires a META TOPIC or TOPICS"
       end
     end
 

data/lib/openc3/interfaces/mqtt_stream_interface.rb

@@ -0,0 +1,78 @@
+# encoding: ascii-8bit
+
+# Copyright 2023 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/interfaces/stream_interface'
+require 'openc3/streams/mqtt_stream'
+
+module OpenC3
+  class MqttStreamInterface < StreamInterface
+    # @param hostname [String] MQTT server to connect to
+    # @param port [Integer] MQTT port
+    # @param ssl [Boolean] Use SSL true/false
+    def initialize(hostname, port = 1883, ssl = false, write_topic = nil, read_topic = nil, protocol_type = nil, *protocol_args)
+      super(protocol_type, protocol_args)
+      @hostname = hostname
+      @port = Integer(port)
+      @ssl = ConfigParser.handle_true_false(ssl)
+      @write_topic = ConfigParser.handle_nil(write_topic)
+      @read_topic = ConfigParser.handle_nil(read_topic)
+      @username = nil
+      @password = nil
+      @cert = nil
+      @key = nil
+      @ca_file = nil
+    end
+
+    # Creates a new {SerialStream} using the parameters passed in the constructor
+    def connect
+      @stream = MqttStream.new(@hostname, @port, @ssl, @write_topic, @read_topic)
+      @stream.username = @username if @username
+      @stream.password = @password if @password
+      @stream.cert = @cert if @cert
+      @stream.key = @key if @key
+      @stream.ca_file = @ca_file if @ca_file
+      super()
+    end
+
+    # Supported Options
+    # USERNAME - Username for Mqtt Server
+    # PASSWORD - Password for Mqtt Server
+    # CERT - Public Key for Client Cert Auth
+    # KEY - Private Key for Client Cert Auth
+    # CA_FILE - Certificate Authority for Client Cert Auth
+    # (see Interface#set_option)
+    def set_option(option_name, option_values)
+      super(option_name, option_values)
+      case option_name.upcase
+      when 'USERNAME'
+        @username = option_values[0]
+      when 'PASSWORD'
+        @password = option_values[0]
+      when 'CERT'
+        @cert = option_values[0]
+      when 'KEY'
+        @key = option_values[0]
+      when 'CA_FILE'
+        # CA_FILE must be given as a file
+        @ca_file = Tempfile.new('ca_file')
+        @ca_file.write(option_values[0])
+        @ca_file.close
+      end
+    end
+  end
+end

data/lib/openc3/interfaces/protocols/cmd_response_protocol.rb

@@ -0,0 +1,116 @@
+# encoding: ascii-8bit
+
+# Copyright 2024 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/config/config_parser'
+require 'openc3/interfaces/protocols/protocol'
+require 'thread' # For Queue
+require 'timeout' # For Timeout::Error
+
+module OpenC3
+  # Protocol that waits for a response for any commands with a defined response packet.
+  # The response packet is identified but not defined by the protocol.
+  class CmdResponseProtocol < Protocol
+    # @param response_timeout [Float] Number of seconds to wait before timing out
+    #   when waiting for a response
+    # @param response_polling_period [Float] Number of seconds to wait between polling
+    #   for a response
+    # @param raise_exceptions [String] Whether to raise exceptions when errors
+    #   occur in the protocol like unexpected responses or response timeouts.
+    # @param allow_empty_data [true/false/nil] See Protocol#initialize
+    def initialize(
+      response_timeout = 5.0,
+      response_polling_period = 0.02,
+      raise_exceptions = false,
+      allow_empty_data = nil
+    )
+      super(allow_empty_data)
+      @response_timeout = ConfigParser.handle_nil(response_timeout)
+      @response_timeout = @response_timeout.to_f if @response_timeout
+      @response_polling_period = response_polling_period.to_f
+      @raise_exceptions = ConfigParser.handle_true_false(raise_exceptions)
+      @write_block_queue = Queue.new
+      @response_packet = nil
+    end
+
+    def connect_reset
+      super()
+      @write_block_queue.clear
+    end
+
+    def disconnect_reset
+      super()
+      @write_block_queue << nil # Unblock the write block queue
+    end
+
+    def read_packet(packet)
+      if @response_packet
+        # Grab the response packet specified in the command
+        result_packet = System.telemetry.packet(@response_packet[0], @response_packet[1]).clone
+        result_packet.buffer = packet.buffer
+        result_packet.received_time = nil
+        result_packet.stored = packet.stored
+        result_packet.extra = packet.extra
+
+        # Release the write
+        @write_block_queue << nil
+
+        # This returns the fully identified and defined packet
+        # Received time is handled by the interface microservice
+        return result_packet
+      else
+        return packet
+      end
+    end
+
+    def write_packet(packet)
+      # Setup the response packet (if there is one)
+      # This primes waiting for the response in post_write_interface
+      @response_packet = packet.response
+
+      return packet
+    end
+
+    def post_write_interface(packet, data, extra = nil)
+      if @response_packet
+        if @response_timeout
+          response_timeout_time = Time.now + @response_timeout
+        else
+          response_timeout_time = nil
+        end
+
+        # Block the write until the response is received
+        begin
+          @write_block_queue.pop(true)
+        rescue
+          sleep(@response_polling_period)
+          retry if !response_timeout_time
+          retry if response_timeout_time and Time.now < response_timeout_time
+          handle_error("#{@interface ? @interface.name : ""}: Timeout waiting for response")
+        end
+
+        @response_packet = nil
+      end
+      return super(packet, data, extra)
+    end
+
+    def handle_error(msg)
+      Logger.error(msg)
+      raise msg if @raise_exceptions
+    end
+  end
+end

data/lib/openc3/interfaces/tcpip_client_interface.rb

@@ -67,6 +67,10 @@ module OpenC3
         @write_timeout,
         @read_timeout
       )
+      # Pass down options to the stream
+      @options.each do |option_name, option_values|
+        @stream.set_option(option_name, option_values)
+      end
       super()
     end
   end

data/lib/openc3/interfaces/tcpip_server_interface.rb

@@ -391,6 +391,11 @@ module OpenC3
       read_socket = socket if listen_read
       stream = TcpipSocketStream.new(write_socket, read_socket, @write_timeout, @read_timeout)
 
+      # Pass down options to the stream
+      @options.each do |option_name, option_values|
+        stream.set_option(option_name, option_values)
+      end
+
       interface = StreamInterface.new
       interface.target_names = @target_names
       interface.cmd_target_names = @cmd_target_names

data/lib/openc3/interfaces.rb

@@ -42,7 +42,7 @@ module OpenC3
   autoload(:PreidentifiedProtocol, 'openc3/interfaces/protocols/preidentified_protocol.rb')
   autoload(:TemplateProtocol, 'openc3/interfaces/protocols/template_protocol.rb')
   autoload(:TerminatedProtocol, 'openc3/interfaces/protocols/terminated_protocol.rb')
-
+  autoload(:CmdResponseProtocol, 'openc3/interfaces/protocols/cmd_response_protocol.rb')
   autoload(:CrcProtocol, 'openc3/interfaces/protocols/crc_protocol.rb')
   autoload(:IgnorePacketProtocol, 'openc3/interfaces/protocols/ignore_packet_protocol.rb')
 end

data/lib/openc3/logs/packet_log_reader.rb

@@ -298,12 +298,12 @@ module OpenC3
       next_offset = 12
       received_time_nsec_since_epoch = time_nsec_since_epoch
       if includes_received_time
-        received_time_nsec_since_epoch = entry[next_offset..(next_offset + OPENC3_RECEIVED_TIME_FIXED_SIZE - 1)].unpack(OPENC3_RECEIVED_TIME_PACK_DIRECTIVE)
+        received_time_nsec_since_epoch = entry[next_offset..(next_offset + OPENC3_RECEIVED_TIME_FIXED_SIZE - 1)].unpack(OPENC3_RECEIVED_TIME_PACK_DIRECTIVE)[0]
         next_offset += OPENC3_RECEIVED_TIME_FIXED_SIZE
       end
       extra = nil
       if includes_extra
-        extra_length = entry[next_offset..(next_offset + OPENC3_EXTRA_LENGTH_FIXED_SIZE - 1)].unpack(OPENC3_EXTRA_LENGTH_PACK_DIRECTIVE)
+        extra_length = entry[next_offset..(next_offset + OPENC3_EXTRA_LENGTH_FIXED_SIZE - 1)].unpack(OPENC3_EXTRA_LENGTH_PACK_DIRECTIVE)[0]
         next_offset += OPENC3_EXTRA_LENGTH_FIXED_SIZE
         extra_encoded = entry[next_offset..(next_offset + extra_length - 1)]
         next_offset += extra_length

data/lib/openc3/logs/text_log_writer.rb

@@ -78,8 +78,9 @@ module OpenC3
       begin
         # Need to write the OFFSET_MARKER for each packet
         @last_offsets.each do |redis_topic, last_offset|
-          time = Time.now
-          data = { time: time.to_nsec_from_epoch, '@timestamp' => time.xmlschema(3), severity: 'INFO', "microservice_name" => Logger.microservice_name, "container_name" => @container_name, "last_offset" => last_offset, "redis_topic" => redis_topic, "type" => "offset" }
+          time = Time.now.utc
+          # timestamp iso8601 with 6 decimal places to match the python output format
+          data = { time: time.to_nsec_from_epoch, '@timestamp' => time.iso8601(6), level: 'INFO', "microservice_name" => Logger.microservice_name, "container_name" => @container_name, "last_offset" => last_offset, "redis_topic" => redis_topic, "type" => "offset" }
           write_entry(time.to_nsec_from_epoch, data.as_json(allow_nan: true).to_json(allow_nan: true)) if @file
         end
 

data/lib/openc3/microservices/decom_microservice.rb

@@ -100,6 +100,7 @@ module OpenC3
           packet.extra = extra
         end
         packet.buffer = msg_hash["buffer"]
+        packet.process # Run processors
         packet.check_limits(System.limits_set) # Process all the limits and call the limits_change_callback (as necessary)
 
         TelemetryDecomTopic.write_packet(packet, scope: @scope)

data/lib/openc3/microservices/interface_microservice.rb

@@ -613,7 +613,16 @@ module OpenC3
 
     def connect
       @logger.info "#{@interface.name}: Connecting ..."
-      @interface.connect
+      begin
+        @interface.connect
+      rescue Exception => error
+        begin
+          @interface.disconnect # Ensure disconnect is called at least once on a partial connect
+        rescue Exception
+          # We want to report any connect errors, not disconnect in this case
+        end
+        raise error
+      end
       @interface.state = 'CONNECTED'
       if @interface_or_router == 'INTERFACE'
         InterfaceStatusModel.set(@interface.as_json(:allow_nan => true), scope: @scope)

data/lib/openc3/microservices/trigger_group_microservice.rb

@@ -333,7 +333,8 @@ module OpenC3
       )
       return nil if packet.nil?
       _, limit = packet.read_with_limits_state(operand[ITEM_TYPE], operand[ITEM_VALUE_TYPE].intern)
-      return limit
+      # Convert limit symbol to string since we'll be comparing with strings
+      return limit.to_s
     end
 
     # extract the value outlined in the operand to get the packet item value

data/lib/openc3/models/cvt_model.rb

@@ -52,12 +52,14 @@ module OpenC3
 
     # Get the hash for packet in the CVT
     # Note: Does not apply overrides
-    def self.get(target_name:, packet_name:, cache_timeout: 0.1, scope: $openc3_scope)
+    def self.get(target_name:, packet_name:, cache_timeout: nil, scope: $openc3_scope)
       key = "#{scope}__tlm__#{target_name}"
       tgt_pkt_key = key + "__#{packet_name}"
-      cache_time, hash = @@packet_cache[tgt_pkt_key]
       now = Time.now
-      return hash if hash and (now - cache_time) < cache_timeout
+      if cache_timeout
+        cache_time, hash = @@packet_cache[tgt_pkt_key]
+        return hash if hash and (now - cache_time) < cache_timeout
+      end
       packet = Store.hget(key, packet_name)
       raise "Packet '#{target_name} #{packet_name}' does not exist" unless packet
       hash = JSON.parse(packet, :allow_nan => true, :create_additions => true)
@@ -67,7 +69,7 @@ module OpenC3
 
     # Set an item in the current value table
     def self.set_item(target_name, packet_name, item_name, value, type:, scope: $openc3_scope)
-      hash = get(target_name: target_name, packet_name: packet_name, cache_timeout: 0.0, scope: scope)
+      hash = get(target_name: target_name, packet_name: packet_name, cache_timeout: nil, scope: scope)
       case type
       when :WITH_UNITS
         hash["#{item_name}__U"] = value.to_s # WITH_UNITS should always be a string
@@ -89,10 +91,10 @@ module OpenC3
     end
 
     # Get an item from the current value table
-    def self.get_item(target_name, packet_name, item_name, type:, cache_timeout: 0.1, scope: $openc3_scope)
+    def self.get_item(target_name, packet_name, item_name, type:, cache_timeout: nil, scope: $openc3_scope)
       result, types = self._handle_item_override(target_name, packet_name, item_name, type: type, cache_timeout: cache_timeout, scope: scope)
       return result if result
-      hash = get(target_name: target_name, packet_name: packet_name, scope: scope)
+      hash = get(target_name: target_name, packet_name: packet_name, cache_timeout: cache_timeout, scope: scope)
       hash.values_at(*types).each do |result|
         if result
           if type == :FORMATTED or type == :WITH_UNITS
@@ -109,7 +111,7 @@ module OpenC3
     # @param items [Array<String>] Items to return. Must be formatted as TGT__PKT__ITEM__TYPE
     # @param stale_time [Integer] Time in seconds from Time.now that value will be marked stale
     # @return [Array] Array of values
-    def self.get_tlm_values(items, stale_time: 30, cache_timeout: 0.1, scope: $openc3_scope)
+    def self.get_tlm_values(items, stale_time: 30, cache_timeout: nil, scope: $openc3_scope)
       now = Time.now
       results = []
       lookups = []
@@ -246,7 +248,7 @@ module OpenC3
       end
     end
 
-    def self.determine_latest_packet_for_item(target_name, item_name, cache_timeout: 0.1, scope: $openc3_scope)
+    def self.determine_latest_packet_for_item(target_name, item_name, cache_timeout: nil, scope: $openc3_scope)
       item_map = TargetModel.get_item_to_packet_map(target_name, scope: scope)
       packet_names = item_map[item_name]
       raise "Item '#{target_name} LATEST #{item_name}' does not exist for scope: #{scope}" unless packet_names
@@ -293,10 +295,12 @@ module OpenC3
     end
 
     def self._get_overrides(now, tgt_pkt_key, overrides, target_name, packet_name, cache_timeout:, scope:)
-      cache_time, hash = @@override_cache[tgt_pkt_key]
-      if hash and (now - cache_time) < cache_timeout
-        overrides[tgt_pkt_key] = hash
-        return hash
+      if cache_timeout
+        cache_time, hash = @@override_cache[tgt_pkt_key]
+        if hash and (now - cache_time) < cache_timeout
+          overrides[tgt_pkt_key] = hash
+          return hash
+        end
       end
       override_data = Store.hget("#{scope}__override__#{target_name}", packet_name)
       if override_data

data/lib/openc3/models/gem_model.rb

@@ -39,11 +39,17 @@ module OpenC3
     include Api
 
     def self.names
-      result = Pathname.new("#{ENV['GEM_HOME']}/gems").children.select { |c| c.directory? }.collect { |p| File.basename(p) + '.gem' }
+      if Dir.exist?("#{ENV['GEM_HOME']}/gems")
+        result = Pathname.new("#{ENV['GEM_HOME']}/gems").children.select { |c| c.directory? }.collect { |p| File.basename(p) + '.gem' }
+      else
+        result = []
+      end
       return result.sort
     end
 
     def self.get(name)
+      path = "#{ENV['GEM_HOME']}/cosmoscache/#{name}"
+      return path if File.exist?(path)
       path = "#{ENV['GEM_HOME']}/cache/#{name}"
       return path if File.exist?(path)
       raise "Gem #{name} not found"
@@ -52,8 +58,9 @@ module OpenC3
     def self.put(gem_file_path, gem_install: true, scope:)
       if File.file?(gem_file_path)
         gem_filename = File.basename(gem_file_path)
-        FileUtils.mkdir_p("#{ENV['GEM_HOME']}/cache") unless Dir.exist?("#{ENV['GEM_HOME']}/cache")
-        FileUtils.cp(gem_file_path, "#{ENV['GEM_HOME']}/cache/#{File.basename(gem_file_path)}")
+        # Put into cosmoscache folder that we control
+        FileUtils.mkdir_p("#{ENV['GEM_HOME']}/cosmoscache") unless Dir.exist?("#{ENV['GEM_HOME']}/cosmoscache")
+        FileUtils.cp(gem_file_path, "#{ENV['GEM_HOME']}/cosmoscache/#{File.basename(gem_file_path)}")
         if gem_install
           Logger.info "Installing gem: #{gem_filename}"
           result = OpenC3::ProcessManager.instance.spawn(["ruby", "/openc3/bin/openc3cli", "geminstall", gem_filename, scope], "package_install", gem_filename, Time.now + 3600.0, scope: scope)
@@ -118,5 +125,15 @@ module OpenC3
       version = File.basename(split_name[-1], '.gem')
       return gem_name, version
     end
+
+    def self.destroy_all_other_versions(name)
+      keep_gem_name, keep_gem_version = GemModel.extract_name_and_version(name)
+      GemModel.names.each do |gem_full_name|
+        gem_name, gem_version = GemModel.extract_name_and_version(gem_full_name)
+        if gem_name == keep_gem_name and gem_version != keep_gem_version
+          GemModel.destroy(gem_full_name)
+        end
+      end
+    end
   end
 end

data/lib/openc3/models/microservice_model.rb

@@ -185,7 +185,7 @@ module OpenC3
         @topics << parameters[0]
       when 'TARGET_NAME'
         parser.verify_num_parameters(1, 1, "#{keyword} <Target Name>")
-        @target_names << parameters[0]
+        @target_names << parameters[0].upcase
       when 'CMD'
         parser.verify_num_parameters(1, nil, "#{keyword} <Args>")
         @cmd = parameters.dup

data/lib/openc3/models/plugin_model.rb

@@ -158,6 +158,11 @@ module OpenC3
           gem_file_path = OpenC3::GemModel.get(gem_name)
         end
 
+        # Attempt to remove all older versions of this same plugin before install to prevent version conflicts
+        # Especially on downgrades
+        # Leave the same version if it already exists
+        OpenC3::GemModel.destroy_all_other_versions(File.basename(gem_file_path))
+
         # Actually install the gem now (slow)
         OpenC3::GemModel.install(gem_file_path, scope: scope) unless validate_only
 
@@ -296,10 +301,15 @@ module OpenC3
 
     # Undeploy all models associated with this plugin
     def undeploy
+      errors = []
       microservice_count = 0
       microservices = MicroserviceModel.find_all_by_plugin(plugin: @name, scope: @scope)
       microservices.each do |name, model_instance|
-        model_instance.destroy
+        begin
+          model_instance.destroy
+        rescue Exception => error
+          errors << error
+        end
         microservice_count += 1
       end
       # Wait for the operator to wake up and remove the microservice processes
@@ -308,21 +318,49 @@ module OpenC3
       # Save TargetModel for last as it has the most to cleanup
       [InterfaceModel, RouterModel, ToolModel, WidgetModel, TargetModel].each do |model|
         model.find_all_by_plugin(plugin: @name, scope: @scope).each do |name, model_instance|
-          model_instance.destroy
+          begin
+            model_instance.destroy
+          rescue Exception => error
+            errors << error
+          end
         end
       end
       # Cleanup Redis stuff that might have been left by microservices
       microservices.each do |name, model_instance|
-        model_instance.cleanup
+        begin
+          model_instance.cleanup
+        rescue Exception => error
+          errors << error
+        end
+      end
+      # Raise all the errors at once
+      if errors.length > 0
+        message = ''
+        errors.each do |error|
+          message += "\n#{error.formatted}\n"
+        end
+        raise message
       end
     rescue Exception => error
-      Logger.error("Error undeploying plugin model #{@name} in scope #{@scope} due to #{error}")
+      Logger.error("Error undeploying plugin model #{@name} in scope #{@scope} due to: #{error}")
+    ensure
+      # Double check everything is gone
+      found = []
+      [MicroserviceModel, InterfaceModel, RouterModel, ToolModel, WidgetModel, TargetModel].each do |model|
+        model.find_all_by_plugin(plugin: @name, scope: @scope).each do |name, model_instance|
+          found << model_instance
+        end
+      end
+      if found.length > 0
+        # If undeploy failed we need to not move forward with anything else
+        Logger.error("Error undeploying plugin model #{@name} in scope #{@scope} due to: Plugin submodels still exist after undeploy = #{found.length}")
+        raise "Plugin #{@name} submodels still exist after undeploy = #{found.length}"
+      end
     end
 
     # Reinstall
     def restore
       plugin_hash = self.as_json(:allow_nan => true)
-      plugin_hash['name'] = plugin_hash['name'].split("__")[0]
       OpenC3::PluginModel.install_phase2(plugin_hash, scope: @scope)
       @destroyed = false
     end