openc3 7.3.0 → 7.4.0

data/lib/openc3/interfaces/interface.rb

@@ -15,6 +15,7 @@
 # This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
+require 'openc3/packets/packet'
 require 'openc3/api/api'
 require 'openc3/logs/stream_log_pair'
 require 'openc3/utilities/secrets'
@@ -433,8 +434,8 @@ module OpenC3
       config['name'] = @name
       config['state'] = @state
       config['clients'] = self.num_clients
-      config['txsize'] = @write_queue_size
-      config['rxsize'] = @read_queue_size
+      config['txsize'] = self.write_queue_size
+      config['rxsize'] = self.read_queue_size
       config['txbytes'] = @bytes_written
       config['rxbytes'] = @bytes_read
       config['txcnt'] = @write_count
@@ -677,12 +678,12 @@ module OpenC3
       result['read_raw_data'] = @read_raw_data
       result['written_raw_data'] = @written_raw_data
       if @read_raw_data_time
-        result['read_raw_data_time'] = @read_raw_data_time.iso8601
+        result['read_raw_data_time'] = @read_raw_data_time.getutc.iso8601(6)
       else
         result['read_raw_data_time'] = nil
       end
       if @written_raw_data_time
-        result['written_raw_data_time'] = @written_raw_data_time.iso8601
+        result['written_raw_data_time'] = @written_raw_data_time.getutc.iso8601(6)
       else
         result['written_raw_data_time'] = nil
       end

data/lib/openc3/interfaces/mqtt_stream_interface.rb

@@ -49,6 +49,8 @@ module OpenC3
 
     # Creates a new {SerialStream} using the parameters passed in the constructor
     def connect
+      # Cleanly close any existing stream rather than leaving it open
+      @stream.disconnect if @stream
       @stream = MqttStream.new(@hostname, @port, @ssl, @write_topic, @read_topic, @ack_timeout)
       @stream.username = @username if @username
       @stream.password = @password if @password

data/lib/openc3/interfaces/protocols/burst_protocol.rb

@@ -23,6 +23,14 @@ module OpenC3
   # Reads all data available on the interface and creates a packet
   # with that data.
   class BurstProtocol < Protocol
+    # Maximum number of bytes which will be buffered while waiting for a
+    # complete packet. This bounds the memory a peer can cause the interface
+    # to allocate by declaring an enormous packet length (LENGTH, PREIDENTIFIED)
+    # or by never sending a terminator (TERMINATED). Deliberately much larger
+    # than any realistic packet so it does not affect existing configurations.
+    # Override with the OPENC3_PROTOCOL_MAX_BUFFER_SIZE environment variable.
+    DEFAULT_MAX_BUFFER_SIZE = 100_000_000
+
     # @param discard_leading_bytes [Integer] The number of bytes to discard
     #   from the binary data after reading. Note that this is often
     #   used to remove a sync pattern from the final packet data.
@@ -37,6 +45,7 @@ module OpenC3
       @sync_pattern = ConfigParser.handle_nil(sync_pattern)
       @sync_pattern = @sync_pattern.hex_to_byte_string if @sync_pattern
       @fill_fields = ConfigParser.handle_true_false(fill_fields)
+      @max_buffer_size = Integer(ENV.fetch('OPENC3_PROTOCOL_MAX_BUFFER_SIZE', DEFAULT_MAX_BUFFER_SIZE))
     end
 
     def reset
@@ -60,6 +69,7 @@ module OpenC3
       # @return [String|nil] Data for a packet consisting of the bytes read
       def read_data(data, extra = nil)
         @data << data
+        check_buffer_size()
         @extra = extra unless (data.length == 0 and extra.nil?) # Maintain extra from last read read_data
 
         while true
@@ -192,6 +202,21 @@ module OpenC3
       return super(data, extra)
     end
 
+    # Raises if the accumulation buffer has grown past @max_buffer_size. Called
+    # from read_data by both the Ruby and the C implementations. Raising causes
+    # Interface#read to log the error and disconnect, which drops the offending
+    # peer and resets this protocol.
+    def check_buffer_size
+      length = @data.length
+      return if length <= @max_buffer_size
+
+      # Release the buffer and clear any partially reduced state now rather than
+      # waiting for the interface to disconnect and reset() us
+      reset()
+      raise "#{@interface ? @interface.name : ""}: Protocol buffer of #{length} bytes exceeds maximum of #{@max_buffer_size} bytes. " \
+            "Increase OPENC3_PROTOCOL_MAX_BUFFER_SIZE."
+    end
+
     def log_discard(length, found)
       Logger.error("#{@interface ? @interface.name : ""}: Sync #{'not ' unless found}found. Discarding #{length} bytes of data.")
       if @data.length >= 0
@@ -210,6 +235,7 @@ module OpenC3
       result['discard_leading_bytes'] = @discard_leading_bytes
       result['sync_pattern'] = @sync_pattern.inspect
       result['fill_fields'] = @fill_fields
+      result['max_buffer_size'] = @max_buffer_size
       return result
     end
 
@@ -218,6 +244,7 @@ module OpenC3
       result['discard_leading_bytes'] = @discard_leading_bytes
       result['sync_pattern'] = @sync_pattern.inspect
       result['fill_fields'] = @fill_fields
+      result['max_buffer_size'] = @max_buffer_size
       return result
     end
   end

data/lib/openc3/interfaces/protocols/length_protocol.rb

@@ -168,6 +168,13 @@ module OpenC3
       raise "Length value received larger than max_length: #{length} > #{@max_length}" if @max_length and length > @max_length
 
       packet_length = (length * @length_bytes_per_count) + @length_value_offset
+      # Reject up front rather than buffering the declared number of bytes first
+      if packet_length > @max_buffer_size
+        reset() # Drop the partial frame rather than holding it until disconnect
+        raise "Calculated packet length of #{packet_length} bytes exceeds maximum buffer size of #{@max_buffer_size} bytes. " \
+              "Increase OPENC3_PROTOCOL_MAX_BUFFER_SIZE."
+      end
+
       # Ensure the calculated packet length is long enough to support the location of the length field
       # without overlap into the next packet
       if (packet_length * 8) < (@length_bit_offset + @length_bit_size)

data/lib/openc3/interfaces/protocols/preidentified_protocol.rb

@@ -106,6 +106,13 @@ module OpenC3
       when 4
         string_length = string_length.unpack('N')[0] # UINT32
         raise "Length value received larger than max_length: #{string_length} > #{@max_length}" if @max_length and string_length > @max_length
+
+        # Reject up front rather than buffering the declared number of bytes first
+        if string_length > @max_buffer_size
+          reset() # Drop the partial frame rather than holding it until disconnect
+          raise "Length value received of #{string_length} bytes exceeds maximum buffer size of #{@max_buffer_size} bytes. " \
+                "Increase OPENC3_PROTOCOL_MAX_BUFFER_SIZE."
+        end
       else
         raise "Unsupported length given to read_length_field_followed_by_string: #{length_num_bytes}"
       end

data/lib/openc3/interfaces/protocols/protocol.rb

@@ -131,13 +131,13 @@ module OpenC3
     def write_details
       result = {'name' => self.class.name.to_s.split("::")[-1]}
       if @write_data_input_time
-        result['write_data_input_time'] = @write_data_input_time.iso8601
+        result['write_data_input_time'] = @write_data_input_time.getutc.iso8601(6)
       else
         result['write_data_input_time'] = nil
       end
       result['write_data_input'] = @write_data_input
       if @write_data_output_time
-        result['write_data_output_time'] = @write_data_output_time.iso8601
+        result['write_data_output_time'] = @write_data_output_time.getutc.iso8601(6)
       else
         result['write_data_output_time'] = nil
       end
@@ -148,13 +148,13 @@ module OpenC3
     def read_details
       result = {'name' => self.class.name.to_s.split("::")[-1]}
       if @read_data_input_time
-        result['read_data_input_time'] = @read_data_input_time.iso8601
+        result['read_data_input_time'] = @read_data_input_time.getutc.iso8601(6)
       else
         result['read_data_input_time'] = nil
       end
       result['read_data_input'] = @read_data_input
       if @read_data_output_time
-        result['read_data_output_time'] = @read_data_output_time.iso8601
+        result['read_data_output_time'] = @read_data_output_time.getutc.iso8601(6)
       else
         result['read_data_output_time'] = nil
       end

data/lib/openc3/interfaces/protocols/template_protocol.rb

@@ -266,7 +266,7 @@ module OpenC3
       result['response_timeout'] = @response_timeout
       result['response_polling_period'] = @response_polling_period
       if @connect_complete_time
-        result['connect_complete_time'] = @connect_complete_time.iso8601
+        result['connect_complete_time'] = @connect_complete_time.getutc.iso8601(6)
       else
         result['connect_complete_time'] = nil
       end
@@ -285,7 +285,7 @@ module OpenC3
       result['response_timeout'] = @response_timeout
       result['response_polling_period'] = @response_polling_period
       if @connect_complete_time
-        result['connect_complete_time'] = @connect_complete_time.iso8601
+        result['connect_complete_time'] = @connect_complete_time.getutc.iso8601(6)
       else
         result['connect_complete_time'] = nil
       end

data/lib/openc3/interfaces/serial_interface.rb

@@ -77,6 +77,8 @@ module OpenC3
 
     # Creates a new {SerialStream} using the parameters passed in the constructor
     def connect
+      # Cleanly close any existing stream rather than leaving it open
+      @stream.disconnect if @stream
       @stream = SerialStream.new(
         @write_port_name,
         @read_port_name,

data/lib/openc3/interfaces/tcpip_client_interface.rb

@@ -76,6 +76,8 @@ module OpenC3
     # Connects the {TcpipClientStream} by passing the
     # initialization parameters to the {TcpipClientStream}.
     def connect
+      # Cleanly close any existing stream rather than leaving it open
+      @stream.disconnect if @stream
       @stream = TcpipClientStream.new(
         @hostname,
         @write_port,

data/lib/openc3/interfaces/udp_interface.rb

@@ -23,6 +23,7 @@ module OpenC3
   # Base class for interfaces that send and receive messages over UDP
   class UdpInterface < Interface
     HOST_127_0_0_1 = '127.0.0.1'
+    HOST_0_0_0_0 = '0.0.0.0'
 
     # @param hostname [String] Machine to connect to
     # @param write_dest_port [Integer] Port to write commands to
@@ -77,8 +78,11 @@ module OpenC3
       @read_timeout = ConfigParser.handle_nil(read_timeout)
       @read_timeout = @read_timeout.to_f if @read_timeout
       @bind_address = ConfigParser.handle_nil(bind_address)
-      if @bind_address && @bind_address.casecmp('LOCALHOST').zero?
-        @bind_address = HOST_127_0_0_1
+      if @bind_address
+        @bind_address = HOST_127_0_0_1 if @bind_address.casecmp('LOCALHOST').zero?
+      else
+        # nil means all local addresses which is what 0.0.0.0 means
+        @bind_address = HOST_0_0_0_0
       end
       @write_socket = nil
       @read_socket = nil
@@ -91,9 +95,9 @@ module OpenC3
       result = ''
       result += " #{@hostname}:#{@write_dest_port} (write dest port)" if @write_dest_port
       result += " #{@write_src_port} (write src port)" if @write_src_port
-      result += " #{@hostname}:#{@read_port} (read)" if @read_port
+      result += " #{@bind_address}:#{@read_port} (read)" if @read_port
       result += " #{@interface_address} (interface addr)" if @interface_address
-      result += " #{@bind_address} (bind addr)" if @bind_address != '0.0.0.0'
+      result += " #{@bind_address} (bind addr)" if @bind_address != HOST_0_0_0_0
       return result.strip
     end
 
@@ -102,13 +106,20 @@ module OpenC3
     # the constructor.
     def connect
       if @read_port and @write_dest_port and @write_src_port and (@read_port == @write_src_port)
+        # read_multicast is false because this socket is not connected and thus
+        # doesn't filter by peer address. Joining the multicast group we write to
+        # would deliver our own commands back to us as telemetry (multicast
+        # loopback is enabled by default).
         @read_socket = UdpReadWriteSocket.new(
           @read_port,
           @bind_address,
           @write_dest_port,
           @hostname,
           @interface_address,
-          @ttl
+          @ttl,
+          false, # read_multicast
+          true,  # write_multicast
+          false  # connect_socket
         )
         @write_socket = @read_socket
       else

data/lib/openc3/io/json_api.rb

@@ -35,8 +35,9 @@ module OpenC3
     # pull openc3-cosmos-script-runner-api url from environment variables
     def _generate_url(microservice_name:, prefix:, schema: 'http', hostname: nil, port:, scope: $openc3_scope)
       prefix = '/' + prefix unless prefix[0] == '/'
-      if ENV['OPENC3_OPERATOR_HOSTNAME']
-        hostname = ENV['OPENC3_OPERATOR_HOSTNAME'] unless hostname
+      operator_hostname = ENV.fetch('OPENC3_OPERATOR_HOSTNAME', nil)
+      if operator_hostname
+        hostname = operator_hostname unless hostname
         return "#{schema}://#{hostname}:#{port.to_i}#{prefix}"
       else
         if ENV['KUBERNETES_SERVICE_HOST']

data/lib/openc3/io/json_drb.rb

@@ -282,10 +282,26 @@ module OpenC3
               response = JsonRpcSuccessResponse.new(result, request.id)
             end
           rescue Exception => e
-            # Filter out the framework stack trace (rails, rack, puma etc)
-            lines = e.formatted.split("\n")
-            i = lines.find_index { |row| row.include?('actionpack') || row.include?('activesupport') }
-            Logger.error lines[0...i].join("\n")
+            error_name = e.class.name.split('::')[-1]
+            method_name = request.method.downcase()
+            case e
+            when AuthError, ForbiddenError
+              # Auth failures are expected (e.g. browser requests before login or with an
+              # expired session token) so log a single line without the stack trace
+              Logger.warn "#{error_name} : #{e.message} calling #{method_name}"
+            when HazardousError, CriticalCmdError
+              # Not errors at all: these are the signal that the UI should raise an
+              # operator confirmation dialog, and the user retries with the hazardous
+              # or critical flag set. A stack trace per prompt is pure noise, and
+              # HazardousError#to_s embeds the formatted command with a newline,
+              # so log just the error name and the method that raised it.
+              Logger.info "#{error_name} calling #{method_name}"
+            else
+              # Filter out the framework stack trace (rails, rack, puma etc)
+              lines = e.formatted.split("\n")
+              i = lines.find_index { |row| row.include?('actionpack') || row.include?('activesupport') }
+              Logger.error lines[0...(i || lines.length)].join("\n")
+            end
 
             if request.id
               if NoMethodError === e

data/lib/openc3/io/udp_sockets.rb

@@ -27,6 +27,9 @@ module OpenC3
   class UdpReadWriteSocket
     HOST_0_0_0_0 = '0.0.0.0'
 
+    # MSG_DONTWAIT is not defined on Windows where sends are left blocking
+    WRITE_FLAGS = Socket.const_defined?('MSG_DONTWAIT') ? Socket::MSG_DONTWAIT : 0
+
     # @param bind_port [Integer[ Port to write data out from and receive data on (0 = randomly assigned)
     # @param bind_address [String] Local address to bind to (0.0.0.0 = All local addresses)
     # @param external_port [Integer] External port to write to
@@ -35,6 +38,8 @@ module OpenC3
     # @param ttl [Integer] Time To Live for outgoing multicast packets
     # @param read_multicast [Boolean] Whether or not to try to read from the external address as multicast
     # @param write_multicast [Boolean] Whether or not to write to the external address as multicast
+    # @param connect_socket [Boolean] Whether to connect the socket to the external address. If false,
+    #   writes explicitly address each datagram so reads can accept datagrams from any source.
     def initialize(
       bind_port = 0,
       bind_address = HOST_0_0_0_0,
@@ -43,10 +48,23 @@ module OpenC3
       multicast_interface_address = nil,
       ttl = 1,
       read_multicast = true,
-      write_multicast = true
+      write_multicast = true,
+      connect_socket = true
     )
 
       @socket = UDPSocket.new
+      @external_address = external_address
+      @external_port = external_port
+      @connect_socket = connect_socket
+      # Resolve the destination once at creation time so writes don't pay for a
+      # (potentially blocking) name lookup on every datagram
+      @external_sockaddr = nil
+      if !connect_socket and external_address and external_port
+        # Explicitly resolve IPv4 because UDPSocket is an AF_INET socket and
+        # sockaddr_in would otherwise hand back an IPv6 address for names like localhost
+        ip_address = Socket.getaddrinfo(external_address, nil, Socket::AF_INET, Socket::SOCK_DGRAM)[0][3]
+        @external_sockaddr = Socket.sockaddr_in(external_port, ip_address)
+      end
 
       # Basic setup to reuse address
       @socket.setsockopt(Socket::SOL_SOCKET, Socket::SO_REUSEADDR, 1)
@@ -55,10 +73,10 @@ module OpenC3
       @socket.bind(bind_address, bind_port) if bind_address and bind_port
 
       # Default send to the specified address and port
-      @socket.connect(external_address, external_port) if external_address and external_port
+      @socket.connect(external_address, external_port) if connect_socket and external_address and external_port
 
       # Handle multicast
-      if UdpReadWriteSocket.multicast?(external_address, external_port)
+      if UdpReadWriteSocket.multicast?(external_address)
         if write_multicast
           # Basic setup set time to live
           @socket.setsockopt(Socket::IPPROTO_IP, Socket::IP_MULTICAST_TTL, ttl.to_i)
@@ -90,7 +108,13 @@ module OpenC3
 
       loop do
         begin
-          bytes_sent = @socket.write_nonblock(data_to_send)
+          if @external_sockaddr
+            # send is used rather than sendmsg_nonblock because sendmsg is not
+            # implemented on Windows
+            bytes_sent = @socket.send(data_to_send, WRITE_FLAGS, @external_sockaddr)
+          else
+            bytes_sent = @socket.write_nonblock(data_to_send)
+          end
         rescue Errno::EAGAIN, Errno::EWOULDBLOCK
           result = IO.fast_select(nil, [@socket], nil, write_timeout)
           if result
@@ -129,12 +153,18 @@ module OpenC3
     end
 
     # @param host [String] Machine name or IP address
-    # @param port [String] Port
+    # @param port [String] Port (unused, kept for backwards compatibility)
     # @return [Boolean] Whether the hostname is multicast
-    def self.multicast?(host, port)
-      return false if host.nil? || port.nil?
+    def self.multicast?(host, port = nil)
+      return false if host.nil?
 
-      Addrinfo.udp(host, port).ipv4_multicast?
+      begin
+        Addrinfo.udp(host, 0).ipv4_multicast?
+      rescue SocketError, ArgumentError
+        # Hostname isn't resolvable so it can't be a multicast address we handle.
+        # Reads don't need the hostname at all so don't fail creating the socket.
+        false
+      end
     end
   end
 

data/lib/openc3/io/win32_serial_driver.rb

@@ -16,6 +16,7 @@
 # if purchased from OpenC3, Inc.
 
 require 'openc3/win32/win32'
+require 'openc3/io/serial_driver'
 require 'timeout' # For Timeout::Error
 
 module OpenC3

data/lib/openc3/logs/packet_log_reader.rb

@@ -142,7 +142,7 @@ module OpenC3
         end
       elsif flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_RAW_PACKET_ENTRY_TYPE_MASK
         packet_index, time_nsec_since_epoch = entry[2..11].unpack('nQ>')
-        received_time_nsec_since_epoch, _extra, packet_data = handle_received_time_extra_and_data(entry, time_nsec_since_epoch, includes_received_time, includes_extra, cbor)
+        received_time_nsec_since_epoch, extra, packet_data = handle_received_time_extra_and_data(entry, time_nsec_since_epoch, includes_received_time, includes_extra, cbor)
         lookup_cmd_or_tlm, target_name, packet_name, _id = @packets[packet_index]
         if cmd_or_tlm != lookup_cmd_or_tlm
           raise "Packet type mismatch, packet:#{cmd_or_tlm}, lookup:#{lookup_cmd_or_tlm}"
@@ -159,6 +159,10 @@ module OpenC3
         packet.set_received_time_fast(received_time)
         packet.cmd_or_tlm = cmd_or_tlm
         packet.stored = stored
+        # Assigned unconditionally because identify_and_define_packet_data
+        # returns the shared System packet, so a nil extra here must clear
+        # the extra left over from a previous read of the same packet
+        packet.extra = extra
         packet.received_count += 1
         return packet
       elsif flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_TARGET_DECLARATION_ENTRY_TYPE_MASK
@@ -304,7 +308,24 @@ module OpenC3
         if cbor
           extra = CBOR.decode(extra_encoded)
         else
-          extra = JSON.parse(extra_encoded, allow_nan: true, create_additions: true)
+          begin
+            extra = JSON.parse(extra_encoded, allow_nan: true, create_additions: true)
+          rescue JSON::ParserError => e
+            # PacketLogWriter up to version 7.3.0 CBOR encoded the extra on
+            # RAW_PACKET entries without setting OPENC3_CBOR_FLAG_MASK, so the
+            # flag claims JSON while the bytes are CBOR. Retry as CBOR so those
+            # existing files can still be read. extra is always a Hash, so a
+            # decode to anything else means these bytes aren't the legacy
+            # encoding either and the original JSON error is the useful one.
+            decoded = begin
+              CBOR.decode(extra_encoded)
+            rescue StandardError
+              raise e
+            end
+            raise e unless Hash === decoded
+
+            extra = decoded
+          end
         end
       end
       data = entry[next_offset..-1]

data/lib/openc3/logs/packet_log_writer.rb

@@ -311,6 +311,11 @@ module OpenC3
           extra = JSON.parse(extra, allow_nan: true, create_additions: true) if String === extra
           length += OPENC3_EXTRA_LENGTH_FIXED_SIZE
           if @data_format == :CBOR
+            # The reader uses this same flag to pick the extra decoder, so it
+            # must be set here as well as in the JSON_PACKET data branch above.
+            # RAW_PACKET entries never reach that branch and would otherwise
+            # write CBOR extra that the reader tries to JSON.parse.
+            flags |= OPENC3_CBOR_FLAG_MASK
             extra_encoded = extra.as_json.to_cbor
           else
             extra_encoded = JSON.generate(extra.as_json, allow_nan: true)

data/lib/openc3/logs/stream_log.rb

@@ -47,7 +47,7 @@ module OpenC3
       raise "log_type must be :READ or :WRITE" unless LOG_TYPES.include? log_type
 
       super(
-        "#{ENV['OPENC3_SCOPE']}/stream_logs/",
+        "#{ENV.fetch('OPENC3_SCOPE', 'DEFAULT')}/stream_logs/",
         true, # Start with logging enabled
         cycle_time,
         cycle_size,

data/lib/openc3/microservices/interface_decom_common.rb

@@ -33,7 +33,8 @@ module OpenC3
       end
       stored = inject_tlm_hash.fetch('stored', false)
       packet.stored = stored.to_s.downcase == 'true'
-      packet.received_time = Time.now.sys
+      received_time = inject_tlm_hash['received_time']
+      packet.received_time = received_time.nil? ? Time.now.sys : Time.from_nsec_from_epoch(received_time).sys
       packet.received_count = TargetModel.increment_telemetry_count(packet.target_name, packet.packet_name, 1, scope: @scope)
       TelemetryTopic.write_packet(packet, scope: @scope)
     end