openc3 5.10.0 → 5.11.0

data/lib/openc3/interfaces/protocols/terminated_protocol.rb

@@ -57,14 +57,14 @@ module OpenC3
       super(discard_leading_bytes, sync_pattern, fill_fields, allow_empty_data)
     end
 
-    def write_data(data)
+    def write_data(data, extra = nil)
       raise "Packet contains termination characters!" if data.index(@write_termination_characters)
 
-      data = super(data)
+      data, extra = super(data, extra)
       @write_termination_characters.each_byte do |byte|
         data << byte
       end
-      return data
+      return data, extra
     end
 
     protected
@@ -88,7 +88,7 @@ module OpenC3
           end
         end
         @data.replace(@data[(index + @read_termination_characters.length)..-1])
-        return packet_data
+        return packet_data, @extra
       else
         return :STOP
       end

data/lib/openc3/interfaces/simulated_target_interface.rb

@@ -54,6 +54,7 @@ module OpenC3
       # Save the current time + delta as the next expected tick time
       @next_tick_time = Time.now.sys + @sim_target.tick_period_seconds
 
+      super()
       @connected = true
     end
 
@@ -150,6 +151,7 @@ module OpenC3
     # Disconnect from the simulator
     def disconnect
       @connected = false
+      super()
     end
 
     protected

data/lib/openc3/interfaces/stream_interface.rb

@@ -72,13 +72,15 @@ module OpenC3
         return nil
       end
 
-      read_interface_base(data)
-      data
+      extra = nil
+      read_interface_base(data, extra)
+      return data, extra
     end
 
-    def write_interface(data)
-      write_interface_base(data)
+    def write_interface(data, extra = nil)
+      write_interface_base(data, extra)
       @stream.write(data)
+      return data, extra
     end
   end
 end

data/lib/openc3/interfaces/tcpip_server_interface.rb

@@ -156,6 +156,7 @@ module OpenC3
         @write_thread = nil
         @write_raw_thread = nil
       end
+      super()
       @connected = true
     end
 
@@ -204,6 +205,7 @@ module OpenC3
 
       shutdown_interfaces(@write_interface_infos)
       @connected = false
+      super()
     end
 
     # Gracefully kill all the threads

data/lib/openc3/interfaces/udp_interface.rb

@@ -121,6 +121,7 @@ module OpenC3
         ) if @write_dest_port
       end
       @thread_sleeper = nil
+      super()
     end
 
     # @return [Boolean] Whether the active ports (read and/or write) have
@@ -146,6 +147,7 @@ module OpenC3
       @read_socket = nil
       @thread_sleeper.cancel if @thread_sleeper
       @thread_sleeper = nil
+      super()
     end
 
     def read
@@ -161,18 +163,19 @@ module OpenC3
     def read_interface
       data = @read_socket.read(@read_timeout)
       Logger.info "#{@name}: Udp read returned 0 bytes (stream closed)" if data.length <= 0
-      read_interface_base(data)
-      return data
+      extra = nil
+      read_interface_base(data, extra)
+      return data, extra
     rescue IOError # Disconnected
       return nil
     end
 
     # Writes to the socket
     # @param data [String] Raw packet data
-    def write_interface(data)
-      write_interface_base(data)
+    def write_interface(data, extra = nil)
+      write_interface_base(data, extra)
       @write_socket.write(data, @write_timeout)
-      data
+      return data, extra
     end
   end
 end

data/lib/openc3/interfaces.rb

@@ -22,6 +22,8 @@
 
 module OpenC3
   autoload(:Interface, 'openc3/interfaces/interface.rb')
+  autoload(:HttpClientInterface, 'openc3/interfaces/http_client_interface.rb')
+  autoload(:HttpServerInterface, 'openc3/interfaces/http_server_interface.rb')
   autoload(:MqttInterface, 'openc3/interfaces/mqtt_interface.rb')
   autoload(:StreamInterface, 'openc3/interfaces/stream_interface.rb')
   autoload(:SerialInterface, 'openc3/interfaces/serial_interface.rb')

data/lib/openc3/logs/buffered_packet_log_writer.rb

@@ -75,23 +75,22 @@ module OpenC3
     # @param data [String] Binary string of data
     # @param id [Integer] Target ID
     # @param redis_offset [Integer] The offset of this packet in its Redis stream
-    def buffered_write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0')
+    def buffered_write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0', received_time_nsec_since_epoch: nil, extra: nil)
       case entry_type
       when :RAW_PACKET, :JSON_PACKET
-        @buffer << [entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset]
+        @buffer << [entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset, received_time_nsec_since_epoch, extra]
         @buffer.sort! {|entry1, entry2| entry1[4] <=> entry2[4] }
         if @buffer.length >= @buffer_depth
           entry = @buffer.shift
-          write(*entry)
+          write(*entry[0..-3], received_time_nsec_since_epoch: entry[-2], extra: entry[-1])
         end
       else
-        write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset)
+        write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: extra)
       end
     end
 
     def buffered_first_time_nsec
-      time = first_time()
-      return time.to_nsec_from_epoch if time
+      return @first_time if @first_time
       return @buffer[0][4] if @buffer[0]
       return nil
     end
@@ -115,7 +114,7 @@ module OpenC3
     def write_buffer
       begin
         @buffer.each do |entry|
-          write(*entry, allow_new_file: false, take_mutex: false)
+          write(*entry[0..-3], allow_new_file: false, take_mutex: false, received_time_nsec_since_epoch: entry[-2], extra: entry[-1])
         end
       rescue => err
         Logger.instance.error "Error writing out buffer : #{err.formatted}"

data/lib/openc3/logs/log_writer.rb

@@ -350,20 +350,12 @@ module OpenC3
       '.log'.freeze
     end
 
-    def first_time
-      Time.from_nsec_from_epoch(@first_time)
-    end
-
-    def last_time
-      Time.from_nsec_from_epoch(@last_time)
-    end
-
     def first_timestamp
-      first_time().to_timestamp # "YYYYMMDDHHmmSSNNNNNNNNN"
+      Time.from_nsec_from_epoch(@first_time).to_timestamp # "YYYYMMDDHHmmSSNNNNNNNNN"
     end
 
     def last_timestamp
-      last_time().to_timestamp # "YYYYMMDDHHmmSSNNNNNNNNN"
+      Time.from_nsec_from_epoch(@last_time).to_timestamp # "YYYYMMDDHHmmSSNNNNNNNNN"
     end
   end
 end

data/lib/openc3/logs/packet_log_constants.rb

@@ -17,7 +17,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 module OpenC3
@@ -38,6 +38,8 @@ module OpenC3
     OPENC3_JSON_PACKET_ENTRY_TYPE_MASK = 0x4000
     OPENC3_OFFSET_MARKER_ENTRY_TYPE_MASK = 0x5000
     OPENC3_KEY_MAP_ENTRY_TYPE_MASK = 0x6000
+    OPENC3_RECEIVED_TIME_FLAG_MASK = 0x0040
+    OPENC3_EXTRA_FLAG_MASK = 0x0080
     OPENC3_CBOR_FLAG_MASK = 0x0100
     OPENC3_ID_FLAG_MASK = 0x0200
     OPENC3_STORED_FLAG_MASK = 0x0400
@@ -47,8 +49,8 @@ module OpenC3
     OPENC3_MAX_PACKET_INDEX = 65535
     OPENC3_MAX_TARGET_INDEX = 65535
 
-    OPENC3_PRIMARY_FIXED_SIZE = 2
-    OPENC3_TARGET_DECLARATION_SECONDARY_FIXED_SIZE = 0
+    OPENC3_PRIMARY_FIXED_SIZE = 2 # 2 bytes for flags - Size of length field is not included in length value
+    OPENC3_TARGET_DECLARATION_SECONDARY_FIXED_SIZE = 0 # No additional data beyond 'Nn' (Length, Flags)
     OPENC3_TARGET_DECLARATION_PACK_DIRECTIVE = 'Nn'.freeze
     OPENC3_TARGET_DECLARATION_PACK_ITEMS = 2 # Useful for testing
 
@@ -67,5 +69,13 @@ module OpenC3
     OPENC3_PACKET_SECONDARY_FIXED_SIZE = 10
     OPENC3_PACKET_PACK_DIRECTIVE = 'NnnQ>'.freeze
     OPENC3_PACKET_PACK_ITEMS = 4 # Useful for testing
+
+    OPENC3_RECEIVED_TIME_FIXED_SIZE = 8
+    OPENC3_RECEIVED_TIME_PACK_DIRECTIVE = 'Q>'.freeze
+    OPENC3_RECEIVED_TIME_PACK_ITEMS = 1 # Useful for testing
+
+    OPENC3_EXTRA_LENGTH_FIXED_SIZE = 4
+    OPENC3_EXTRA_LENGTH_PACK_DIRECTIVE = 'N'.freeze
+    OPENC3_EXTRA_LENGTH_PACK_ITEMS = 1 # Useful for testing
   end
 end

data/lib/openc3/logs/packet_log_reader.rb

@@ -127,35 +127,41 @@ module OpenC3
       id = true if flags & OPENC3_ID_FLAG_MASK == OPENC3_ID_FLAG_MASK
       cbor = false
       cbor = true if flags & OPENC3_CBOR_FLAG_MASK == OPENC3_CBOR_FLAG_MASK
+      includes_received_time = false
+      includes_received_time = true if flags & OPENC3_RECEIVED_TIME_FLAG_MASK == OPENC3_RECEIVED_TIME_FLAG_MASK
+      includes_extra = false
+      includes_extra = true if flags & OPENC3_EXTRA_FLAG_MASK == OPENC3_EXTRA_FLAG_MASK
 
       if flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_JSON_PACKET_ENTRY_TYPE_MASK
         packet_index, time_nsec_since_epoch = entry[2..11].unpack('nQ>')
-        json_data = entry[12..-1]
+        next_offset = 12
+        received_time_nsec_since_epoch, extra, json_data = handle_received_time_extra_and_data(entry, time_nsec_since_epoch, includes_received_time, includes_extra, cbor)
         lookup_cmd_or_tlm, target_name, packet_name, id, key_map = @packets[packet_index]
         if cmd_or_tlm != lookup_cmd_or_tlm
           raise "Packet type mismatch, packet:#{cmd_or_tlm}, lookup:#{lookup_cmd_or_tlm}"
         end
 
         if cbor
-          return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, CBOR.decode(json_data), key_map)
+          return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, CBOR.decode(json_data), key_map, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: extra)
         else
-          return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, json_data, key_map)
+          return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, json_data, key_map, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: extra)
         end
       elsif flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_RAW_PACKET_ENTRY_TYPE_MASK
         packet_index, time_nsec_since_epoch = entry[2..11].unpack('nQ>')
-        packet_data = entry[12..-1]
+        received_time_nsec_since_epoch, extra, packet_data = handle_received_time_extra_and_data(entry, time_nsec_since_epoch, includes_received_time, includes_extra, cbor)
         lookup_cmd_or_tlm, target_name, packet_name, id = @packets[packet_index]
         if cmd_or_tlm != lookup_cmd_or_tlm
           raise "Packet type mismatch, packet:#{cmd_or_tlm}, lookup:#{lookup_cmd_or_tlm}"
         end
 
-        received_time = Time.from_nsec_from_epoch(time_nsec_since_epoch)
         if identify_and_define
-          packet = identify_and_define_packet_data(cmd_or_tlm, target_name, packet_name, received_time, packet_data)
+          packet = identify_and_define_packet_data(cmd_or_tlm, target_name, packet_name, packet_data)
         else
           # Build Packet
           packet = Packet.new(target_name, packet_name, :BIG_ENDIAN, nil, packet_data)
         end
+        packet.packet_time = Time.from_nsec_from_epoch(time_nsec_since_epoch)
+        received_time = Time.from_nsec_from_epoch(received_time_nsec_since_epoch)
         packet.set_received_time_fast(received_time)
         packet.cmd_or_tlm = cmd_or_tlm
         packet.stored = stored
@@ -219,95 +225,6 @@ module OpenC3
       raise err
     end
 
-    # TODO: Currently not used
-    # Returns an analysis of the log file by reading all the packets and
-    # returning information about each packet. This information maps directly
-    # to the parameters need by the {#read_at_offset} method and thus should be
-    # called before using {#read_at_offset}.
-    #
-    # @param filename [String] The filename to analyze
-    # @param progress_callback [Proc] Callback that should receive a single
-    #   floating point parameter which is the percentage done
-    # @return [Array<Array<Integer, Integer, String, String, Time, Time>] Array
-    #   of arrays for each packet found in the log file consisting of:
-    #   [File position, length, target name, packet name, time formatted,
-    #   received time].
-    # def packet_offsets(filename, progress_callback = nil)
-    #   open(filename)
-    #   offsets = []
-    #   filesize = size().to_f
-
-    #   while true
-    #     current_pos = @file.pos
-    #     packet = read(false)
-    #     break unless packet
-    #     offsets << current_pos
-    #     if progress_callback
-    #       break if progress_callback.call(current_pos / filesize)
-    #     end
-    #   end
-
-    #   return offsets
-    # ensure
-    #   close()
-    # end
-
-    # TODO: Currently not used
-    # Reads a packet from the opened log file. Should only be used in
-    # conjunction with {#packet_offsets}.
-    #
-    # @param file_offset [Integer] Byte offset into the log file to start
-    #   reading
-    # @param identify_and_define (see #each)
-    # @return [Packet]
-    # def read_at_offset(file_offset, identify_and_define = true)
-    #   @file.seek(file_offset, IO::SEEK_SET)
-    #   return read(identify_and_define)
-    # rescue => err
-    #   close()
-    #   raise err
-    # end
-
-    # TODO: Currently not used
-    # Read the first packet from the log file and reset the file position back
-    # to the current position. This allows the client to call read multiple
-    # times to return packets, call first, and continue calling read which will
-    # return the next packet in the file.
-    #
-    # @return [Packet]
-    # def first
-    #   original_position = @file.pos
-    #   @file.seek(0, IO::SEEK_SET)
-    #   read_file_header()
-    #   packet = read()
-    #   raise "No first packet found" unless packet
-    #   @file.seek(original_position, IO::SEEK_SET)
-    #   packet.clone
-    # rescue => err
-    #   close()
-    #   raise err
-    # end
-
-    # TODO: Currently not used
-    # Read the last packet from the log file and reset the file position back
-    # to the current position. This allows the client to call read multiple
-    # times to return packets, call last, and continue calling read which will
-    # return the next packet in the file.
-    #
-    # @return [Packet]
-    # def last
-    #   raise "TODO: Implement me - Need to add end of file entry to support"
-    #   original_position = @file.pos
-    #   @file.seek(-1, IO::SEEK_END)
-    #   packet = search(-1)
-    #   raise "No last packet found" unless packet
-    #   @file.seek(original_position, IO::SEEK_SET)
-    #   packet.clone
-    # rescue => err
-    #   close()
-    #   raise err
-    # end
-
     # @return [Integer] The size of the log file being processed
     def size
       @file.stat.size
@@ -333,7 +250,7 @@ module OpenC3
     end
 
     # This is best effort. May return unidentified/undefined packets
-    def identify_and_define_packet_data(cmd_or_tlm, target_name, packet_name, received_time, packet_data)
+    def identify_and_define_packet_data(cmd_or_tlm, target_name, packet_name, packet_data)
       packet = nil
       unless target_name and packet_name
         if cmd_or_tlm == :CMD
@@ -376,8 +293,28 @@ module OpenC3
       end
     end
 
-    def seek_to_time(time)
-      raise "TODO: Implement me - Use index file or offsets"
+    # Handle common optional fields in raw and JSON packets
+    def handle_received_time_extra_and_data(entry, time_nsec_since_epoch, includes_received_time, includes_extra, cbor)
+      next_offset = 12
+      received_time_nsec_since_epoch = time_nsec_since_epoch
+      if includes_received_time
+        received_time_nsec_since_epoch = entry[next_offset..(next_offset + OPENC3_RECEIVED_TIME_FIXED_SIZE - 1)].unpack(OPENC3_RECEIVED_TIME_PACK_DIRECTIVE)
+        next_offset += OPENC3_RECEIVED_TIME_FIXED_SIZE
+      end
+      extra = nil
+      if includes_extra
+        extra_length = entry[next_offset..(next_offset + OPENC3_EXTRA_LENGTH_FIXED_SIZE - 1)].unpack(OPENC3_EXTRA_LENGTH_PACK_DIRECTIVE)
+        next_offset += OPENC3_EXTRA_LENGTH_FIXED_SIZE
+        extra_encoded = entry[next_offset..(next_offset + extra_length - 1)]
+        next_offset += extra_length
+        if cbor
+          extra = CBOR.decode(extra_encoded)
+        else
+          extra = JSON.parse(extra_encode, allow_nan: true, create_additions: true)
+        end
+      end
+      data = entry[next_offset..-1]
+      return received_time_nsec_since_epoch, extra, data
     end
   end
 end

data/lib/openc3/logs/packet_log_writer.rb

@@ -64,22 +64,15 @@ module OpenC3
         enforce_time_order
       )
       @label = label
-      @index_file = nil
-      @index_filename = nil
       @cmd_packet_table = {}
       @tlm_packet_table = {}
       @key_map_table = {}
       @target_dec_entries = []
       @packet_dec_entries = []
-      @key_map_entries = []
       @next_packet_index = 0
       @target_indexes = {}
       @next_target_index = 0
       @data_format = :CBOR # Default to CBOR for improved compression
-
-      # This is an optimization to avoid creating a new entry object
-      # each time we create an entry which we do a LOT!
-      @index_entry = String.new
     end
 
     # Write a packet to the log file.
@@ -97,13 +90,13 @@ module OpenC3
     # @param data [String] Binary string of data
     # @param id [Integer] Target ID
     # @param redis_offset [Integer] The offset of this packet in its Redis stream
-    def write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0', take_mutex: true, allow_new_file: true)
+    def write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0', take_mutex: true, allow_new_file: true, received_time_nsec_since_epoch: nil, extra: nil)
       return if !@logging_enabled
 
       @mutex.lock if take_mutex
       begin
         prepare_write(time_nsec_since_epoch, data.length, redis_topic, redis_offset, allow_new_file: allow_new_file)
-        write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id) if @file
+        write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: extra) if @file
       ensure
         @mutex.unlock if take_mutex
       end
@@ -112,7 +105,7 @@ module OpenC3
       OpenC3.handle_critical_exception(err)
     end
 
-    # Starting a new index file is a critical operation so the entire method is
+    # Starting a new file is a critical operation so the entire method is
     # wrapped with a rescue and handled with handle_critical_exception
     # Assumes mutex has already been taken
     def start_new_file
@@ -120,10 +113,6 @@ module OpenC3
       @file.write(OPENC3_FILE_HEADER)
       @file_size += OPENC3_FILE_HEADER.length
 
-      # Start index log file
-      @index_filename = create_unique_filename('.idx'.freeze)
-      @index_file = File.new(@index_filename, 'wb')
-      @index_file.write(OPENC3_INDEX_HEADER)
       @cmd_packet_table = {}
       @tlm_packet_table = {}
       @key_map_table = {}
@@ -132,8 +121,6 @@ module OpenC3
       @next_target_index = 0
       @target_dec_entries = []
       @packet_dec_entries = []
-      @key_map_entries = []
-      Logger.debug "Index Log File Opened : #{@index_filename}"
     rescue => err
       Logger.error "Error starting new log file: #{err.formatted}"
       @logging_enabled = false
@@ -151,22 +138,6 @@ module OpenC3
           write_entry(:OFFSET_MARKER, nil, nil, nil, nil, nil, last_offset + ',' + redis_topic, nil) if @file
         end
 
-        if @index_file
-          begin
-            write_index_file_footer()
-            @index_file.close unless @index_file.closed?
-            Logger.debug "Index Log File Closed : #{@index_filename}"
-            date = first_timestamp[0..7] # YYYYMMDD
-            bucket_key = File.join(@remote_log_directory, date, "#{first_timestamp}__#{last_timestamp}__#{@label}.idx")
-            threads << BucketUtilities.move_log_file_to_bucket(@index_filename, bucket_key)
-          rescue Exception => err
-            Logger.instance.error "Error closing #{@index_filename} : #{err.formatted}"
-          end
-
-          @index_file = nil
-          @index_filename = nil
-        end
-
         threads.concat(super(false))
 
       ensure
@@ -241,7 +212,7 @@ module OpenC3
       return packet_index
     end
 
-    def write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id)
+    def write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, received_time_nsec_since_epoch: nil, extra: nil)
       raise ArgumentError.new("Length of id must be 64, got #{id.length}") if id and id.length != 64 # 64 hex digits, gets packed to 32 bytes with .pack('H*')
 
       length = OPENC3_PRIMARY_FIXED_SIZE
@@ -283,7 +254,6 @@ module OpenC3
         packet_index = get_packet_index(cmd_or_tlm, target_name, packet_name, entry_type, data)
         @entry.clear
         @entry << [length, flags, packet_index].pack(OPENC3_KEY_MAP_PACK_DIRECTIVE) << data
-        @key_map_entries << @entry.dup
       when :OFFSET_MARKER
         flags |= OPENC3_OFFSET_MARKER_ENTRY_TYPE_MASK
         length += OPENC3_OFFSET_MARKER_SECONDARY_FIXED_SIZE + data.length
@@ -318,13 +288,28 @@ module OpenC3
         if cmd_or_tlm == :CMD
           flags |= OPENC3_CMD_FLAG_MASK
         end
+        if received_time_nsec_since_epoch
+          flags |= OPENC3_RECEIVED_TIME_FLAG_MASK
+          length += OPENC3_RECEIVED_TIME_FIXED_SIZE
+        end
+        extra_encoded = nil
+        if extra
+          flags |= OPENC3_EXTRA_FLAG_MASK
+          extra = JSON.parse(extra, :allow_nan => true, :create_additions => true) if String === extra
+          length += OPENC3_EXTRA_LENGTH_FIXED_SIZE
+          if @data_format == :CBOR
+            extra_encoded = extra.as_json.to_cbor
+          else
+            extra_encoded = JSON.generate(extra.as_json, :allow_nan => true)
+          end
+          length += extra_encoded.length
+        end
         length += OPENC3_PACKET_SECONDARY_FIXED_SIZE + data.length
         @entry.clear
-        @index_entry.clear
-        @index_entry << [length, flags, packet_index, time_nsec_since_epoch].pack(OPENC3_PACKET_PACK_DIRECTIVE)
-        @entry << @index_entry << data
-        @index_entry << [@file_size].pack('Q>')
-        @index_file.write(@index_entry)
+        @entry << [length, flags, packet_index, time_nsec_since_epoch].pack(OPENC3_PACKET_PACK_DIRECTIVE)
+        @entry << [received_time_nsec_since_epoch].pack(OPENC3_RECEIVED_TIME_PACK_DIRECTIVE) if received_time_nsec_since_epoch
+        @entry << [extra_encoded.length].pack(OPENC3_EXTRA_LENGTH_PACK_DIRECTIVE) << extra_encoded if extra_encoded
+        @entry << data
         @first_time = time_nsec_since_epoch if !@first_time or time_nsec_since_epoch < @first_time
         @last_time = time_nsec_since_epoch if !@last_time or time_nsec_since_epoch > @last_time
       else
@@ -334,29 +319,6 @@ module OpenC3
       @file_size += @entry.length
     end
 
-    def write_index_file_footer
-      footer_length = 4 # Includes length of length field at end
-      @index_file.write([@target_dec_entries.length].pack('n'))
-      footer_length += 2
-      @target_dec_entries.each do |target_dec_entry|
-        @index_file.write(target_dec_entry)
-        footer_length += target_dec_entry.length
-      end
-      @index_file.write([@packet_dec_entries.length].pack('n'))
-      footer_length += 2
-      @packet_dec_entries.each do |packet_dec_entry|
-        @index_file.write(packet_dec_entry)
-        footer_length += packet_dec_entry.length
-      end
-      @index_file.write([@key_map_entries.length].pack('n'))
-      footer_length += 2
-      @key_map_entries.each do |key_map_entry|
-        @index_file.write(key_map_entry)
-        footer_length += key_map_entry.length
-      end
-      @index_file.write([footer_length].pack('N'))
-    end
-
     def bucket_filename
       "#{first_timestamp}__#{last_timestamp}__#{@label}" + extension
     end

data/lib/openc3/logs/text_log_writer.rb

@@ -17,15 +17,22 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'openc3/logs/log_writer'
+require 'socket'
 
 module OpenC3
   # Creates a text log. Can automatically cycle the log based on an elasped
   # time period or when the log file reaches a predefined size.
   class TextLogWriter < LogWriter
+    NEWLINE = "\n".freeze
+    def initialize(*args)
+      super(*args)
+      @container_name = Socket.gethostname
+    end
+
     # Write to the log file.
     #
     # If no log file currently exists in the filesystem, a new file will be
@@ -47,11 +54,9 @@ module OpenC3
     end
 
     def write_entry(time_nsec_since_epoch, data)
-      @entry.clear
-      @entry << "#{time_nsec_since_epoch}\t"
-      @entry << "#{data}\n"
-      @file.write(@entry)
-      @file_size += @entry.length
+      @file.write(data)
+      @file.write(NEWLINE)
+      @file_size += (data.length + NEWLINE.length)
       @first_time = time_nsec_since_epoch if !@first_time or time_nsec_since_epoch < @first_time
       @last_time = time_nsec_since_epoch if !@last_time or time_nsec_since_epoch > @last_time
     end
@@ -65,6 +70,27 @@ module OpenC3
       "#{first_timestamp}__#{last_timestamp}__#{topic_name}" + extension
     end
 
+    # Closing a log file isn't critical so we just log an error
+    # Returns threads that moves log to bucket
+    def close_file(take_mutex = true)
+      threads = []
+      @mutex.lock if take_mutex
+      begin
+        # Need to write the OFFSET_MARKER for each packet
+        @last_offsets.each do |redis_topic, last_offset|
+          time = Time.now
+          data = { time: time.to_nsec_from_epoch, '@timestamp' => time.xmlschema(3), severity: 'INFO', "microservice_name" => Logger.microservice_name, "container_name" => @container_name, "last_offset" => last_offset, "redis_topic" => redis_topic, "type" => "offset" }
+          write_entry(time.to_nsec_from_epoch, data.as_json(allow_nan: true).to_json(allow_nan: true)) if @file
+        end
+
+        threads.concat(super(false))
+
+      ensure
+        @mutex.unlock if take_mutex
+      end
+      return threads
+    end
+
     def extension
       '.txt'.freeze
     end