openc3 5.0.11 → 5.1.0

data/lib/openc3/logs/buffered_packet_log_reader.rb

@@ -0,0 +1,65 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/logs/packet_log_reader'
+require 'openc3/logs/packet_log_writer'
+
+module OpenC3
+  # Buffers files so small time differences can be read in time order
+  class BufferedPacketLogReader < PacketLogReader
+
+    attr_reader :bucket_file
+
+    def initialize(bucket_file = nil, buffer_depth = 10)
+      super()
+      @bucket_file = bucket_file
+      @buffer_depth = buffer_depth
+    end
+
+    def next_packet_time
+      fill_buffer()
+      packet = @buffer[0]
+      return packet.packet_time if packet
+      return nil
+    end
+
+    def buffered_read(identify_and_define = true)
+      fill_buffer(identify_and_define)
+      return @buffer.shift
+    end
+
+    protected
+
+    def fill_buffer(identify_and_define = true)
+      while true
+        break if @buffer.length >= @buffer_depth
+        packet = read(identify_and_define)
+        break unless packet
+        packet = packet.dup if identify_and_define
+        @buffer << packet
+        @buffer.sort! {|pkt1, pkt2| pkt1.packet_time <=> pkt2.packet_time }
+      end
+    end
+
+    def reset
+      super()
+      @buffer = []
+    end
+
+  end
+end

data/lib/openc3/logs/buffered_packet_log_writer.rb

@@ -0,0 +1,118 @@
+# encoding: ascii-8bit
+
+# Copyright 2022 OpenC3, Inc.
+# All Rights Reserved.
+#
+# This program is free software; you can modify and/or redistribute it
+# under the terms of the GNU Affero General Public License
+# as published by the Free Software Foundation; version 3 with
+# attribution addendums as found in the LICENSE.txt
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
+
+require 'openc3/logs/packet_log_writer'
+
+module OpenC3
+  # Creates a packet log. Can automatically cycle the log based on an elasped
+  # time period or when the log file reaches a predefined size.
+  class BufferedPacketLogWriter < PacketLogWriter
+    # @param remote_log_directory [String] The path to store the log files
+    # @param label [String] Label to apply to the log filename
+    # @param logging_enabled [Boolean] Whether to start with logging enabled
+    # @param cycle_time [Integer] The amount of time in seconds before creating
+    #   a new log file. This can be combined with cycle_size but is better used
+    #   independently.
+    # @param cycle_size [Integer] The size in bytes before creating a new log
+    #   file. This can be combined with cycle_time but is better used
+    #   independently.
+    # @param cycle_hour [Integer] The time at which to cycle the log. Combined with
+    #   cycle_minute to cycle the log daily at the specified time. If nil, the log
+    #   will be cycled hourly at the specified cycle_minute.
+    # @param cycle_minute [Integer] The time at which to cycle the log. See cycle_hour
+    #   for more information.
+    # @param buffer_depth [Integer] Number of packets to buffer before writing to file
+    def initialize(
+      remote_log_directory,
+      label,
+      logging_enabled = true,
+      cycle_time = nil,
+      cycle_size = 1_000_000_000,
+      cycle_hour = nil,
+      cycle_minute = nil,
+      buffer_depth = 10
+    )
+      super(
+        remote_log_directory,
+        label,
+        logging_enabled,
+        cycle_time,
+        cycle_size,
+        cycle_hour,
+        cycle_minute
+      )
+      @buffer_depth = Integer(buffer_depth)
+      @buffer = []
+    end
+
+    # Write a packet to the log file.
+    #
+    # If no log file currently exists in the filesystem, a new file will be
+    # created.
+    #
+    # @param entry_type [Symbol] Type of entry to write. Must be one of
+    #   :TARGET_DECLARATION, :PACKET_DECLARATION, :RAW_PACKET, :JSON_PACKET, :OFFSET_MARKER, :KEY_MAP
+    # @param cmd_or_tlm [Symbol] One of :CMD or :TLM
+    # @param target_name [String] Name of the target
+    # @param packet_name [String] Name of the packet
+    # @param time_nsec_since_epoch [Integer] 64 bit integer nsecs since EPOCH
+    # @param stored [Boolean] Whether this data is stored telemetry
+    # @param data [String] Binary string of data
+    # @param id [Integer] Target ID
+    # @param redis_offset [Integer] The offset of this packet in its Redis stream
+    def buffered_write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0')
+      case entry_type
+      when :RAW_PACKET, :JSON_PACKET
+        @buffer << [entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset]
+        @buffer.sort! {|entry1, entry2| entry1[4] <=> entry2[4] }
+        if @buffer.length >= @buffer_depth
+          entry = @buffer.shift
+          write(*entry)
+        end
+      else
+        write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset)
+      end
+    end
+
+    def buffered_first_time_nsec
+      time = first_time()
+      return time.to_nsec_from_epoch if time
+      return @buffer[0][4] if @buffer[0]
+      return nil
+    end
+
+    def start_new_file(empty_buffer = false)
+      write_buffer() if empty_buffer
+      super()
+    end
+
+    def write_buffer
+      @buffer.each do |entry|
+        write(*entry)
+      end
+      @buffer = []
+    end
+
+    # Need to write out all remaining buffer entries and then shutdown
+    # Returns thread that moves final log to bucket
+    def shutdown
+      write_buffer()
+      return super()
+    end
+  end
+end

data/lib/openc3/logs/log_writer.rb

@@ -16,11 +16,14 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 require 'thread'
 require 'openc3/config/config_parser'
 require 'openc3/topics/topic'
-require 'openc3/utilities/s3'
+require 'openc3/utilities/bucket_utilities'
 
 module OpenC3
   # Creates a log. Can automatically cycle the log based on an elasped
@@ -52,10 +55,19 @@ module OpenC3
     # @return [Mutex] Instance mutex protecting file
     attr_reader :mutex
 
+    # Redis offsets for each topic to cleanup
+    attr_accessor :cleanup_offsets
+
+    # Time at which to cleanup
+    attr_accessor :cleanup_times
+
     # The cycle time interval. Cycle times are only checked at this level of
     # granularity.
     CYCLE_TIME_INTERVAL = 10
 
+    # Delay in seconds before trimming Redis streams
+    CLEANUP_DELAY = 60
+
     # Mutex protecting class variables
     @@mutex = Mutex.new
 
@@ -68,7 +80,7 @@ module OpenC3
     # Sleeper used to delay cycle thread
     @@cycle_sleeper = nil
 
-    # @param remote_log_directory [String] The s3 path to store the log files
+    # @param remote_log_directory [String] The path to store the log files
     # @param logging_enabled [Boolean] Whether to start with logging enabled
     # @param cycle_time [Integer] The amount of time in seconds before creating
     #   a new log file. This can be combined with cycle_size but is better used
@@ -81,16 +93,13 @@ module OpenC3
     #   will be cycled hourly at the specified cycle_minute.
     # @param cycle_minute [Integer] The time at which to cycle the log. See cycle_hour
     #   for more information.
-    # @param redis_topic [String] The key of the Redis stream to trim when files are
-    #   moved to S3
     def initialize(
       remote_log_directory,
       logging_enabled = true,
       cycle_time = nil,
       cycle_size = 1000000000,
       cycle_hour = nil,
-      cycle_minute = nil,
-      redis_topic: nil
+      cycle_minute = nil
     )
       @remote_log_directory = remote_log_directory
       @logging_enabled = ConfigParser.handle_true_false(logging_enabled)
@@ -113,22 +122,22 @@ module OpenC3
       @first_time = nil
       @last_time = nil
       @cancel_threads = false
-      @last_offset = nil
-      @previous_file_redis_offset = nil
-      @redis_topic = redis_topic
+      @last_offsets = {}
+      @cleanup_offsets = []
+      @cleanup_times = []
+      @previous_time_nsec_since_epoch = nil
 
       # This is an optimization to avoid creating a new entry object
       # each time we create an entry which we do a LOT!
       @entry = String.new
 
-      if @cycle_time or @cycle_hour or @cycle_minute
-        @@mutex.synchronize do
-          @@instances << self
+      # Always make sure there is a cycle thread - (because it does trimming)
+      @@mutex.synchronize do
+        @@instances << self
 
-          unless @@cycle_thread
-            @@cycle_thread = OpenC3.safe_thread("Log cycle") do
-              cycle_thread_body()
-            end
+        unless @@cycle_thread
+          @@cycle_thread = OpenC3.safe_thread("Log cycle") do
+            cycle_thread_body()
           end
         end
       end
@@ -143,12 +152,14 @@ module OpenC3
 
     # Stops all logging and closes the current log file.
     def stop
-      @mutex.synchronize { @logging_enabled = false; close_file(false) }
+      threads = nil
+      @mutex.synchronize { @logging_enabled = false; threads = close_file(false) }
+      return threads
     end
 
     # Stop all logging, close the current log file, and kill the logging threads.
     def shutdown
-      stop()
+      threads = stop()
       @@mutex.synchronize do
         @@instances.delete(self)
         if @@instances.length <= 0
@@ -157,6 +168,7 @@ module OpenC3
           @@cycle_thread = nil
         end
       end
+      return threads
     end
 
     def graceful_kill
@@ -192,18 +204,41 @@ module OpenC3
             instance.mutex.synchronize do
               utc_now = Time.now.utc
               # Logger.debug("start:#{@start_time.to_f} now:#{utc_now.to_f} cycle:#{@cycle_time} new:#{(utc_now - @start_time) > @cycle_time}")
-              if instance.logging_enabled and
-                (
-                  # Cycle based on total time logging
-                  (instance.cycle_time and (utc_now - instance.start_time) > instance.cycle_time) or
-
-                  # Cycle daily at a specific time
-                  (instance.cycle_hour and instance.cycle_minute and utc_now.hour == instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.yday != utc_now.yday) or
-
-                  # Cycle hourly at a specific time
-                  (instance.cycle_minute and not instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.hour != utc_now.hour)
-                )
-                instance.close_file(false)
+              if instance.logging_enabled and instance.filename # Logging and file opened
+                # Cycle based on total time logging
+                if (instance.cycle_time and (utc_now - instance.start_time) > instance.cycle_time)
+                  Logger.debug("Log writer start new file due to cycle time")
+                  instance.close_file(false)
+                # Cycle daily at a specific time
+                elsif (instance.cycle_hour and instance.cycle_minute and utc_now.hour == instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.yday != utc_now.yday)
+                  Logger.debug("Log writer start new file daily")
+                  instance.close_file(false)
+                # Cycle hourly at a specific time
+                elsif (instance.cycle_minute and not instance.cycle_hour and utc_now.min == instance.cycle_minute and instance.start_time.hour != utc_now.hour)
+                  Logger.debug("Log writer start new file hourly")
+                  instance.close_file(false)
+                end
+              end
+
+              # Check for cleanup time
+              indexes_to_clear = []
+              instance.cleanup_times.each_with_index do |cleanup_time, index|
+                if cleanup_time <= utc_now
+                  # Now that the file is in S3, trim the Redis stream up until the previous file.
+                  # This keeps one minute of data in Redis
+                  instance.cleanup_offsets[index].each do |redis_topic, cleanup_offset|
+                    Topic.trim_topic(redis_topic, cleanup_offset)
+                  end
+                  indexes_to_clear << index
+                end
+              end
+              if indexes_to_clear.length > 0
+                indexes_to_clear.each do |index|
+                  instance.cleanup_offsets[index] = nil
+                  instance.cleanup_times[index] = nil
+                end
+                instance.cleanup_offsets.compact!
+                instance.cleanup_times.compact!
               end
             end
           end
@@ -231,6 +266,7 @@ module OpenC3
       @start_time = Time.now.utc
       @first_time = nil
       @last_time = nil
+      @previous_time_nsec_since_epoch = nil
       Logger.debug "Log File Opened : #{@filename}"
     rescue => err
       Logger.error "Error starting new log file: #{err.formatted}"
@@ -238,17 +274,30 @@ module OpenC3
       OpenC3.handle_critical_exception(err)
     end
 
-    def prepare_write(time_nsec_since_epoch, data_length, redis_offset)
+    def prepare_write(time_nsec_since_epoch, data_length, redis_topic, redis_offset)
       # This check includes logging_enabled again because it might have changed since we acquired the mutex
-      if @logging_enabled and (!@file or (@cycle_size and (@file_size + data_length) > @cycle_size))
-        start_new_file()
+      # Ensures new files based on size, and ensures always increasing time order in files
+      if @logging_enabled
+        if !@file
+          Logger.debug("Log writer start new file because no file opened")
+          start_new_file()
+        elsif @cycle_size and ((@file_size + data_length) > @cycle_size)
+          Logger.debug("Log writer start new file due to cycle size #{@cycle_size}")
+          start_new_file()
+        elsif @previous_time_nsec_since_epoch and (@previous_time_nsec_since_epoch > time_nsec_since_epoch)
+          Logger.debug("Log writer start new file due to out of order time: #{Time.from_nsec_from_epoch(@previous_time_nsec_since_epoch)} #{Time.from_nsec_from_epoch(time_nsec_since_epoch)}")
+          start_new_file()
+        end
       end
-      @last_offset = redis_offset # This is needed for the redis offset marker entry at the end of the log file
+      @last_offsets[redis_topic] = redis_offset if redis_topic and redis_offset # This is needed for the redis offset marker entry at the end of the log file
+      @previous_time_nsec_since_epoch = time_nsec_since_epoch
     end
 
     # Closing a log file isn't critical so we just log an error. NOTE: This also trims the Redis stream
     # to keep a full file's worth of data in the stream. This is what prevents continuous stream growth.
+    # Returns thread that moves log to bucket
     def close_file(take_mutex = true)
+      thread = nil
       @mutex.lock if take_mutex
       begin
         if @file
@@ -256,12 +305,15 @@ module OpenC3
             @file.close unless @file.closed?
             Logger.debug "Log File Closed : #{@filename}"
             date = first_timestamp[0..7] # YYYYMMDD
-            s3_key = File.join(@remote_log_directory, date, s3_filename)
-            S3Utilities.move_log_file_to_s3(@filename, s3_key)
-            # Now that the file is in S3, trim the Redis stream up until the previous file.
-            # This keeps one file worth of data in Redis as a safety buffer
-            Topic.trim_topic(@redis_topic, @previous_file_redis_offset) if @redis_topic and @previous_file_redis_offset
-            @previous_file_redis_offset = @last_offset
+            bucket_key = File.join(@remote_log_directory, date, bucket_filename())
+            thread = BucketUtilities.move_log_file_to_bucket(@filename, bucket_key)
+            # Now that the file is in storage, trim the Redis stream after a delay
+            @cleanup_offsets << {}
+            @last_offsets.each do |redis_topic, last_offset|
+              @cleanup_offsets[-1][redis_topic] = last_offset
+            end
+            @cleanup_times << Time.now + CLEANUP_DELAY
+            @last_offsets.clear
           rescue Exception => err
             Logger.instance.error "Error closing #{@filename} : #{err.formatted}"
           end
@@ -273,9 +325,10 @@ module OpenC3
       ensure
         @mutex.unlock if take_mutex
       end
+      return thread
     end
 
-    def s3_filename
+    def bucket_filename
       "#{first_timestamp}__#{last_timestamp}" + extension
     end
 

data/lib/openc3/logs/packet_log_constants.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 module OpenC3
   module PacketLogConstants
@@ -34,6 +37,8 @@ module OpenC3
     OPENC3_RAW_PACKET_ENTRY_TYPE_MASK = 0x3000
     OPENC3_JSON_PACKET_ENTRY_TYPE_MASK = 0x4000
     OPENC3_OFFSET_MARKER_ENTRY_TYPE_MASK = 0x5000
+    OPENC3_KEY_MAP_ENTRY_TYPE_MASK = 0x6000
+    OPENC3_CBOR_FLAG_MASK = 0x0100
     OPENC3_ID_FLAG_MASK = 0x0200
     OPENC3_STORED_FLAG_MASK = 0x0400
     OPENC3_CMD_FLAG_MASK = 0x0800
@@ -55,6 +60,10 @@ module OpenC3
     OPENC3_OFFSET_MARKER_PACK_DIRECTIVE = 'Nn'.freeze
     OPENC3_OFFSET_MARKER_PACK_ITEMS = 2 # Useful for testing
 
+    OPENC3_KEY_MAP_SECONDARY_FIXED_SIZE = 2
+    OPENC3_KEY_MAP_PACK_DIRECTIVE = 'Nnn'.freeze
+    OPENC3_KEY_MAP_PACK_ITEMS = 3 # Useful for testing
+
     OPENC3_PACKET_SECONDARY_FIXED_SIZE = 10
     OPENC3_PACKET_PACK_DIRECTIVE = 'NnnQ>'.freeze
     OPENC3_PACKET_PACK_ITEMS = 4 # Useful for testing

data/lib/openc3/logs/packet_log_reader.rb

@@ -16,12 +16,16 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/core_ext/io'
 require 'openc3/packets/packet'
 require 'openc3/packets/json_packet'
 require 'openc3/io/buffered_file'
 require 'openc3/logs/packet_log_constants'
+require 'cbor'
 
 module OpenC3
   # Reads a packet log of either commands or telemetry.
@@ -29,6 +33,8 @@ module OpenC3
     include PacketLogConstants
 
     attr_reader :redis_offset
+    attr_reader :last_offsets
+    attr_reader :filename
 
     MAX_READ_SIZE = 1000000000
 
@@ -119,16 +125,22 @@ module OpenC3
       stored = true if flags & OPENC3_STORED_FLAG_MASK == OPENC3_STORED_FLAG_MASK
       id = false
       id = true if flags & OPENC3_ID_FLAG_MASK == OPENC3_ID_FLAG_MASK
+      cbor = false
+      cbor = true if flags & OPENC3_CBOR_FLAG_MASK == OPENC3_CBOR_FLAG_MASK
 
       if flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_JSON_PACKET_ENTRY_TYPE_MASK
         packet_index, time_nsec_since_epoch = entry[2..11].unpack('nQ>')
         json_data = entry[12..-1]
-        lookup_cmd_or_tlm, target_name, packet_name, id = @packets[packet_index]
+        lookup_cmd_or_tlm, target_name, packet_name, id, key_map = @packets[packet_index]
         if cmd_or_tlm != lookup_cmd_or_tlm
           raise "Packet type mismatch, packet:#{cmd_or_tlm}, lookup:#{lookup_cmd_or_tlm}"
         end
 
-        return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, json_data)
+        if cbor
+          return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, CBOR.decode(json_data), key_map)
+        else
+          return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, json_data, key_map)
+        end
       elsif flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_RAW_PACKET_ENTRY_TYPE_MASK
         packet_index, time_nsec_since_epoch = entry[2..11].unpack('nQ>')
         packet_data = entry[12..-1]
@@ -171,8 +183,26 @@ module OpenC3
         end
         @packets << [cmd_or_tlm, target_name, packet_name, id]
         return read(identify_and_define)
+      elsif flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_KEY_MAP_ENTRY_TYPE_MASK
+        packet_index = entry[2..3].unpack('n')[0]
+        key_map_length = length - OPENC3_PRIMARY_FIXED_SIZE - OPENC3_KEY_MAP_SECONDARY_FIXED_SIZE
+        if cbor
+          key_map = CBOR.decode(entry[4..(key_map_length + 3)])
+        else
+          key_map = JSON.parse(entry[4..(key_map_length + 3)], :allow_nan => true, :create_additions => true)
+        end
+        @packets[packet_index] << key_map
+        return read(identify_and_define)
       elsif flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_OFFSET_MARKER_ENTRY_TYPE_MASK
-        @redis_offset = entry[2..-1]
+        data = entry[2..-1]
+        split_data = data.split(',')
+        redis_offset = split_data[0]
+        redis_topic = split_data[1]
+        if redis_topic
+          @last_offsets[redis_topic] = redis_offset
+        else
+          @redis_offset = redis_offset
+        end
         return read(identify_and_define)
       else
         raise "Invalid Entry Flags: #{flags}"
@@ -292,6 +322,7 @@ module OpenC3
       @packets = []
       @packet_ids = []
       @redis_offset = nil
+      @last_offsets = {}
     end
 
     # This is best effort. May return unidentified/undefined packets