openc3 5.0.11 → 5.1.1

data/lib/openc3/logs/packet_log_writer.rb

@@ -16,9 +16,13 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 require 'openc3/logs/log_writer'
 require 'openc3/logs/packet_log_constants'
+require 'cbor'
 
 module OpenC3
   # Creates a packet log. Can automatically cycle the log based on an elasped
@@ -26,7 +30,9 @@ module OpenC3
   class PacketLogWriter < LogWriter
     include PacketLogConstants
 
-    # @param remote_log_directory [String] The s3 path to store the log files
+    attr_accessor :data_format
+
+    # @param remote_log_directory [String] The path to store the log files
     # @param label [String] Label to apply to the log filename
     # @param logging_enabled [Boolean] Whether to start with logging enabled
     # @param cycle_time [Integer] The amount of time in seconds before creating
@@ -40,8 +46,6 @@ module OpenC3
     #   will be cycled hourly at the specified cycle_minute.
     # @param cycle_minute [Integer] The time at which to cycle the log. See cycle_hour
     #   for more information.
-    # @param redis_topic [String] The key of the Redis stream to trim when files are
-    #   moved to S3
     def initialize(
       remote_log_directory,
       label,
@@ -50,7 +54,7 @@ module OpenC3
       cycle_size = 1_000_000_000,
       cycle_hour = nil,
       cycle_minute = nil,
-      redis_topic: nil
+      enforce_time_order = true
     )
       super(
         remote_log_directory,
@@ -59,18 +63,21 @@ module OpenC3
         cycle_size,
         cycle_hour,
         cycle_minute,
-        redis_topic: redis_topic
+        enforce_time_order
       )
       @label = label
       @index_file = nil
       @index_filename = nil
       @cmd_packet_table = {}
       @tlm_packet_table = {}
+      @key_map_table = {}
       @target_dec_entries = []
       @packet_dec_entries = []
+      @key_map_entries = []
       @next_packet_index = 0
       @target_indexes = {}
       @next_target_index = 0
+      @data_format = :CBOR # Default to CBOR for improved compression
 
       # This is an optimization to avoid creating a new entry object
       # each time we create an entry which we do a LOT!
@@ -83,7 +90,7 @@ module OpenC3
     # created.
     #
     # @param entry_type [Symbol] Type of entry to write. Must be one of
-    #   :TARGET_DECLARATION, :PACKET_DECLARATION, :RAW_PACKET, :JSON_PACKET
+    #   :TARGET_DECLARATION, :PACKET_DECLARATION, :RAW_PACKET, :JSON_PACKET, :OFFSET_MARKER, :KEY_MAP
     # @param cmd_or_tlm [Symbol] One of :CMD or :TLM
     # @param target_name [String] Name of the target
     # @param packet_name [String] Name of the packet
@@ -92,11 +99,11 @@ module OpenC3
     # @param data [String] Binary string of data
     # @param id [Integer] Target ID
     # @param redis_offset [Integer] The offset of this packet in its Redis stream
-    def write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_offset = '0-0')
+    def write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0')
       return if !@logging_enabled
 
       @mutex.synchronize do
-        prepare_write(time_nsec_since_epoch, data.length, redis_offset)
+        prepare_write(time_nsec_since_epoch, data.length, redis_topic, redis_offset)
         write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id) if @file
       end
     rescue => err
@@ -119,10 +126,12 @@ module OpenC3
 
       @cmd_packet_table = {}
       @tlm_packet_table = {}
+      @key_map_table = {}
       @next_packet_index = 0
       @target_indexes = {}
       @target_dec_entries = []
       @packet_dec_entries = []
+      @key_map_entries = []
       Logger.debug "Index Log File Opened : #{@index_filename}"
     rescue => err
       Logger.error "Error starting new log file: #{err.formatted}"
@@ -131,20 +140,26 @@ module OpenC3
     end
 
     # Closing a log file isn't critical so we just log an error
+    # Returns threads that moves log to bucket
     def close_file(take_mutex = true)
-      write_entry(:OFFSET_MARKER, nil, nil, nil, nil, nil, nil, nil) if @file
-      super
-
+      threads = []
       @mutex.lock if take_mutex
       begin
+        # Need to write the OFFSET_MARKER for each packet
+        @last_offsets.each do |redis_topic, last_offset|
+          write_entry(:OFFSET_MARKER, nil, nil, nil, nil, nil, last_offset + ',' + redis_topic, nil) if @file
+        end
+
+        threads.concat(super(false))
+
         if @index_file
           begin
             write_index_file_footer()
             @index_file.close unless @index_file.closed?
             Logger.debug "Index Log File Closed : #{@index_filename}"
             date = first_timestamp[0..7] # YYYYMMDD
-            s3_key = File.join(@remote_log_directory, date, "#{first_timestamp}__#{last_timestamp}__#{@label}.idx")
-            S3Utilities.move_log_file_to_s3(@index_filename, s3_key)
+            bucket_key = File.join(@remote_log_directory, date, "#{first_timestamp}__#{last_timestamp}__#{@label}.idx")
+            threads << BucketUtilities.move_log_file_to_bucket(@index_filename, bucket_key)
           rescue Exception => err
             Logger.instance.error "Error closing #{@index_filename} : #{err.formatted}"
           end
@@ -155,9 +170,10 @@ module OpenC3
       ensure
         @mutex.unlock if take_mutex
       end
+      return threads
     end
 
-    def get_packet_index(cmd_or_tlm, target_name, packet_name)
+    def get_packet_index(cmd_or_tlm, target_name, packet_name, entry_type, data)
       if cmd_or_tlm == :CMD
         target_table = @cmd_packet_table[target_name]
       else
@@ -200,6 +216,26 @@ module OpenC3
         # No packet def
       end
       write_entry(:PACKET_DECLARATION, cmd_or_tlm, target_name, packet_name, nil, nil, nil, id)
+      if entry_type == :JSON_PACKET
+        key_map = @key_map_table[packet_index]
+        unless key_map
+          parsed = data
+          parsed = JSON.parse(data, :allow_nan => true, :create_additions => true) if String === parsed
+          keys = parsed.keys
+          key_map = {}
+          reverse_key_map = {}
+          keys.each_with_index do |key, index|
+            key_map[index.to_s] = key
+            reverse_key_map[key] = index.to_s
+          end
+          @key_map_table[packet_index] = reverse_key_map
+          if @data_format == :CBOR
+            write_entry(:KEY_MAP, cmd_or_tlm, target_name, packet_name, nil, nil, key_map.to_cbor, nil)
+          else # JSON
+            write_entry(:KEY_MAP, cmd_or_tlm, target_name, packet_name, nil, nil, JSON.generate(key_map, :allow_nan => true), nil)
+          end
+        end
+      end
       return packet_index
     end
 
@@ -238,19 +274,44 @@ module OpenC3
         @entry << [length, flags, target_index].pack(OPENC3_PACKET_DECLARATION_PACK_DIRECTIVE) << packet_name
         @entry << [id].pack('H*') if id
         @packet_dec_entries << @entry.dup
+      when :KEY_MAP
+        flags |= OPENC3_KEY_MAP_ENTRY_TYPE_MASK
+        flags |= OPENC3_CBOR_FLAG_MASK if @data_format == :CBOR
+        length += OPENC3_KEY_MAP_SECONDARY_FIXED_SIZE + data.length
+        packet_index = get_packet_index(cmd_or_tlm, target_name, packet_name, entry_type, data)
+        @entry.clear
+        @entry << [length, flags, packet_index].pack(OPENC3_KEY_MAP_PACK_DIRECTIVE) << data
+        @key_map_entries << @entry.dup
       when :OFFSET_MARKER
         flags |= OPENC3_OFFSET_MARKER_ENTRY_TYPE_MASK
-        length += OPENC3_OFFSET_MARKER_SECONDARY_FIXED_SIZE + @last_offset.length
+        length += OPENC3_OFFSET_MARKER_SECONDARY_FIXED_SIZE + data.length
         @entry.clear
-        @entry << [length, flags].pack(OPENC3_OFFSET_MARKER_PACK_DIRECTIVE) << @last_offset
+        @entry << [length, flags].pack(OPENC3_OFFSET_MARKER_PACK_DIRECTIVE) << data
       when :RAW_PACKET, :JSON_PACKET
         target_name = 'UNKNOWN'.freeze unless target_name
         packet_name = 'UNKNOWN'.freeze unless packet_name
-        packet_index = get_packet_index(cmd_or_tlm, target_name, packet_name)
+        packet_index = get_packet_index(cmd_or_tlm, target_name, packet_name, entry_type, data)
         if entry_type == :RAW_PACKET
           flags |= OPENC3_RAW_PACKET_ENTRY_TYPE_MASK
         else
           flags |= OPENC3_JSON_PACKET_ENTRY_TYPE_MASK
+          key_map = @key_map_table[packet_index]
+          if key_map
+            # Compress data using key map
+            data = JSON.parse(data, :allow_nan => true, :create_additions => true) if String === data
+            compressed = {}
+            data.each do |key, value|
+              compressed_key = key_map[key]
+              compressed_key = key unless compressed_key
+              compressed[compressed_key] = value
+            end
+            if @data_format == :CBOR
+              flags |= OPENC3_CBOR_FLAG_MASK
+              data = compressed.to_cbor
+            else
+              data = JSON.generate(compressed, :allow_nan => true)
+            end
+          end
         end
         if cmd_or_tlm == :CMD
           flags |= OPENC3_CMD_FLAG_MASK
@@ -285,10 +346,16 @@ module OpenC3
         @index_file.write(packet_dec_entry)
         footer_length += packet_dec_entry.length
       end
+      @index_file.write([@key_map_entries.length].pack('n'))
+      footer_length += 2
+      @key_map_entries.each do |key_map_entry|
+        @index_file.write(key_map_entry)
+        footer_length += key_map_entry.length
+      end
       @index_file.write([footer_length].pack('N'))
     end
 
-    def s3_filename
+    def bucket_filename
       "#{first_timestamp}__#{last_timestamp}__#{@label}" + extension
     end
 

data/lib/openc3/logs/text_log_writer.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/logs/log_writer'
 
@@ -31,11 +34,11 @@ module OpenC3
     # @param time_nsec_since_epoch [Integer] 64 bit integer nsecs since EPOCH
     # @param data [String] String of data
     # @param redis_offset [Integer] The offset of this packet in its Redis stream
-    def write(time_nsec_since_epoch, data, redis_offset)
+    def write(time_nsec_since_epoch, data, redis_topic, redis_offset)
       return if !@logging_enabled
 
       @mutex.synchronize do
-        prepare_write(time_nsec_since_epoch, data.length, redis_offset)
+        prepare_write(time_nsec_since_epoch, data.length, redis_topic, redis_offset)
         write_entry(time_nsec_since_epoch, data) if @file
       end
     rescue => err
@@ -53,11 +56,12 @@ module OpenC3
       @last_time = time_nsec_since_epoch if !@last_time or time_nsec_since_epoch > @last_time
     end
 
-    def s3_filename
+    def bucket_filename
       # Put the name of the redis topic in the filename, but remove the scope
       # because we're already in a directory with the scope name
-      split_index = @redis_topic.index("__") + 2
-      topic_name = @redis_topic[split_index, @redis_topic.length - split_index]
+      redis_topic = @last_offsets.keys[0].to_s
+      split_index = redis_topic.index("__") + 2
+      topic_name = redis_topic[split_index, redis_topic.length - split_index]
       "#{first_timestamp}__#{last_timestamp}__#{topic_name}" + extension
     end
 

data/lib/openc3/logs.rb

@@ -16,10 +16,16 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 module OpenC3
-  autoload(:PacketLogWriter, 'openc3/logs/packet_log_writer.rb')
-  autoload(:PacketLogWriterPair, 'openc3/logs/packet_log_writer_pair.rb')
+  autoload(:BufferedPacketLogReader, 'openc3/logs/buffered_packet_log_reader.rb')
+  autoload(:BufferedPacketLogWriter, 'openc3/logs/buffered_packet_log_writer.rb')
+  autoload(:LogWriter, 'openc3/logs/log_writer.rb')
+  autoload(:PacketLogConstants, 'openc3/logs/packet_log_constants.rb')
   autoload(:PacketLogReader, 'openc3/logs/packet_log_reader.rb')
+  autoload(:PacketLogWriter, 'openc3/logs/packet_log_writer.rb')
   autoload(:TextLogWriter, 'openc3/logs/text_log_writer.rb')
 end

data/lib/openc3/microservices/cleanup_microservice.rb

@@ -16,10 +16,14 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license 
+# if purchased from OpenC3, Inc.
 
 require 'openc3/models/target_model'
 require 'openc3/microservices/microservice'
-require 'openc3/utilities/s3'
+require 'openc3/utilities/bucket'
+require 'openc3/utilities/bucket_utilities'
 
 module OpenC3
   class CleanupMicroservice < Microservice
@@ -28,33 +32,29 @@ module OpenC3
       target_name = split_name[-1]
       target = TargetModel.get_model(name: target_name, scope: @scope)
 
-      rubys3_client = Aws::S3::Client.new
+      bucket = Bucket.getClient()
       while true
         break if @cancel_thread
 
         @state = 'GETTING_OBJECTS'
         start_time = Time.now
         [
-         ["#{@scope}/raw_logs/cmd/#{target_name}/", target.cmd_log_retain_time],
-         ["#{@scope}/decom_logs/cmd/#{target_name}/", target.cmd_decom_log_retain_time],
-         ["#{@scope}/raw_logs/tlm/#{target_name}/", target.tlm_log_retain_time],
-         ["#{@scope}/decom_logs/tlm/#{target_name}/", target.tlm_decom_log_retain_time],
-         ["#{@scope}/reduced_minute_logs/tlm/#{target_name}/", target.reduced_minute_log_retain_time],
-         ["#{@scope}/reduced_hour_logs/tlm/#{target_name}/", target.reduced_hour_log_retain_time],
-         ["#{@scope}/reduced_day_logs/tlm/#{target_name}/", target.reduced_day_log_retain_time],
+         ["#{@scope}/raw_logs/cmd/#{target_name}", target.cmd_log_retain_time],
+         ["#{@scope}/decom_logs/cmd/#{target_name}", target.cmd_decom_log_retain_time],
+         ["#{@scope}/raw_logs/tlm/#{target_name}", target.tlm_log_retain_time],
+         ["#{@scope}/decom_logs/tlm/#{target_name}", target.tlm_decom_log_retain_time],
+         ["#{@scope}/reduced_minute_logs/tlm/#{target_name}", target.reduced_minute_log_retain_time],
+         ["#{@scope}/reduced_hour_logs/tlm/#{target_name}", target.reduced_hour_log_retain_time],
+         ["#{@scope}/reduced_day_logs/tlm/#{target_name}", target.reduced_day_log_retain_time],
         ].each do |prefix, retain_time|
           next unless retain_time
           time = start_time - retain_time
-          total_size, oldest_list = S3Utilities.list_files_before_time('logs', prefix, time)
-          delete_items = []
-          oldest_list.each do |item|
-            delete_items << { :key => item.key }
-          end
-          if delete_items.length > 0
+          oldest_list = BucketUtilities.list_files_before_time(ENV['OPENC3_LOGS_BUCKET'], prefix, time)
+          if oldest_list.length > 0
             @state = 'DELETING_OBJECTS'
-            delete_items.each_slice(1000) do |delete_slice|
-              rubys3_client.delete_objects({ bucket: 'logs', delete: { objects: delete_slice } })
-              Logger.info("Deleted #{delete_slice.length} #{target_name} log files")
+            oldest_list.each_slice(1000) do |slice|
+              bucket.delete_objects(bucket: ENV['OPENC3_LOGS_BUCKET'], keys: slice)
+              Logger.info("Deleted #{slice.length} #{target_name} log files")
             end
           end
         end

data/lib/openc3/microservices/decom_microservice.rb

@@ -16,6 +16,9 @@
 # Modified by OpenC3, Inc.
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
+#
+# This file may also be used under the terms of a commercial license
+# if purchased from OpenC3, Inc.
 
 require 'openc3/microservices/microservice'
 require 'openc3/topics/telemetry_decom_topic'
@@ -33,6 +36,7 @@ module OpenC3
       super(*args)
       Topic.update_topic_offsets(@topics)
       System.telemetry.limits_change_callback = method(:limits_change_callback)
+      LimitsEventTopic.sync_system(scope: @scope)
     end
 
     def run
@@ -40,12 +44,15 @@ module OpenC3
         break if @cancel_thread
 
         begin
-          Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
-            break if @cancel_thread
+          OpenC3.in_span("read_topics") do
+            Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
+              break if @cancel_thread
 
-            decom_packet(topic, msg_id, msg_hash, redis)
-            @count += 1
+              decom_packet(topic, msg_id, msg_hash, redis)
+              @count += 1
+            end
           end
+          LimitsEventTopic.sync_system_thread_body(scope: @scope)
         rescue => e
           @error = e
           Logger.error("Decom error: #{e.formatted}")
@@ -54,23 +61,23 @@ module OpenC3
     end
 
     def decom_packet(topic, _msg_id, msg_hash, _redis)
-      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      target_name = msg_hash["target_name"]
-      packet_name = msg_hash["packet_name"]
-
-      current_limits_set = LimitsEventTopic.current_set(scope: @scope)
-
-      packet = System.telemetry.packet(target_name, packet_name)
-      packet.stored = ConfigParser.handle_true_false(msg_hash["stored"])
-      packet.received_time = Time.from_nsec_from_epoch(msg_hash["time"].to_i)
-      packet.received_count = msg_hash["received_count"].to_i
-      packet.buffer = msg_hash["buffer"]
-      packet.check_limits(current_limits_set.intern) # Process all the limits and call the limits_change_callback (as necessary)
-
-      TelemetryDecomTopic.write_packet(packet, scope: @scope)
-      diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
-      metric_labels = { "packet" => packet_name, "target" => target_name }
-      @metric.add_sample(name: DECOM_METRIC_NAME, value: diff, labels: metric_labels)
+      OpenC3.in_span("decom_packet") do
+        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        target_name = msg_hash["target_name"]
+        packet_name = msg_hash["packet_name"]
+
+        packet = System.telemetry.packet(target_name, packet_name)
+        packet.stored = ConfigParser.handle_true_false(msg_hash["stored"])
+        packet.received_time = Time.from_nsec_from_epoch(msg_hash["time"].to_i)
+        packet.received_count = msg_hash["received_count"].to_i
+        packet.buffer = msg_hash["buffer"]
+        packet.check_limits(System.limits_set) # Process all the limits and call the limits_change_callback (as necessary)
+
+        TelemetryDecomTopic.write_packet(packet, scope: @scope)
+        diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
+        metric_labels = { "packet" => packet_name, "target" => target_name }
+        @metric.add_sample(name: DECOM_METRIC_NAME, value: diff, labels: metric_labels)
+      end
     end
 
     # Called when an item in any packet changes limits states.
@@ -96,7 +103,6 @@ module OpenC3
         when :YELLOW, :YELLOW_LOW, :YELLOW_HIGH
           Logger.warn message
         when :RED, :RED_LOW, :RED_HIGH
-          # TODO: Is this necessary? The LimitsEventTopic is what communicates with LimitsMonitor
           notification = NotificationModel.new(
             time: time_nsec,
             severity: "critical",
@@ -111,8 +117,8 @@ module OpenC3
 
       # The openc3_limits_events topic can be listened to for all limits events, it is a continuous stream
       event = { type: :LIMITS_CHANGE, target_name: packet.target_name, packet_name: packet.packet_name,
-                item_name: item.name, old_limits_state: old_limits_state, new_limits_state: item.limits.state,
-                time_nsec: time_nsec, message: message }
+                item_name: item.name, old_limits_state: old_limits_state.to_s, new_limits_state: item.limits.state.to_s,
+                time_nsec: time_nsec, message: message.to_s }
       LimitsEventTopic.write(event, scope: @scope)
 
       if item.limits.response