openc3 5.1.0 → 5.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: afcf1342f6607ba15d11c94601916717928eda05dfcb845afabd3e81e77bbdd0
-  data.tar.gz: 95b2f282fd4253019e42ae0ae785906ade6f4910e6f0860279beef3c967cecbd
+  metadata.gz: fa2f9aabf07f0b9c9237f2f44644c8da055ca3069d46399db837b3b9bb22fdb3
+  data.tar.gz: 16afda4302a384d554872ff416ba76f3c822cf36f986687256f71e6c2e729370
 SHA512:
-  metadata.gz: 8bd2a2fd4f3db741d0de89627ff45910a99c6464e8c9b8308e0c6926af9ac1569680a3456dc50b8163e806938c7707336475754f2219aa6e9b553dcab86870cd
-  data.tar.gz: 539582ced510f20b5770d99d02a885b9ad19bc923e65fc46c6ed602cb6ddec1b948859ae07389da603a5a9af7fcef001099438428d2cb3265f0be4aec939078e
+  metadata.gz: 394108166336ca206f0d56caa9d805e3c694a8c80b47915e3552fab228e8a9418bb61bd616ac897127e01b920d7da28d7e3c2bb0673e3e28e45283f44d0c9eda
+  data.tar.gz: ce284cea11dc3bf1c788cb89bfced3ff783b892902ec0f0dcc1922ba6edffda0ef9fe0bc5ef358fec95949e15425f8241e66a36f7a5f9472a39167e0cc4f9b53

data/lib/openc3/io/json_rpc.rb

@@ -17,11 +17,12 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'json'
 require 'date'
+require 'openc3/core_ext/string'
 
 class Object
   def as_json(options = nil) #:nodoc:
@@ -57,10 +58,11 @@ class String
   NON_ASCII_PRINTABLE = /[^\x21-\x7e\s]/
 
   def as_json(options = nil)
-    if NON_ASCII_PRINTABLE.match?(self)
-      self.to_json_raw_object
+    as_utf8 = self.dup.force_encoding('UTF-8')
+    if as_utf8.valid_encoding?
+      return as_utf8
     else
-      self
+      return self.to_json_raw_object
     end
   end #:nodoc:
 end

data/lib/openc3/logs/buffered_packet_log_writer.rb

@@ -45,6 +45,7 @@ module OpenC3
       cycle_size = 1_000_000_000,
       cycle_hour = nil,
       cycle_minute = nil,
+      enforce_time_order = true,
       buffer_depth = 10
     )
       super(
@@ -54,7 +55,8 @@ module OpenC3
         cycle_time,
         cycle_size,
         cycle_hour,
-        cycle_minute
+        cycle_minute,
+        enforce_time_order
       )
       @buffer_depth = Integer(buffer_depth)
       @buffer = []

data/lib/openc3/logs/log_writer.rb

@@ -99,7 +99,8 @@ module OpenC3
       cycle_time = nil,
       cycle_size = 1000000000,
       cycle_hour = nil,
-      cycle_minute = nil
+      cycle_minute = nil,
+      enforce_time_order = true
     )
       @remote_log_directory = remote_log_directory
       @logging_enabled = ConfigParser.handle_true_false(logging_enabled)
@@ -114,6 +115,7 @@ module OpenC3
       @cycle_hour = Integer(@cycle_hour) if @cycle_hour
       @cycle_minute = ConfigParser.handle_nil(cycle_minute)
       @cycle_minute = Integer(@cycle_minute) if @cycle_minute
+      @enforce_time_order = ConfigParser.handle_true_false(enforce_time_order)
       @mutex = Mutex.new
       @file = nil
       @file_size = 0
@@ -284,7 +286,7 @@ module OpenC3
         elsif @cycle_size and ((@file_size + data_length) > @cycle_size)
           Logger.debug("Log writer start new file due to cycle size #{@cycle_size}")
           start_new_file()
-        elsif @previous_time_nsec_since_epoch and (@previous_time_nsec_since_epoch > time_nsec_since_epoch)
+        elsif @enforce_time_order and @previous_time_nsec_since_epoch and (@previous_time_nsec_since_epoch > time_nsec_since_epoch)
           Logger.debug("Log writer start new file due to out of order time: #{Time.from_nsec_from_epoch(@previous_time_nsec_since_epoch)} #{Time.from_nsec_from_epoch(time_nsec_since_epoch)}")
           start_new_file()
         end
@@ -297,7 +299,7 @@ module OpenC3
     # to keep a full file's worth of data in the stream. This is what prevents continuous stream growth.
     # Returns thread that moves log to bucket
     def close_file(take_mutex = true)
-      thread = nil
+      threads = []
       @mutex.lock if take_mutex
       begin
         if @file
@@ -306,7 +308,7 @@ module OpenC3
             Logger.debug "Log File Closed : #{@filename}"
             date = first_timestamp[0..7] # YYYYMMDD
             bucket_key = File.join(@remote_log_directory, date, bucket_filename())
-            thread = BucketUtilities.move_log_file_to_bucket(@filename, bucket_key)
+            threads << BucketUtilities.move_log_file_to_bucket(@filename, bucket_key)
             # Now that the file is in storage, trim the Redis stream after a delay
             @cleanup_offsets << {}
             @last_offsets.each do |redis_topic, last_offset|
@@ -325,7 +327,7 @@ module OpenC3
       ensure
         @mutex.unlock if take_mutex
       end
-      return thread
+      return threads
     end
 
     def bucket_filename

data/lib/openc3/logs/packet_log_writer.rb

@@ -53,7 +53,8 @@ module OpenC3
       cycle_time = nil,
       cycle_size = 1_000_000_000,
       cycle_hour = nil,
-      cycle_minute = nil
+      cycle_minute = nil,
+      enforce_time_order = true
     )
       super(
         remote_log_directory,
@@ -61,7 +62,8 @@ module OpenC3
         cycle_time,
         cycle_size,
         cycle_hour,
-        cycle_minute
+        cycle_minute,
+        enforce_time_order
       )
       @label = label
       @index_file = nil
@@ -148,7 +150,7 @@ module OpenC3
           write_entry(:OFFSET_MARKER, nil, nil, nil, nil, nil, last_offset + ',' + redis_topic, nil) if @file
         end
 
-        threads << super(false)
+        threads.concat(super(false))
 
         if @index_file
           begin

data/lib/openc3/microservices/log_microservice.rb

@@ -79,8 +79,8 @@ module OpenC3
         rt_label = "#{scope}__#{target_name}__ALL__rt__#{type}"
         stored_label = "#{scope}__#{target_name}__ALL__stored__#{type}"
         @plws[target_name] ||= {
-          :RT => BufferedPacketLogWriter.new(remote_log_directory, rt_label, true, @cycle_time, @cycle_size, nil, nil, @buffer_depth),
-          :STORED => BufferedPacketLogWriter.new(remote_log_directory, stored_label, true, @cycle_time, @cycle_size, nil, nil, @buffer_depth)
+          :RT => BufferedPacketLogWriter.new(remote_log_directory, rt_label, true, @cycle_time, @cycle_size, nil, nil, true, @buffer_depth),
+          :STORED => BufferedPacketLogWriter.new(remote_log_directory, stored_label, true, @cycle_time, @cycle_size, nil, nil, true, @buffer_depth)
         }
       end
     end

data/lib/openc3/microservices/reducer_microservice.rb

@@ -208,7 +208,7 @@ module OpenC3
         # 20220101204857274290500__20220101205857276524900__DEFAULT__INST__ALL__rt__reduced_minute.bin
         remote_log_directory = "#{scope}/reduced_#{type}_logs/tlm/#{target_name}"
         label = "#{scope}__#{target_name}__ALL__#{rt_or_stored}__reduced_#{type}"
-        plw = BufferedPacketLogWriter.new(remote_log_directory, label, true, nil, 1_000_000_000, nil, nil, @buffer_depth)
+        plw = BufferedPacketLogWriter.new(remote_log_directory, label, true, nil, 1_000_000_000, nil, nil, true, @buffer_depth)
         @packet_logs["#{scope}__#{target_name}__#{rt_or_stored}__#{type}"] = plw
       end
 

data/lib/openc3/microservices/text_log_microservice.rb

@@ -17,7 +17,7 @@
 # All changes Copyright 2022, OpenC3, Inc.
 # All Rights Reserved
 #
-# This file may also be used under the terms of a commercial license 
+# This file may also be used under the terms of a commercial license
 # if purchased from OpenC3, Inc.
 
 require 'openc3/microservices/microservice'
@@ -44,36 +44,35 @@ module OpenC3
     end
 
     def run
-      tlws = setup_tlws
+      setup_tlws()
       while true
         break if @cancel_thread
 
         Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
           break if @cancel_thread
 
-          log_data(tlws, topic, msg_id, msg_hash, redis)
+          log_data(topic, msg_id, msg_hash, redis)
         end
       end
     end
 
     def setup_tlws
-      tlws = {}
+      @tlws = {}
       @topics.each do |topic|
         topic_split = topic.gsub(/{|}/, '').split("__") # Remove the redis hashtag curly braces
         scope = topic_split[0]
         log_name = topic_split[1]
         remote_log_directory = "#{scope}/text_logs/#{log_name}"
-        tlws[topic] = TextLogWriter.new(remote_log_directory, true, @cycle_time, @cycle_size)
+        @tlws[topic] = TextLogWriter.new(remote_log_directory, true, @cycle_time, @cycle_size, nil, nil, false)
       end
-      return tlws
     end
 
-    def log_data(tlws, topic, msg_id, msg_hash, redis)
+    def log_data(topic, msg_id, msg_hash, redis)
       start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
       keys = msg_hash.keys
       keys.delete("time")
       entry = keys.reduce("") { |data, key| data + "#{key}: #{msg_hash[key]}\t" }
-      tlws[topic].write(msg_hash["time"].to_i, entry, topic, msg_id)
+      @tlws[topic].write(msg_hash["time"].to_i, entry, topic, msg_id)
       @count += 1
       diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
       @metric.add_sample(name: "log_duration_seconds", value: diff, labels: {})
@@ -81,6 +80,19 @@ module OpenC3
       @error = err
       Logger.error("#{@name} error: #{err.formatted}")
     end
+
+    def shutdown
+      # Make sure all the existing logs are properly closed down
+      threads = []
+      @tlws.each do |topic, tlw|
+        threads.concat(tlw.shutdown)
+      end
+      # Wait for all the logging threads to move files to buckets
+      threads.flatten.compact.each do |thread|
+        thread.join
+      end
+      super()
+    end
   end
 end
 

data/lib/openc3/version.rb

@@ -1,14 +1,14 @@
 # encoding: ascii-8bit
 
-OPENC3_VERSION = '5.1.0'
+OPENC3_VERSION = '5.1.1'
 module OpenC3
   module Version
     MAJOR = '5'
     MINOR = '1'
-    PATCH = '0'
+    PATCH = '1'
     OTHER = ''
-    BUILD = '7bd7fb9fdfa75abefb1b6ef01a55931c7b1d98dd'
+    BUILD = 'c94738fcddee73959df8a60ac8746ab38a16c17c'
   end
-  VERSION = '5.1.0'
-  GEM_VERSION = '5.1.0'
+  VERSION = '5.1.1'
+  GEM_VERSION = '5.1.1'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: openc3
 version: !ruby/object:Gem::Version
-  version: 5.1.0
+  version: 5.1.1
 platform: ruby
 authors:
 - Ryan Melton
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-11-17 00:00:00.000000000 Z
+date: 2022-11-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler