openc3 5.1.0 → 5.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/openc3/io/json_rpc.rb +6 -4
- data/lib/openc3/logs/buffered_packet_log_writer.rb +3 -1
- data/lib/openc3/logs/log_writer.rb +7 -5
- data/lib/openc3/logs/packet_log_writer.rb +5 -3
- data/lib/openc3/microservices/log_microservice.rb +2 -2
- data/lib/openc3/microservices/reducer_microservice.rb +1 -1
- data/lib/openc3/microservices/text_log_microservice.rb +20 -8
- data/lib/openc3/version.rb +5 -5
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fa2f9aabf07f0b9c9237f2f44644c8da055ca3069d46399db837b3b9bb22fdb3
|
4
|
+
data.tar.gz: 16afda4302a384d554872ff416ba76f3c822cf36f986687256f71e6c2e729370
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 394108166336ca206f0d56caa9d805e3c694a8c80b47915e3552fab228e8a9418bb61bd616ac897127e01b920d7da28d7e3c2bb0673e3e28e45283f44d0c9eda
|
7
|
+
data.tar.gz: ce284cea11dc3bf1c788cb89bfced3ff783b892902ec0f0dcc1922ba6edffda0ef9fe0bc5ef358fec95949e15425f8241e66a36f7a5f9472a39167e0cc4f9b53
|
data/lib/openc3/io/json_rpc.rb
CHANGED
@@ -17,11 +17,12 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
require 'json'
|
24
24
|
require 'date'
|
25
|
+
require 'openc3/core_ext/string'
|
25
26
|
|
26
27
|
class Object
|
27
28
|
def as_json(options = nil) #:nodoc:
|
@@ -57,10 +58,11 @@ class String
|
|
57
58
|
NON_ASCII_PRINTABLE = /[^\x21-\x7e\s]/
|
58
59
|
|
59
60
|
def as_json(options = nil)
|
60
|
-
|
61
|
-
|
61
|
+
as_utf8 = self.dup.force_encoding('UTF-8')
|
62
|
+
if as_utf8.valid_encoding?
|
63
|
+
return as_utf8
|
62
64
|
else
|
63
|
-
self
|
65
|
+
return self.to_json_raw_object
|
64
66
|
end
|
65
67
|
end #:nodoc:
|
66
68
|
end
|
@@ -45,6 +45,7 @@ module OpenC3
|
|
45
45
|
cycle_size = 1_000_000_000,
|
46
46
|
cycle_hour = nil,
|
47
47
|
cycle_minute = nil,
|
48
|
+
enforce_time_order = true,
|
48
49
|
buffer_depth = 10
|
49
50
|
)
|
50
51
|
super(
|
@@ -54,7 +55,8 @@ module OpenC3
|
|
54
55
|
cycle_time,
|
55
56
|
cycle_size,
|
56
57
|
cycle_hour,
|
57
|
-
cycle_minute
|
58
|
+
cycle_minute,
|
59
|
+
enforce_time_order
|
58
60
|
)
|
59
61
|
@buffer_depth = Integer(buffer_depth)
|
60
62
|
@buffer = []
|
@@ -99,7 +99,8 @@ module OpenC3
|
|
99
99
|
cycle_time = nil,
|
100
100
|
cycle_size = 1000000000,
|
101
101
|
cycle_hour = nil,
|
102
|
-
cycle_minute = nil
|
102
|
+
cycle_minute = nil,
|
103
|
+
enforce_time_order = true
|
103
104
|
)
|
104
105
|
@remote_log_directory = remote_log_directory
|
105
106
|
@logging_enabled = ConfigParser.handle_true_false(logging_enabled)
|
@@ -114,6 +115,7 @@ module OpenC3
|
|
114
115
|
@cycle_hour = Integer(@cycle_hour) if @cycle_hour
|
115
116
|
@cycle_minute = ConfigParser.handle_nil(cycle_minute)
|
116
117
|
@cycle_minute = Integer(@cycle_minute) if @cycle_minute
|
118
|
+
@enforce_time_order = ConfigParser.handle_true_false(enforce_time_order)
|
117
119
|
@mutex = Mutex.new
|
118
120
|
@file = nil
|
119
121
|
@file_size = 0
|
@@ -284,7 +286,7 @@ module OpenC3
|
|
284
286
|
elsif @cycle_size and ((@file_size + data_length) > @cycle_size)
|
285
287
|
Logger.debug("Log writer start new file due to cycle size #{@cycle_size}")
|
286
288
|
start_new_file()
|
287
|
-
elsif @previous_time_nsec_since_epoch and (@previous_time_nsec_since_epoch > time_nsec_since_epoch)
|
289
|
+
elsif @enforce_time_order and @previous_time_nsec_since_epoch and (@previous_time_nsec_since_epoch > time_nsec_since_epoch)
|
288
290
|
Logger.debug("Log writer start new file due to out of order time: #{Time.from_nsec_from_epoch(@previous_time_nsec_since_epoch)} #{Time.from_nsec_from_epoch(time_nsec_since_epoch)}")
|
289
291
|
start_new_file()
|
290
292
|
end
|
@@ -297,7 +299,7 @@ module OpenC3
|
|
297
299
|
# to keep a full file's worth of data in the stream. This is what prevents continuous stream growth.
|
298
300
|
# Returns thread that moves log to bucket
|
299
301
|
def close_file(take_mutex = true)
|
300
|
-
|
302
|
+
threads = []
|
301
303
|
@mutex.lock if take_mutex
|
302
304
|
begin
|
303
305
|
if @file
|
@@ -306,7 +308,7 @@ module OpenC3
|
|
306
308
|
Logger.debug "Log File Closed : #{@filename}"
|
307
309
|
date = first_timestamp[0..7] # YYYYMMDD
|
308
310
|
bucket_key = File.join(@remote_log_directory, date, bucket_filename())
|
309
|
-
|
311
|
+
threads << BucketUtilities.move_log_file_to_bucket(@filename, bucket_key)
|
310
312
|
# Now that the file is in storage, trim the Redis stream after a delay
|
311
313
|
@cleanup_offsets << {}
|
312
314
|
@last_offsets.each do |redis_topic, last_offset|
|
@@ -325,7 +327,7 @@ module OpenC3
|
|
325
327
|
ensure
|
326
328
|
@mutex.unlock if take_mutex
|
327
329
|
end
|
328
|
-
return
|
330
|
+
return threads
|
329
331
|
end
|
330
332
|
|
331
333
|
def bucket_filename
|
@@ -53,7 +53,8 @@ module OpenC3
|
|
53
53
|
cycle_time = nil,
|
54
54
|
cycle_size = 1_000_000_000,
|
55
55
|
cycle_hour = nil,
|
56
|
-
cycle_minute = nil
|
56
|
+
cycle_minute = nil,
|
57
|
+
enforce_time_order = true
|
57
58
|
)
|
58
59
|
super(
|
59
60
|
remote_log_directory,
|
@@ -61,7 +62,8 @@ module OpenC3
|
|
61
62
|
cycle_time,
|
62
63
|
cycle_size,
|
63
64
|
cycle_hour,
|
64
|
-
cycle_minute
|
65
|
+
cycle_minute,
|
66
|
+
enforce_time_order
|
65
67
|
)
|
66
68
|
@label = label
|
67
69
|
@index_file = nil
|
@@ -148,7 +150,7 @@ module OpenC3
|
|
148
150
|
write_entry(:OFFSET_MARKER, nil, nil, nil, nil, nil, last_offset + ',' + redis_topic, nil) if @file
|
149
151
|
end
|
150
152
|
|
151
|
-
threads
|
153
|
+
threads.concat(super(false))
|
152
154
|
|
153
155
|
if @index_file
|
154
156
|
begin
|
@@ -79,8 +79,8 @@ module OpenC3
|
|
79
79
|
rt_label = "#{scope}__#{target_name}__ALL__rt__#{type}"
|
80
80
|
stored_label = "#{scope}__#{target_name}__ALL__stored__#{type}"
|
81
81
|
@plws[target_name] ||= {
|
82
|
-
:RT => BufferedPacketLogWriter.new(remote_log_directory, rt_label, true, @cycle_time, @cycle_size, nil, nil, @buffer_depth),
|
83
|
-
:STORED => BufferedPacketLogWriter.new(remote_log_directory, stored_label, true, @cycle_time, @cycle_size, nil, nil, @buffer_depth)
|
82
|
+
:RT => BufferedPacketLogWriter.new(remote_log_directory, rt_label, true, @cycle_time, @cycle_size, nil, nil, true, @buffer_depth),
|
83
|
+
:STORED => BufferedPacketLogWriter.new(remote_log_directory, stored_label, true, @cycle_time, @cycle_size, nil, nil, true, @buffer_depth)
|
84
84
|
}
|
85
85
|
end
|
86
86
|
end
|
@@ -208,7 +208,7 @@ module OpenC3
|
|
208
208
|
# 20220101204857274290500__20220101205857276524900__DEFAULT__INST__ALL__rt__reduced_minute.bin
|
209
209
|
remote_log_directory = "#{scope}/reduced_#{type}_logs/tlm/#{target_name}"
|
210
210
|
label = "#{scope}__#{target_name}__ALL__#{rt_or_stored}__reduced_#{type}"
|
211
|
-
plw = BufferedPacketLogWriter.new(remote_log_directory, label, true, nil, 1_000_000_000, nil, nil, @buffer_depth)
|
211
|
+
plw = BufferedPacketLogWriter.new(remote_log_directory, label, true, nil, 1_000_000_000, nil, nil, true, @buffer_depth)
|
212
212
|
@packet_logs["#{scope}__#{target_name}__#{rt_or_stored}__#{type}"] = plw
|
213
213
|
end
|
214
214
|
|
@@ -17,7 +17,7 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
require 'openc3/microservices/microservice'
|
@@ -44,36 +44,35 @@ module OpenC3
|
|
44
44
|
end
|
45
45
|
|
46
46
|
def run
|
47
|
-
|
47
|
+
setup_tlws()
|
48
48
|
while true
|
49
49
|
break if @cancel_thread
|
50
50
|
|
51
51
|
Topic.read_topics(@topics) do |topic, msg_id, msg_hash, redis|
|
52
52
|
break if @cancel_thread
|
53
53
|
|
54
|
-
log_data(
|
54
|
+
log_data(topic, msg_id, msg_hash, redis)
|
55
55
|
end
|
56
56
|
end
|
57
57
|
end
|
58
58
|
|
59
59
|
def setup_tlws
|
60
|
-
tlws = {}
|
60
|
+
@tlws = {}
|
61
61
|
@topics.each do |topic|
|
62
62
|
topic_split = topic.gsub(/{|}/, '').split("__") # Remove the redis hashtag curly braces
|
63
63
|
scope = topic_split[0]
|
64
64
|
log_name = topic_split[1]
|
65
65
|
remote_log_directory = "#{scope}/text_logs/#{log_name}"
|
66
|
-
tlws[topic] = TextLogWriter.new(remote_log_directory, true, @cycle_time, @cycle_size)
|
66
|
+
@tlws[topic] = TextLogWriter.new(remote_log_directory, true, @cycle_time, @cycle_size, nil, nil, false)
|
67
67
|
end
|
68
|
-
return tlws
|
69
68
|
end
|
70
69
|
|
71
|
-
def log_data(
|
70
|
+
def log_data(topic, msg_id, msg_hash, redis)
|
72
71
|
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
73
72
|
keys = msg_hash.keys
|
74
73
|
keys.delete("time")
|
75
74
|
entry = keys.reduce("") { |data, key| data + "#{key}: #{msg_hash[key]}\t" }
|
76
|
-
tlws[topic].write(msg_hash["time"].to_i, entry, topic, msg_id)
|
75
|
+
@tlws[topic].write(msg_hash["time"].to_i, entry, topic, msg_id)
|
77
76
|
@count += 1
|
78
77
|
diff = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start # seconds as a float
|
79
78
|
@metric.add_sample(name: "log_duration_seconds", value: diff, labels: {})
|
@@ -81,6 +80,19 @@ module OpenC3
|
|
81
80
|
@error = err
|
82
81
|
Logger.error("#{@name} error: #{err.formatted}")
|
83
82
|
end
|
83
|
+
|
84
|
+
def shutdown
|
85
|
+
# Make sure all the existing logs are properly closed down
|
86
|
+
threads = []
|
87
|
+
@tlws.each do |topic, tlw|
|
88
|
+
threads.concat(tlw.shutdown)
|
89
|
+
end
|
90
|
+
# Wait for all the logging threads to move files to buckets
|
91
|
+
threads.flatten.compact.each do |thread|
|
92
|
+
thread.join
|
93
|
+
end
|
94
|
+
super()
|
95
|
+
end
|
84
96
|
end
|
85
97
|
end
|
86
98
|
|
data/lib/openc3/version.rb
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# encoding: ascii-8bit
|
2
2
|
|
3
|
-
OPENC3_VERSION = '5.1.
|
3
|
+
OPENC3_VERSION = '5.1.1'
|
4
4
|
module OpenC3
|
5
5
|
module Version
|
6
6
|
MAJOR = '5'
|
7
7
|
MINOR = '1'
|
8
|
-
PATCH = '
|
8
|
+
PATCH = '1'
|
9
9
|
OTHER = ''
|
10
|
-
BUILD = '
|
10
|
+
BUILD = 'c94738fcddee73959df8a60ac8746ab38a16c17c'
|
11
11
|
end
|
12
|
-
VERSION = '5.1.
|
13
|
-
GEM_VERSION = '5.1.
|
12
|
+
VERSION = '5.1.1'
|
13
|
+
GEM_VERSION = '5.1.1'
|
14
14
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: openc3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.1.
|
4
|
+
version: 5.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Melton
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-11-
|
12
|
+
date: 2022-11-19 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|