openc3 5.10.1 → 5.11.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Rakefile +3 -2
- data/data/config/target.yaml +9 -0
- data/ext/openc3/ext/packet/packet.c +3 -0
- data/ext/openc3/ext/reducer_microservice/extconf.rb +13 -0
- data/ext/openc3/ext/reducer_microservice/reducer_microservice.c +165 -0
- data/ext/openc3/ext/structure/structure.c +7 -9
- data/lib/openc3/accessors/accessor.rb +53 -3
- data/lib/openc3/accessors/binary_accessor.rb +16 -0
- data/lib/openc3/accessors/cbor_accessor.rb +3 -3
- data/lib/openc3/accessors/form_accessor.rb +78 -0
- data/lib/openc3/accessors/http_accessor.rb +145 -0
- data/lib/openc3/accessors/json_accessor.rb +19 -3
- data/lib/openc3/accessors/xml_accessor.rb +18 -1
- data/lib/openc3/accessors.rb +3 -1
- data/lib/openc3/config/config_parser.rb +7 -5
- data/lib/openc3/config/meta_config_parser.rb +1 -1
- data/lib/openc3/core_ext/string.rb +16 -1
- data/lib/openc3/interfaces/http_client_interface.rb +202 -0
- data/lib/openc3/interfaces/http_server_interface.rb +183 -0
- data/lib/openc3/interfaces/interface.rb +86 -16
- data/lib/openc3/interfaces/mqtt_interface.rb +6 -5
- data/lib/openc3/interfaces/protocols/burst_protocol.rb +11 -11
- data/lib/openc3/interfaces/protocols/cobs_protocol.rb +7 -7
- data/lib/openc3/interfaces/protocols/crc_protocol.rb +7 -7
- data/lib/openc3/interfaces/protocols/length_protocol.rb +6 -6
- data/lib/openc3/interfaces/protocols/preidentified_protocol.rb +9 -5
- data/lib/openc3/interfaces/protocols/protocol.rb +8 -6
- data/lib/openc3/interfaces/protocols/slip_protocol.rb +8 -8
- data/lib/openc3/interfaces/protocols/template_protocol.rb +6 -7
- data/lib/openc3/interfaces/protocols/terminated_protocol.rb +4 -4
- data/lib/openc3/interfaces/simulated_target_interface.rb +2 -0
- data/lib/openc3/interfaces/stream_interface.rb +6 -4
- data/lib/openc3/interfaces/tcpip_server_interface.rb +2 -0
- data/lib/openc3/interfaces/udp_interface.rb +8 -5
- data/lib/openc3/interfaces.rb +2 -0
- data/lib/openc3/logs/buffered_packet_log_writer.rb +6 -7
- data/lib/openc3/logs/log_writer.rb +2 -10
- data/lib/openc3/logs/packet_log_constants.rb +13 -3
- data/lib/openc3/logs/packet_log_reader.rb +35 -98
- data/lib/openc3/logs/packet_log_writer.rb +24 -62
- data/lib/openc3/logs/text_log_writer.rb +32 -6
- data/lib/openc3/microservices/cleanup_microservice.rb +23 -16
- data/lib/openc3/microservices/decom_microservice.rb +8 -20
- data/lib/openc3/microservices/log_microservice.rb +3 -1
- data/lib/openc3/microservices/reaction_microservice.rb +22 -11
- data/lib/openc3/microservices/reducer_microservice.rb +174 -130
- data/lib/openc3/{models/notification_model.rb → microservices/scope_cleanup_microservice.rb} +20 -21
- data/lib/openc3/microservices/text_log_microservice.rb +2 -5
- data/lib/openc3/microservices/timeline_microservice.rb +0 -1
- data/lib/openc3/microservices/trigger_group_microservice.rb +0 -1
- data/lib/openc3/migrations/20230915000002_no_scope_log_messages.rb +44 -0
- data/lib/openc3/models/microservice_model.rb +1 -1
- data/lib/openc3/models/scope_model.rb +65 -34
- data/lib/openc3/models/target_model.rb +25 -5
- data/lib/openc3/models/tool_model.rb +14 -2
- data/lib/openc3/models/widget_model.rb +1 -1
- data/lib/openc3/packets/json_packet.rb +10 -2
- data/lib/openc3/packets/packet.rb +30 -9
- data/lib/openc3/packets/packet_config.rb +6 -2
- data/lib/openc3/packets/parsers/packet_item_parser.rb +11 -6
- data/lib/openc3/packets/structure.rb +19 -12
- data/lib/openc3/script/storage.rb +1 -1
- data/lib/openc3/script/web_socket_api.rb +17 -14
- data/lib/openc3/topics/telemetry_topic.rb +2 -1
- data/lib/openc3/utilities/bucket_utilities.rb +2 -0
- data/lib/openc3/utilities/cli_generator.rb +1 -1
- data/lib/openc3/utilities/logger.rb +62 -47
- data/lib/openc3/utilities/metric.rb +19 -1
- data/lib/openc3/utilities/sleeper.rb +3 -1
- data/lib/openc3/utilities/throttle.rb +76 -0
- data/lib/openc3/version.rb +6 -6
- data/templates/tool_angular/package.json +20 -20
- data/templates/tool_angular/yarn.lock +112 -106
- data/templates/tool_react/package.json +16 -18
- data/templates/tool_react/yarn.lock +977 -664
- data/templates/tool_svelte/.prettierrc.js +5 -0
- data/templates/tool_svelte/package.json +18 -18
- data/templates/tool_svelte/src/services/cable.js +1 -1
- data/templates/tool_svelte/src/services/openc3-api.js +173 -173
- data/templates/tool_svelte/yarn.lock +767 -665
- data/templates/tool_vue/package.json +10 -10
- data/templates/tool_vue/yarn.lock +225 -43
- data/templates/widget/package.json +10 -10
- data/templates/widget/yarn.lock +223 -46
- metadata +41 -4
- data/lib/openc3/topics/notifications_topic.rb +0 -31
@@ -57,14 +57,14 @@ module OpenC3
|
|
57
57
|
super(discard_leading_bytes, sync_pattern, fill_fields, allow_empty_data)
|
58
58
|
end
|
59
59
|
|
60
|
-
def write_data(data)
|
60
|
+
def write_data(data, extra = nil)
|
61
61
|
raise "Packet contains termination characters!" if data.index(@write_termination_characters)
|
62
62
|
|
63
|
-
data = super(data)
|
63
|
+
data, extra = super(data, extra)
|
64
64
|
@write_termination_characters.each_byte do |byte|
|
65
65
|
data << byte
|
66
66
|
end
|
67
|
-
return data
|
67
|
+
return data, extra
|
68
68
|
end
|
69
69
|
|
70
70
|
protected
|
@@ -88,7 +88,7 @@ module OpenC3
|
|
88
88
|
end
|
89
89
|
end
|
90
90
|
@data.replace(@data[(index + @read_termination_characters.length)..-1])
|
91
|
-
return packet_data
|
91
|
+
return packet_data, @extra
|
92
92
|
else
|
93
93
|
return :STOP
|
94
94
|
end
|
@@ -54,6 +54,7 @@ module OpenC3
|
|
54
54
|
# Save the current time + delta as the next expected tick time
|
55
55
|
@next_tick_time = Time.now.sys + @sim_target.tick_period_seconds
|
56
56
|
|
57
|
+
super()
|
57
58
|
@connected = true
|
58
59
|
end
|
59
60
|
|
@@ -150,6 +151,7 @@ module OpenC3
|
|
150
151
|
# Disconnect from the simulator
|
151
152
|
def disconnect
|
152
153
|
@connected = false
|
154
|
+
super()
|
153
155
|
end
|
154
156
|
|
155
157
|
protected
|
@@ -72,13 +72,15 @@ module OpenC3
|
|
72
72
|
return nil
|
73
73
|
end
|
74
74
|
|
75
|
-
|
76
|
-
data
|
75
|
+
extra = nil
|
76
|
+
read_interface_base(data, extra)
|
77
|
+
return data, extra
|
77
78
|
end
|
78
79
|
|
79
|
-
def write_interface(data)
|
80
|
-
write_interface_base(data)
|
80
|
+
def write_interface(data, extra = nil)
|
81
|
+
write_interface_base(data, extra)
|
81
82
|
@stream.write(data)
|
83
|
+
return data, extra
|
82
84
|
end
|
83
85
|
end
|
84
86
|
end
|
@@ -156,6 +156,7 @@ module OpenC3
|
|
156
156
|
@write_thread = nil
|
157
157
|
@write_raw_thread = nil
|
158
158
|
end
|
159
|
+
super()
|
159
160
|
@connected = true
|
160
161
|
end
|
161
162
|
|
@@ -204,6 +205,7 @@ module OpenC3
|
|
204
205
|
|
205
206
|
shutdown_interfaces(@write_interface_infos)
|
206
207
|
@connected = false
|
208
|
+
super()
|
207
209
|
end
|
208
210
|
|
209
211
|
# Gracefully kill all the threads
|
@@ -121,6 +121,7 @@ module OpenC3
|
|
121
121
|
) if @write_dest_port
|
122
122
|
end
|
123
123
|
@thread_sleeper = nil
|
124
|
+
super()
|
124
125
|
end
|
125
126
|
|
126
127
|
# @return [Boolean] Whether the active ports (read and/or write) have
|
@@ -146,6 +147,7 @@ module OpenC3
|
|
146
147
|
@read_socket = nil
|
147
148
|
@thread_sleeper.cancel if @thread_sleeper
|
148
149
|
@thread_sleeper = nil
|
150
|
+
super()
|
149
151
|
end
|
150
152
|
|
151
153
|
def read
|
@@ -161,18 +163,19 @@ module OpenC3
|
|
161
163
|
def read_interface
|
162
164
|
data = @read_socket.read(@read_timeout)
|
163
165
|
Logger.info "#{@name}: Udp read returned 0 bytes (stream closed)" if data.length <= 0
|
164
|
-
|
165
|
-
|
166
|
+
extra = nil
|
167
|
+
read_interface_base(data, extra)
|
168
|
+
return data, extra
|
166
169
|
rescue IOError # Disconnected
|
167
170
|
return nil
|
168
171
|
end
|
169
172
|
|
170
173
|
# Writes to the socket
|
171
174
|
# @param data [String] Raw packet data
|
172
|
-
def write_interface(data)
|
173
|
-
write_interface_base(data)
|
175
|
+
def write_interface(data, extra = nil)
|
176
|
+
write_interface_base(data, extra)
|
174
177
|
@write_socket.write(data, @write_timeout)
|
175
|
-
data
|
178
|
+
return data, extra
|
176
179
|
end
|
177
180
|
end
|
178
181
|
end
|
data/lib/openc3/interfaces.rb
CHANGED
@@ -22,6 +22,8 @@
|
|
22
22
|
|
23
23
|
module OpenC3
|
24
24
|
autoload(:Interface, 'openc3/interfaces/interface.rb')
|
25
|
+
autoload(:HttpClientInterface, 'openc3/interfaces/http_client_interface.rb')
|
26
|
+
autoload(:HttpServerInterface, 'openc3/interfaces/http_server_interface.rb')
|
25
27
|
autoload(:MqttInterface, 'openc3/interfaces/mqtt_interface.rb')
|
26
28
|
autoload(:StreamInterface, 'openc3/interfaces/stream_interface.rb')
|
27
29
|
autoload(:SerialInterface, 'openc3/interfaces/serial_interface.rb')
|
@@ -75,23 +75,22 @@ module OpenC3
|
|
75
75
|
# @param data [String] Binary string of data
|
76
76
|
# @param id [Integer] Target ID
|
77
77
|
# @param redis_offset [Integer] The offset of this packet in its Redis stream
|
78
|
-
def buffered_write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0')
|
78
|
+
def buffered_write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0', received_time_nsec_since_epoch: nil, extra: nil)
|
79
79
|
case entry_type
|
80
80
|
when :RAW_PACKET, :JSON_PACKET
|
81
|
-
@buffer << [entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset]
|
81
|
+
@buffer << [entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset, received_time_nsec_since_epoch, extra]
|
82
82
|
@buffer.sort! {|entry1, entry2| entry1[4] <=> entry2[4] }
|
83
83
|
if @buffer.length >= @buffer_depth
|
84
84
|
entry = @buffer.shift
|
85
|
-
write(*entry)
|
85
|
+
write(*entry[0..-3], received_time_nsec_since_epoch: entry[-2], extra: entry[-1])
|
86
86
|
end
|
87
87
|
else
|
88
|
-
write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset)
|
88
|
+
write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, redis_topic, redis_offset, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: extra)
|
89
89
|
end
|
90
90
|
end
|
91
91
|
|
92
92
|
def buffered_first_time_nsec
|
93
|
-
|
94
|
-
return time.to_nsec_from_epoch if time
|
93
|
+
return @first_time if @first_time
|
95
94
|
return @buffer[0][4] if @buffer[0]
|
96
95
|
return nil
|
97
96
|
end
|
@@ -115,7 +114,7 @@ module OpenC3
|
|
115
114
|
def write_buffer
|
116
115
|
begin
|
117
116
|
@buffer.each do |entry|
|
118
|
-
write(*entry, allow_new_file: false, take_mutex: false)
|
117
|
+
write(*entry[0..-3], allow_new_file: false, take_mutex: false, received_time_nsec_since_epoch: entry[-2], extra: entry[-1])
|
119
118
|
end
|
120
119
|
rescue => err
|
121
120
|
Logger.instance.error "Error writing out buffer : #{err.formatted}"
|
@@ -350,20 +350,12 @@ module OpenC3
|
|
350
350
|
'.log'.freeze
|
351
351
|
end
|
352
352
|
|
353
|
-
def first_time
|
354
|
-
Time.from_nsec_from_epoch(@first_time)
|
355
|
-
end
|
356
|
-
|
357
|
-
def last_time
|
358
|
-
Time.from_nsec_from_epoch(@last_time)
|
359
|
-
end
|
360
|
-
|
361
353
|
def first_timestamp
|
362
|
-
first_time
|
354
|
+
Time.from_nsec_from_epoch(@first_time).to_timestamp # "YYYYMMDDHHmmSSNNNNNNNNN"
|
363
355
|
end
|
364
356
|
|
365
357
|
def last_timestamp
|
366
|
-
last_time
|
358
|
+
Time.from_nsec_from_epoch(@last_time).to_timestamp # "YYYYMMDDHHmmSSNNNNNNNNN"
|
367
359
|
end
|
368
360
|
end
|
369
361
|
end
|
@@ -17,7 +17,7 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
module OpenC3
|
@@ -38,6 +38,8 @@ module OpenC3
|
|
38
38
|
OPENC3_JSON_PACKET_ENTRY_TYPE_MASK = 0x4000
|
39
39
|
OPENC3_OFFSET_MARKER_ENTRY_TYPE_MASK = 0x5000
|
40
40
|
OPENC3_KEY_MAP_ENTRY_TYPE_MASK = 0x6000
|
41
|
+
OPENC3_RECEIVED_TIME_FLAG_MASK = 0x0040
|
42
|
+
OPENC3_EXTRA_FLAG_MASK = 0x0080
|
41
43
|
OPENC3_CBOR_FLAG_MASK = 0x0100
|
42
44
|
OPENC3_ID_FLAG_MASK = 0x0200
|
43
45
|
OPENC3_STORED_FLAG_MASK = 0x0400
|
@@ -47,8 +49,8 @@ module OpenC3
|
|
47
49
|
OPENC3_MAX_PACKET_INDEX = 65535
|
48
50
|
OPENC3_MAX_TARGET_INDEX = 65535
|
49
51
|
|
50
|
-
OPENC3_PRIMARY_FIXED_SIZE = 2
|
51
|
-
OPENC3_TARGET_DECLARATION_SECONDARY_FIXED_SIZE = 0
|
52
|
+
OPENC3_PRIMARY_FIXED_SIZE = 2 # 2 bytes for flags - Size of length field is not included in length value
|
53
|
+
OPENC3_TARGET_DECLARATION_SECONDARY_FIXED_SIZE = 0 # No additional data beyond 'Nn' (Length, Flags)
|
52
54
|
OPENC3_TARGET_DECLARATION_PACK_DIRECTIVE = 'Nn'.freeze
|
53
55
|
OPENC3_TARGET_DECLARATION_PACK_ITEMS = 2 # Useful for testing
|
54
56
|
|
@@ -67,5 +69,13 @@ module OpenC3
|
|
67
69
|
OPENC3_PACKET_SECONDARY_FIXED_SIZE = 10
|
68
70
|
OPENC3_PACKET_PACK_DIRECTIVE = 'NnnQ>'.freeze
|
69
71
|
OPENC3_PACKET_PACK_ITEMS = 4 # Useful for testing
|
72
|
+
|
73
|
+
OPENC3_RECEIVED_TIME_FIXED_SIZE = 8
|
74
|
+
OPENC3_RECEIVED_TIME_PACK_DIRECTIVE = 'Q>'.freeze
|
75
|
+
OPENC3_RECEIVED_TIME_PACK_ITEMS = 1 # Useful for testing
|
76
|
+
|
77
|
+
OPENC3_EXTRA_LENGTH_FIXED_SIZE = 4
|
78
|
+
OPENC3_EXTRA_LENGTH_PACK_DIRECTIVE = 'N'.freeze
|
79
|
+
OPENC3_EXTRA_LENGTH_PACK_ITEMS = 1 # Useful for testing
|
70
80
|
end
|
71
81
|
end
|
@@ -127,35 +127,41 @@ module OpenC3
|
|
127
127
|
id = true if flags & OPENC3_ID_FLAG_MASK == OPENC3_ID_FLAG_MASK
|
128
128
|
cbor = false
|
129
129
|
cbor = true if flags & OPENC3_CBOR_FLAG_MASK == OPENC3_CBOR_FLAG_MASK
|
130
|
+
includes_received_time = false
|
131
|
+
includes_received_time = true if flags & OPENC3_RECEIVED_TIME_FLAG_MASK == OPENC3_RECEIVED_TIME_FLAG_MASK
|
132
|
+
includes_extra = false
|
133
|
+
includes_extra = true if flags & OPENC3_EXTRA_FLAG_MASK == OPENC3_EXTRA_FLAG_MASK
|
130
134
|
|
131
135
|
if flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_JSON_PACKET_ENTRY_TYPE_MASK
|
132
136
|
packet_index, time_nsec_since_epoch = entry[2..11].unpack('nQ>')
|
133
|
-
|
137
|
+
next_offset = 12
|
138
|
+
received_time_nsec_since_epoch, extra, json_data = handle_received_time_extra_and_data(entry, time_nsec_since_epoch, includes_received_time, includes_extra, cbor)
|
134
139
|
lookup_cmd_or_tlm, target_name, packet_name, id, key_map = @packets[packet_index]
|
135
140
|
if cmd_or_tlm != lookup_cmd_or_tlm
|
136
141
|
raise "Packet type mismatch, packet:#{cmd_or_tlm}, lookup:#{lookup_cmd_or_tlm}"
|
137
142
|
end
|
138
143
|
|
139
144
|
if cbor
|
140
|
-
return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, CBOR.decode(json_data), key_map)
|
145
|
+
return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, CBOR.decode(json_data), key_map, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: extra)
|
141
146
|
else
|
142
|
-
return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, json_data, key_map)
|
147
|
+
return JsonPacket.new(cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, json_data, key_map, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: extra)
|
143
148
|
end
|
144
149
|
elsif flags & OPENC3_ENTRY_TYPE_MASK == OPENC3_RAW_PACKET_ENTRY_TYPE_MASK
|
145
150
|
packet_index, time_nsec_since_epoch = entry[2..11].unpack('nQ>')
|
146
|
-
packet_data = entry
|
151
|
+
received_time_nsec_since_epoch, extra, packet_data = handle_received_time_extra_and_data(entry, time_nsec_since_epoch, includes_received_time, includes_extra, cbor)
|
147
152
|
lookup_cmd_or_tlm, target_name, packet_name, id = @packets[packet_index]
|
148
153
|
if cmd_or_tlm != lookup_cmd_or_tlm
|
149
154
|
raise "Packet type mismatch, packet:#{cmd_or_tlm}, lookup:#{lookup_cmd_or_tlm}"
|
150
155
|
end
|
151
156
|
|
152
|
-
received_time = Time.from_nsec_from_epoch(time_nsec_since_epoch)
|
153
157
|
if identify_and_define
|
154
|
-
packet = identify_and_define_packet_data(cmd_or_tlm, target_name, packet_name,
|
158
|
+
packet = identify_and_define_packet_data(cmd_or_tlm, target_name, packet_name, packet_data)
|
155
159
|
else
|
156
160
|
# Build Packet
|
157
161
|
packet = Packet.new(target_name, packet_name, :BIG_ENDIAN, nil, packet_data)
|
158
162
|
end
|
163
|
+
packet.packet_time = Time.from_nsec_from_epoch(time_nsec_since_epoch)
|
164
|
+
received_time = Time.from_nsec_from_epoch(received_time_nsec_since_epoch)
|
159
165
|
packet.set_received_time_fast(received_time)
|
160
166
|
packet.cmd_or_tlm = cmd_or_tlm
|
161
167
|
packet.stored = stored
|
@@ -219,95 +225,6 @@ module OpenC3
|
|
219
225
|
raise err
|
220
226
|
end
|
221
227
|
|
222
|
-
# TODO: Currently not used
|
223
|
-
# Returns an analysis of the log file by reading all the packets and
|
224
|
-
# returning information about each packet. This information maps directly
|
225
|
-
# to the parameters need by the {#read_at_offset} method and thus should be
|
226
|
-
# called before using {#read_at_offset}.
|
227
|
-
#
|
228
|
-
# @param filename [String] The filename to analyze
|
229
|
-
# @param progress_callback [Proc] Callback that should receive a single
|
230
|
-
# floating point parameter which is the percentage done
|
231
|
-
# @return [Array<Array<Integer, Integer, String, String, Time, Time>] Array
|
232
|
-
# of arrays for each packet found in the log file consisting of:
|
233
|
-
# [File position, length, target name, packet name, time formatted,
|
234
|
-
# received time].
|
235
|
-
# def packet_offsets(filename, progress_callback = nil)
|
236
|
-
# open(filename)
|
237
|
-
# offsets = []
|
238
|
-
# filesize = size().to_f
|
239
|
-
|
240
|
-
# while true
|
241
|
-
# current_pos = @file.pos
|
242
|
-
# packet = read(false)
|
243
|
-
# break unless packet
|
244
|
-
# offsets << current_pos
|
245
|
-
# if progress_callback
|
246
|
-
# break if progress_callback.call(current_pos / filesize)
|
247
|
-
# end
|
248
|
-
# end
|
249
|
-
|
250
|
-
# return offsets
|
251
|
-
# ensure
|
252
|
-
# close()
|
253
|
-
# end
|
254
|
-
|
255
|
-
# TODO: Currently not used
|
256
|
-
# Reads a packet from the opened log file. Should only be used in
|
257
|
-
# conjunction with {#packet_offsets}.
|
258
|
-
#
|
259
|
-
# @param file_offset [Integer] Byte offset into the log file to start
|
260
|
-
# reading
|
261
|
-
# @param identify_and_define (see #each)
|
262
|
-
# @return [Packet]
|
263
|
-
# def read_at_offset(file_offset, identify_and_define = true)
|
264
|
-
# @file.seek(file_offset, IO::SEEK_SET)
|
265
|
-
# return read(identify_and_define)
|
266
|
-
# rescue => err
|
267
|
-
# close()
|
268
|
-
# raise err
|
269
|
-
# end
|
270
|
-
|
271
|
-
# TODO: Currently not used
|
272
|
-
# Read the first packet from the log file and reset the file position back
|
273
|
-
# to the current position. This allows the client to call read multiple
|
274
|
-
# times to return packets, call first, and continue calling read which will
|
275
|
-
# return the next packet in the file.
|
276
|
-
#
|
277
|
-
# @return [Packet]
|
278
|
-
# def first
|
279
|
-
# original_position = @file.pos
|
280
|
-
# @file.seek(0, IO::SEEK_SET)
|
281
|
-
# read_file_header()
|
282
|
-
# packet = read()
|
283
|
-
# raise "No first packet found" unless packet
|
284
|
-
# @file.seek(original_position, IO::SEEK_SET)
|
285
|
-
# packet.clone
|
286
|
-
# rescue => err
|
287
|
-
# close()
|
288
|
-
# raise err
|
289
|
-
# end
|
290
|
-
|
291
|
-
# TODO: Currently not used
|
292
|
-
# Read the last packet from the log file and reset the file position back
|
293
|
-
# to the current position. This allows the client to call read multiple
|
294
|
-
# times to return packets, call last, and continue calling read which will
|
295
|
-
# return the next packet in the file.
|
296
|
-
#
|
297
|
-
# @return [Packet]
|
298
|
-
# def last
|
299
|
-
# raise "TODO: Implement me - Need to add end of file entry to support"
|
300
|
-
# original_position = @file.pos
|
301
|
-
# @file.seek(-1, IO::SEEK_END)
|
302
|
-
# packet = search(-1)
|
303
|
-
# raise "No last packet found" unless packet
|
304
|
-
# @file.seek(original_position, IO::SEEK_SET)
|
305
|
-
# packet.clone
|
306
|
-
# rescue => err
|
307
|
-
# close()
|
308
|
-
# raise err
|
309
|
-
# end
|
310
|
-
|
311
228
|
# @return [Integer] The size of the log file being processed
|
312
229
|
def size
|
313
230
|
@file.stat.size
|
@@ -333,7 +250,7 @@ module OpenC3
|
|
333
250
|
end
|
334
251
|
|
335
252
|
# This is best effort. May return unidentified/undefined packets
|
336
|
-
def identify_and_define_packet_data(cmd_or_tlm, target_name, packet_name,
|
253
|
+
def identify_and_define_packet_data(cmd_or_tlm, target_name, packet_name, packet_data)
|
337
254
|
packet = nil
|
338
255
|
unless target_name and packet_name
|
339
256
|
if cmd_or_tlm == :CMD
|
@@ -376,8 +293,28 @@ module OpenC3
|
|
376
293
|
end
|
377
294
|
end
|
378
295
|
|
379
|
-
|
380
|
-
|
296
|
+
# Handle common optional fields in raw and JSON packets
|
297
|
+
def handle_received_time_extra_and_data(entry, time_nsec_since_epoch, includes_received_time, includes_extra, cbor)
|
298
|
+
next_offset = 12
|
299
|
+
received_time_nsec_since_epoch = time_nsec_since_epoch
|
300
|
+
if includes_received_time
|
301
|
+
received_time_nsec_since_epoch = entry[next_offset..(next_offset + OPENC3_RECEIVED_TIME_FIXED_SIZE - 1)].unpack(OPENC3_RECEIVED_TIME_PACK_DIRECTIVE)
|
302
|
+
next_offset += OPENC3_RECEIVED_TIME_FIXED_SIZE
|
303
|
+
end
|
304
|
+
extra = nil
|
305
|
+
if includes_extra
|
306
|
+
extra_length = entry[next_offset..(next_offset + OPENC3_EXTRA_LENGTH_FIXED_SIZE - 1)].unpack(OPENC3_EXTRA_LENGTH_PACK_DIRECTIVE)
|
307
|
+
next_offset += OPENC3_EXTRA_LENGTH_FIXED_SIZE
|
308
|
+
extra_encoded = entry[next_offset..(next_offset + extra_length - 1)]
|
309
|
+
next_offset += extra_length
|
310
|
+
if cbor
|
311
|
+
extra = CBOR.decode(extra_encoded)
|
312
|
+
else
|
313
|
+
extra = JSON.parse(extra_encode, allow_nan: true, create_additions: true)
|
314
|
+
end
|
315
|
+
end
|
316
|
+
data = entry[next_offset..-1]
|
317
|
+
return received_time_nsec_since_epoch, extra, data
|
381
318
|
end
|
382
319
|
end
|
383
320
|
end
|
@@ -64,22 +64,15 @@ module OpenC3
|
|
64
64
|
enforce_time_order
|
65
65
|
)
|
66
66
|
@label = label
|
67
|
-
@index_file = nil
|
68
|
-
@index_filename = nil
|
69
67
|
@cmd_packet_table = {}
|
70
68
|
@tlm_packet_table = {}
|
71
69
|
@key_map_table = {}
|
72
70
|
@target_dec_entries = []
|
73
71
|
@packet_dec_entries = []
|
74
|
-
@key_map_entries = []
|
75
72
|
@next_packet_index = 0
|
76
73
|
@target_indexes = {}
|
77
74
|
@next_target_index = 0
|
78
75
|
@data_format = :CBOR # Default to CBOR for improved compression
|
79
|
-
|
80
|
-
# This is an optimization to avoid creating a new entry object
|
81
|
-
# each time we create an entry which we do a LOT!
|
82
|
-
@index_entry = String.new
|
83
76
|
end
|
84
77
|
|
85
78
|
# Write a packet to the log file.
|
@@ -97,13 +90,13 @@ module OpenC3
|
|
97
90
|
# @param data [String] Binary string of data
|
98
91
|
# @param id [Integer] Target ID
|
99
92
|
# @param redis_offset [Integer] The offset of this packet in its Redis stream
|
100
|
-
def write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0', take_mutex: true, allow_new_file: true)
|
93
|
+
def write(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id = nil, redis_topic = nil, redis_offset = '0-0', take_mutex: true, allow_new_file: true, received_time_nsec_since_epoch: nil, extra: nil)
|
101
94
|
return if !@logging_enabled
|
102
95
|
|
103
96
|
@mutex.lock if take_mutex
|
104
97
|
begin
|
105
98
|
prepare_write(time_nsec_since_epoch, data.length, redis_topic, redis_offset, allow_new_file: allow_new_file)
|
106
|
-
write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id) if @file
|
99
|
+
write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, received_time_nsec_since_epoch: received_time_nsec_since_epoch, extra: extra) if @file
|
107
100
|
ensure
|
108
101
|
@mutex.unlock if take_mutex
|
109
102
|
end
|
@@ -112,7 +105,7 @@ module OpenC3
|
|
112
105
|
OpenC3.handle_critical_exception(err)
|
113
106
|
end
|
114
107
|
|
115
|
-
# Starting a new
|
108
|
+
# Starting a new file is a critical operation so the entire method is
|
116
109
|
# wrapped with a rescue and handled with handle_critical_exception
|
117
110
|
# Assumes mutex has already been taken
|
118
111
|
def start_new_file
|
@@ -120,10 +113,6 @@ module OpenC3
|
|
120
113
|
@file.write(OPENC3_FILE_HEADER)
|
121
114
|
@file_size += OPENC3_FILE_HEADER.length
|
122
115
|
|
123
|
-
# Start index log file
|
124
|
-
@index_filename = create_unique_filename('.idx'.freeze)
|
125
|
-
@index_file = File.new(@index_filename, 'wb')
|
126
|
-
@index_file.write(OPENC3_INDEX_HEADER)
|
127
116
|
@cmd_packet_table = {}
|
128
117
|
@tlm_packet_table = {}
|
129
118
|
@key_map_table = {}
|
@@ -132,8 +121,6 @@ module OpenC3
|
|
132
121
|
@next_target_index = 0
|
133
122
|
@target_dec_entries = []
|
134
123
|
@packet_dec_entries = []
|
135
|
-
@key_map_entries = []
|
136
|
-
Logger.debug "Index Log File Opened : #{@index_filename}"
|
137
124
|
rescue => err
|
138
125
|
Logger.error "Error starting new log file: #{err.formatted}"
|
139
126
|
@logging_enabled = false
|
@@ -151,22 +138,6 @@ module OpenC3
|
|
151
138
|
write_entry(:OFFSET_MARKER, nil, nil, nil, nil, nil, last_offset + ',' + redis_topic, nil) if @file
|
152
139
|
end
|
153
140
|
|
154
|
-
if @index_file
|
155
|
-
begin
|
156
|
-
write_index_file_footer()
|
157
|
-
@index_file.close unless @index_file.closed?
|
158
|
-
Logger.debug "Index Log File Closed : #{@index_filename}"
|
159
|
-
date = first_timestamp[0..7] # YYYYMMDD
|
160
|
-
bucket_key = File.join(@remote_log_directory, date, "#{first_timestamp}__#{last_timestamp}__#{@label}.idx")
|
161
|
-
threads << BucketUtilities.move_log_file_to_bucket(@index_filename, bucket_key)
|
162
|
-
rescue Exception => err
|
163
|
-
Logger.instance.error "Error closing #{@index_filename} : #{err.formatted}"
|
164
|
-
end
|
165
|
-
|
166
|
-
@index_file = nil
|
167
|
-
@index_filename = nil
|
168
|
-
end
|
169
|
-
|
170
141
|
threads.concat(super(false))
|
171
142
|
|
172
143
|
ensure
|
@@ -241,7 +212,7 @@ module OpenC3
|
|
241
212
|
return packet_index
|
242
213
|
end
|
243
214
|
|
244
|
-
def write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id)
|
215
|
+
def write_entry(entry_type, cmd_or_tlm, target_name, packet_name, time_nsec_since_epoch, stored, data, id, received_time_nsec_since_epoch: nil, extra: nil)
|
245
216
|
raise ArgumentError.new("Length of id must be 64, got #{id.length}") if id and id.length != 64 # 64 hex digits, gets packed to 32 bytes with .pack('H*')
|
246
217
|
|
247
218
|
length = OPENC3_PRIMARY_FIXED_SIZE
|
@@ -283,7 +254,6 @@ module OpenC3
|
|
283
254
|
packet_index = get_packet_index(cmd_or_tlm, target_name, packet_name, entry_type, data)
|
284
255
|
@entry.clear
|
285
256
|
@entry << [length, flags, packet_index].pack(OPENC3_KEY_MAP_PACK_DIRECTIVE) << data
|
286
|
-
@key_map_entries << @entry.dup
|
287
257
|
when :OFFSET_MARKER
|
288
258
|
flags |= OPENC3_OFFSET_MARKER_ENTRY_TYPE_MASK
|
289
259
|
length += OPENC3_OFFSET_MARKER_SECONDARY_FIXED_SIZE + data.length
|
@@ -318,13 +288,28 @@ module OpenC3
|
|
318
288
|
if cmd_or_tlm == :CMD
|
319
289
|
flags |= OPENC3_CMD_FLAG_MASK
|
320
290
|
end
|
291
|
+
if received_time_nsec_since_epoch
|
292
|
+
flags |= OPENC3_RECEIVED_TIME_FLAG_MASK
|
293
|
+
length += OPENC3_RECEIVED_TIME_FIXED_SIZE
|
294
|
+
end
|
295
|
+
extra_encoded = nil
|
296
|
+
if extra
|
297
|
+
flags |= OPENC3_EXTRA_FLAG_MASK
|
298
|
+
extra = JSON.parse(extra, :allow_nan => true, :create_additions => true) if String === extra
|
299
|
+
length += OPENC3_EXTRA_LENGTH_FIXED_SIZE
|
300
|
+
if @data_format == :CBOR
|
301
|
+
extra_encoded = extra.as_json.to_cbor
|
302
|
+
else
|
303
|
+
extra_encoded = JSON.generate(extra.as_json, :allow_nan => true)
|
304
|
+
end
|
305
|
+
length += extra_encoded.length
|
306
|
+
end
|
321
307
|
length += OPENC3_PACKET_SECONDARY_FIXED_SIZE + data.length
|
322
308
|
@entry.clear
|
323
|
-
@
|
324
|
-
@
|
325
|
-
@entry <<
|
326
|
-
@
|
327
|
-
@index_file.write(@index_entry)
|
309
|
+
@entry << [length, flags, packet_index, time_nsec_since_epoch].pack(OPENC3_PACKET_PACK_DIRECTIVE)
|
310
|
+
@entry << [received_time_nsec_since_epoch].pack(OPENC3_RECEIVED_TIME_PACK_DIRECTIVE) if received_time_nsec_since_epoch
|
311
|
+
@entry << [extra_encoded.length].pack(OPENC3_EXTRA_LENGTH_PACK_DIRECTIVE) << extra_encoded if extra_encoded
|
312
|
+
@entry << data
|
328
313
|
@first_time = time_nsec_since_epoch if !@first_time or time_nsec_since_epoch < @first_time
|
329
314
|
@last_time = time_nsec_since_epoch if !@last_time or time_nsec_since_epoch > @last_time
|
330
315
|
else
|
@@ -334,29 +319,6 @@ module OpenC3
|
|
334
319
|
@file_size += @entry.length
|
335
320
|
end
|
336
321
|
|
337
|
-
def write_index_file_footer
|
338
|
-
footer_length = 4 # Includes length of length field at end
|
339
|
-
@index_file.write([@target_dec_entries.length].pack('n'))
|
340
|
-
footer_length += 2
|
341
|
-
@target_dec_entries.each do |target_dec_entry|
|
342
|
-
@index_file.write(target_dec_entry)
|
343
|
-
footer_length += target_dec_entry.length
|
344
|
-
end
|
345
|
-
@index_file.write([@packet_dec_entries.length].pack('n'))
|
346
|
-
footer_length += 2
|
347
|
-
@packet_dec_entries.each do |packet_dec_entry|
|
348
|
-
@index_file.write(packet_dec_entry)
|
349
|
-
footer_length += packet_dec_entry.length
|
350
|
-
end
|
351
|
-
@index_file.write([@key_map_entries.length].pack('n'))
|
352
|
-
footer_length += 2
|
353
|
-
@key_map_entries.each do |key_map_entry|
|
354
|
-
@index_file.write(key_map_entry)
|
355
|
-
footer_length += key_map_entry.length
|
356
|
-
end
|
357
|
-
@index_file.write([footer_length].pack('N'))
|
358
|
-
end
|
359
|
-
|
360
322
|
def bucket_filename
|
361
323
|
"#{first_timestamp}__#{last_timestamp}__#{@label}" + extension
|
362
324
|
end
|
@@ -17,15 +17,22 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
require 'openc3/logs/log_writer'
|
24
|
+
require 'socket'
|
24
25
|
|
25
26
|
module OpenC3
|
26
27
|
# Creates a text log. Can automatically cycle the log based on an elasped
|
27
28
|
# time period or when the log file reaches a predefined size.
|
28
29
|
class TextLogWriter < LogWriter
|
30
|
+
NEWLINE = "\n".freeze
|
31
|
+
def initialize(*args)
|
32
|
+
super(*args)
|
33
|
+
@container_name = Socket.gethostname
|
34
|
+
end
|
35
|
+
|
29
36
|
# Write to the log file.
|
30
37
|
#
|
31
38
|
# If no log file currently exists in the filesystem, a new file will be
|
@@ -47,11 +54,9 @@ module OpenC3
|
|
47
54
|
end
|
48
55
|
|
49
56
|
def write_entry(time_nsec_since_epoch, data)
|
50
|
-
@
|
51
|
-
@
|
52
|
-
@
|
53
|
-
@file.write(@entry)
|
54
|
-
@file_size += @entry.length
|
57
|
+
@file.write(data)
|
58
|
+
@file.write(NEWLINE)
|
59
|
+
@file_size += (data.length + NEWLINE.length)
|
55
60
|
@first_time = time_nsec_since_epoch if !@first_time or time_nsec_since_epoch < @first_time
|
56
61
|
@last_time = time_nsec_since_epoch if !@last_time or time_nsec_since_epoch > @last_time
|
57
62
|
end
|
@@ -65,6 +70,27 @@ module OpenC3
|
|
65
70
|
"#{first_timestamp}__#{last_timestamp}__#{topic_name}" + extension
|
66
71
|
end
|
67
72
|
|
73
|
+
# Closing a log file isn't critical so we just log an error
|
74
|
+
# Returns threads that moves log to bucket
|
75
|
+
def close_file(take_mutex = true)
|
76
|
+
threads = []
|
77
|
+
@mutex.lock if take_mutex
|
78
|
+
begin
|
79
|
+
# Need to write the OFFSET_MARKER for each packet
|
80
|
+
@last_offsets.each do |redis_topic, last_offset|
|
81
|
+
time = Time.now
|
82
|
+
data = { time: time.to_nsec_from_epoch, '@timestamp' => time.xmlschema(3), severity: 'INFO', "microservice_name" => Logger.microservice_name, "container_name" => @container_name, "last_offset" => last_offset, "redis_topic" => redis_topic, "type" => "offset" }
|
83
|
+
write_entry(time.to_nsec_from_epoch, data.as_json(allow_nan: true).to_json(allow_nan: true)) if @file
|
84
|
+
end
|
85
|
+
|
86
|
+
threads.concat(super(false))
|
87
|
+
|
88
|
+
ensure
|
89
|
+
@mutex.unlock if take_mutex
|
90
|
+
end
|
91
|
+
return threads
|
92
|
+
end
|
93
|
+
|
68
94
|
def extension
|
69
95
|
'.txt'.freeze
|
70
96
|
end
|