openc3 5.1.1 → 5.2.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/openc3cli +48 -9
- data/data/config/interface_modifiers.yaml +14 -0
- data/data/config/parameter_modifiers.yaml +5 -3
- data/data/config/screen.yaml +12 -8
- data/data/config/target.yaml +33 -0
- data/ext/openc3/ext/config_parser/config_parser.c +66 -63
- data/ext/openc3/ext/packet/packet.c +1 -4
- data/lib/openc3/api/README.md +5 -0
- data/lib/openc3/api/api.rb +3 -1
- data/lib/openc3/api/cmd_api.rb +43 -112
- data/lib/openc3/api/interface_api.rb +3 -3
- data/lib/openc3/api/offline_access_api.rb +78 -0
- data/lib/openc3/api/settings_api.rb +3 -1
- data/lib/openc3/api/stash_api.rb +63 -0
- data/lib/openc3/api/target_api.rb +4 -5
- data/lib/openc3/config/config_parser.rb +47 -47
- data/lib/openc3/interfaces/interface.rb +11 -1
- data/lib/openc3/interfaces/protocols/burst_protocol.rb +30 -16
- data/lib/openc3/interfaces/protocols/fixed_protocol.rb +8 -2
- data/lib/openc3/interfaces/protocols/ignore_packet_protocol.rb +2 -2
- data/lib/openc3/interfaces/protocols/override_protocol.rb +2 -2
- data/lib/openc3/interfaces/tcpip_server_interface.rb +3 -1
- data/lib/openc3/io/json_api_object.rb +30 -9
- data/lib/openc3/io/json_drb.rb +6 -1
- data/lib/openc3/io/json_drb_object.rb +18 -9
- data/lib/openc3/io/json_rpc.rb +5 -3
- data/lib/openc3/logs/buffered_packet_log_writer.rb +1 -1
- data/lib/openc3/logs/log_writer.rb +8 -2
- data/lib/openc3/microservices/cleanup_microservice.rb +3 -3
- data/lib/openc3/microservices/decom_microservice.rb +8 -8
- data/lib/openc3/microservices/interface_microservice.rb +86 -71
- data/lib/openc3/microservices/log_microservice.rb +5 -3
- data/lib/openc3/microservices/microservice.rb +18 -14
- data/lib/openc3/microservices/multi_microservice.rb +62 -0
- data/lib/openc3/microservices/periodic_microservice.rb +58 -0
- data/lib/openc3/microservices/reaction_microservice.rb +61 -47
- data/lib/openc3/microservices/reducer_microservice.rb +64 -40
- data/lib/openc3/microservices/router_microservice.rb +4 -4
- data/lib/openc3/microservices/text_log_microservice.rb +2 -2
- data/lib/openc3/microservices/timeline_microservice.rb +44 -30
- data/lib/openc3/microservices/trigger_group_microservice.rb +39 -36
- data/lib/openc3/migrations/20221202214600_add_target_names.rb +30 -0
- data/lib/openc3/migrations/20221210174900_convert_to_multi.rb +65 -0
- data/lib/openc3/models/cvt_model.rb +1 -1
- data/lib/openc3/models/gem_model.rb +24 -20
- data/lib/openc3/models/interface_model.rb +69 -35
- data/lib/openc3/models/metadata_model.rb +1 -1
- data/lib/openc3/models/microservice_model.rb +7 -24
- data/lib/openc3/models/migration_model.rb +52 -0
- data/lib/openc3/models/model.rb +2 -7
- data/lib/openc3/models/note_model.rb +1 -1
- data/lib/openc3/models/offline_access_model.rb +55 -0
- data/lib/openc3/models/plugin_model.rb +12 -3
- data/lib/openc3/models/reaction_model.rb +6 -2
- data/lib/openc3/models/scope_model.rb +89 -13
- data/lib/openc3/models/settings_model.rb +1 -1
- data/lib/openc3/models/stash_model.rb +53 -0
- data/lib/openc3/models/target_model.rb +301 -130
- data/lib/openc3/models/tool_model.rb +1 -12
- data/lib/openc3/models/widget_model.rb +1 -6
- data/lib/openc3/operators/microservice_operator.rb +45 -6
- data/lib/openc3/operators/operator.rb +27 -5
- data/lib/openc3/packets/commands.rb +1 -25
- data/lib/openc3/packets/limits.rb +0 -75
- data/lib/openc3/packets/packet.rb +0 -28
- data/lib/openc3/packets/packet_item.rb +23 -0
- data/lib/openc3/packets/packet_item_limits.rb +2 -2
- data/lib/openc3/packets/parsers/state_parser.rb +10 -6
- data/lib/openc3/packets/telemetry.rb +1 -45
- data/lib/openc3/script/commands.rb +41 -71
- data/lib/openc3/script/extract.rb +15 -1
- data/lib/openc3/script/{calendar.rb → metadata.rb} +42 -17
- data/lib/openc3/script/script.rb +13 -5
- data/lib/openc3/script/storage.rb +3 -1
- data/lib/openc3/system/system.rb +19 -17
- data/lib/openc3/tools/cmd_tlm_server/interface_thread.rb +4 -4
- data/lib/openc3/top_level.rb +3 -3
- data/lib/openc3/topics/command_decom_topic.rb +2 -2
- data/lib/openc3/topics/command_topic.rb +7 -6
- data/lib/openc3/topics/interface_topic.rb +2 -2
- data/lib/openc3/topics/router_topic.rb +1 -1
- data/lib/openc3/topics/telemetry_topic.rb +2 -1
- data/lib/openc3/utilities/authentication.rb +35 -14
- data/lib/openc3/utilities/aws_bucket.rb +4 -3
- data/lib/openc3/utilities/bucket.rb +4 -2
- data/lib/openc3/utilities/bucket_file_cache.rb +3 -8
- data/lib/openc3/utilities/bucket_utilities.rb +77 -15
- data/lib/openc3/utilities/local_mode.rb +12 -9
- data/lib/openc3/utilities/logger.rb +17 -9
- data/lib/openc3/utilities/message_log.rb +6 -5
- data/lib/openc3/utilities/migration.rb +22 -0
- data/lib/openc3/utilities/store_autoload.rb +7 -5
- data/lib/openc3/utilities/target_file.rb +9 -7
- data/lib/openc3/version.rb +6 -6
- data/lib/openc3.rb +2 -1
- metadata +14 -3
@@ -17,7 +17,7 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
require 'openc3/version'
|
@@ -60,6 +60,8 @@ module OpenC3
|
|
60
60
|
|
61
61
|
REFRESH_OFFSET_SECONDS = 60
|
62
62
|
|
63
|
+
attr_reader :refresh_token
|
64
|
+
|
63
65
|
# @param url [String] The url of the openc3 or keycloak in the cluster
|
64
66
|
def initialize(url)
|
65
67
|
@url = url
|
@@ -72,7 +74,7 @@ module OpenC3
|
|
72
74
|
end
|
73
75
|
|
74
76
|
# Load the token from the environment
|
75
|
-
def token
|
77
|
+
def token
|
76
78
|
@auth_mutex.synchronize do
|
77
79
|
@log = [nil, nil]
|
78
80
|
current_time = Time.now.to_i
|
@@ -87,23 +89,42 @@ module OpenC3
|
|
87
89
|
"Bearer #{@token}"
|
88
90
|
end
|
89
91
|
|
92
|
+
def get_token_from_refresh_token(refresh_token)
|
93
|
+
current_time = Time.now.to_i
|
94
|
+
begin
|
95
|
+
@refresh_token = refresh_token
|
96
|
+
_refresh_token(current_time)
|
97
|
+
return @token
|
98
|
+
rescue OpenC3AuthenticationError
|
99
|
+
return nil
|
100
|
+
end
|
101
|
+
return nil
|
102
|
+
end
|
103
|
+
|
90
104
|
private
|
91
105
|
|
92
106
|
# Make the token and save token to instance
|
93
107
|
def _make_token(current_time)
|
94
108
|
client_id = ENV['OPENC3_API_CLIENT'] || 'api'
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
|
103
|
-
|
104
|
-
|
105
|
-
|
106
|
-
|
109
|
+
if ENV['OPENC3_API_USER'] and ENV['OPENC3_API_PASSWORD']
|
110
|
+
# Username and password
|
111
|
+
data = "username=#{ENV['OPENC3_API_USER']}&password=#{ENV['OPENC3_API_PASSWORD']}"
|
112
|
+
data << "&client_id=#{client_id}"
|
113
|
+
data << '&grant_type=password&scope=openid'
|
114
|
+
headers = {
|
115
|
+
'Content-Type' => 'application/x-www-form-urlencoded',
|
116
|
+
'User-Agent' => "OpenC3KeycloakAuthorization / #{OPENC3_VERSION} (ruby/openc3/lib/utilities/authentication)",
|
117
|
+
}
|
118
|
+
oath = _make_request(headers, data)
|
119
|
+
@token = oath['access_token']
|
120
|
+
@refresh_token = oath['refresh_token']
|
121
|
+
@expires_at = current_time + oath['expires_in'] - REFRESH_OFFSET_SECONDS
|
122
|
+
@refresh_expires_at = current_time + oath['refresh_expires_in'] - REFRESH_OFFSET_SECONDS
|
123
|
+
else
|
124
|
+
# Offline Access Token
|
125
|
+
@refresh_token ||= ENV['OPENC3_API_TOKEN']
|
126
|
+
_refresh_token(current_time)
|
127
|
+
end
|
107
128
|
end
|
108
129
|
|
109
130
|
# Refresh the token and save token to instance
|
@@ -13,7 +13,7 @@
|
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
14
|
# GNU Affero General Public License for more details.
|
15
15
|
#
|
16
|
-
# This file may also be used under the terms of a commercial license
|
16
|
+
# This file may also be used under the terms of a commercial license
|
17
17
|
# if purchased from OpenC3, Inc.
|
18
18
|
|
19
19
|
require 'openc3/utilities/bucket'
|
@@ -107,12 +107,13 @@ module OpenC3
|
|
107
107
|
nil
|
108
108
|
end
|
109
109
|
|
110
|
-
def list_objects(bucket:, prefix: nil)
|
110
|
+
def list_objects(bucket:, prefix: nil, max_request: 1000, max_total: 100_000)
|
111
111
|
token = nil
|
112
112
|
result = []
|
113
113
|
while true
|
114
|
-
resp = @client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys:
|
114
|
+
resp = @client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys: max_request)
|
115
115
|
result.concat(resp.contents)
|
116
|
+
break if result.length >= max_total
|
116
117
|
break unless resp.is_truncated
|
117
118
|
token = resp.next_continuation_token
|
118
119
|
end
|
@@ -13,9 +13,11 @@
|
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
14
|
# GNU Affero General Public License for more details.
|
15
15
|
#
|
16
|
-
# This file may also be used under the terms of a commercial license
|
16
|
+
# This file may also be used under the terms of a commercial license
|
17
17
|
# if purchased from OpenC3, Inc.
|
18
18
|
|
19
|
+
ENV['OPENC3_CLOUD'] ||= 'local'
|
20
|
+
|
19
21
|
# Interface class implemented by each cloud provider: AWS, GCS, Azure
|
20
22
|
module OpenC3
|
21
23
|
class Bucket
|
@@ -51,7 +53,7 @@ module OpenC3
|
|
51
53
|
raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
|
52
54
|
end
|
53
55
|
|
54
|
-
def list_objects(bucket:, prefix: nil)
|
56
|
+
def list_objects(bucket:, prefix: nil, max_request: nil, max_total: nil)
|
55
57
|
raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
|
56
58
|
end
|
57
59
|
|
@@ -17,7 +17,7 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
require 'fileutils'
|
@@ -38,11 +38,8 @@ class BucketFile
|
|
38
38
|
attr_reader :init_time
|
39
39
|
attr_accessor :priority
|
40
40
|
|
41
|
-
def initialize(bucket_path
|
42
|
-
@bucket =
|
43
|
-
unless @bucket
|
44
|
-
@bucket = OpenC3::Bucket.getClient()
|
45
|
-
end
|
41
|
+
def initialize(bucket_path)
|
42
|
+
@bucket = OpenC3::Bucket.getClient()
|
46
43
|
@bucket_path = bucket_path
|
47
44
|
@local_path = nil
|
48
45
|
@reservation_count = 0
|
@@ -149,8 +146,6 @@ class BucketFileCache
|
|
149
146
|
end
|
150
147
|
|
151
148
|
def initialize
|
152
|
-
@bucket = OpenC3::Bucket.getClient()
|
153
|
-
|
154
149
|
# Create local file cache location
|
155
150
|
@cache_dir = Dir.mktmpdir
|
156
151
|
FileUtils.mkdir_p(@cache_dir)
|
@@ -17,7 +17,7 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
require 'openc3/utilities/bucket'
|
@@ -26,8 +26,19 @@ require 'zlib'
|
|
26
26
|
|
27
27
|
module OpenC3
|
28
28
|
class BucketUtilities
|
29
|
-
|
30
|
-
|
29
|
+
FILE_TIMESTAMP_FORMAT = "%Y%m%d%H%M%S%N"
|
30
|
+
DIRECTORY_TIMESTAMP_FORMAT = "%Y%m%d"
|
31
|
+
|
32
|
+
# @param bucket [String] Name of the bucket to list
|
33
|
+
# @param prefix [String] Prefix to filter all files by
|
34
|
+
# @param start_time [Time|nil] Ruby time to find files after. nil means no start (first file on).
|
35
|
+
# @param end_time [Time|nil] Ruby time to find files before. nil means no end (up to last file).
|
36
|
+
# @param overlap [Boolean] Whether to include files which overlap the start and end time
|
37
|
+
# @param max_request [Integer] How many files to request in each API call
|
38
|
+
# @param max_total [Integer] Total number of files before stopping API requests
|
39
|
+
def self.files_between_time(bucket, prefix, start_time, end_time, file_suffix: nil,
|
40
|
+
overlap: false, max_request: 1000, max_total: 100_000)
|
41
|
+
client = Bucket.getClient()
|
31
42
|
oldest_list = []
|
32
43
|
|
33
44
|
# Return nothing if bucket doesn't exist (it won't at the very beginning)
|
@@ -35,23 +46,19 @@ module OpenC3
|
|
35
46
|
return oldest_list
|
36
47
|
end
|
37
48
|
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
oldest_list << item.key
|
45
|
-
else
|
46
|
-
break
|
47
|
-
end
|
49
|
+
directories = client.list_directories(bucket: bucket, path: prefix)
|
50
|
+
filtered_directories = filter_directories_to_time_range(directories, start_time, end_time)
|
51
|
+
filtered_directories.each do |directory|
|
52
|
+
directory_files = client.list_objects(bucket: bucket, prefix: "#{prefix}/#{directory}", max_request: max_request, max_total: max_total)
|
53
|
+
files = filter_files_to_time_range(directory_files, start_time, end_time, file_suffix: file_suffix, overlap: overlap)
|
54
|
+
oldest_list.concat(files)
|
48
55
|
end
|
49
56
|
return oldest_list
|
50
57
|
end
|
51
58
|
|
52
59
|
def self.move_log_file_to_bucket(filename, bucket_key, metadata: {})
|
53
60
|
Thread.new do
|
54
|
-
client = Bucket.getClient
|
61
|
+
client = Bucket.getClient()
|
55
62
|
|
56
63
|
zipped = compress_file(filename)
|
57
64
|
bucket_key = bucket_key + '.gz'
|
@@ -73,7 +80,7 @@ module OpenC3
|
|
73
80
|
has_version_number = /(-|_|\.)\d+(-|_|\.)\d+(-|_|\.)\d+\./.match(filename)
|
74
81
|
has_content_hash = /\.[a-f0-9]{20}\./.match(filename)
|
75
82
|
return nil if has_version_number or has_content_hash
|
76
|
-
return 'no-
|
83
|
+
return 'no-store'
|
77
84
|
end
|
78
85
|
|
79
86
|
def self.compress_file(filename, chunk_size = 50_000_000)
|
@@ -105,5 +112,60 @@ module OpenC3
|
|
105
112
|
|
106
113
|
return unzipped
|
107
114
|
end
|
115
|
+
|
116
|
+
# Private methods
|
117
|
+
|
118
|
+
def self.filter_directories_to_time_range(directories, start_time, end_time)
|
119
|
+
result = []
|
120
|
+
directories.each do |directory|
|
121
|
+
result << directory if directory_in_time_range(directory, start_time, end_time)
|
122
|
+
end
|
123
|
+
return result
|
124
|
+
end
|
125
|
+
|
126
|
+
def self.directory_in_time_range(directory, start_time, end_time)
|
127
|
+
basename = File.basename(directory)
|
128
|
+
directory_start_time = DateTime.strptime(basename, DIRECTORY_TIMESTAMP_FORMAT).to_time
|
129
|
+
directory_end_time = directory_start_time + Time::SEC_PER_DAY
|
130
|
+
if (not start_time or start_time < directory_end_time) and (not end_time or end_time >= directory_start_time)
|
131
|
+
return true
|
132
|
+
else
|
133
|
+
return false
|
134
|
+
end
|
135
|
+
end
|
136
|
+
|
137
|
+
def self.filter_files_to_time_range(files, start_time, end_time, file_suffix: nil, overlap: false)
|
138
|
+
result = []
|
139
|
+
files.each do |file|
|
140
|
+
file_key = file.key.to_s
|
141
|
+
next if file_suffix and not file_key.end_with?(file_suffix)
|
142
|
+
if file_in_time_range(file_key, start_time, end_time, overlap: overlap)
|
143
|
+
result << file_key
|
144
|
+
end
|
145
|
+
end
|
146
|
+
return result
|
147
|
+
end
|
148
|
+
|
149
|
+
def self.file_in_time_range(bucket_path, start_time, end_time, overlap:)
|
150
|
+
file_start_time, file_end_time = get_file_times(bucket_path)
|
151
|
+
if overlap
|
152
|
+
if (not start_time or start_time <= file_end_time) and (not end_time or end_time >= file_start_time)
|
153
|
+
return true
|
154
|
+
end
|
155
|
+
else
|
156
|
+
if (not start_time or start_time <= file_start_time) and (not end_time or end_time >= file_end_time)
|
157
|
+
return true
|
158
|
+
end
|
159
|
+
end
|
160
|
+
return false
|
161
|
+
end
|
162
|
+
|
163
|
+
def self.get_file_times(bucket_path)
|
164
|
+
basename = File.basename(bucket_path)
|
165
|
+
file_start_timestamp, file_end_timestamp, other = basename.split("__")
|
166
|
+
file_start_time = DateTime.strptime(file_start_timestamp, FILE_TIMESTAMP_FORMAT).to_time
|
167
|
+
file_end_time = DateTime.strptime(file_end_timestamp, FILE_TIMESTAMP_FORMAT).to_time
|
168
|
+
return file_start_time, file_end_time
|
169
|
+
end
|
108
170
|
end
|
109
171
|
end
|
@@ -13,7 +13,7 @@
|
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
14
|
# GNU Affero General Public License for more details.
|
15
15
|
#
|
16
|
-
# This file may also be used under the terms of a commercial license
|
16
|
+
# This file may also be used under the terms of a commercial license
|
17
17
|
# if purchased from OpenC3, Inc.
|
18
18
|
|
19
19
|
require 'fileutils'
|
@@ -201,8 +201,9 @@ module OpenC3
|
|
201
201
|
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
202
202
|
variables = plugin_hash['variables']
|
203
203
|
if variables
|
204
|
-
|
205
|
-
|
204
|
+
PluginModel::RESERVED_VARIABLE_NAMES.each do |name|
|
205
|
+
variables.delete(name)
|
206
|
+
end
|
206
207
|
end
|
207
208
|
if plugin_file_path =~ Regexp.new("^#{OPENC3_LOCAL_MODE_PATH}/#{scope}/")
|
208
209
|
# From local init - Always just update the exact one
|
@@ -400,14 +401,16 @@ module OpenC3
|
|
400
401
|
files << split_key[2..-1].join('/') if include_temp
|
401
402
|
next
|
402
403
|
end
|
403
|
-
|
404
|
-
|
405
|
-
|
406
|
-
|
407
|
-
|
404
|
+
if path_matchers
|
405
|
+
found = false
|
406
|
+
path_matchers.each do |path|
|
407
|
+
if split_key.include?(path)
|
408
|
+
found = true
|
409
|
+
break
|
410
|
+
end
|
408
411
|
end
|
412
|
+
next unless found
|
409
413
|
end
|
410
|
-
next unless found
|
411
414
|
files << split_key[2..-1].join('/')
|
412
415
|
end
|
413
416
|
return files.sort
|
@@ -17,7 +17,7 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
require 'openc3/core_ext/class'
|
@@ -41,11 +41,8 @@ module OpenC3
|
|
41
41
|
# @return [String] Additional detail to add to messages
|
42
42
|
instance_attr_accessor :detail_string
|
43
43
|
|
44
|
-
# @return [String] Fluent tag
|
45
|
-
instance_attr_accessor :tag
|
46
|
-
|
47
44
|
# @return [String] Microservice name
|
48
|
-
|
45
|
+
attr_reader :microservice_name
|
49
46
|
|
50
47
|
# @return [String] Scope
|
51
48
|
instance_attr_accessor :scope
|
@@ -78,11 +75,22 @@ module OpenC3
|
|
78
75
|
@detail_string = nil
|
79
76
|
@container_name = Socket.gethostname
|
80
77
|
@microservice_name = nil
|
81
|
-
@tag = @container_name + ".log"
|
82
|
-
@mutex = Mutex.new
|
83
78
|
@no_store = ENV['OPENC3_NO_STORE']
|
84
79
|
end
|
85
80
|
|
81
|
+
# Only set the microservice name once (to help with multi microservices)
|
82
|
+
def microservice_name=(name)
|
83
|
+
@microservice_name = name unless @microservice_name
|
84
|
+
end
|
85
|
+
|
86
|
+
def self.microservice_name
|
87
|
+
self.instance.microservice_name
|
88
|
+
end
|
89
|
+
|
90
|
+
def self.microservice_name=(name)
|
91
|
+
self.instance.microservice_name = name
|
92
|
+
end
|
93
|
+
|
86
94
|
# @param message [String] The message to print if the log level is at or
|
87
95
|
# below the method name log level.
|
88
96
|
# @param block [Proc] Block to call which should return a string to append
|
@@ -164,11 +172,11 @@ module OpenC3
|
|
164
172
|
protected
|
165
173
|
|
166
174
|
def log_message(severity_string, message, scope:, user:)
|
167
|
-
|
175
|
+
@@mutex.synchronize do
|
168
176
|
data = { time: Time.now.to_nsec_from_epoch, '@timestamp' => Time.now.xmlschema(3), severity: severity_string }
|
169
177
|
data[:microservice_name] = @microservice_name if @microservice_name
|
170
178
|
data[:detail] = @detail_string if @detail_string
|
171
|
-
data[:user] = user['
|
179
|
+
data[:user] = user['username'] || 'Unknown' if user # EE: If a user is passed, put its name ('Unknown' if it doesn't have a name). Don't include user data if no user was passed
|
172
180
|
if block_given?
|
173
181
|
message = yield
|
174
182
|
end
|
@@ -17,7 +17,7 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
require 'openc3/config/config_parser'
|
@@ -36,9 +36,10 @@ module OpenC3
|
|
36
36
|
# @param tool_name [String] The name of the tool creating the message log.
|
37
37
|
# This will be inserted into the message log filename to help identify it.
|
38
38
|
# @param log_dir [String] The filesystem path to store the message log file.
|
39
|
-
|
39
|
+
# @param tags [Array<String>] Array of strings to put into the filename
|
40
|
+
def initialize(tool_name, log_dir, tags: ['messages'], scope:)
|
40
41
|
@remote_log_directory = "#{scope}/tool_logs/#{tool_name}/"
|
41
|
-
@
|
42
|
+
@tags = tags.unshift(tool_name)
|
42
43
|
@log_dir = log_dir
|
43
44
|
@filename = ''
|
44
45
|
@file = nil
|
@@ -82,9 +83,9 @@ module OpenC3
|
|
82
83
|
def start(take_mutex = true)
|
83
84
|
@mutex.lock if take_mutex
|
84
85
|
# Prevent starting files too fast
|
85
|
-
sleep(0.1) until !File.exist?(File.join(@log_dir, File.build_timestamped_filename(
|
86
|
+
sleep(0.1) until !File.exist?(File.join(@log_dir, File.build_timestamped_filename(@tags)))
|
86
87
|
stop(false)
|
87
|
-
timed_filename = File.build_timestamped_filename(
|
88
|
+
timed_filename = File.build_timestamped_filename(@tags)
|
88
89
|
@start_day = timed_filename[0..9].gsub("_", "") # YYYYMMDD
|
89
90
|
@filename = File.join(@log_dir, timed_filename)
|
90
91
|
@file = File.open(@filename, 'a')
|
@@ -0,0 +1,22 @@
|
|
1
|
+
# encoding: ascii-8bit
|
2
|
+
|
3
|
+
# Copyright 2022 OpenC3, Inc.
|
4
|
+
# All Rights Reserved.
|
5
|
+
#
|
6
|
+
# This program is free software; you can modify and/or redistribute it
|
7
|
+
# under the terms of the GNU Affero General Public License
|
8
|
+
# as published by the Free Software Foundation; version 3 with
|
9
|
+
# attribution addendums as found in the LICENSE.txt
|
10
|
+
#
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
# GNU Affero General Public License for more details.
|
15
|
+
#
|
16
|
+
# This file may also be used under the terms of a commercial license
|
17
|
+
# if purchased from OpenC3, Inc.
|
18
|
+
|
19
|
+
module OpenC3
|
20
|
+
class Migration
|
21
|
+
end
|
22
|
+
end
|
@@ -80,7 +80,6 @@ module OpenC3
|
|
80
80
|
@redis_key = ENV['OPENC3_REDIS_PASSWORD']
|
81
81
|
@redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME']}:#{ENV['OPENC3_REDIS_PORT']}"
|
82
82
|
@redis_pool = ConnectionPool.new(size: pool_size) { build_redis() }
|
83
|
-
@topic_offsets = {}
|
84
83
|
end
|
85
84
|
|
86
85
|
unless $openc3_redis_cluster
|
@@ -143,14 +142,16 @@ module OpenC3
|
|
143
142
|
topics.each do |topic|
|
144
143
|
# Normally we will just be grabbing the topic offset
|
145
144
|
# this allows xread to get everything past this point
|
146
|
-
|
145
|
+
Thread.current[:topic_offsets] ||= {}
|
146
|
+
topic_offsets = Thread.current[:topic_offsets]
|
147
|
+
last_id = topic_offsets[topic]
|
147
148
|
if last_id
|
148
149
|
offsets << last_id
|
149
150
|
else
|
150
151
|
# If there is no topic offset this is the first call.
|
151
152
|
# Get the last offset ID so we'll start getting everything from now on
|
152
153
|
offsets << get_last_offset(topic)
|
153
|
-
|
154
|
+
topic_offsets[topic] = offsets[-1]
|
154
155
|
end
|
155
156
|
end
|
156
157
|
return offsets
|
@@ -158,6 +159,8 @@ module OpenC3
|
|
158
159
|
|
159
160
|
unless $openc3_redis_cluster
|
160
161
|
def read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil)
|
162
|
+
Thread.current[:topic_offsets] ||= {}
|
163
|
+
topic_offsets = Thread.current[:topic_offsets]
|
161
164
|
begin
|
162
165
|
# Logger.debug "read_topics: #{topics}, #{offsets} pool:#{@redis_pool}"
|
163
166
|
@redis_pool.with do |redis|
|
@@ -166,7 +169,7 @@ module OpenC3
|
|
166
169
|
if result and result.length > 0
|
167
170
|
result.each do |topic, messages|
|
168
171
|
messages.each do |msg_id, msg_hash|
|
169
|
-
|
172
|
+
topic_offsets[topic] = msg_id
|
170
173
|
yield topic, msg_id, msg_hash, redis if block_given?
|
171
174
|
end
|
172
175
|
end
|
@@ -199,7 +202,6 @@ module OpenC3
|
|
199
202
|
# @return [String] the entry id
|
200
203
|
def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true')
|
201
204
|
id = '*' if id.nil?
|
202
|
-
# Logger.debug "write_topic topic:#{topic} id:#{id} hash:#{msg_hash}"
|
203
205
|
@redis_pool.with do |redis|
|
204
206
|
return redis.xadd(topic, msg_hash, id: id, maxlen: maxlen, approximate: approximate)
|
205
207
|
end
|
@@ -13,7 +13,7 @@
|
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
14
|
# GNU Affero General Public License for more details.
|
15
15
|
#
|
16
|
-
# This file may also be used under the terms of a commercial license
|
16
|
+
# This file may also be used under the terms of a commercial license
|
17
17
|
# if purchased from OpenC3, Inc.
|
18
18
|
|
19
19
|
require 'fileutils'
|
@@ -44,14 +44,16 @@ module OpenC3
|
|
44
44
|
next
|
45
45
|
end
|
46
46
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
47
|
+
if path_matchers
|
48
|
+
found = false
|
49
|
+
path_matchers.each do |path|
|
50
|
+
if split_key.include?(path)
|
51
|
+
found = true
|
52
|
+
break
|
53
|
+
end
|
52
54
|
end
|
55
|
+
next unless found
|
53
56
|
end
|
54
|
-
next unless found
|
55
57
|
result_no_scope_or_target_folder = split_key[2..-1].join('/')
|
56
58
|
if object.key.include?("#{scope}/targets_modified")
|
57
59
|
modified << result_no_scope_or_target_folder
|
data/lib/openc3/version.rb
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# encoding: ascii-8bit
|
2
2
|
|
3
|
-
OPENC3_VERSION = '5.
|
3
|
+
OPENC3_VERSION = '5.2.0'
|
4
4
|
module OpenC3
|
5
5
|
module Version
|
6
6
|
MAJOR = '5'
|
7
|
-
MINOR = '
|
8
|
-
PATCH = '
|
7
|
+
MINOR = '2'
|
8
|
+
PATCH = '0'
|
9
9
|
OTHER = ''
|
10
|
-
BUILD = '
|
10
|
+
BUILD = '49c62512b75f45a8cbfc20a6fb5ed114bab35e23'
|
11
11
|
end
|
12
|
-
VERSION = '5.
|
13
|
-
GEM_VERSION = '5.
|
12
|
+
VERSION = '5.2.0'
|
13
|
+
GEM_VERSION = '5.2.0'
|
14
14
|
end
|
data/lib/openc3.rb
CHANGED
@@ -17,7 +17,7 @@
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
18
18
|
# All Rights Reserved
|
19
19
|
#
|
20
|
-
# This file may also be used under the terms of a commercial license
|
20
|
+
# This file may also be used under the terms of a commercial license
|
21
21
|
# if purchased from OpenC3, Inc.
|
22
22
|
|
23
23
|
# This file sets up using the OpenC3 framework
|
@@ -51,3 +51,4 @@ require 'openc3/system'
|
|
51
51
|
# OpenC3 services need to die if something goes wrong so they can be restarted
|
52
52
|
require 'thread'
|
53
53
|
Thread.abort_on_exception = true
|
54
|
+
Thread.report_on_exception = true
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: openc3
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 5.
|
4
|
+
version: 5.2.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Ryan Melton
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2022-
|
12
|
+
date: 2022-12-13 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: bundler
|
@@ -711,14 +711,17 @@ files:
|
|
711
711
|
- lib/openc3/accessors/html_accessor.rb
|
712
712
|
- lib/openc3/accessors/json_accessor.rb
|
713
713
|
- lib/openc3/accessors/xml_accessor.rb
|
714
|
+
- lib/openc3/api/README.md
|
714
715
|
- lib/openc3/api/api.rb
|
715
716
|
- lib/openc3/api/authorized_api.rb
|
716
717
|
- lib/openc3/api/cmd_api.rb
|
717
718
|
- lib/openc3/api/config_api.rb
|
718
719
|
- lib/openc3/api/interface_api.rb
|
719
720
|
- lib/openc3/api/limits_api.rb
|
721
|
+
- lib/openc3/api/offline_access_api.rb
|
720
722
|
- lib/openc3/api/router_api.rb
|
721
723
|
- lib/openc3/api/settings_api.rb
|
724
|
+
- lib/openc3/api/stash_api.rb
|
722
725
|
- lib/openc3/api/target_api.rb
|
723
726
|
- lib/openc3/api/tlm_api.rb
|
724
727
|
- lib/openc3/bridge/bridge.rb
|
@@ -810,6 +813,8 @@ files:
|
|
810
813
|
- lib/openc3/microservices/interface_microservice.rb
|
811
814
|
- lib/openc3/microservices/log_microservice.rb
|
812
815
|
- lib/openc3/microservices/microservice.rb
|
816
|
+
- lib/openc3/microservices/multi_microservice.rb
|
817
|
+
- lib/openc3/microservices/periodic_microservice.rb
|
813
818
|
- lib/openc3/microservices/plugin_microservice.rb
|
814
819
|
- lib/openc3/microservices/reaction_microservice.rb
|
815
820
|
- lib/openc3/microservices/reducer_microservice.rb
|
@@ -817,6 +822,8 @@ files:
|
|
817
822
|
- lib/openc3/microservices/text_log_microservice.rb
|
818
823
|
- lib/openc3/microservices/timeline_microservice.rb
|
819
824
|
- lib/openc3/microservices/trigger_group_microservice.rb
|
825
|
+
- lib/openc3/migrations/20221202214600_add_target_names.rb
|
826
|
+
- lib/openc3/migrations/20221210174900_convert_to_multi.rb
|
820
827
|
- lib/openc3/models/activity_model.rb
|
821
828
|
- lib/openc3/models/auth_model.rb
|
822
829
|
- lib/openc3/models/cvt_model.rb
|
@@ -829,9 +836,11 @@ files:
|
|
829
836
|
- lib/openc3/models/metric_model.rb
|
830
837
|
- lib/openc3/models/microservice_model.rb
|
831
838
|
- lib/openc3/models/microservice_status_model.rb
|
839
|
+
- lib/openc3/models/migration_model.rb
|
832
840
|
- lib/openc3/models/model.rb
|
833
841
|
- lib/openc3/models/note_model.rb
|
834
842
|
- lib/openc3/models/notification_model.rb
|
843
|
+
- lib/openc3/models/offline_access_model.rb
|
835
844
|
- lib/openc3/models/ping_model.rb
|
836
845
|
- lib/openc3/models/plugin_model.rb
|
837
846
|
- lib/openc3/models/process_status_model.rb
|
@@ -842,6 +851,7 @@ files:
|
|
842
851
|
- lib/openc3/models/scope_model.rb
|
843
852
|
- lib/openc3/models/settings_model.rb
|
844
853
|
- lib/openc3/models/sorted_model.rb
|
854
|
+
- lib/openc3/models/stash_model.rb
|
845
855
|
- lib/openc3/models/target_model.rb
|
846
856
|
- lib/openc3/models/timeline_model.rb
|
847
857
|
- lib/openc3/models/tool_config_model.rb
|
@@ -878,11 +888,11 @@ files:
|
|
878
888
|
- lib/openc3/processors/watermark_processor.rb
|
879
889
|
- lib/openc3/script.rb
|
880
890
|
- lib/openc3/script/api_shared.rb
|
881
|
-
- lib/openc3/script/calendar.rb
|
882
891
|
- lib/openc3/script/commands.rb
|
883
892
|
- lib/openc3/script/exceptions.rb
|
884
893
|
- lib/openc3/script/extract.rb
|
885
894
|
- lib/openc3/script/limits.rb
|
895
|
+
- lib/openc3/script/metadata.rb
|
886
896
|
- lib/openc3/script/script.rb
|
887
897
|
- lib/openc3/script/script_runner.rb
|
888
898
|
- lib/openc3/script/storage.rb
|
@@ -937,6 +947,7 @@ files:
|
|
937
947
|
- lib/openc3/utilities/logger.rb
|
938
948
|
- lib/openc3/utilities/message_log.rb
|
939
949
|
- lib/openc3/utilities/metric.rb
|
950
|
+
- lib/openc3/utilities/migration.rb
|
940
951
|
- lib/openc3/utilities/open_telemetry.rb
|
941
952
|
- lib/openc3/utilities/process_manager.rb
|
942
953
|
- lib/openc3/utilities/quaternion.rb
|