openc3 5.1.1 → 5.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/bin/openc3cli +48 -9
- data/data/config/interface_modifiers.yaml +14 -0
- data/data/config/parameter_modifiers.yaml +5 -3
- data/data/config/screen.yaml +12 -8
- data/data/config/target.yaml +33 -0
- data/ext/openc3/ext/config_parser/config_parser.c +66 -63
- data/ext/openc3/ext/packet/packet.c +1 -4
- data/lib/openc3/api/README.md +5 -0
- data/lib/openc3/api/api.rb +3 -1
- data/lib/openc3/api/cmd_api.rb +43 -112
- data/lib/openc3/api/interface_api.rb +3 -3
- data/lib/openc3/api/offline_access_api.rb +78 -0
- data/lib/openc3/api/settings_api.rb +3 -1
- data/lib/openc3/api/stash_api.rb +63 -0
- data/lib/openc3/api/target_api.rb +4 -5
- data/lib/openc3/config/config_parser.rb +47 -47
- data/lib/openc3/interfaces/interface.rb +11 -1
- data/lib/openc3/interfaces/protocols/burst_protocol.rb +30 -16
- data/lib/openc3/interfaces/protocols/fixed_protocol.rb +8 -2
- data/lib/openc3/interfaces/protocols/ignore_packet_protocol.rb +2 -2
- data/lib/openc3/interfaces/protocols/override_protocol.rb +2 -2
- data/lib/openc3/interfaces/tcpip_server_interface.rb +3 -1
- data/lib/openc3/io/json_api_object.rb +30 -9
- data/lib/openc3/io/json_drb.rb +6 -1
- data/lib/openc3/io/json_drb_object.rb +18 -9
- data/lib/openc3/io/json_rpc.rb +5 -3
- data/lib/openc3/logs/buffered_packet_log_writer.rb +1 -1
- data/lib/openc3/logs/log_writer.rb +8 -2
- data/lib/openc3/microservices/cleanup_microservice.rb +3 -3
- data/lib/openc3/microservices/decom_microservice.rb +8 -8
- data/lib/openc3/microservices/interface_microservice.rb +86 -71
- data/lib/openc3/microservices/log_microservice.rb +5 -3
- data/lib/openc3/microservices/microservice.rb +18 -14
- data/lib/openc3/microservices/multi_microservice.rb +62 -0
- data/lib/openc3/microservices/periodic_microservice.rb +58 -0
- data/lib/openc3/microservices/reaction_microservice.rb +61 -47
- data/lib/openc3/microservices/reducer_microservice.rb +64 -40
- data/lib/openc3/microservices/router_microservice.rb +4 -4
- data/lib/openc3/microservices/text_log_microservice.rb +2 -2
- data/lib/openc3/microservices/timeline_microservice.rb +44 -30
- data/lib/openc3/microservices/trigger_group_microservice.rb +39 -36
- data/lib/openc3/migrations/20221202214600_add_target_names.rb +30 -0
- data/lib/openc3/migrations/20221210174900_convert_to_multi.rb +65 -0
- data/lib/openc3/models/cvt_model.rb +1 -1
- data/lib/openc3/models/gem_model.rb +24 -20
- data/lib/openc3/models/interface_model.rb +69 -35
- data/lib/openc3/models/metadata_model.rb +1 -1
- data/lib/openc3/models/microservice_model.rb +7 -24
- data/lib/openc3/models/migration_model.rb +52 -0
- data/lib/openc3/models/model.rb +2 -7
- data/lib/openc3/models/note_model.rb +1 -1
- data/lib/openc3/models/offline_access_model.rb +55 -0
- data/lib/openc3/models/plugin_model.rb +12 -3
- data/lib/openc3/models/reaction_model.rb +6 -2
- data/lib/openc3/models/scope_model.rb +89 -13
- data/lib/openc3/models/settings_model.rb +1 -1
- data/lib/openc3/models/stash_model.rb +53 -0
- data/lib/openc3/models/target_model.rb +301 -130
- data/lib/openc3/models/tool_model.rb +1 -12
- data/lib/openc3/models/widget_model.rb +1 -6
- data/lib/openc3/operators/microservice_operator.rb +45 -6
- data/lib/openc3/operators/operator.rb +27 -5
- data/lib/openc3/packets/commands.rb +1 -25
- data/lib/openc3/packets/limits.rb +0 -75
- data/lib/openc3/packets/packet.rb +0 -28
- data/lib/openc3/packets/packet_item.rb +23 -0
- data/lib/openc3/packets/packet_item_limits.rb +2 -2
- data/lib/openc3/packets/parsers/state_parser.rb +10 -6
- data/lib/openc3/packets/telemetry.rb +1 -45
- data/lib/openc3/script/commands.rb +41 -71
- data/lib/openc3/script/extract.rb +15 -1
- data/lib/openc3/script/{calendar.rb → metadata.rb} +42 -17
- data/lib/openc3/script/script.rb +13 -5
- data/lib/openc3/script/storage.rb +3 -1
- data/lib/openc3/system/system.rb +19 -17
- data/lib/openc3/tools/cmd_tlm_server/interface_thread.rb +4 -4
- data/lib/openc3/top_level.rb +3 -3
- data/lib/openc3/topics/command_decom_topic.rb +2 -2
- data/lib/openc3/topics/command_topic.rb +7 -6
- data/lib/openc3/topics/interface_topic.rb +2 -2
- data/lib/openc3/topics/router_topic.rb +1 -1
- data/lib/openc3/topics/telemetry_topic.rb +2 -1
- data/lib/openc3/utilities/authentication.rb +35 -14
- data/lib/openc3/utilities/aws_bucket.rb +4 -3
- data/lib/openc3/utilities/bucket.rb +4 -2
- data/lib/openc3/utilities/bucket_file_cache.rb +3 -8
- data/lib/openc3/utilities/bucket_utilities.rb +77 -15
- data/lib/openc3/utilities/local_mode.rb +12 -9
- data/lib/openc3/utilities/logger.rb +17 -9
- data/lib/openc3/utilities/message_log.rb +6 -5
- data/lib/openc3/utilities/migration.rb +22 -0
- data/lib/openc3/utilities/store_autoload.rb +7 -5
- data/lib/openc3/utilities/target_file.rb +9 -7
- data/lib/openc3/version.rb +6 -6
- data/lib/openc3.rb +2 -1
- metadata +14 -3
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
|
18
18
|
# All Rights Reserved
|
|
19
19
|
#
|
|
20
|
-
# This file may also be used under the terms of a commercial license
|
|
20
|
+
# This file may also be used under the terms of a commercial license
|
|
21
21
|
# if purchased from OpenC3, Inc.
|
|
22
22
|
|
|
23
23
|
require 'openc3/version'
|
|
@@ -60,6 +60,8 @@ module OpenC3
|
|
|
60
60
|
|
|
61
61
|
REFRESH_OFFSET_SECONDS = 60
|
|
62
62
|
|
|
63
|
+
attr_reader :refresh_token
|
|
64
|
+
|
|
63
65
|
# @param url [String] The url of the openc3 or keycloak in the cluster
|
|
64
66
|
def initialize(url)
|
|
65
67
|
@url = url
|
|
@@ -72,7 +74,7 @@ module OpenC3
|
|
|
72
74
|
end
|
|
73
75
|
|
|
74
76
|
# Load the token from the environment
|
|
75
|
-
def token
|
|
77
|
+
def token
|
|
76
78
|
@auth_mutex.synchronize do
|
|
77
79
|
@log = [nil, nil]
|
|
78
80
|
current_time = Time.now.to_i
|
|
@@ -87,23 +89,42 @@ module OpenC3
|
|
|
87
89
|
"Bearer #{@token}"
|
|
88
90
|
end
|
|
89
91
|
|
|
92
|
+
def get_token_from_refresh_token(refresh_token)
|
|
93
|
+
current_time = Time.now.to_i
|
|
94
|
+
begin
|
|
95
|
+
@refresh_token = refresh_token
|
|
96
|
+
_refresh_token(current_time)
|
|
97
|
+
return @token
|
|
98
|
+
rescue OpenC3AuthenticationError
|
|
99
|
+
return nil
|
|
100
|
+
end
|
|
101
|
+
return nil
|
|
102
|
+
end
|
|
103
|
+
|
|
90
104
|
private
|
|
91
105
|
|
|
92
106
|
# Make the token and save token to instance
|
|
93
107
|
def _make_token(current_time)
|
|
94
108
|
client_id = ENV['OPENC3_API_CLIENT'] || 'api'
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
109
|
+
if ENV['OPENC3_API_USER'] and ENV['OPENC3_API_PASSWORD']
|
|
110
|
+
# Username and password
|
|
111
|
+
data = "username=#{ENV['OPENC3_API_USER']}&password=#{ENV['OPENC3_API_PASSWORD']}"
|
|
112
|
+
data << "&client_id=#{client_id}"
|
|
113
|
+
data << '&grant_type=password&scope=openid'
|
|
114
|
+
headers = {
|
|
115
|
+
'Content-Type' => 'application/x-www-form-urlencoded',
|
|
116
|
+
'User-Agent' => "OpenC3KeycloakAuthorization / #{OPENC3_VERSION} (ruby/openc3/lib/utilities/authentication)",
|
|
117
|
+
}
|
|
118
|
+
oath = _make_request(headers, data)
|
|
119
|
+
@token = oath['access_token']
|
|
120
|
+
@refresh_token = oath['refresh_token']
|
|
121
|
+
@expires_at = current_time + oath['expires_in'] - REFRESH_OFFSET_SECONDS
|
|
122
|
+
@refresh_expires_at = current_time + oath['refresh_expires_in'] - REFRESH_OFFSET_SECONDS
|
|
123
|
+
else
|
|
124
|
+
# Offline Access Token
|
|
125
|
+
@refresh_token ||= ENV['OPENC3_API_TOKEN']
|
|
126
|
+
_refresh_token(current_time)
|
|
127
|
+
end
|
|
107
128
|
end
|
|
108
129
|
|
|
109
130
|
# Refresh the token and save token to instance
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
14
|
# GNU Affero General Public License for more details.
|
|
15
15
|
#
|
|
16
|
-
# This file may also be used under the terms of a commercial license
|
|
16
|
+
# This file may also be used under the terms of a commercial license
|
|
17
17
|
# if purchased from OpenC3, Inc.
|
|
18
18
|
|
|
19
19
|
require 'openc3/utilities/bucket'
|
|
@@ -107,12 +107,13 @@ module OpenC3
|
|
|
107
107
|
nil
|
|
108
108
|
end
|
|
109
109
|
|
|
110
|
-
def list_objects(bucket:, prefix: nil)
|
|
110
|
+
def list_objects(bucket:, prefix: nil, max_request: 1000, max_total: 100_000)
|
|
111
111
|
token = nil
|
|
112
112
|
result = []
|
|
113
113
|
while true
|
|
114
|
-
resp = @client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys:
|
|
114
|
+
resp = @client.list_objects_v2(bucket: bucket, prefix: prefix, max_keys: max_request)
|
|
115
115
|
result.concat(resp.contents)
|
|
116
|
+
break if result.length >= max_total
|
|
116
117
|
break unless resp.is_truncated
|
|
117
118
|
token = resp.next_continuation_token
|
|
118
119
|
end
|
|
@@ -13,9 +13,11 @@
|
|
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
14
|
# GNU Affero General Public License for more details.
|
|
15
15
|
#
|
|
16
|
-
# This file may also be used under the terms of a commercial license
|
|
16
|
+
# This file may also be used under the terms of a commercial license
|
|
17
17
|
# if purchased from OpenC3, Inc.
|
|
18
18
|
|
|
19
|
+
ENV['OPENC3_CLOUD'] ||= 'local'
|
|
20
|
+
|
|
19
21
|
# Interface class implemented by each cloud provider: AWS, GCS, Azure
|
|
20
22
|
module OpenC3
|
|
21
23
|
class Bucket
|
|
@@ -51,7 +53,7 @@ module OpenC3
|
|
|
51
53
|
raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
|
|
52
54
|
end
|
|
53
55
|
|
|
54
|
-
def list_objects(bucket:, prefix: nil)
|
|
56
|
+
def list_objects(bucket:, prefix: nil, max_request: nil, max_total: nil)
|
|
55
57
|
raise NotImplementedError, "#{self.class} has not implemented method '#{__method__}'"
|
|
56
58
|
end
|
|
57
59
|
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
|
18
18
|
# All Rights Reserved
|
|
19
19
|
#
|
|
20
|
-
# This file may also be used under the terms of a commercial license
|
|
20
|
+
# This file may also be used under the terms of a commercial license
|
|
21
21
|
# if purchased from OpenC3, Inc.
|
|
22
22
|
|
|
23
23
|
require 'fileutils'
|
|
@@ -38,11 +38,8 @@ class BucketFile
|
|
|
38
38
|
attr_reader :init_time
|
|
39
39
|
attr_accessor :priority
|
|
40
40
|
|
|
41
|
-
def initialize(bucket_path
|
|
42
|
-
@bucket =
|
|
43
|
-
unless @bucket
|
|
44
|
-
@bucket = OpenC3::Bucket.getClient()
|
|
45
|
-
end
|
|
41
|
+
def initialize(bucket_path)
|
|
42
|
+
@bucket = OpenC3::Bucket.getClient()
|
|
46
43
|
@bucket_path = bucket_path
|
|
47
44
|
@local_path = nil
|
|
48
45
|
@reservation_count = 0
|
|
@@ -149,8 +146,6 @@ class BucketFileCache
|
|
|
149
146
|
end
|
|
150
147
|
|
|
151
148
|
def initialize
|
|
152
|
-
@bucket = OpenC3::Bucket.getClient()
|
|
153
|
-
|
|
154
149
|
# Create local file cache location
|
|
155
150
|
@cache_dir = Dir.mktmpdir
|
|
156
151
|
FileUtils.mkdir_p(@cache_dir)
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
|
18
18
|
# All Rights Reserved
|
|
19
19
|
#
|
|
20
|
-
# This file may also be used under the terms of a commercial license
|
|
20
|
+
# This file may also be used under the terms of a commercial license
|
|
21
21
|
# if purchased from OpenC3, Inc.
|
|
22
22
|
|
|
23
23
|
require 'openc3/utilities/bucket'
|
|
@@ -26,8 +26,19 @@ require 'zlib'
|
|
|
26
26
|
|
|
27
27
|
module OpenC3
|
|
28
28
|
class BucketUtilities
|
|
29
|
-
|
|
30
|
-
|
|
29
|
+
FILE_TIMESTAMP_FORMAT = "%Y%m%d%H%M%S%N"
|
|
30
|
+
DIRECTORY_TIMESTAMP_FORMAT = "%Y%m%d"
|
|
31
|
+
|
|
32
|
+
# @param bucket [String] Name of the bucket to list
|
|
33
|
+
# @param prefix [String] Prefix to filter all files by
|
|
34
|
+
# @param start_time [Time|nil] Ruby time to find files after. nil means no start (first file on).
|
|
35
|
+
# @param end_time [Time|nil] Ruby time to find files before. nil means no end (up to last file).
|
|
36
|
+
# @param overlap [Boolean] Whether to include files which overlap the start and end time
|
|
37
|
+
# @param max_request [Integer] How many files to request in each API call
|
|
38
|
+
# @param max_total [Integer] Total number of files before stopping API requests
|
|
39
|
+
def self.files_between_time(bucket, prefix, start_time, end_time, file_suffix: nil,
|
|
40
|
+
overlap: false, max_request: 1000, max_total: 100_000)
|
|
41
|
+
client = Bucket.getClient()
|
|
31
42
|
oldest_list = []
|
|
32
43
|
|
|
33
44
|
# Return nothing if bucket doesn't exist (it won't at the very beginning)
|
|
@@ -35,23 +46,19 @@ module OpenC3
|
|
|
35
46
|
return oldest_list
|
|
36
47
|
end
|
|
37
48
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
oldest_list << item.key
|
|
45
|
-
else
|
|
46
|
-
break
|
|
47
|
-
end
|
|
49
|
+
directories = client.list_directories(bucket: bucket, path: prefix)
|
|
50
|
+
filtered_directories = filter_directories_to_time_range(directories, start_time, end_time)
|
|
51
|
+
filtered_directories.each do |directory|
|
|
52
|
+
directory_files = client.list_objects(bucket: bucket, prefix: "#{prefix}/#{directory}", max_request: max_request, max_total: max_total)
|
|
53
|
+
files = filter_files_to_time_range(directory_files, start_time, end_time, file_suffix: file_suffix, overlap: overlap)
|
|
54
|
+
oldest_list.concat(files)
|
|
48
55
|
end
|
|
49
56
|
return oldest_list
|
|
50
57
|
end
|
|
51
58
|
|
|
52
59
|
def self.move_log_file_to_bucket(filename, bucket_key, metadata: {})
|
|
53
60
|
Thread.new do
|
|
54
|
-
client = Bucket.getClient
|
|
61
|
+
client = Bucket.getClient()
|
|
55
62
|
|
|
56
63
|
zipped = compress_file(filename)
|
|
57
64
|
bucket_key = bucket_key + '.gz'
|
|
@@ -73,7 +80,7 @@ module OpenC3
|
|
|
73
80
|
has_version_number = /(-|_|\.)\d+(-|_|\.)\d+(-|_|\.)\d+\./.match(filename)
|
|
74
81
|
has_content_hash = /\.[a-f0-9]{20}\./.match(filename)
|
|
75
82
|
return nil if has_version_number or has_content_hash
|
|
76
|
-
return 'no-
|
|
83
|
+
return 'no-store'
|
|
77
84
|
end
|
|
78
85
|
|
|
79
86
|
def self.compress_file(filename, chunk_size = 50_000_000)
|
|
@@ -105,5 +112,60 @@ module OpenC3
|
|
|
105
112
|
|
|
106
113
|
return unzipped
|
|
107
114
|
end
|
|
115
|
+
|
|
116
|
+
# Private methods
|
|
117
|
+
|
|
118
|
+
def self.filter_directories_to_time_range(directories, start_time, end_time)
|
|
119
|
+
result = []
|
|
120
|
+
directories.each do |directory|
|
|
121
|
+
result << directory if directory_in_time_range(directory, start_time, end_time)
|
|
122
|
+
end
|
|
123
|
+
return result
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
def self.directory_in_time_range(directory, start_time, end_time)
|
|
127
|
+
basename = File.basename(directory)
|
|
128
|
+
directory_start_time = DateTime.strptime(basename, DIRECTORY_TIMESTAMP_FORMAT).to_time
|
|
129
|
+
directory_end_time = directory_start_time + Time::SEC_PER_DAY
|
|
130
|
+
if (not start_time or start_time < directory_end_time) and (not end_time or end_time >= directory_start_time)
|
|
131
|
+
return true
|
|
132
|
+
else
|
|
133
|
+
return false
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
|
|
137
|
+
def self.filter_files_to_time_range(files, start_time, end_time, file_suffix: nil, overlap: false)
|
|
138
|
+
result = []
|
|
139
|
+
files.each do |file|
|
|
140
|
+
file_key = file.key.to_s
|
|
141
|
+
next if file_suffix and not file_key.end_with?(file_suffix)
|
|
142
|
+
if file_in_time_range(file_key, start_time, end_time, overlap: overlap)
|
|
143
|
+
result << file_key
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
return result
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
def self.file_in_time_range(bucket_path, start_time, end_time, overlap:)
|
|
150
|
+
file_start_time, file_end_time = get_file_times(bucket_path)
|
|
151
|
+
if overlap
|
|
152
|
+
if (not start_time or start_time <= file_end_time) and (not end_time or end_time >= file_start_time)
|
|
153
|
+
return true
|
|
154
|
+
end
|
|
155
|
+
else
|
|
156
|
+
if (not start_time or start_time <= file_start_time) and (not end_time or end_time >= file_end_time)
|
|
157
|
+
return true
|
|
158
|
+
end
|
|
159
|
+
end
|
|
160
|
+
return false
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def self.get_file_times(bucket_path)
|
|
164
|
+
basename = File.basename(bucket_path)
|
|
165
|
+
file_start_timestamp, file_end_timestamp, other = basename.split("__")
|
|
166
|
+
file_start_time = DateTime.strptime(file_start_timestamp, FILE_TIMESTAMP_FORMAT).to_time
|
|
167
|
+
file_end_time = DateTime.strptime(file_end_timestamp, FILE_TIMESTAMP_FORMAT).to_time
|
|
168
|
+
return file_start_time, file_end_time
|
|
169
|
+
end
|
|
108
170
|
end
|
|
109
171
|
end
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
14
|
# GNU Affero General Public License for more details.
|
|
15
15
|
#
|
|
16
|
-
# This file may also be used under the terms of a commercial license
|
|
16
|
+
# This file may also be used under the terms of a commercial license
|
|
17
17
|
# if purchased from OpenC3, Inc.
|
|
18
18
|
|
|
19
19
|
require 'fileutils'
|
|
@@ -201,8 +201,9 @@ module OpenC3
|
|
|
201
201
|
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
|
202
202
|
variables = plugin_hash['variables']
|
|
203
203
|
if variables
|
|
204
|
-
|
|
205
|
-
|
|
204
|
+
PluginModel::RESERVED_VARIABLE_NAMES.each do |name|
|
|
205
|
+
variables.delete(name)
|
|
206
|
+
end
|
|
206
207
|
end
|
|
207
208
|
if plugin_file_path =~ Regexp.new("^#{OPENC3_LOCAL_MODE_PATH}/#{scope}/")
|
|
208
209
|
# From local init - Always just update the exact one
|
|
@@ -400,14 +401,16 @@ module OpenC3
|
|
|
400
401
|
files << split_key[2..-1].join('/') if include_temp
|
|
401
402
|
next
|
|
402
403
|
end
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
404
|
+
if path_matchers
|
|
405
|
+
found = false
|
|
406
|
+
path_matchers.each do |path|
|
|
407
|
+
if split_key.include?(path)
|
|
408
|
+
found = true
|
|
409
|
+
break
|
|
410
|
+
end
|
|
408
411
|
end
|
|
412
|
+
next unless found
|
|
409
413
|
end
|
|
410
|
-
next unless found
|
|
411
414
|
files << split_key[2..-1].join('/')
|
|
412
415
|
end
|
|
413
416
|
return files.sort
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
|
18
18
|
# All Rights Reserved
|
|
19
19
|
#
|
|
20
|
-
# This file may also be used under the terms of a commercial license
|
|
20
|
+
# This file may also be used under the terms of a commercial license
|
|
21
21
|
# if purchased from OpenC3, Inc.
|
|
22
22
|
|
|
23
23
|
require 'openc3/core_ext/class'
|
|
@@ -41,11 +41,8 @@ module OpenC3
|
|
|
41
41
|
# @return [String] Additional detail to add to messages
|
|
42
42
|
instance_attr_accessor :detail_string
|
|
43
43
|
|
|
44
|
-
# @return [String] Fluent tag
|
|
45
|
-
instance_attr_accessor :tag
|
|
46
|
-
|
|
47
44
|
# @return [String] Microservice name
|
|
48
|
-
|
|
45
|
+
attr_reader :microservice_name
|
|
49
46
|
|
|
50
47
|
# @return [String] Scope
|
|
51
48
|
instance_attr_accessor :scope
|
|
@@ -78,11 +75,22 @@ module OpenC3
|
|
|
78
75
|
@detail_string = nil
|
|
79
76
|
@container_name = Socket.gethostname
|
|
80
77
|
@microservice_name = nil
|
|
81
|
-
@tag = @container_name + ".log"
|
|
82
|
-
@mutex = Mutex.new
|
|
83
78
|
@no_store = ENV['OPENC3_NO_STORE']
|
|
84
79
|
end
|
|
85
80
|
|
|
81
|
+
# Only set the microservice name once (to help with multi microservices)
|
|
82
|
+
def microservice_name=(name)
|
|
83
|
+
@microservice_name = name unless @microservice_name
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def self.microservice_name
|
|
87
|
+
self.instance.microservice_name
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def self.microservice_name=(name)
|
|
91
|
+
self.instance.microservice_name = name
|
|
92
|
+
end
|
|
93
|
+
|
|
86
94
|
# @param message [String] The message to print if the log level is at or
|
|
87
95
|
# below the method name log level.
|
|
88
96
|
# @param block [Proc] Block to call which should return a string to append
|
|
@@ -164,11 +172,11 @@ module OpenC3
|
|
|
164
172
|
protected
|
|
165
173
|
|
|
166
174
|
def log_message(severity_string, message, scope:, user:)
|
|
167
|
-
|
|
175
|
+
@@mutex.synchronize do
|
|
168
176
|
data = { time: Time.now.to_nsec_from_epoch, '@timestamp' => Time.now.xmlschema(3), severity: severity_string }
|
|
169
177
|
data[:microservice_name] = @microservice_name if @microservice_name
|
|
170
178
|
data[:detail] = @detail_string if @detail_string
|
|
171
|
-
data[:user] = user['
|
|
179
|
+
data[:user] = user['username'] || 'Unknown' if user # EE: If a user is passed, put its name ('Unknown' if it doesn't have a name). Don't include user data if no user was passed
|
|
172
180
|
if block_given?
|
|
173
181
|
message = yield
|
|
174
182
|
end
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
|
18
18
|
# All Rights Reserved
|
|
19
19
|
#
|
|
20
|
-
# This file may also be used under the terms of a commercial license
|
|
20
|
+
# This file may also be used under the terms of a commercial license
|
|
21
21
|
# if purchased from OpenC3, Inc.
|
|
22
22
|
|
|
23
23
|
require 'openc3/config/config_parser'
|
|
@@ -36,9 +36,10 @@ module OpenC3
|
|
|
36
36
|
# @param tool_name [String] The name of the tool creating the message log.
|
|
37
37
|
# This will be inserted into the message log filename to help identify it.
|
|
38
38
|
# @param log_dir [String] The filesystem path to store the message log file.
|
|
39
|
-
|
|
39
|
+
# @param tags [Array<String>] Array of strings to put into the filename
|
|
40
|
+
def initialize(tool_name, log_dir, tags: ['messages'], scope:)
|
|
40
41
|
@remote_log_directory = "#{scope}/tool_logs/#{tool_name}/"
|
|
41
|
-
@
|
|
42
|
+
@tags = tags.unshift(tool_name)
|
|
42
43
|
@log_dir = log_dir
|
|
43
44
|
@filename = ''
|
|
44
45
|
@file = nil
|
|
@@ -82,9 +83,9 @@ module OpenC3
|
|
|
82
83
|
def start(take_mutex = true)
|
|
83
84
|
@mutex.lock if take_mutex
|
|
84
85
|
# Prevent starting files too fast
|
|
85
|
-
sleep(0.1) until !File.exist?(File.join(@log_dir, File.build_timestamped_filename(
|
|
86
|
+
sleep(0.1) until !File.exist?(File.join(@log_dir, File.build_timestamped_filename(@tags)))
|
|
86
87
|
stop(false)
|
|
87
|
-
timed_filename = File.build_timestamped_filename(
|
|
88
|
+
timed_filename = File.build_timestamped_filename(@tags)
|
|
88
89
|
@start_day = timed_filename[0..9].gsub("_", "") # YYYYMMDD
|
|
89
90
|
@filename = File.join(@log_dir, timed_filename)
|
|
90
91
|
@file = File.open(@filename, 'a')
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# encoding: ascii-8bit
|
|
2
|
+
|
|
3
|
+
# Copyright 2022 OpenC3, Inc.
|
|
4
|
+
# All Rights Reserved.
|
|
5
|
+
#
|
|
6
|
+
# This program is free software; you can modify and/or redistribute it
|
|
7
|
+
# under the terms of the GNU Affero General Public License
|
|
8
|
+
# as published by the Free Software Foundation; version 3 with
|
|
9
|
+
# attribution addendums as found in the LICENSE.txt
|
|
10
|
+
#
|
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
|
+
# GNU Affero General Public License for more details.
|
|
15
|
+
#
|
|
16
|
+
# This file may also be used under the terms of a commercial license
|
|
17
|
+
# if purchased from OpenC3, Inc.
|
|
18
|
+
|
|
19
|
+
module OpenC3
|
|
20
|
+
class Migration
|
|
21
|
+
end
|
|
22
|
+
end
|
|
@@ -80,7 +80,6 @@ module OpenC3
|
|
|
80
80
|
@redis_key = ENV['OPENC3_REDIS_PASSWORD']
|
|
81
81
|
@redis_url = "redis://#{ENV['OPENC3_REDIS_HOSTNAME']}:#{ENV['OPENC3_REDIS_PORT']}"
|
|
82
82
|
@redis_pool = ConnectionPool.new(size: pool_size) { build_redis() }
|
|
83
|
-
@topic_offsets = {}
|
|
84
83
|
end
|
|
85
84
|
|
|
86
85
|
unless $openc3_redis_cluster
|
|
@@ -143,14 +142,16 @@ module OpenC3
|
|
|
143
142
|
topics.each do |topic|
|
|
144
143
|
# Normally we will just be grabbing the topic offset
|
|
145
144
|
# this allows xread to get everything past this point
|
|
146
|
-
|
|
145
|
+
Thread.current[:topic_offsets] ||= {}
|
|
146
|
+
topic_offsets = Thread.current[:topic_offsets]
|
|
147
|
+
last_id = topic_offsets[topic]
|
|
147
148
|
if last_id
|
|
148
149
|
offsets << last_id
|
|
149
150
|
else
|
|
150
151
|
# If there is no topic offset this is the first call.
|
|
151
152
|
# Get the last offset ID so we'll start getting everything from now on
|
|
152
153
|
offsets << get_last_offset(topic)
|
|
153
|
-
|
|
154
|
+
topic_offsets[topic] = offsets[-1]
|
|
154
155
|
end
|
|
155
156
|
end
|
|
156
157
|
return offsets
|
|
@@ -158,6 +159,8 @@ module OpenC3
|
|
|
158
159
|
|
|
159
160
|
unless $openc3_redis_cluster
|
|
160
161
|
def read_topics(topics, offsets = nil, timeout_ms = 1000, count = nil)
|
|
162
|
+
Thread.current[:topic_offsets] ||= {}
|
|
163
|
+
topic_offsets = Thread.current[:topic_offsets]
|
|
161
164
|
begin
|
|
162
165
|
# Logger.debug "read_topics: #{topics}, #{offsets} pool:#{@redis_pool}"
|
|
163
166
|
@redis_pool.with do |redis|
|
|
@@ -166,7 +169,7 @@ module OpenC3
|
|
|
166
169
|
if result and result.length > 0
|
|
167
170
|
result.each do |topic, messages|
|
|
168
171
|
messages.each do |msg_id, msg_hash|
|
|
169
|
-
|
|
172
|
+
topic_offsets[topic] = msg_id
|
|
170
173
|
yield topic, msg_id, msg_hash, redis if block_given?
|
|
171
174
|
end
|
|
172
175
|
end
|
|
@@ -199,7 +202,6 @@ module OpenC3
|
|
|
199
202
|
# @return [String] the entry id
|
|
200
203
|
def write_topic(topic, msg_hash, id = '*', maxlen = nil, approximate = 'true')
|
|
201
204
|
id = '*' if id.nil?
|
|
202
|
-
# Logger.debug "write_topic topic:#{topic} id:#{id} hash:#{msg_hash}"
|
|
203
205
|
@redis_pool.with do |redis|
|
|
204
206
|
return redis.xadd(topic, msg_hash, id: id, maxlen: maxlen, approximate: approximate)
|
|
205
207
|
end
|
|
@@ -13,7 +13,7 @@
|
|
|
13
13
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
14
14
|
# GNU Affero General Public License for more details.
|
|
15
15
|
#
|
|
16
|
-
# This file may also be used under the terms of a commercial license
|
|
16
|
+
# This file may also be used under the terms of a commercial license
|
|
17
17
|
# if purchased from OpenC3, Inc.
|
|
18
18
|
|
|
19
19
|
require 'fileutils'
|
|
@@ -44,14 +44,16 @@ module OpenC3
|
|
|
44
44
|
next
|
|
45
45
|
end
|
|
46
46
|
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
47
|
+
if path_matchers
|
|
48
|
+
found = false
|
|
49
|
+
path_matchers.each do |path|
|
|
50
|
+
if split_key.include?(path)
|
|
51
|
+
found = true
|
|
52
|
+
break
|
|
53
|
+
end
|
|
52
54
|
end
|
|
55
|
+
next unless found
|
|
53
56
|
end
|
|
54
|
-
next unless found
|
|
55
57
|
result_no_scope_or_target_folder = split_key[2..-1].join('/')
|
|
56
58
|
if object.key.include?("#{scope}/targets_modified")
|
|
57
59
|
modified << result_no_scope_or_target_folder
|
data/lib/openc3/version.rb
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
# encoding: ascii-8bit
|
|
2
2
|
|
|
3
|
-
OPENC3_VERSION = '5.
|
|
3
|
+
OPENC3_VERSION = '5.2.0'
|
|
4
4
|
module OpenC3
|
|
5
5
|
module Version
|
|
6
6
|
MAJOR = '5'
|
|
7
|
-
MINOR = '
|
|
8
|
-
PATCH = '
|
|
7
|
+
MINOR = '2'
|
|
8
|
+
PATCH = '0'
|
|
9
9
|
OTHER = ''
|
|
10
|
-
BUILD = '
|
|
10
|
+
BUILD = '49c62512b75f45a8cbfc20a6fb5ed114bab35e23'
|
|
11
11
|
end
|
|
12
|
-
VERSION = '5.
|
|
13
|
-
GEM_VERSION = '5.
|
|
12
|
+
VERSION = '5.2.0'
|
|
13
|
+
GEM_VERSION = '5.2.0'
|
|
14
14
|
end
|
data/lib/openc3.rb
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
# All changes Copyright 2022, OpenC3, Inc.
|
|
18
18
|
# All Rights Reserved
|
|
19
19
|
#
|
|
20
|
-
# This file may also be used under the terms of a commercial license
|
|
20
|
+
# This file may also be used under the terms of a commercial license
|
|
21
21
|
# if purchased from OpenC3, Inc.
|
|
22
22
|
|
|
23
23
|
# This file sets up using the OpenC3 framework
|
|
@@ -51,3 +51,4 @@ require 'openc3/system'
|
|
|
51
51
|
# OpenC3 services need to die if something goes wrong so they can be restarted
|
|
52
52
|
require 'thread'
|
|
53
53
|
Thread.abort_on_exception = true
|
|
54
|
+
Thread.report_on_exception = true
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: openc3
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 5.
|
|
4
|
+
version: 5.2.0
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Ryan Melton
|
|
@@ -9,7 +9,7 @@ authors:
|
|
|
9
9
|
autorequire:
|
|
10
10
|
bindir: bin
|
|
11
11
|
cert_chain: []
|
|
12
|
-
date: 2022-
|
|
12
|
+
date: 2022-12-13 00:00:00.000000000 Z
|
|
13
13
|
dependencies:
|
|
14
14
|
- !ruby/object:Gem::Dependency
|
|
15
15
|
name: bundler
|
|
@@ -711,14 +711,17 @@ files:
|
|
|
711
711
|
- lib/openc3/accessors/html_accessor.rb
|
|
712
712
|
- lib/openc3/accessors/json_accessor.rb
|
|
713
713
|
- lib/openc3/accessors/xml_accessor.rb
|
|
714
|
+
- lib/openc3/api/README.md
|
|
714
715
|
- lib/openc3/api/api.rb
|
|
715
716
|
- lib/openc3/api/authorized_api.rb
|
|
716
717
|
- lib/openc3/api/cmd_api.rb
|
|
717
718
|
- lib/openc3/api/config_api.rb
|
|
718
719
|
- lib/openc3/api/interface_api.rb
|
|
719
720
|
- lib/openc3/api/limits_api.rb
|
|
721
|
+
- lib/openc3/api/offline_access_api.rb
|
|
720
722
|
- lib/openc3/api/router_api.rb
|
|
721
723
|
- lib/openc3/api/settings_api.rb
|
|
724
|
+
- lib/openc3/api/stash_api.rb
|
|
722
725
|
- lib/openc3/api/target_api.rb
|
|
723
726
|
- lib/openc3/api/tlm_api.rb
|
|
724
727
|
- lib/openc3/bridge/bridge.rb
|
|
@@ -810,6 +813,8 @@ files:
|
|
|
810
813
|
- lib/openc3/microservices/interface_microservice.rb
|
|
811
814
|
- lib/openc3/microservices/log_microservice.rb
|
|
812
815
|
- lib/openc3/microservices/microservice.rb
|
|
816
|
+
- lib/openc3/microservices/multi_microservice.rb
|
|
817
|
+
- lib/openc3/microservices/periodic_microservice.rb
|
|
813
818
|
- lib/openc3/microservices/plugin_microservice.rb
|
|
814
819
|
- lib/openc3/microservices/reaction_microservice.rb
|
|
815
820
|
- lib/openc3/microservices/reducer_microservice.rb
|
|
@@ -817,6 +822,8 @@ files:
|
|
|
817
822
|
- lib/openc3/microservices/text_log_microservice.rb
|
|
818
823
|
- lib/openc3/microservices/timeline_microservice.rb
|
|
819
824
|
- lib/openc3/microservices/trigger_group_microservice.rb
|
|
825
|
+
- lib/openc3/migrations/20221202214600_add_target_names.rb
|
|
826
|
+
- lib/openc3/migrations/20221210174900_convert_to_multi.rb
|
|
820
827
|
- lib/openc3/models/activity_model.rb
|
|
821
828
|
- lib/openc3/models/auth_model.rb
|
|
822
829
|
- lib/openc3/models/cvt_model.rb
|
|
@@ -829,9 +836,11 @@ files:
|
|
|
829
836
|
- lib/openc3/models/metric_model.rb
|
|
830
837
|
- lib/openc3/models/microservice_model.rb
|
|
831
838
|
- lib/openc3/models/microservice_status_model.rb
|
|
839
|
+
- lib/openc3/models/migration_model.rb
|
|
832
840
|
- lib/openc3/models/model.rb
|
|
833
841
|
- lib/openc3/models/note_model.rb
|
|
834
842
|
- lib/openc3/models/notification_model.rb
|
|
843
|
+
- lib/openc3/models/offline_access_model.rb
|
|
835
844
|
- lib/openc3/models/ping_model.rb
|
|
836
845
|
- lib/openc3/models/plugin_model.rb
|
|
837
846
|
- lib/openc3/models/process_status_model.rb
|
|
@@ -842,6 +851,7 @@ files:
|
|
|
842
851
|
- lib/openc3/models/scope_model.rb
|
|
843
852
|
- lib/openc3/models/settings_model.rb
|
|
844
853
|
- lib/openc3/models/sorted_model.rb
|
|
854
|
+
- lib/openc3/models/stash_model.rb
|
|
845
855
|
- lib/openc3/models/target_model.rb
|
|
846
856
|
- lib/openc3/models/timeline_model.rb
|
|
847
857
|
- lib/openc3/models/tool_config_model.rb
|
|
@@ -878,11 +888,11 @@ files:
|
|
|
878
888
|
- lib/openc3/processors/watermark_processor.rb
|
|
879
889
|
- lib/openc3/script.rb
|
|
880
890
|
- lib/openc3/script/api_shared.rb
|
|
881
|
-
- lib/openc3/script/calendar.rb
|
|
882
891
|
- lib/openc3/script/commands.rb
|
|
883
892
|
- lib/openc3/script/exceptions.rb
|
|
884
893
|
- lib/openc3/script/extract.rb
|
|
885
894
|
- lib/openc3/script/limits.rb
|
|
895
|
+
- lib/openc3/script/metadata.rb
|
|
886
896
|
- lib/openc3/script/script.rb
|
|
887
897
|
- lib/openc3/script/script_runner.rb
|
|
888
898
|
- lib/openc3/script/storage.rb
|
|
@@ -937,6 +947,7 @@ files:
|
|
|
937
947
|
- lib/openc3/utilities/logger.rb
|
|
938
948
|
- lib/openc3/utilities/message_log.rb
|
|
939
949
|
- lib/openc3/utilities/metric.rb
|
|
950
|
+
- lib/openc3/utilities/migration.rb
|
|
940
951
|
- lib/openc3/utilities/open_telemetry.rb
|
|
941
952
|
- lib/openc3/utilities/process_manager.rb
|
|
942
953
|
- lib/openc3/utilities/quaternion.rb
|