openc3 5.0.8 → 5.0.9
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/openc3cli +128 -95
- data/data/config/widgets.yaml +49 -73
- data/lib/openc3/api/cmd_api.rb +5 -2
- data/lib/openc3/api/interface_api.rb +15 -0
- data/lib/openc3/api/limits_api.rb +3 -3
- data/lib/openc3/core_ext/file.rb +5 -0
- data/lib/openc3/io/json_rpc.rb +3 -3
- data/lib/openc3/microservices/cleanup_microservice.rb +7 -5
- data/lib/openc3/models/interface_model.rb +41 -0
- data/lib/openc3/models/microservice_model.rb +7 -1
- data/lib/openc3/models/plugin_model.rb +1 -1
- data/lib/openc3/models/target_model.rb +85 -68
- data/lib/openc3/packets/parsers/xtce_parser.rb +68 -3
- data/lib/openc3/script/storage.rb +18 -1
- data/lib/openc3/utilities/local_mode.rb +518 -0
- data/lib/openc3/utilities/target_file.rb +146 -0
- data/lib/openc3/version.rb +5 -5
- metadata +9 -7
@@ -0,0 +1,518 @@
|
|
1
|
+
# encoding: ascii-8bit
|
2
|
+
|
3
|
+
# Copyright 2022 OpenC3, Inc.
|
4
|
+
# All Rights Reserved.
|
5
|
+
#
|
6
|
+
# This program is free software; you can modify and/or redistribute it
|
7
|
+
# under the terms of the GNU Affero General Public License
|
8
|
+
# as published by the Free Software Foundation; version 3 with
|
9
|
+
# attribution addendums as found in the LICENSE.txt
|
10
|
+
#
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
# GNU Affero General Public License for more details.
|
15
|
+
|
16
|
+
require 'fileutils'
|
17
|
+
require 'json'
|
18
|
+
require 'openc3/core_ext/file'
|
19
|
+
# require 'openc3/models/gem_model' # These are used but also create circular dependency
|
20
|
+
# require 'openc3/models/plugin_model' # These are used but also create circular dependency
|
21
|
+
require 'openc3/utilities/s3'
|
22
|
+
|
23
|
+
module OpenC3
|
24
|
+
module LocalMode
|
25
|
+
OPENC3_LOCAL_MODE_PATH = ENV['OPENC3_LOCAL_MODE_PATH'] || "/plugins"
|
26
|
+
|
27
|
+
DEFAULT_PLUGINS = [
|
28
|
+
'openc3-tool-admin',
|
29
|
+
'openc3-tool-autonomic',
|
30
|
+
'openc3-tool-base',
|
31
|
+
'openc3-tool-calendar',
|
32
|
+
'openc3-tool-cmdsender',
|
33
|
+
'openc3-tool-cmdtlmserver',
|
34
|
+
'openc3-tool-dataextractor',
|
35
|
+
'openc3-tool-dataviewer',
|
36
|
+
'openc3-tool-handbooks',
|
37
|
+
'openc3-tool-limitsmonitor',
|
38
|
+
'openc3-tool-packetviewer',
|
39
|
+
'openc3-tool-scriptrunner',
|
40
|
+
'openc3-tool-tablemanager',
|
41
|
+
'openc3-tool-tlmgrapher',
|
42
|
+
'openc3-tool-tlmviewer',
|
43
|
+
'openc3-enterprise-tool-base',
|
44
|
+
'openc3-enterprise-tool-admin',
|
45
|
+
]
|
46
|
+
|
47
|
+
# Install plugins from local plugins folder
|
48
|
+
# Can only be used from openc3cli because calls top_level load_plugin
|
49
|
+
def self.local_init
|
50
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
51
|
+
puts "Local init running: #{OPENC3_LOCAL_MODE_PATH} exists"
|
52
|
+
Dir.each_child(OPENC3_LOCAL_MODE_PATH).each do |scope_dir|
|
53
|
+
next unless File.directory?("#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}")
|
54
|
+
puts "Local init found scope: #{scope_dir}"
|
55
|
+
Dir.each_child("#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}") do |plugin_dir|
|
56
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}/#{plugin_dir}"
|
57
|
+
next if plugin_dir == "targets_modified" or not File.directory?(full_folder_path)
|
58
|
+
puts "Local init found plugin_dir: #{full_folder_path}"
|
59
|
+
gems, plugin_instance = scan_plugin_dir(full_folder_path)
|
60
|
+
|
61
|
+
if gems.length > 1
|
62
|
+
puts "Local plugin folder contains more than one gem - skipping: #{full_folder_path}"
|
63
|
+
next
|
64
|
+
end
|
65
|
+
|
66
|
+
if gems.length == 1 and plugin_instance
|
67
|
+
# If one gem file and plugin_instance.json - Install instance
|
68
|
+
load_plugin(gems[0], scope: scope_dir.upcase, plugin_hash_file: plugin_instance)
|
69
|
+
elsif gems.length == 1
|
70
|
+
# Else If just gem - Install with default settings
|
71
|
+
load_plugin(gems[0], scope: scope_dir.upcase)
|
72
|
+
else
|
73
|
+
puts "Local plugin folder contains no gem file - skipping: #{full_folder_path}"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
sync_targets_modified()
|
78
|
+
puts "Local init complete"
|
79
|
+
else
|
80
|
+
puts "Local init canceled: Local mode not enabled or #{OPENC3_LOCAL_MODE_PATH} does not exist"
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def self.scan_plugin_dir(path)
|
85
|
+
gems = []
|
86
|
+
plugin_instance = nil
|
87
|
+
|
88
|
+
Dir.each_child(path) do |filename|
|
89
|
+
full_path = "#{path}/#{filename}"
|
90
|
+
if not File.directory?(full_path)
|
91
|
+
if File.extname(filename) == '.gem'
|
92
|
+
gems << full_path
|
93
|
+
elsif filename == 'plugin_instance.json'
|
94
|
+
plugin_instance = full_path
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
return gems, plugin_instance
|
100
|
+
end
|
101
|
+
|
102
|
+
def self.scan_local_mode
|
103
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
104
|
+
local_plugins = {}
|
105
|
+
|
106
|
+
Dir.each_child(OPENC3_LOCAL_MODE_PATH) do |scope_dir|
|
107
|
+
full_scope_dir = "#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}"
|
108
|
+
next unless File.directory?(full_scope_dir)
|
109
|
+
local_plugins[scope_dir] ||= {}
|
110
|
+
Dir.each_child("#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}") do |plugin_dir|
|
111
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}/#{plugin_dir}"
|
112
|
+
next if plugin_dir == "targets_modified" or not File.directory?(full_folder_path)
|
113
|
+
gems, plugin_instance = scan_plugin_dir(full_folder_path)
|
114
|
+
local_plugins[scope_dir][full_folder_path] = {gems: gems, plugin_instance: plugin_instance}
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
return local_plugins
|
119
|
+
end
|
120
|
+
return {}
|
121
|
+
end
|
122
|
+
|
123
|
+
def self.analyze_local_mode(plugin_name:, scope:)
|
124
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
125
|
+
# We already know a plugin with this name doesn't exist in the models
|
126
|
+
# Now need to determine if there is a highly likely candidate that has been
|
127
|
+
# updated, so that we don't do an erroneous extra plugin install
|
128
|
+
|
129
|
+
gem_name = plugin_name.split("__")[0].split('-')[0..-2].join('-')
|
130
|
+
|
131
|
+
local_plugins = scan_local_mode()
|
132
|
+
scope_plugins = local_plugins[scope]
|
133
|
+
if scope_plugins
|
134
|
+
# Scan models for same gem
|
135
|
+
found_models = {}
|
136
|
+
models = OpenC3::PluginModel.all(scope: scope)
|
137
|
+
models.each do |name, details|
|
138
|
+
model_gem_name = name.split("__")[0].split('-')[0..-2].join('-')
|
139
|
+
found_models[name] = details if gem_name == model_gem_name
|
140
|
+
end
|
141
|
+
|
142
|
+
# Definitely new install if no found models
|
143
|
+
return nil if found_models.length == 0
|
144
|
+
|
145
|
+
# Scan local for same gem and try to match pairs
|
146
|
+
found_local_plugins = {}
|
147
|
+
scope_plugins.each do |folder_path, details|
|
148
|
+
gems = details[:gems]
|
149
|
+
plugin_instance = details[:plugin_instance]
|
150
|
+
next unless plugin_instance
|
151
|
+
next if gems.length != 1
|
152
|
+
|
153
|
+
local_gem_name = File.basename(gems[0]).split('-')[0..-2].join('-')
|
154
|
+
if gem_name == local_gem_name
|
155
|
+
# Gems match - Do the names match?
|
156
|
+
data = File.read(plugin_instance)
|
157
|
+
json = JSON.parse(data, :allow_nan => true, :create_additions => true)
|
158
|
+
|
159
|
+
found = false
|
160
|
+
found_models.each do |name, model_details|
|
161
|
+
if json["name"] == name
|
162
|
+
# Matched pair
|
163
|
+
found = true
|
164
|
+
break
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
if found # names match
|
169
|
+
# Remove from the list because we have a matched set
|
170
|
+
# (local plugin_instance name and plugin model name)
|
171
|
+
found_models.delete(json["name"])
|
172
|
+
else
|
173
|
+
# Found a local plugin with the right gem, but a different name
|
174
|
+
found_local_plugins[folder_path] = details
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
# At this point we only have unmatched plugins in found_models
|
180
|
+
|
181
|
+
# Not a local mode install if no found local plugins
|
182
|
+
return nil if found_local_plugins.length == 0
|
183
|
+
|
184
|
+
# If we have any unmatched models, assume this should match the first
|
185
|
+
# (found_models are existing installed plugins with the same gem but
|
186
|
+
# a different name)
|
187
|
+
found_models.each do |name, model_details|
|
188
|
+
puts "Choosing #{name} for update from local plugins"
|
189
|
+
return model_details
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
return nil
|
194
|
+
end
|
195
|
+
|
196
|
+
# If old_plugin_name then this is an online upgrade
|
197
|
+
def self.update_local_plugin(plugin_file_path, plugin_hash, old_plugin_name: nil, scope:)
|
198
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
199
|
+
variables = plugin_hash['variables']
|
200
|
+
if variables
|
201
|
+
variables.delete("target_name")
|
202
|
+
variables.delete("microservice_name")
|
203
|
+
end
|
204
|
+
if plugin_file_path =~ Regexp.new("^#{OPENC3_LOCAL_MODE_PATH}/#{scope}/")
|
205
|
+
# From local init - Always just update the exact one
|
206
|
+
File.open(File.join(File.dirname(plugin_file_path), 'plugin_instance.json'), 'wb') do |file|
|
207
|
+
file.write(JSON.pretty_generate(plugin_hash, :allow_nan => true))
|
208
|
+
end
|
209
|
+
else
|
210
|
+
# From online install / update
|
211
|
+
# Or init install of container plugin
|
212
|
+
# Try to find an existing local folder for this plugin
|
213
|
+
found = false
|
214
|
+
|
215
|
+
gem_name = File.basename(plugin_file_path).split('-')[0..-2].join('-')
|
216
|
+
FileUtils.mkdir_p("#{OPENC3_LOCAL_MODE_PATH}/#{scope}")
|
217
|
+
|
218
|
+
Dir.each_child("#{OPENC3_LOCAL_MODE_PATH}/#{scope}") do |plugin_dir|
|
219
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/#{plugin_dir}"
|
220
|
+
next if plugin_dir == "targets_modified" or not File.directory?(full_folder_path)
|
221
|
+
|
222
|
+
gems, plugin_instance = scan_plugin_dir(full_folder_path)
|
223
|
+
next if gems.length > 1
|
224
|
+
|
225
|
+
if gems.length == 1
|
226
|
+
found_gem_name = File.basename(gems[0]).split('-')[0..-2].join('-')
|
227
|
+
if found_gem_name == gem_name
|
228
|
+
# Same gem at least - Now see if same instance
|
229
|
+
if plugin_instance
|
230
|
+
if old_plugin_name
|
231
|
+
# And we're updating a plugin
|
232
|
+
data = File.read(plugin_instance)
|
233
|
+
json = JSON.parse(data, :allow_nan => true, :create_additions => true)
|
234
|
+
if json["name"] == old_plugin_name
|
235
|
+
# Found plugin to update
|
236
|
+
found = true
|
237
|
+
update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
|
238
|
+
end
|
239
|
+
else
|
240
|
+
# New install of same plugin - Leave it alone
|
241
|
+
end
|
242
|
+
else
|
243
|
+
# No exiting instance.json, but we found the same gem
|
244
|
+
# This shouldn't happen without users using this wrong
|
245
|
+
# We will update
|
246
|
+
found = true
|
247
|
+
update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
|
248
|
+
end
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
unless found
|
254
|
+
# Then we will make a local version
|
255
|
+
# Create a folder for this plugin and add gem and plugin_instance.json
|
256
|
+
folder_name = gem_name
|
257
|
+
count = 1
|
258
|
+
while File.exist?("#{OPENC3_LOCAL_MODE_PATH}/#{scope}/#{folder_name}")
|
259
|
+
folder_name = gem_name + "-" + count.to_s
|
260
|
+
count += 1
|
261
|
+
end
|
262
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/#{folder_name}"
|
263
|
+
update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
def self.update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
|
270
|
+
return if DEFAULT_PLUGINS.include?(gem_name)
|
271
|
+
puts "Updating local plugin files: #{full_folder_path}"
|
272
|
+
FileUtils.mkdir_p(full_folder_path)
|
273
|
+
gems, plugin_instance = scan_plugin_dir(full_folder_path)
|
274
|
+
gems.each do |gem|
|
275
|
+
File.delete(gem)
|
276
|
+
end
|
277
|
+
temp_dir = Dir.mktmpdir
|
278
|
+
begin
|
279
|
+
unless File.exists?(plugin_file_path)
|
280
|
+
plugin_file_path = OpenC3::GemModel.get(temp_dir, plugin_file_path)
|
281
|
+
end
|
282
|
+
File.open(File.join(full_folder_path, File.basename(plugin_file_path)), 'wb') do |file|
|
283
|
+
data = File.read(plugin_file_path)
|
284
|
+
file.write(data)
|
285
|
+
end
|
286
|
+
File.open(File.join(full_folder_path, 'plugin_instance.json'), 'wb') do |file|
|
287
|
+
file.write(JSON.pretty_generate(plugin_hash, :allow_nan => true))
|
288
|
+
end
|
289
|
+
ensure
|
290
|
+
FileUtils.remove_entry(temp_dir) if temp_dir and File.exist?(temp_dir)
|
291
|
+
end
|
292
|
+
end
|
293
|
+
|
294
|
+
def self.remove_local_plugin(plugin_name, scope:)
|
295
|
+
local_plugins = scan_local_mode()
|
296
|
+
scope_local_plugins = local_plugins[scope]
|
297
|
+
if scope_local_plugins
|
298
|
+
scope_local_plugins.each do |full_folder_path, details|
|
299
|
+
gems = details[:gems]
|
300
|
+
plugin_instance = details[:plugin_instance]
|
301
|
+
if gems.length == 1 and plugin_instance
|
302
|
+
data = File.read(plugin_instance)
|
303
|
+
json = JSON.parse(data, :allow_nan => true, :create_additions => true)
|
304
|
+
instance_name = json['name']
|
305
|
+
if plugin_name == instance_name
|
306
|
+
puts "Removing local plugin files: #{full_folder_path}"
|
307
|
+
File.delete(gems[0])
|
308
|
+
File.delete(plugin_instance)
|
309
|
+
break
|
310
|
+
end
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|
314
|
+
end
|
315
|
+
|
316
|
+
def self.sync_targets_modified
|
317
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
318
|
+
rubys3_client = Aws::S3::Client.new
|
319
|
+
|
320
|
+
# Ensure config bucket exists
|
321
|
+
begin
|
322
|
+
rubys3_client.head_bucket(bucket: 'config')
|
323
|
+
rescue Aws::S3::Errors::NotFound
|
324
|
+
rubys3_client.create_bucket(bucket: 'config')
|
325
|
+
end
|
326
|
+
|
327
|
+
scopes = ScopeModel.names()
|
328
|
+
scopes.each do |scope|
|
329
|
+
sync_with_minio(rubys3_client, scope: scope)
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
def self.modified_targets(scope:)
|
335
|
+
targets = {}
|
336
|
+
local_catalog = build_local_catalog(scope: scope)
|
337
|
+
local_catalog.each do |key, size|
|
338
|
+
split_key = key.split('/') # scope/targets_modified/target_name/*
|
339
|
+
target_name = split_key[2]
|
340
|
+
if target_name
|
341
|
+
targets[target_name] = true
|
342
|
+
end
|
343
|
+
end
|
344
|
+
return targets.keys.sort
|
345
|
+
end
|
346
|
+
|
347
|
+
def self.modified_files(target_name, scope:)
|
348
|
+
modified = []
|
349
|
+
local_catalog = build_local_catalog(scope: scope)
|
350
|
+
local_catalog.each do |key, size|
|
351
|
+
split_key = key.split('/') # scope/targets_modified/target_name/*
|
352
|
+
local_target_name = split_key[2]
|
353
|
+
if target_name == local_target_name
|
354
|
+
modified << split_key[3..-1].join('/')
|
355
|
+
end
|
356
|
+
end
|
357
|
+
# Paths do not include target name
|
358
|
+
return modified.sort
|
359
|
+
end
|
360
|
+
|
361
|
+
def self.delete_modified(target_name, scope:)
|
362
|
+
full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified/#{target_name}"
|
363
|
+
FileUtils.rm_rf(full_path)
|
364
|
+
end
|
365
|
+
|
366
|
+
def self.zip_target(target_name, zip, scope:)
|
367
|
+
modified = modified_files(target_name, scope: scope)
|
368
|
+
modified.each do |file_path|
|
369
|
+
full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified/#{target_name}/#{file_path}"
|
370
|
+
zip.add(file_path, full_path)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
def self.put_target_file(path, io_or_string, scope:)
|
375
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{path}"
|
376
|
+
FileUtils.mkdir_p(File.dirname(full_folder_path))
|
377
|
+
File.open(full_folder_path, 'wb') do |file|
|
378
|
+
if String === io_or_string
|
379
|
+
data = io_or_string
|
380
|
+
else
|
381
|
+
data = io_or_string.read
|
382
|
+
end
|
383
|
+
file.write(data)
|
384
|
+
end
|
385
|
+
end
|
386
|
+
|
387
|
+
def self.open_local_file(path, scope:)
|
388
|
+
full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified/#{path}"
|
389
|
+
return File.open(full_path, 'rb') if File.exist?(full_path)
|
390
|
+
return nil
|
391
|
+
end
|
392
|
+
|
393
|
+
def self.local_target_files(scope:, path_matchers:)
|
394
|
+
files = []
|
395
|
+
local_catalog = build_local_catalog(scope: scope)
|
396
|
+
local_catalog.each do |key, size|
|
397
|
+
split_key = key.split('/')
|
398
|
+
found = false
|
399
|
+
path_matchers.each do |path|
|
400
|
+
if split_key.include?(path)
|
401
|
+
found = true
|
402
|
+
break
|
403
|
+
end
|
404
|
+
end
|
405
|
+
next unless found
|
406
|
+
files << split_key[2..-1].join('/')
|
407
|
+
end
|
408
|
+
return files.sort
|
409
|
+
end
|
410
|
+
|
411
|
+
# Helper methods
|
412
|
+
|
413
|
+
def self.sync_remote_to_local(rubys3_client, key)
|
414
|
+
local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
|
415
|
+
FileUtils.mkdir_p(File.dirname(local_path))
|
416
|
+
rubys3_client.get_object(bucket: 'config', key: key, response_target: local_path)
|
417
|
+
end
|
418
|
+
|
419
|
+
def self.sync_local_to_remote(rubys3_client, key)
|
420
|
+
local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
|
421
|
+
File.open(local_path, 'rb') do |read_file|
|
422
|
+
rubys3_client.put_object(bucket: 'config', key: key, body: read_file)
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
426
|
+
def self.delete_local(key)
|
427
|
+
local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
|
428
|
+
File.delete(local_path) if File.exist?(local_path)
|
429
|
+
nil
|
430
|
+
end
|
431
|
+
|
432
|
+
def self.delete_remote(rubys3_client, key)
|
433
|
+
rubys3_client.delete_object(bucket: 'config', key: key)
|
434
|
+
end
|
435
|
+
|
436
|
+
# Returns equivalent names and sizes to remote catalog
|
437
|
+
# {"scope/targets_modified/target_name/file" => size}
|
438
|
+
def self.build_local_catalog(scope:)
|
439
|
+
local_catalog = {}
|
440
|
+
local_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified"
|
441
|
+
prefix_length = "#{OPENC3_LOCAL_MODE_PATH}/".length
|
442
|
+
FileUtils.mkdir_p(local_folder_path)
|
443
|
+
Dir.glob(local_folder_path + "/**/*").each do |filename|
|
444
|
+
next if File.directory?(filename)
|
445
|
+
mod_filename = filename[prefix_length..-1]
|
446
|
+
local_catalog[mod_filename] = File.size(filename)
|
447
|
+
end
|
448
|
+
return local_catalog
|
449
|
+
end
|
450
|
+
|
451
|
+
# Returns keys and sizes from remote catalog
|
452
|
+
# {"scope/targets_modified/target_name/file" => size}
|
453
|
+
def self.build_remote_catalog(rubys3_client, scope:)
|
454
|
+
remote_catalog = {}
|
455
|
+
bucket = 'config'
|
456
|
+
prefix = "#{scope}/targets_modified"
|
457
|
+
token = nil
|
458
|
+
while true
|
459
|
+
resp = rubys3_client.list_objects_v2({
|
460
|
+
bucket: bucket,
|
461
|
+
max_keys: 1000,
|
462
|
+
prefix: prefix,
|
463
|
+
continuation_token: token
|
464
|
+
})
|
465
|
+
|
466
|
+
resp.contents.each do |item|
|
467
|
+
remote_catalog[item.key] = item.size
|
468
|
+
end
|
469
|
+
break unless resp.is_truncated
|
470
|
+
token = resp.next_continuation_token
|
471
|
+
end
|
472
|
+
return remote_catalog
|
473
|
+
end
|
474
|
+
|
475
|
+
def self.sync_with_minio(rubys3_client, scope:)
|
476
|
+
# Build catalogs
|
477
|
+
local_catalog = build_local_catalog(scope: scope)
|
478
|
+
remote_catalog = build_remote_catalog(rubys3_client, scope: scope)
|
479
|
+
|
480
|
+
# Find and Handle Differences
|
481
|
+
local_catalog.each do |key, size|
|
482
|
+
remote_size = remote_catalog[key]
|
483
|
+
if remote_size
|
484
|
+
# Both files exist
|
485
|
+
if ENV['OPENC3_LOCAL_MODE_SECONDARY']
|
486
|
+
sync_remote_to_local(rubys3_client, key) if size != remote_size or ENV['OPENC3_LOCAL_MODE_FORCE_SYNC']
|
487
|
+
else
|
488
|
+
sync_local_to_remote(rubys3_client, key) if size != remote_size or ENV['OPENC3_LOCAL_MODE_FORCE_SYNC']
|
489
|
+
end
|
490
|
+
else
|
491
|
+
# Remote is missing local file
|
492
|
+
if ENV['OPENC3_LOCAL_MODE_SECONDARY'] and ENV['OPENC3_LOCAL_MODE_SYNC_REMOVE']
|
493
|
+
delete_local(key)
|
494
|
+
else
|
495
|
+
# Go ahead and copy up to get in sync
|
496
|
+
sync_local_to_remote(rubys3_client, key)
|
497
|
+
end
|
498
|
+
end
|
499
|
+
end
|
500
|
+
|
501
|
+
remote_catalog.each do |key, size|
|
502
|
+
local_size = local_catalog[key]
|
503
|
+
if local_size
|
504
|
+
# Both files exist - Handled earlier
|
505
|
+
else
|
506
|
+
# Local is missing remote file
|
507
|
+
if not ENV['OPENC3_LOCAL_MODE_SECONDARY'] and ENV['OPENC3_LOCAL_MODE_SYNC_REMOVE']
|
508
|
+
delete_remote(rubys3_client, key)
|
509
|
+
else
|
510
|
+
# Go ahead and copy down to get in sync
|
511
|
+
sync_remote_to_local(rubys3_client, key)
|
512
|
+
end
|
513
|
+
end
|
514
|
+
end
|
515
|
+
end
|
516
|
+
|
517
|
+
end
|
518
|
+
end
|
@@ -0,0 +1,146 @@
|
|
1
|
+
# encoding: ascii-8bit
|
2
|
+
|
3
|
+
# Copyright 2022 OpenC3, Inc.
|
4
|
+
# All Rights Reserved.
|
5
|
+
#
|
6
|
+
# This program is free software; you can modify and/or redistribute it
|
7
|
+
# under the terms of the GNU Affero General Public License
|
8
|
+
# as published by the Free Software Foundation; version 3 with
|
9
|
+
# attribution addendums as found in the LICENSE.txt
|
10
|
+
#
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
# GNU Affero General Public License for more details.
|
15
|
+
|
16
|
+
require 'fileutils'
|
17
|
+
require 'json'
|
18
|
+
require 'openc3/utilities/local_mode'
|
19
|
+
require 'openc3/utilities/s3'
|
20
|
+
|
21
|
+
module OpenC3
|
22
|
+
class TargetFile
|
23
|
+
|
24
|
+
DEFAULT_BUCKET_NAME = 'config'
|
25
|
+
|
26
|
+
def self.all(scope, path_matchers)
|
27
|
+
result = []
|
28
|
+
modified = []
|
29
|
+
|
30
|
+
rubys3_client = Aws::S3::Client.new
|
31
|
+
token = nil
|
32
|
+
while true
|
33
|
+
resp = rubys3_client.list_objects_v2({
|
34
|
+
bucket: DEFAULT_BUCKET_NAME,
|
35
|
+
prefix: "#{scope}/targets",
|
36
|
+
max_keys: 1000,
|
37
|
+
continuation_token: token
|
38
|
+
})
|
39
|
+
|
40
|
+
resp.contents.each do |object|
|
41
|
+
split_key = object.key.split('/')
|
42
|
+
found = false
|
43
|
+
path_matchers.each do |path|
|
44
|
+
if split_key.include?(path)
|
45
|
+
found = true
|
46
|
+
break
|
47
|
+
end
|
48
|
+
end
|
49
|
+
next unless found
|
50
|
+
result_no_scope_or_target_folder = split_key[2..-1].join('/')
|
51
|
+
if object.key.include?("#{scope}/targets_modified")
|
52
|
+
modified << result_no_scope_or_target_folder
|
53
|
+
else
|
54
|
+
result << result_no_scope_or_target_folder
|
55
|
+
end
|
56
|
+
end
|
57
|
+
break unless resp.is_truncated
|
58
|
+
token = resp.next_continuation_token
|
59
|
+
end
|
60
|
+
|
61
|
+
# Add in local targets_modified if present
|
62
|
+
if ENV['OPENC3_LOCAL_MODE']
|
63
|
+
local_modified = OpenC3::LocalMode.local_target_files(scope: scope, path_matchers: path_matchers)
|
64
|
+
local_modified.each do |filename|
|
65
|
+
modified << filename unless modified.include?(filename)
|
66
|
+
result << filename unless result.include?(filename)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Determine if there are any modified files and mark them with '*'
|
71
|
+
result.map! do |file|
|
72
|
+
if modified.include?(file)
|
73
|
+
modified.delete(file)
|
74
|
+
"#{file}*"
|
75
|
+
else
|
76
|
+
file
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
# Concat any remaining modified files (new files not in original target)
|
81
|
+
result.concat(modified)
|
82
|
+
result.sort
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.body(scope, name)
|
86
|
+
name = name.split('*')[0] # Split '*' that indicates modified
|
87
|
+
rubys3_client = Aws::S3::Client.new
|
88
|
+
begin
|
89
|
+
# First try opening a potentially modified version by looking for the modified target
|
90
|
+
if ENV['OPENC3_LOCAL_MODE']
|
91
|
+
local_file = OpenC3::LocalMode.open_local_file(name, scope: scope)
|
92
|
+
return local_file.read if local_file
|
93
|
+
end
|
94
|
+
|
95
|
+
resp =
|
96
|
+
rubys3_client.get_object(
|
97
|
+
bucket: DEFAULT_BUCKET_NAME,
|
98
|
+
key: "#{scope}/targets_modified/#{name}",
|
99
|
+
)
|
100
|
+
rescue Aws::S3::Errors::NoSuchKey
|
101
|
+
# Now try the original
|
102
|
+
resp =
|
103
|
+
rubys3_client.get_object(
|
104
|
+
bucket: DEFAULT_BUCKET_NAME,
|
105
|
+
key: "#{scope}/targets/#{name}",
|
106
|
+
)
|
107
|
+
end
|
108
|
+
if File.extname(name) == ".bin"
|
109
|
+
resp.body.binmode
|
110
|
+
end
|
111
|
+
resp.body.read
|
112
|
+
end
|
113
|
+
|
114
|
+
def self.create(scope, name, text, content_type: 'text/plain')
|
115
|
+
return false unless text
|
116
|
+
if ENV['OPENC3_LOCAL_MODE']
|
117
|
+
OpenC3::LocalMode.put_target_file("#{scope}/targets_modified/#{name}", text, scope: scope)
|
118
|
+
end
|
119
|
+
OpenC3::S3Utilities.put_object_and_check(
|
120
|
+
# Use targets_modified to save modifications
|
121
|
+
# This keeps the original target clean (read-only)
|
122
|
+
key: "#{scope}/targets_modified/#{name}",
|
123
|
+
body: text,
|
124
|
+
bucket: DEFAULT_BUCKET_NAME,
|
125
|
+
content_type: content_type,
|
126
|
+
)
|
127
|
+
true
|
128
|
+
end
|
129
|
+
|
130
|
+
def self.destroy(scope, name)
|
131
|
+
rubys3_client = Aws::S3::Client.new
|
132
|
+
|
133
|
+
if ENV['OPENC3_LOCAL_MODE']
|
134
|
+
OpenC3::LocalMode.delete_local("#{scope}/targets_modified/#{name}")
|
135
|
+
end
|
136
|
+
|
137
|
+
# Only delete file from the modified target directory
|
138
|
+
rubys3_client.delete_object(
|
139
|
+
key: "#{scope}/targets_modified/#{name}",
|
140
|
+
bucket: DEFAULT_BUCKET_NAME,
|
141
|
+
)
|
142
|
+
true
|
143
|
+
end
|
144
|
+
|
145
|
+
end
|
146
|
+
end
|
data/lib/openc3/version.rb
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# encoding: ascii-8bit
|
2
2
|
|
3
|
-
OPENC3_VERSION = '5.0.
|
3
|
+
OPENC3_VERSION = '5.0.9'
|
4
4
|
module OpenC3
|
5
5
|
module Version
|
6
6
|
MAJOR = '5'
|
7
7
|
MINOR = '0'
|
8
|
-
PATCH = '
|
8
|
+
PATCH = '9'
|
9
9
|
OTHER = ''
|
10
|
-
BUILD = '
|
10
|
+
BUILD = '7c04d727e4664a45e6b1021bea58c06e2c046b10'
|
11
11
|
end
|
12
|
-
VERSION = '5.0.
|
13
|
-
GEM_VERSION = '5.0.
|
12
|
+
VERSION = '5.0.9'
|
13
|
+
GEM_VERSION = '5.0.9'
|
14
14
|
end
|