openc3 5.0.8 → 5.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of openc3 might be problematic. Click here for more details.
- checksums.yaml +4 -4
- data/bin/openc3cli +128 -95
- data/data/config/widgets.yaml +49 -73
- data/lib/openc3/api/cmd_api.rb +5 -2
- data/lib/openc3/api/interface_api.rb +15 -0
- data/lib/openc3/api/limits_api.rb +3 -3
- data/lib/openc3/core_ext/file.rb +5 -0
- data/lib/openc3/io/json_rpc.rb +3 -3
- data/lib/openc3/microservices/cleanup_microservice.rb +7 -5
- data/lib/openc3/models/interface_model.rb +41 -0
- data/lib/openc3/models/microservice_model.rb +7 -1
- data/lib/openc3/models/plugin_model.rb +1 -1
- data/lib/openc3/models/target_model.rb +85 -68
- data/lib/openc3/packets/parsers/xtce_parser.rb +68 -3
- data/lib/openc3/script/storage.rb +18 -1
- data/lib/openc3/utilities/local_mode.rb +518 -0
- data/lib/openc3/utilities/target_file.rb +146 -0
- data/lib/openc3/version.rb +5 -5
- metadata +9 -7
@@ -0,0 +1,518 @@
|
|
1
|
+
# encoding: ascii-8bit
|
2
|
+
|
3
|
+
# Copyright 2022 OpenC3, Inc.
|
4
|
+
# All Rights Reserved.
|
5
|
+
#
|
6
|
+
# This program is free software; you can modify and/or redistribute it
|
7
|
+
# under the terms of the GNU Affero General Public License
|
8
|
+
# as published by the Free Software Foundation; version 3 with
|
9
|
+
# attribution addendums as found in the LICENSE.txt
|
10
|
+
#
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
# GNU Affero General Public License for more details.
|
15
|
+
|
16
|
+
require 'fileutils'
|
17
|
+
require 'json'
|
18
|
+
require 'openc3/core_ext/file'
|
19
|
+
# require 'openc3/models/gem_model' # These are used but also create circular dependency
|
20
|
+
# require 'openc3/models/plugin_model' # These are used but also create circular dependency
|
21
|
+
require 'openc3/utilities/s3'
|
22
|
+
|
23
|
+
module OpenC3
|
24
|
+
module LocalMode
|
25
|
+
OPENC3_LOCAL_MODE_PATH = ENV['OPENC3_LOCAL_MODE_PATH'] || "/plugins"
|
26
|
+
|
27
|
+
DEFAULT_PLUGINS = [
|
28
|
+
'openc3-tool-admin',
|
29
|
+
'openc3-tool-autonomic',
|
30
|
+
'openc3-tool-base',
|
31
|
+
'openc3-tool-calendar',
|
32
|
+
'openc3-tool-cmdsender',
|
33
|
+
'openc3-tool-cmdtlmserver',
|
34
|
+
'openc3-tool-dataextractor',
|
35
|
+
'openc3-tool-dataviewer',
|
36
|
+
'openc3-tool-handbooks',
|
37
|
+
'openc3-tool-limitsmonitor',
|
38
|
+
'openc3-tool-packetviewer',
|
39
|
+
'openc3-tool-scriptrunner',
|
40
|
+
'openc3-tool-tablemanager',
|
41
|
+
'openc3-tool-tlmgrapher',
|
42
|
+
'openc3-tool-tlmviewer',
|
43
|
+
'openc3-enterprise-tool-base',
|
44
|
+
'openc3-enterprise-tool-admin',
|
45
|
+
]
|
46
|
+
|
47
|
+
# Install plugins from local plugins folder
|
48
|
+
# Can only be used from openc3cli because calls top_level load_plugin
|
49
|
+
def self.local_init
|
50
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
51
|
+
puts "Local init running: #{OPENC3_LOCAL_MODE_PATH} exists"
|
52
|
+
Dir.each_child(OPENC3_LOCAL_MODE_PATH).each do |scope_dir|
|
53
|
+
next unless File.directory?("#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}")
|
54
|
+
puts "Local init found scope: #{scope_dir}"
|
55
|
+
Dir.each_child("#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}") do |plugin_dir|
|
56
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}/#{plugin_dir}"
|
57
|
+
next if plugin_dir == "targets_modified" or not File.directory?(full_folder_path)
|
58
|
+
puts "Local init found plugin_dir: #{full_folder_path}"
|
59
|
+
gems, plugin_instance = scan_plugin_dir(full_folder_path)
|
60
|
+
|
61
|
+
if gems.length > 1
|
62
|
+
puts "Local plugin folder contains more than one gem - skipping: #{full_folder_path}"
|
63
|
+
next
|
64
|
+
end
|
65
|
+
|
66
|
+
if gems.length == 1 and plugin_instance
|
67
|
+
# If one gem file and plugin_instance.json - Install instance
|
68
|
+
load_plugin(gems[0], scope: scope_dir.upcase, plugin_hash_file: plugin_instance)
|
69
|
+
elsif gems.length == 1
|
70
|
+
# Else If just gem - Install with default settings
|
71
|
+
load_plugin(gems[0], scope: scope_dir.upcase)
|
72
|
+
else
|
73
|
+
puts "Local plugin folder contains no gem file - skipping: #{full_folder_path}"
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
77
|
+
sync_targets_modified()
|
78
|
+
puts "Local init complete"
|
79
|
+
else
|
80
|
+
puts "Local init canceled: Local mode not enabled or #{OPENC3_LOCAL_MODE_PATH} does not exist"
|
81
|
+
end
|
82
|
+
end
|
83
|
+
|
84
|
+
def self.scan_plugin_dir(path)
|
85
|
+
gems = []
|
86
|
+
plugin_instance = nil
|
87
|
+
|
88
|
+
Dir.each_child(path) do |filename|
|
89
|
+
full_path = "#{path}/#{filename}"
|
90
|
+
if not File.directory?(full_path)
|
91
|
+
if File.extname(filename) == '.gem'
|
92
|
+
gems << full_path
|
93
|
+
elsif filename == 'plugin_instance.json'
|
94
|
+
plugin_instance = full_path
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
98
|
+
|
99
|
+
return gems, plugin_instance
|
100
|
+
end
|
101
|
+
|
102
|
+
def self.scan_local_mode
|
103
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
104
|
+
local_plugins = {}
|
105
|
+
|
106
|
+
Dir.each_child(OPENC3_LOCAL_MODE_PATH) do |scope_dir|
|
107
|
+
full_scope_dir = "#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}"
|
108
|
+
next unless File.directory?(full_scope_dir)
|
109
|
+
local_plugins[scope_dir] ||= {}
|
110
|
+
Dir.each_child("#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}") do |plugin_dir|
|
111
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope_dir}/#{plugin_dir}"
|
112
|
+
next if plugin_dir == "targets_modified" or not File.directory?(full_folder_path)
|
113
|
+
gems, plugin_instance = scan_plugin_dir(full_folder_path)
|
114
|
+
local_plugins[scope_dir][full_folder_path] = {gems: gems, plugin_instance: plugin_instance}
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
118
|
+
return local_plugins
|
119
|
+
end
|
120
|
+
return {}
|
121
|
+
end
|
122
|
+
|
123
|
+
def self.analyze_local_mode(plugin_name:, scope:)
|
124
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
125
|
+
# We already know a plugin with this name doesn't exist in the models
|
126
|
+
# Now need to determine if there is a highly likely candidate that has been
|
127
|
+
# updated, so that we don't do an erroneous extra plugin install
|
128
|
+
|
129
|
+
gem_name = plugin_name.split("__")[0].split('-')[0..-2].join('-')
|
130
|
+
|
131
|
+
local_plugins = scan_local_mode()
|
132
|
+
scope_plugins = local_plugins[scope]
|
133
|
+
if scope_plugins
|
134
|
+
# Scan models for same gem
|
135
|
+
found_models = {}
|
136
|
+
models = OpenC3::PluginModel.all(scope: scope)
|
137
|
+
models.each do |name, details|
|
138
|
+
model_gem_name = name.split("__")[0].split('-')[0..-2].join('-')
|
139
|
+
found_models[name] = details if gem_name == model_gem_name
|
140
|
+
end
|
141
|
+
|
142
|
+
# Definitely new install if no found models
|
143
|
+
return nil if found_models.length == 0
|
144
|
+
|
145
|
+
# Scan local for same gem and try to match pairs
|
146
|
+
found_local_plugins = {}
|
147
|
+
scope_plugins.each do |folder_path, details|
|
148
|
+
gems = details[:gems]
|
149
|
+
plugin_instance = details[:plugin_instance]
|
150
|
+
next unless plugin_instance
|
151
|
+
next if gems.length != 1
|
152
|
+
|
153
|
+
local_gem_name = File.basename(gems[0]).split('-')[0..-2].join('-')
|
154
|
+
if gem_name == local_gem_name
|
155
|
+
# Gems match - Do the names match?
|
156
|
+
data = File.read(plugin_instance)
|
157
|
+
json = JSON.parse(data, :allow_nan => true, :create_additions => true)
|
158
|
+
|
159
|
+
found = false
|
160
|
+
found_models.each do |name, model_details|
|
161
|
+
if json["name"] == name
|
162
|
+
# Matched pair
|
163
|
+
found = true
|
164
|
+
break
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
if found # names match
|
169
|
+
# Remove from the list because we have a matched set
|
170
|
+
# (local plugin_instance name and plugin model name)
|
171
|
+
found_models.delete(json["name"])
|
172
|
+
else
|
173
|
+
# Found a local plugin with the right gem, but a different name
|
174
|
+
found_local_plugins[folder_path] = details
|
175
|
+
end
|
176
|
+
end
|
177
|
+
end
|
178
|
+
|
179
|
+
# At this point we only have unmatched plugins in found_models
|
180
|
+
|
181
|
+
# Not a local mode install if no found local plugins
|
182
|
+
return nil if found_local_plugins.length == 0
|
183
|
+
|
184
|
+
# If we have any unmatched models, assume this should match the first
|
185
|
+
# (found_models are existing installed plugins with the same gem but
|
186
|
+
# a different name)
|
187
|
+
found_models.each do |name, model_details|
|
188
|
+
puts "Choosing #{name} for update from local plugins"
|
189
|
+
return model_details
|
190
|
+
end
|
191
|
+
end
|
192
|
+
end
|
193
|
+
return nil
|
194
|
+
end
|
195
|
+
|
196
|
+
# If old_plugin_name then this is an online upgrade
|
197
|
+
def self.update_local_plugin(plugin_file_path, plugin_hash, old_plugin_name: nil, scope:)
|
198
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
199
|
+
variables = plugin_hash['variables']
|
200
|
+
if variables
|
201
|
+
variables.delete("target_name")
|
202
|
+
variables.delete("microservice_name")
|
203
|
+
end
|
204
|
+
if plugin_file_path =~ Regexp.new("^#{OPENC3_LOCAL_MODE_PATH}/#{scope}/")
|
205
|
+
# From local init - Always just update the exact one
|
206
|
+
File.open(File.join(File.dirname(plugin_file_path), 'plugin_instance.json'), 'wb') do |file|
|
207
|
+
file.write(JSON.pretty_generate(plugin_hash, :allow_nan => true))
|
208
|
+
end
|
209
|
+
else
|
210
|
+
# From online install / update
|
211
|
+
# Or init install of container plugin
|
212
|
+
# Try to find an existing local folder for this plugin
|
213
|
+
found = false
|
214
|
+
|
215
|
+
gem_name = File.basename(plugin_file_path).split('-')[0..-2].join('-')
|
216
|
+
FileUtils.mkdir_p("#{OPENC3_LOCAL_MODE_PATH}/#{scope}")
|
217
|
+
|
218
|
+
Dir.each_child("#{OPENC3_LOCAL_MODE_PATH}/#{scope}") do |plugin_dir|
|
219
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/#{plugin_dir}"
|
220
|
+
next if plugin_dir == "targets_modified" or not File.directory?(full_folder_path)
|
221
|
+
|
222
|
+
gems, plugin_instance = scan_plugin_dir(full_folder_path)
|
223
|
+
next if gems.length > 1
|
224
|
+
|
225
|
+
if gems.length == 1
|
226
|
+
found_gem_name = File.basename(gems[0]).split('-')[0..-2].join('-')
|
227
|
+
if found_gem_name == gem_name
|
228
|
+
# Same gem at least - Now see if same instance
|
229
|
+
if plugin_instance
|
230
|
+
if old_plugin_name
|
231
|
+
# And we're updating a plugin
|
232
|
+
data = File.read(plugin_instance)
|
233
|
+
json = JSON.parse(data, :allow_nan => true, :create_additions => true)
|
234
|
+
if json["name"] == old_plugin_name
|
235
|
+
# Found plugin to update
|
236
|
+
found = true
|
237
|
+
update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
|
238
|
+
end
|
239
|
+
else
|
240
|
+
# New install of same plugin - Leave it alone
|
241
|
+
end
|
242
|
+
else
|
243
|
+
# No exiting instance.json, but we found the same gem
|
244
|
+
# This shouldn't happen without users using this wrong
|
245
|
+
# We will update
|
246
|
+
found = true
|
247
|
+
update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
|
248
|
+
end
|
249
|
+
end
|
250
|
+
end
|
251
|
+
end
|
252
|
+
|
253
|
+
unless found
|
254
|
+
# Then we will make a local version
|
255
|
+
# Create a folder for this plugin and add gem and plugin_instance.json
|
256
|
+
folder_name = gem_name
|
257
|
+
count = 1
|
258
|
+
while File.exist?("#{OPENC3_LOCAL_MODE_PATH}/#{scope}/#{folder_name}")
|
259
|
+
folder_name = gem_name + "-" + count.to_s
|
260
|
+
count += 1
|
261
|
+
end
|
262
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/#{folder_name}"
|
263
|
+
update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
|
264
|
+
end
|
265
|
+
end
|
266
|
+
end
|
267
|
+
end
|
268
|
+
|
269
|
+
def self.update_local_plugin_files(full_folder_path, plugin_file_path, plugin_hash, gem_name)
|
270
|
+
return if DEFAULT_PLUGINS.include?(gem_name)
|
271
|
+
puts "Updating local plugin files: #{full_folder_path}"
|
272
|
+
FileUtils.mkdir_p(full_folder_path)
|
273
|
+
gems, plugin_instance = scan_plugin_dir(full_folder_path)
|
274
|
+
gems.each do |gem|
|
275
|
+
File.delete(gem)
|
276
|
+
end
|
277
|
+
temp_dir = Dir.mktmpdir
|
278
|
+
begin
|
279
|
+
unless File.exists?(plugin_file_path)
|
280
|
+
plugin_file_path = OpenC3::GemModel.get(temp_dir, plugin_file_path)
|
281
|
+
end
|
282
|
+
File.open(File.join(full_folder_path, File.basename(plugin_file_path)), 'wb') do |file|
|
283
|
+
data = File.read(plugin_file_path)
|
284
|
+
file.write(data)
|
285
|
+
end
|
286
|
+
File.open(File.join(full_folder_path, 'plugin_instance.json'), 'wb') do |file|
|
287
|
+
file.write(JSON.pretty_generate(plugin_hash, :allow_nan => true))
|
288
|
+
end
|
289
|
+
ensure
|
290
|
+
FileUtils.remove_entry(temp_dir) if temp_dir and File.exist?(temp_dir)
|
291
|
+
end
|
292
|
+
end
|
293
|
+
|
294
|
+
def self.remove_local_plugin(plugin_name, scope:)
|
295
|
+
local_plugins = scan_local_mode()
|
296
|
+
scope_local_plugins = local_plugins[scope]
|
297
|
+
if scope_local_plugins
|
298
|
+
scope_local_plugins.each do |full_folder_path, details|
|
299
|
+
gems = details[:gems]
|
300
|
+
plugin_instance = details[:plugin_instance]
|
301
|
+
if gems.length == 1 and plugin_instance
|
302
|
+
data = File.read(plugin_instance)
|
303
|
+
json = JSON.parse(data, :allow_nan => true, :create_additions => true)
|
304
|
+
instance_name = json['name']
|
305
|
+
if plugin_name == instance_name
|
306
|
+
puts "Removing local plugin files: #{full_folder_path}"
|
307
|
+
File.delete(gems[0])
|
308
|
+
File.delete(plugin_instance)
|
309
|
+
break
|
310
|
+
end
|
311
|
+
end
|
312
|
+
end
|
313
|
+
end
|
314
|
+
end
|
315
|
+
|
316
|
+
def self.sync_targets_modified
|
317
|
+
if ENV['OPENC3_LOCAL_MODE'] and Dir.exist?(OPENC3_LOCAL_MODE_PATH)
|
318
|
+
rubys3_client = Aws::S3::Client.new
|
319
|
+
|
320
|
+
# Ensure config bucket exists
|
321
|
+
begin
|
322
|
+
rubys3_client.head_bucket(bucket: 'config')
|
323
|
+
rescue Aws::S3::Errors::NotFound
|
324
|
+
rubys3_client.create_bucket(bucket: 'config')
|
325
|
+
end
|
326
|
+
|
327
|
+
scopes = ScopeModel.names()
|
328
|
+
scopes.each do |scope|
|
329
|
+
sync_with_minio(rubys3_client, scope: scope)
|
330
|
+
end
|
331
|
+
end
|
332
|
+
end
|
333
|
+
|
334
|
+
def self.modified_targets(scope:)
|
335
|
+
targets = {}
|
336
|
+
local_catalog = build_local_catalog(scope: scope)
|
337
|
+
local_catalog.each do |key, size|
|
338
|
+
split_key = key.split('/') # scope/targets_modified/target_name/*
|
339
|
+
target_name = split_key[2]
|
340
|
+
if target_name
|
341
|
+
targets[target_name] = true
|
342
|
+
end
|
343
|
+
end
|
344
|
+
return targets.keys.sort
|
345
|
+
end
|
346
|
+
|
347
|
+
def self.modified_files(target_name, scope:)
|
348
|
+
modified = []
|
349
|
+
local_catalog = build_local_catalog(scope: scope)
|
350
|
+
local_catalog.each do |key, size|
|
351
|
+
split_key = key.split('/') # scope/targets_modified/target_name/*
|
352
|
+
local_target_name = split_key[2]
|
353
|
+
if target_name == local_target_name
|
354
|
+
modified << split_key[3..-1].join('/')
|
355
|
+
end
|
356
|
+
end
|
357
|
+
# Paths do not include target name
|
358
|
+
return modified.sort
|
359
|
+
end
|
360
|
+
|
361
|
+
def self.delete_modified(target_name, scope:)
|
362
|
+
full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified/#{target_name}"
|
363
|
+
FileUtils.rm_rf(full_path)
|
364
|
+
end
|
365
|
+
|
366
|
+
def self.zip_target(target_name, zip, scope:)
|
367
|
+
modified = modified_files(target_name, scope: scope)
|
368
|
+
modified.each do |file_path|
|
369
|
+
full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified/#{target_name}/#{file_path}"
|
370
|
+
zip.add(file_path, full_path)
|
371
|
+
end
|
372
|
+
end
|
373
|
+
|
374
|
+
def self.put_target_file(path, io_or_string, scope:)
|
375
|
+
full_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{path}"
|
376
|
+
FileUtils.mkdir_p(File.dirname(full_folder_path))
|
377
|
+
File.open(full_folder_path, 'wb') do |file|
|
378
|
+
if String === io_or_string
|
379
|
+
data = io_or_string
|
380
|
+
else
|
381
|
+
data = io_or_string.read
|
382
|
+
end
|
383
|
+
file.write(data)
|
384
|
+
end
|
385
|
+
end
|
386
|
+
|
387
|
+
def self.open_local_file(path, scope:)
|
388
|
+
full_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified/#{path}"
|
389
|
+
return File.open(full_path, 'rb') if File.exist?(full_path)
|
390
|
+
return nil
|
391
|
+
end
|
392
|
+
|
393
|
+
def self.local_target_files(scope:, path_matchers:)
|
394
|
+
files = []
|
395
|
+
local_catalog = build_local_catalog(scope: scope)
|
396
|
+
local_catalog.each do |key, size|
|
397
|
+
split_key = key.split('/')
|
398
|
+
found = false
|
399
|
+
path_matchers.each do |path|
|
400
|
+
if split_key.include?(path)
|
401
|
+
found = true
|
402
|
+
break
|
403
|
+
end
|
404
|
+
end
|
405
|
+
next unless found
|
406
|
+
files << split_key[2..-1].join('/')
|
407
|
+
end
|
408
|
+
return files.sort
|
409
|
+
end
|
410
|
+
|
411
|
+
# Helper methods
|
412
|
+
|
413
|
+
def self.sync_remote_to_local(rubys3_client, key)
|
414
|
+
local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
|
415
|
+
FileUtils.mkdir_p(File.dirname(local_path))
|
416
|
+
rubys3_client.get_object(bucket: 'config', key: key, response_target: local_path)
|
417
|
+
end
|
418
|
+
|
419
|
+
def self.sync_local_to_remote(rubys3_client, key)
|
420
|
+
local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
|
421
|
+
File.open(local_path, 'rb') do |read_file|
|
422
|
+
rubys3_client.put_object(bucket: 'config', key: key, body: read_file)
|
423
|
+
end
|
424
|
+
end
|
425
|
+
|
426
|
+
def self.delete_local(key)
|
427
|
+
local_path = "#{OPENC3_LOCAL_MODE_PATH}/#{key}"
|
428
|
+
File.delete(local_path) if File.exist?(local_path)
|
429
|
+
nil
|
430
|
+
end
|
431
|
+
|
432
|
+
def self.delete_remote(rubys3_client, key)
|
433
|
+
rubys3_client.delete_object(bucket: 'config', key: key)
|
434
|
+
end
|
435
|
+
|
436
|
+
# Returns equivalent names and sizes to remote catalog
|
437
|
+
# {"scope/targets_modified/target_name/file" => size}
|
438
|
+
def self.build_local_catalog(scope:)
|
439
|
+
local_catalog = {}
|
440
|
+
local_folder_path = "#{OPENC3_LOCAL_MODE_PATH}/#{scope}/targets_modified"
|
441
|
+
prefix_length = "#{OPENC3_LOCAL_MODE_PATH}/".length
|
442
|
+
FileUtils.mkdir_p(local_folder_path)
|
443
|
+
Dir.glob(local_folder_path + "/**/*").each do |filename|
|
444
|
+
next if File.directory?(filename)
|
445
|
+
mod_filename = filename[prefix_length..-1]
|
446
|
+
local_catalog[mod_filename] = File.size(filename)
|
447
|
+
end
|
448
|
+
return local_catalog
|
449
|
+
end
|
450
|
+
|
451
|
+
# Returns keys and sizes from remote catalog
|
452
|
+
# {"scope/targets_modified/target_name/file" => size}
|
453
|
+
def self.build_remote_catalog(rubys3_client, scope:)
|
454
|
+
remote_catalog = {}
|
455
|
+
bucket = 'config'
|
456
|
+
prefix = "#{scope}/targets_modified"
|
457
|
+
token = nil
|
458
|
+
while true
|
459
|
+
resp = rubys3_client.list_objects_v2({
|
460
|
+
bucket: bucket,
|
461
|
+
max_keys: 1000,
|
462
|
+
prefix: prefix,
|
463
|
+
continuation_token: token
|
464
|
+
})
|
465
|
+
|
466
|
+
resp.contents.each do |item|
|
467
|
+
remote_catalog[item.key] = item.size
|
468
|
+
end
|
469
|
+
break unless resp.is_truncated
|
470
|
+
token = resp.next_continuation_token
|
471
|
+
end
|
472
|
+
return remote_catalog
|
473
|
+
end
|
474
|
+
|
475
|
+
def self.sync_with_minio(rubys3_client, scope:)
|
476
|
+
# Build catalogs
|
477
|
+
local_catalog = build_local_catalog(scope: scope)
|
478
|
+
remote_catalog = build_remote_catalog(rubys3_client, scope: scope)
|
479
|
+
|
480
|
+
# Find and Handle Differences
|
481
|
+
local_catalog.each do |key, size|
|
482
|
+
remote_size = remote_catalog[key]
|
483
|
+
if remote_size
|
484
|
+
# Both files exist
|
485
|
+
if ENV['OPENC3_LOCAL_MODE_SECONDARY']
|
486
|
+
sync_remote_to_local(rubys3_client, key) if size != remote_size or ENV['OPENC3_LOCAL_MODE_FORCE_SYNC']
|
487
|
+
else
|
488
|
+
sync_local_to_remote(rubys3_client, key) if size != remote_size or ENV['OPENC3_LOCAL_MODE_FORCE_SYNC']
|
489
|
+
end
|
490
|
+
else
|
491
|
+
# Remote is missing local file
|
492
|
+
if ENV['OPENC3_LOCAL_MODE_SECONDARY'] and ENV['OPENC3_LOCAL_MODE_SYNC_REMOVE']
|
493
|
+
delete_local(key)
|
494
|
+
else
|
495
|
+
# Go ahead and copy up to get in sync
|
496
|
+
sync_local_to_remote(rubys3_client, key)
|
497
|
+
end
|
498
|
+
end
|
499
|
+
end
|
500
|
+
|
501
|
+
remote_catalog.each do |key, size|
|
502
|
+
local_size = local_catalog[key]
|
503
|
+
if local_size
|
504
|
+
# Both files exist - Handled earlier
|
505
|
+
else
|
506
|
+
# Local is missing remote file
|
507
|
+
if not ENV['OPENC3_LOCAL_MODE_SECONDARY'] and ENV['OPENC3_LOCAL_MODE_SYNC_REMOVE']
|
508
|
+
delete_remote(rubys3_client, key)
|
509
|
+
else
|
510
|
+
# Go ahead and copy down to get in sync
|
511
|
+
sync_remote_to_local(rubys3_client, key)
|
512
|
+
end
|
513
|
+
end
|
514
|
+
end
|
515
|
+
end
|
516
|
+
|
517
|
+
end
|
518
|
+
end
|
@@ -0,0 +1,146 @@
|
|
1
|
+
# encoding: ascii-8bit
|
2
|
+
|
3
|
+
# Copyright 2022 OpenC3, Inc.
|
4
|
+
# All Rights Reserved.
|
5
|
+
#
|
6
|
+
# This program is free software; you can modify and/or redistribute it
|
7
|
+
# under the terms of the GNU Affero General Public License
|
8
|
+
# as published by the Free Software Foundation; version 3 with
|
9
|
+
# attribution addendums as found in the LICENSE.txt
|
10
|
+
#
|
11
|
+
# This program is distributed in the hope that it will be useful,
|
12
|
+
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
13
|
+
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
14
|
+
# GNU Affero General Public License for more details.
|
15
|
+
|
16
|
+
require 'fileutils'
|
17
|
+
require 'json'
|
18
|
+
require 'openc3/utilities/local_mode'
|
19
|
+
require 'openc3/utilities/s3'
|
20
|
+
|
21
|
+
module OpenC3
|
22
|
+
class TargetFile
|
23
|
+
|
24
|
+
DEFAULT_BUCKET_NAME = 'config'
|
25
|
+
|
26
|
+
def self.all(scope, path_matchers)
|
27
|
+
result = []
|
28
|
+
modified = []
|
29
|
+
|
30
|
+
rubys3_client = Aws::S3::Client.new
|
31
|
+
token = nil
|
32
|
+
while true
|
33
|
+
resp = rubys3_client.list_objects_v2({
|
34
|
+
bucket: DEFAULT_BUCKET_NAME,
|
35
|
+
prefix: "#{scope}/targets",
|
36
|
+
max_keys: 1000,
|
37
|
+
continuation_token: token
|
38
|
+
})
|
39
|
+
|
40
|
+
resp.contents.each do |object|
|
41
|
+
split_key = object.key.split('/')
|
42
|
+
found = false
|
43
|
+
path_matchers.each do |path|
|
44
|
+
if split_key.include?(path)
|
45
|
+
found = true
|
46
|
+
break
|
47
|
+
end
|
48
|
+
end
|
49
|
+
next unless found
|
50
|
+
result_no_scope_or_target_folder = split_key[2..-1].join('/')
|
51
|
+
if object.key.include?("#{scope}/targets_modified")
|
52
|
+
modified << result_no_scope_or_target_folder
|
53
|
+
else
|
54
|
+
result << result_no_scope_or_target_folder
|
55
|
+
end
|
56
|
+
end
|
57
|
+
break unless resp.is_truncated
|
58
|
+
token = resp.next_continuation_token
|
59
|
+
end
|
60
|
+
|
61
|
+
# Add in local targets_modified if present
|
62
|
+
if ENV['OPENC3_LOCAL_MODE']
|
63
|
+
local_modified = OpenC3::LocalMode.local_target_files(scope: scope, path_matchers: path_matchers)
|
64
|
+
local_modified.each do |filename|
|
65
|
+
modified << filename unless modified.include?(filename)
|
66
|
+
result << filename unless result.include?(filename)
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Determine if there are any modified files and mark them with '*'
|
71
|
+
result.map! do |file|
|
72
|
+
if modified.include?(file)
|
73
|
+
modified.delete(file)
|
74
|
+
"#{file}*"
|
75
|
+
else
|
76
|
+
file
|
77
|
+
end
|
78
|
+
end
|
79
|
+
|
80
|
+
# Concat any remaining modified files (new files not in original target)
|
81
|
+
result.concat(modified)
|
82
|
+
result.sort
|
83
|
+
end
|
84
|
+
|
85
|
+
def self.body(scope, name)
|
86
|
+
name = name.split('*')[0] # Split '*' that indicates modified
|
87
|
+
rubys3_client = Aws::S3::Client.new
|
88
|
+
begin
|
89
|
+
# First try opening a potentially modified version by looking for the modified target
|
90
|
+
if ENV['OPENC3_LOCAL_MODE']
|
91
|
+
local_file = OpenC3::LocalMode.open_local_file(name, scope: scope)
|
92
|
+
return local_file.read if local_file
|
93
|
+
end
|
94
|
+
|
95
|
+
resp =
|
96
|
+
rubys3_client.get_object(
|
97
|
+
bucket: DEFAULT_BUCKET_NAME,
|
98
|
+
key: "#{scope}/targets_modified/#{name}",
|
99
|
+
)
|
100
|
+
rescue Aws::S3::Errors::NoSuchKey
|
101
|
+
# Now try the original
|
102
|
+
resp =
|
103
|
+
rubys3_client.get_object(
|
104
|
+
bucket: DEFAULT_BUCKET_NAME,
|
105
|
+
key: "#{scope}/targets/#{name}",
|
106
|
+
)
|
107
|
+
end
|
108
|
+
if File.extname(name) == ".bin"
|
109
|
+
resp.body.binmode
|
110
|
+
end
|
111
|
+
resp.body.read
|
112
|
+
end
|
113
|
+
|
114
|
+
def self.create(scope, name, text, content_type: 'text/plain')
|
115
|
+
return false unless text
|
116
|
+
if ENV['OPENC3_LOCAL_MODE']
|
117
|
+
OpenC3::LocalMode.put_target_file("#{scope}/targets_modified/#{name}", text, scope: scope)
|
118
|
+
end
|
119
|
+
OpenC3::S3Utilities.put_object_and_check(
|
120
|
+
# Use targets_modified to save modifications
|
121
|
+
# This keeps the original target clean (read-only)
|
122
|
+
key: "#{scope}/targets_modified/#{name}",
|
123
|
+
body: text,
|
124
|
+
bucket: DEFAULT_BUCKET_NAME,
|
125
|
+
content_type: content_type,
|
126
|
+
)
|
127
|
+
true
|
128
|
+
end
|
129
|
+
|
130
|
+
def self.destroy(scope, name)
|
131
|
+
rubys3_client = Aws::S3::Client.new
|
132
|
+
|
133
|
+
if ENV['OPENC3_LOCAL_MODE']
|
134
|
+
OpenC3::LocalMode.delete_local("#{scope}/targets_modified/#{name}")
|
135
|
+
end
|
136
|
+
|
137
|
+
# Only delete file from the modified target directory
|
138
|
+
rubys3_client.delete_object(
|
139
|
+
key: "#{scope}/targets_modified/#{name}",
|
140
|
+
bucket: DEFAULT_BUCKET_NAME,
|
141
|
+
)
|
142
|
+
true
|
143
|
+
end
|
144
|
+
|
145
|
+
end
|
146
|
+
end
|
data/lib/openc3/version.rb
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# encoding: ascii-8bit
|
2
2
|
|
3
|
-
OPENC3_VERSION = '5.0.
|
3
|
+
OPENC3_VERSION = '5.0.9'
|
4
4
|
module OpenC3
|
5
5
|
module Version
|
6
6
|
MAJOR = '5'
|
7
7
|
MINOR = '0'
|
8
|
-
PATCH = '
|
8
|
+
PATCH = '9'
|
9
9
|
OTHER = ''
|
10
|
-
BUILD = '
|
10
|
+
BUILD = '7c04d727e4664a45e6b1021bea58c06e2c046b10'
|
11
11
|
end
|
12
|
-
VERSION = '5.0.
|
13
|
-
GEM_VERSION = '5.0.
|
12
|
+
VERSION = '5.0.9'
|
13
|
+
GEM_VERSION = '5.0.9'
|
14
14
|
end
|