aspera-cli 4.14.0 → 4.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +54 -3
- data/CONTRIBUTING.md +7 -7
- data/README.md +1457 -880
- data/bin/ascli +18 -9
- data/bin/asession +12 -14
- data/examples/proxy.pac +1 -1
- data/lib/aspera/aoc.rb +198 -127
- data/lib/aspera/ascmd.rb +24 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
- data/lib/aspera/cli/error.rb +17 -0
- data/lib/aspera/cli/extended_value.rb +47 -12
- data/lib/aspera/cli/formatter.rb +260 -171
- data/lib/aspera/cli/hints.rb +80 -0
- data/lib/aspera/cli/main.rb +101 -147
- data/lib/aspera/cli/manager.rb +160 -124
- data/lib/aspera/cli/plugin.rb +70 -59
- data/lib/aspera/cli/plugins/alee.rb +0 -1
- data/lib/aspera/cli/plugins/aoc.rb +239 -273
- data/lib/aspera/cli/plugins/ats.rb +8 -5
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +516 -375
- data/lib/aspera/cli/plugins/console.rb +40 -0
- data/lib/aspera/cli/plugins/cos.rb +4 -5
- data/lib/aspera/cli/plugins/faspex.rb +99 -84
- data/lib/aspera/cli/plugins/faspex5.rb +179 -148
- data/lib/aspera/cli/plugins/node.rb +219 -153
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -17
- data/lib/aspera/cli/plugins/preview.rb +46 -32
- data/lib/aspera/cli/plugins/server.rb +57 -17
- data/lib/aspera/cli/plugins/shares.rb +34 -12
- data/lib/aspera/cli/sync_actions.rb +68 -0
- data/lib/aspera/cli/transfer_agent.rb +45 -55
- data/lib/aspera/cli/transfer_progress.rb +74 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -1
- data/lib/aspera/command_line_builder.rb +14 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/environment.rb +17 -6
- data/lib/aspera/fasp/agent_aspera.rb +126 -0
- data/lib/aspera/fasp/agent_base.rb +31 -77
- data/lib/aspera/fasp/agent_connect.rb +21 -22
- data/lib/aspera/fasp/agent_direct.rb +88 -102
- data/lib/aspera/fasp/agent_httpgw.rb +196 -192
- data/lib/aspera/fasp/agent_node.rb +41 -34
- data/lib/aspera/fasp/agent_trsdk.rb +75 -34
- data/lib/aspera/fasp/error_info.rb +2 -2
- data/lib/aspera/fasp/faux_file.rb +52 -0
- data/lib/aspera/fasp/installation.rb +43 -184
- data/lib/aspera/fasp/management.rb +244 -0
- data/lib/aspera/fasp/parameters.rb +59 -26
- data/lib/aspera/fasp/parameters.yaml +75 -8
- data/lib/aspera/fasp/products.rb +162 -0
- data/lib/aspera/fasp/transfer_spec.rb +1 -1
- data/lib/aspera/fasp/uri.rb +4 -4
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +49 -0
- data/lib/aspera/line_logger.rb +23 -0
- data/lib/aspera/log.rb +57 -16
- data/lib/aspera/node.rb +97 -14
- data/lib/aspera/oauth.rb +36 -18
- data/lib/aspera/open_application.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -2
- data/lib/aspera/preview/generator.rb +22 -35
- data/lib/aspera/preview/options.rb +2 -0
- data/lib/aspera/preview/terminal.rb +24 -13
- data/lib/aspera/preview/utils.rb +19 -26
- data/lib/aspera/rest.rb +103 -72
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +15 -14
- data/lib/aspera/rest_errors_aspera.rb +37 -34
- data/lib/aspera/secret_hider.rb +14 -16
- data/lib/aspera/ssh.rb +4 -1
- data/lib/aspera/sync.rb +128 -122
- data/lib/aspera/temp_file_manager.rb +10 -3
- data/lib/aspera/web_auth.rb +10 -7
- data/lib/aspera/web_server_simple.rb +9 -4
- data.tar.gz.sig +0 -0
- metadata +33 -15
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/listener/line_dump.rb +0 -19
- data/lib/aspera/cli/listener/logger.rb +0 -22
- data/lib/aspera/cli/listener/progress.rb +0 -50
- data/lib/aspera/cli/listener/progress_multi.rb +0 -84
- data/lib/aspera/cli/plugins/sync.rb +0 -44
- data/lib/aspera/fasp/listener.rb +0 -13
@@ -1,17 +1,53 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/cli/
|
3
|
+
require 'aspera/cli/basic_auth_plugin'
|
4
|
+
require 'aspera/nagios'
|
4
5
|
require 'xmlsimple'
|
5
6
|
|
6
7
|
module Aspera
|
7
8
|
module Cli
|
8
9
|
module Plugins
|
9
10
|
class Orchestrator < Aspera::Cli::BasicAuthPlugin
|
11
|
+
class << self
|
12
|
+
STANDARD_PATH = '/aspera/orchestrator'
|
13
|
+
def detect(address_or_url)
|
14
|
+
address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
|
15
|
+
urls = [address_or_url]
|
16
|
+
urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
|
17
|
+
urls.each do |base_url|
|
18
|
+
next unless base_url.match?('https?://')
|
19
|
+
api = Rest.new(base_url: base_url)
|
20
|
+
test_endpoint = 'api/remote_node_ping'
|
21
|
+
result = api.read(test_endpoint, {format: :json})
|
22
|
+
next unless result[:data]['remote_orchestrator_info']
|
23
|
+
url = result[:http].uri.to_s
|
24
|
+
return {
|
25
|
+
version: result[:data]['remote_orchestrator_info']['orchestrator-version'],
|
26
|
+
url: url[0..url.index(test_endpoint) - 2]
|
27
|
+
}
|
28
|
+
rescue StandardError => e
|
29
|
+
Log.log.debug{"detect error: #{e}"}
|
30
|
+
end
|
31
|
+
return nil
|
32
|
+
end
|
33
|
+
|
34
|
+
def wizard(object:, private_key_path: nil, pub_key_pem: nil)
|
35
|
+
options = object.options
|
36
|
+
return {
|
37
|
+
preset_value: {
|
38
|
+
url: options.get_option(:url, mandatory: true),
|
39
|
+
username: options.get_option(:username, mandatory: true),
|
40
|
+
password: options.get_option(:password, mandatory: true)
|
41
|
+
},
|
42
|
+
test_args: 'workflow list'
|
43
|
+
}
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
10
47
|
def initialize(env)
|
11
48
|
super(env)
|
12
|
-
options.declare(:
|
13
|
-
options.declare(:
|
14
|
-
options.declare(:synchronous, 'Work step:parameter expected as result', values: :bool, default: :no)
|
49
|
+
options.declare(:result, "Specify result value as: 'work_step:parameter'")
|
50
|
+
options.declare(:synchronous, 'Wait for completion', values: :bool, default: :no)
|
15
51
|
options.declare(:ret_style, 'How return type is requested in api', values: %i[header arg ext], default: :arg)
|
16
52
|
options.declare(:auth_style, 'Authentication type', values: %i[arg_pass head_basic apikey], default: :head_basic)
|
17
53
|
options.parse_options!
|
@@ -64,7 +100,7 @@ module Aspera
|
|
64
100
|
end
|
65
101
|
result = @api_orch.call(call_args)
|
66
102
|
result[:data] = XmlSimple.xml_in(result[:http].body, opt[:xml_opt] || {'ForceArray' => true}) if format.eql?('xml')
|
67
|
-
Log.dump(:data, result[:data])
|
103
|
+
Log.log.debug{Log.dump(:data, result[:data])}
|
68
104
|
return result
|
69
105
|
end
|
70
106
|
|
@@ -74,7 +110,7 @@ module Aspera
|
|
74
110
|
when :arg_pass
|
75
111
|
rest_params[:auth] = {
|
76
112
|
type: :url,
|
77
|
-
|
113
|
+
url_query: {
|
78
114
|
'login' => options.get_option(:username, mandatory: true),
|
79
115
|
'password' => options.get_option(:password, mandatory: true) }}
|
80
116
|
when :head_basic
|
@@ -118,9 +154,9 @@ module Aspera
|
|
118
154
|
end
|
119
155
|
case command
|
120
156
|
when :status
|
121
|
-
|
122
|
-
|
123
|
-
result = call_ao('workflows_status',
|
157
|
+
call_opts = {}
|
158
|
+
call_opts[:id] = wf_id unless wf_id.eql?(ExtendedValue::ALL)
|
159
|
+
result = call_ao('workflows_status', call_opts)[:data]
|
124
160
|
return {type: :object_list, data: result['workflows']['workflow']}
|
125
161
|
when :list
|
126
162
|
result = call_ao('workflows_list', id: 0)[:data]
|
@@ -145,19 +181,18 @@ module Aspera
|
|
145
181
|
}
|
146
182
|
call_params = {format: :json}
|
147
183
|
override_accept = nil
|
148
|
-
#
|
149
|
-
|
150
|
-
self.options.get_option(:params, mandatory: true).each do |name, value|
|
184
|
+
# get external parameters if any
|
185
|
+
options.get_next_argument('external_parameters', mandatory: false, type: Hash, default: {}).each do |name, value|
|
151
186
|
call_params["external_parameters[#{name}]"] = value
|
152
187
|
end
|
153
188
|
# synchronous call ?
|
154
|
-
call_params['synchronous'] = true if
|
189
|
+
call_params['synchronous'] = true if options.get_option(:synchronous, mandatory: true)
|
155
190
|
# expected result for synchro call ?
|
156
|
-
|
157
|
-
unless
|
191
|
+
result_location = options.get_option(:result)
|
192
|
+
unless result_location.nil?
|
158
193
|
result[:type] = :status
|
159
|
-
fields =
|
160
|
-
raise "Expects: work_step:result_name
|
194
|
+
fields = result_location.split(':')
|
195
|
+
raise Cli::BadArgument, "Expects: work_step:result_name : #{result_location}" if fields.length != 2
|
161
196
|
call_params['explicit_output_step'] = fields[0]
|
162
197
|
call_params['explicit_output_variable'] = fields[1]
|
163
198
|
# implicitly, call is synchronous
|
@@ -1,17 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# cspell:ignore trevents
|
3
4
|
require 'aspera/cli/basic_auth_plugin'
|
4
5
|
require 'aspera/preview/generator'
|
5
6
|
require 'aspera/preview/options'
|
6
7
|
require 'aspera/preview/utils'
|
7
8
|
require 'aspera/preview/file_types'
|
9
|
+
require 'aspera/preview/terminal'
|
8
10
|
require 'aspera/fasp/transfer_spec'
|
9
11
|
require 'aspera/persistency_action_once'
|
10
12
|
require 'aspera/node'
|
11
13
|
require 'aspera/hash_ext'
|
12
14
|
require 'aspera/timer_limiter'
|
13
15
|
require 'aspera/id_generator'
|
14
|
-
require 'date'
|
15
16
|
require 'securerandom'
|
16
17
|
|
17
18
|
module Aspera
|
@@ -26,8 +27,11 @@ module Aspera
|
|
26
27
|
PREVIEW_BASENAME = 'preview'
|
27
28
|
# subfolder in system tmp folder
|
28
29
|
TMP_DIR_PREFIX = 'prev_tmp'
|
30
|
+
# same value as in aspera.conf
|
29
31
|
DEFAULT_PREVIEWS_FOLDER = 'previews'
|
32
|
+
# mark that this is used by a particular access key
|
30
33
|
AK_MARKER_FILE = '.aspera_access_key'
|
34
|
+
# URL prefix for local storage
|
31
35
|
PVCL_LOCAL_STORAGE = 'file:///'
|
32
36
|
LOG_LIMITER_SEC = 30.0
|
33
37
|
private_constant :PREV_GEN_TAG,
|
@@ -53,6 +57,8 @@ module Aspera
|
|
53
57
|
@gen_options = Aspera::Preview::Options.new
|
54
58
|
# used to trigger periodic processing
|
55
59
|
@periodic = TimerLimiter.new(LOG_LIMITER_SEC)
|
60
|
+
# Proc
|
61
|
+
@filter_block = nil
|
56
62
|
# link CLI options to gen_info attributes
|
57
63
|
options.declare(
|
58
64
|
:skip_format, 'Skip this preview format (multiple possible)', values: Aspera::Preview::Generator::PREVIEW_FORMATS,
|
@@ -66,7 +72,7 @@ module Aspera
|
|
66
72
|
options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
|
67
73
|
options.declare(:temp_folder, 'Path to temp folder', default: Dir.tmpdir)
|
68
74
|
options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, default: [])
|
69
|
-
options.declare(:
|
75
|
+
options.declare(:base, 'Basename of output for for test')
|
70
76
|
options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
|
71
77
|
options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
|
72
78
|
options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', values: :bool, default: false)
|
@@ -232,6 +238,7 @@ module Aspera
|
|
232
238
|
def get_infos_remote(gen_infos, entry)
|
233
239
|
# store source directly here
|
234
240
|
local_original_filepath = File.join(@tmp_folder, entry['name'])
|
241
|
+
# require 'date'
|
235
242
|
# original_mtime=DateTime.parse(entry['modified_time'])
|
236
243
|
# out: where previews are generated
|
237
244
|
local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
|
@@ -255,9 +262,10 @@ module Aspera
|
|
255
262
|
"#{entry['id']}#{PREVIEW_FOLDER_SUFFIX}"
|
256
263
|
end
|
257
264
|
|
258
|
-
|
259
|
-
|
260
|
-
|
265
|
+
# Generate a file name based on basename and format (extension)
|
266
|
+
def preview_filename(preview_format, base_name=nil)
|
267
|
+
base_name ||= PREVIEW_BASENAME
|
268
|
+
return "#{base_name}.#{preview_format}"
|
261
269
|
end
|
262
270
|
|
263
271
|
# generate preview files for one folder entry (file) if necessary
|
@@ -290,10 +298,13 @@ module Aspera
|
|
290
298
|
next false if gen_info[:preview_newer_than_original]
|
291
299
|
end
|
292
300
|
end
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
301
|
+
begin
|
302
|
+
# need generator for further checks
|
303
|
+
gen_info[:generator] = Aspera::Preview::Generator.new(gen_info[:src], gen_info[:dst], @gen_options, @tmp_folder, entry['content_type'])
|
304
|
+
rescue
|
305
|
+
# no conversion supported
|
306
|
+
next false
|
307
|
+
end
|
297
308
|
# shall we skip it ?
|
298
309
|
next false if @skip_types.include?(gen_info[:generator].conversion_type)
|
299
310
|
# ok we need to generate
|
@@ -307,7 +318,7 @@ module Aspera
|
|
307
318
|
# download original file to temp folder
|
308
319
|
do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
|
309
320
|
end
|
310
|
-
Log.log.info{"source: #{entry['id']}: #{entry['path']}
|
321
|
+
Log.log.info{"source: #{entry['id']}: #{entry['path']}"}
|
311
322
|
gen_infos.each do |gen_info|
|
312
323
|
gen_info[:generator].generate rescue nil
|
313
324
|
end
|
@@ -328,24 +339,22 @@ module Aspera
|
|
328
339
|
end # generate_preview
|
329
340
|
|
330
341
|
# scan all files in provided folder entry
|
331
|
-
# @param
|
332
|
-
def scan_folder_files(top_entry,
|
333
|
-
|
342
|
+
# @param top_path subpath to start folder scan inside
|
343
|
+
def scan_folder_files(top_entry, top_path=nil)
|
344
|
+
unless top_path.nil?
|
334
345
|
# canonical path: start with / and ends with /
|
335
|
-
|
336
|
-
scan_start = "#{scan_start}/" # unless scan_start.end_with?('/')
|
346
|
+
top_path = '/' + top_path.split('/').reject(&:empty?).join('/') + '/'
|
337
347
|
end
|
338
|
-
|
339
|
-
Log.log.debug{"scan: #{top_entry} : #{scan_start}".green}
|
348
|
+
Log.log.debug{"scan: #{top_entry} : #{top_path}".green}
|
340
349
|
# don't use recursive call, use list instead
|
341
350
|
entries_to_process = [top_entry]
|
342
351
|
until entries_to_process.empty?
|
343
352
|
entry = entries_to_process.shift
|
344
|
-
# process this entry only if it is within the
|
353
|
+
# process this entry only if it is within the top_path
|
345
354
|
entry_path_with_slash = entry['path']
|
346
355
|
Log.log.info{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
|
347
356
|
entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
348
|
-
if !
|
357
|
+
if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
|
349
358
|
Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
|
350
359
|
next
|
351
360
|
end
|
@@ -353,7 +362,7 @@ module Aspera
|
|
353
362
|
begin
|
354
363
|
case entry['type']
|
355
364
|
when 'file'
|
356
|
-
if filter_block.call(entry)
|
365
|
+
if @filter_block.call(entry)
|
357
366
|
generate_preview(entry)
|
358
367
|
else
|
359
368
|
Log.log.debug('skip by filter')
|
@@ -386,11 +395,11 @@ module Aspera
|
|
386
395
|
end
|
387
396
|
end
|
388
397
|
|
389
|
-
ACTIONS = %i[scan events trevents check test].freeze
|
398
|
+
ACTIONS = %i[scan events trevents check test show].freeze
|
390
399
|
|
391
400
|
def execute_action
|
392
401
|
command = options.get_next_command(ACTIONS)
|
393
|
-
unless %i[check test].include?(command)
|
402
|
+
unless %i[check test show].include?(command)
|
394
403
|
# this will use node api
|
395
404
|
@api_node = Aspera::Node.new(params: basic_auth_params)
|
396
405
|
@transfer_server_address = URI.parse(@api_node.params[:base_url]).host
|
@@ -408,7 +417,7 @@ module Aspera
|
|
408
417
|
if @access_remote
|
409
418
|
# NOTE: the filter "name", it's why we take the first one
|
410
419
|
@previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
|
411
|
-
raise
|
420
|
+
raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
|
412
421
|
'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
|
413
422
|
else
|
414
423
|
raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
|
@@ -417,9 +426,9 @@ module Aspera
|
|
417
426
|
@local_storage_root = @local_storage_root[PVCL_LOCAL_STORAGE.length..-1] if @local_storage_root.start_with?(PVCL_LOCAL_STORAGE)
|
418
427
|
# TODO: windows could have "C:" ?
|
419
428
|
raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/')
|
420
|
-
raise
|
429
|
+
raise Cli::Error, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
|
421
430
|
@local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
|
422
|
-
raise
|
431
|
+
raise Cli::Error, "Folder #{@local_preview_folder} does not exist locally. " \
|
423
432
|
'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
|
424
433
|
# protection to avoid clash of file id for two different access keys
|
425
434
|
marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
|
@@ -450,9 +459,11 @@ module Aspera
|
|
450
459
|
else
|
451
460
|
@api_node.read("files/#{scan_id}")[:data]
|
452
461
|
end
|
462
|
+
@filter_block = Aspera::Node.file_matcher_from_argument(options)
|
453
463
|
scan_folder_files(folder_info, scan_path)
|
454
464
|
return Main.result_status('scan finished')
|
455
465
|
when :events, :trevents
|
466
|
+
@filter_block = Aspera::Node.file_matcher_from_argument(options)
|
456
467
|
iteration_persistency = nil
|
457
468
|
if options.get_option(:once_only, mandatory: true)
|
458
469
|
iteration_persistency = PersistencyActionOnce.new(
|
@@ -470,15 +481,18 @@ module Aspera
|
|
470
481
|
return Main.result_status("#{command} finished")
|
471
482
|
when :check
|
472
483
|
return Main.result_status('Tools validated')
|
473
|
-
when :test
|
474
|
-
format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS)
|
484
|
+
when :test, :show
|
475
485
|
source = options.get_next_argument('source file')
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
raise "out format #{format} not supported" unless g.supported?
|
486
|
+
format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS, default: :png)
|
487
|
+
generated_file_path = preview_filename(format, options.get_option(:base))
|
488
|
+
g = Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder, nil)
|
480
489
|
g.generate
|
481
|
-
|
490
|
+
if command.eql?(:show)
|
491
|
+
terminal_options = options.get_option(:query, default: {}).symbolize_keys
|
492
|
+
Log.log.debug{"preview: #{generated_file_path}"}
|
493
|
+
formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
|
494
|
+
end
|
495
|
+
return Main.result_status("generated: #{generated_file_path}")
|
482
496
|
else
|
483
497
|
raise 'error'
|
484
498
|
end
|
@@ -1,9 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# cspell:ignore ascmd zmode zuid zgid fasping
|
3
4
|
require 'aspera/cli/basic_auth_plugin'
|
4
|
-
require 'aspera/cli/
|
5
|
-
require 'aspera/ascmd'
|
5
|
+
require 'aspera/cli/sync_actions'
|
6
6
|
require 'aspera/fasp/transfer_spec'
|
7
|
+
require 'aspera/ascmd'
|
7
8
|
require 'aspera/ssh'
|
8
9
|
require 'aspera/nagios'
|
9
10
|
require 'tempfile'
|
@@ -13,17 +14,8 @@ module Aspera
|
|
13
14
|
module Cli
|
14
15
|
module Plugins
|
15
16
|
# implement basic remote access with FASP/SSH
|
16
|
-
class SyncSpecServer
|
17
|
-
def initialize(transfer_spec)
|
18
|
-
@transfer_spec = transfer_spec
|
19
|
-
end
|
20
|
-
|
21
|
-
def transfer_spec(direction, local_path, remote_path)
|
22
|
-
return @transfer_spec
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
17
|
class Server < Aspera::Cli::BasicAuthPlugin
|
18
|
+
include SyncActions
|
27
19
|
SSH_SCHEME = 'ssh'
|
28
20
|
LOCAL_SCHEME = 'local'
|
29
21
|
HTTPS_SCHEME = 'https'
|
@@ -49,11 +41,59 @@ module Aspera
|
|
49
41
|
end
|
50
42
|
end
|
51
43
|
|
44
|
+
class << self
|
45
|
+
def application_name
|
46
|
+
'HSTS Fasp/SSH'
|
47
|
+
end
|
48
|
+
|
49
|
+
def detect(address_or_url)
|
50
|
+
urls = if address_or_url.match?(%r{^[a-z]{1,6}://})
|
51
|
+
[address_or_url]
|
52
|
+
else
|
53
|
+
[
|
54
|
+
"ssh://#{address_or_url}:33001",
|
55
|
+
"ssh://#{address_or_url}:22"
|
56
|
+
]
|
57
|
+
# wss not practical as it requires a token
|
58
|
+
end
|
59
|
+
|
60
|
+
urls.each do |base_url|
|
61
|
+
server_uri = URI.parse(base_url)
|
62
|
+
Log.log.debug{"URI=#{server_uri}, host=#{server_uri.hostname}, port=#{server_uri.port}, scheme=#{server_uri.scheme}"}
|
63
|
+
next unless server_uri.scheme.eql?(SSH_SCHEME)
|
64
|
+
begin
|
65
|
+
socket = TCPSocket.new(server_uri.hostname, server_uri.port)
|
66
|
+
socket.puts('SSH-2.0-Ascli_0.0')
|
67
|
+
version = socket.gets.chomp
|
68
|
+
if version.match?(/^SSH-2.0-/)
|
69
|
+
return {version: version.gsub(/^SSH-2.0-/, ''), url: base_url}
|
70
|
+
end
|
71
|
+
rescue StandardError => e
|
72
|
+
Log.log.debug{"detect error: #{e}"}
|
73
|
+
end
|
74
|
+
end
|
75
|
+
return nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def wizard(object:, private_key_path: nil, pub_key_pem: nil)
|
79
|
+
options = object.options
|
80
|
+
return {
|
81
|
+
preset_value: {
|
82
|
+
url: options.get_option(:url, mandatory: true),
|
83
|
+
username: options.get_option(:username, mandatory: true),
|
84
|
+
password: options.get_option(:password, mandatory: true)
|
85
|
+
},
|
86
|
+
test_args: 'files br /'
|
87
|
+
}
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
52
91
|
def initialize(env)
|
53
92
|
super(env)
|
54
93
|
options.declare(:ssh_keys, 'SSH key path list (Array or single)')
|
55
94
|
options.declare(:passphrase, 'SSH private key passphrase')
|
56
95
|
options.declare(:ssh_options, 'SSH options', types: Hash, default: {})
|
96
|
+
SyncActions.declare_options(options)
|
57
97
|
options.parse_options!
|
58
98
|
@ssh_opts = options.get_option(:ssh_options).symbolize_keys
|
59
99
|
end
|
@@ -138,8 +178,8 @@ module Aspera
|
|
138
178
|
Fasp::TransferSpec.action_to_direction(transfer_spec, command)
|
139
179
|
return Main.result_transfer(transfer.start(transfer_spec))
|
140
180
|
when :sync
|
141
|
-
|
142
|
-
return
|
181
|
+
# lets ignore the arguments provided by execute_sync_action, we just give the transfer spec
|
182
|
+
return execute_sync_action {transfer_spec}
|
143
183
|
end
|
144
184
|
end
|
145
185
|
|
@@ -191,10 +231,10 @@ module Aspera
|
|
191
231
|
when *TRANSFER_COMMANDS
|
192
232
|
return execute_transfer(command, server_transfer_spec)
|
193
233
|
when *Aspera::AsCmd::OPERATIONS
|
194
|
-
|
234
|
+
command_arguments = options.get_next_argument('ascmd command arguments', expected: :multiple, mandatory: false)
|
195
235
|
ascmd = Aspera::AsCmd.new(ascmd_executor)
|
196
236
|
begin
|
197
|
-
result = ascmd.send(:execute_single, command,
|
237
|
+
result = ascmd.send(:execute_single, command, command_arguments)
|
198
238
|
case command
|
199
239
|
when :mkdir, :mv, :cp, :rm
|
200
240
|
return Main.result_success
|
@@ -206,7 +246,7 @@ module Aspera
|
|
206
246
|
return {type: :single_object, data: result.stringify_keys}
|
207
247
|
end
|
208
248
|
rescue Aspera::AsCmd::Error => e
|
209
|
-
raise
|
249
|
+
raise Cli::BadArgument, e.extended_message
|
210
250
|
end
|
211
251
|
else raise 'internal error: unexpected action'
|
212
252
|
end
|
@@ -7,20 +7,43 @@ module Aspera
|
|
7
7
|
module Plugins
|
8
8
|
# Plugin for Aspera Shares v1
|
9
9
|
class Shares < Aspera::Cli::BasicAuthPlugin
|
10
|
+
API_BASE = 'node_api'
|
10
11
|
class << self
|
11
|
-
def detect(
|
12
|
-
|
13
|
-
|
12
|
+
def detect(address_or_url)
|
13
|
+
address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
|
14
|
+
api = Rest.new(base_url: address_or_url, redirect_max: 1)
|
15
|
+
found = false
|
14
16
|
begin
|
15
17
|
# shall fail: shares requires auth, but we check error message
|
16
18
|
# TODO: use ping instead ?
|
17
|
-
api.read(
|
19
|
+
api.read("#{API_BASE}/app")
|
18
20
|
rescue RestCallError => e
|
19
21
|
if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
20
|
-
|
22
|
+
found = true
|
21
23
|
end
|
22
24
|
end
|
23
|
-
nil
|
25
|
+
return nil unless found
|
26
|
+
version = 'unknown'
|
27
|
+
test_page = api.call({ operation: 'GET', subpath: 'login' })
|
28
|
+
if (m = test_page[:http].body.match(/\(v(1\..*)\)/))
|
29
|
+
version = m[1]
|
30
|
+
end
|
31
|
+
return {
|
32
|
+
version: version,
|
33
|
+
url: address_or_url
|
34
|
+
}
|
35
|
+
end
|
36
|
+
|
37
|
+
def wizard(object:, private_key_path: nil, pub_key_pem: nil)
|
38
|
+
options = object.options
|
39
|
+
return {
|
40
|
+
preset_value: {
|
41
|
+
url: options.get_option(:url, mandatory: true),
|
42
|
+
username: options.get_option(:username, mandatory: true),
|
43
|
+
password: options.get_option(:password, mandatory: true)
|
44
|
+
},
|
45
|
+
test_args: 'files br /'
|
46
|
+
}
|
24
47
|
end
|
25
48
|
end
|
26
49
|
|
@@ -44,7 +67,7 @@ module Aspera
|
|
44
67
|
nagios = Nagios.new
|
45
68
|
begin
|
46
69
|
Rest
|
47
|
-
.new(base_url: options.get_option(:url, mandatory: true)
|
70
|
+
.new(base_url: "#{options.get_option(:url, mandatory: true)}/#{API_BASE}")
|
48
71
|
.call(
|
49
72
|
operation: 'GET',
|
50
73
|
subpath: 'ping',
|
@@ -56,9 +79,9 @@ module Aspera
|
|
56
79
|
end
|
57
80
|
return nagios.result
|
58
81
|
when :repository, :files
|
59
|
-
api_shares_node = basic_auth_api(
|
82
|
+
api_shares_node = basic_auth_api(API_BASE)
|
60
83
|
repo_command = options.get_next_command(Node::COMMANDS_SHARES)
|
61
|
-
return Node.new(@agents
|
84
|
+
return Node.new(@agents, api: api_shares_node).execute_action(repo_command)
|
62
85
|
when :admin
|
63
86
|
api_shares_admin = basic_auth_api('api/v1')
|
64
87
|
admin_command = options.get_next_command(%i[user group share node].freeze)
|
@@ -92,7 +115,7 @@ module Aspera
|
|
92
115
|
display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:any)
|
93
116
|
return entity_command(entity_verb, api_shares_admin, entities_path, display_fields: display_fields)
|
94
117
|
when :import
|
95
|
-
return do_bulk_operation(
|
118
|
+
return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
|
96
119
|
entity_parameters = entity_parameters.transform_keys{|k|k.gsub(/\s+/, '_').downcase}
|
97
120
|
raise 'expecting Hash' unless entity_parameters.is_a?(Hash)
|
98
121
|
SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if entity_parameters[p].nil?}
|
@@ -102,8 +125,7 @@ module Aspera
|
|
102
125
|
api_shares_admin.create("#{entities_path}/import", entity_parameters)[:data]
|
103
126
|
end
|
104
127
|
when :add
|
105
|
-
return do_bulk_operation(
|
106
|
-
raise "expecting string (name), have #{entity_name.class}" unless entity_name.is_a?(String)
|
128
|
+
return do_bulk_operation(command: entity_verb, descr: "#{entity_type} name", values: String) do |entity_name|
|
107
129
|
api_shares_admin.create(entities_path, {entity_type=>entity_name})[:data]
|
108
130
|
end
|
109
131
|
when *USR_GRP_SETTINGS
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/sync'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
module Cli
|
7
|
+
# Module for sync actions
|
8
|
+
module SyncActions
|
9
|
+
SIMPLE_ARGUMENTS_SYNC = {
|
10
|
+
direction: Aspera::Sync::DIRECTIONS,
|
11
|
+
local_dir: String,
|
12
|
+
remote_dir: String
|
13
|
+
}.stringify_keys.freeze
|
14
|
+
|
15
|
+
class << self
|
16
|
+
def declare_options(options)
|
17
|
+
options.declare(:sync_info, 'Information for sync instance and sessions', types: Hash)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def execute_sync_action(&block)
|
22
|
+
# options = Aspera::Cli::Manager.new
|
23
|
+
raise 'Internal Error: No block given' unless block
|
24
|
+
command = options.get_next_command(%i[start admin])
|
25
|
+
# try to get 3 arguments as simple arguments
|
26
|
+
case command
|
27
|
+
when :start
|
28
|
+
simple_session_args = {}
|
29
|
+
SIMPLE_ARGUMENTS_SYNC.each do |arg, check|
|
30
|
+
value = options.get_next_argument(
|
31
|
+
arg,
|
32
|
+
type: check.is_a?(Class) ? check : nil,
|
33
|
+
expected: check.is_a?(Class) ? :single : check,
|
34
|
+
mandatory: false)
|
35
|
+
break if value.nil?
|
36
|
+
simple_session_args[arg] = value.to_s
|
37
|
+
end
|
38
|
+
async_params = nil
|
39
|
+
if simple_session_args.empty?
|
40
|
+
async_params = options.get_option(:sync_info, mandatory: true)
|
41
|
+
else
|
42
|
+
raise Cli::BadArgument,
|
43
|
+
"Provide zero or 3 arguments: #{SIMPLE_ARGUMENTS_SYNC.keys.join(',')}" unless simple_session_args.keys.sort == SIMPLE_ARGUMENTS_SYNC.keys.sort
|
44
|
+
async_params = options.get_option(
|
45
|
+
:sync_info,
|
46
|
+
mandatory: false,
|
47
|
+
default: {'sessions' => [{'name' => File.basename(simple_session_args['local_dir'])}]})
|
48
|
+
raise "sync_info shall be a Hash with key 'sessions' with Array of Hash: #{async_params}" unless async_params.is_a?(Hash) &&
|
49
|
+
async_params['sessions']&.is_a?(Array) &&
|
50
|
+
async_params['sessions'].first.is_a?(Hash)
|
51
|
+
async_params['sessions'].first.merge!(simple_session_args)
|
52
|
+
end
|
53
|
+
Log.log.debug{Log.dump('async_params', async_params)}
|
54
|
+
Aspera::Sync.start(async_params, &block)
|
55
|
+
return Main.result_success
|
56
|
+
when :admin
|
57
|
+
command2 = options.get_next_command([:status])
|
58
|
+
case command2
|
59
|
+
when :status
|
60
|
+
sync_session_name = options.get_next_argument('name of sync session', mandatory: false, type: String)
|
61
|
+
async_params = options.get_option(:sync_info, mandatory: true)
|
62
|
+
return {type: :single_object, data: Aspera::Sync.admin_status(async_params, sync_session_name)}
|
63
|
+
end # command2
|
64
|
+
end # command
|
65
|
+
end # execute_action
|
66
|
+
end # SyncActions
|
67
|
+
end # Cli
|
68
|
+
end # Aspera
|