aspera-cli 4.14.0 → 4.15.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +54 -3
- data/CONTRIBUTING.md +7 -7
- data/README.md +1457 -880
- data/bin/ascli +18 -9
- data/bin/asession +12 -14
- data/examples/proxy.pac +1 -1
- data/lib/aspera/aoc.rb +198 -127
- data/lib/aspera/ascmd.rb +24 -14
- data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
- data/lib/aspera/cli/error.rb +17 -0
- data/lib/aspera/cli/extended_value.rb +47 -12
- data/lib/aspera/cli/formatter.rb +260 -171
- data/lib/aspera/cli/hints.rb +80 -0
- data/lib/aspera/cli/main.rb +101 -147
- data/lib/aspera/cli/manager.rb +160 -124
- data/lib/aspera/cli/plugin.rb +70 -59
- data/lib/aspera/cli/plugins/alee.rb +0 -1
- data/lib/aspera/cli/plugins/aoc.rb +239 -273
- data/lib/aspera/cli/plugins/ats.rb +8 -5
- data/lib/aspera/cli/plugins/bss.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +516 -375
- data/lib/aspera/cli/plugins/console.rb +40 -0
- data/lib/aspera/cli/plugins/cos.rb +4 -5
- data/lib/aspera/cli/plugins/faspex.rb +99 -84
- data/lib/aspera/cli/plugins/faspex5.rb +179 -148
- data/lib/aspera/cli/plugins/node.rb +219 -153
- data/lib/aspera/cli/plugins/orchestrator.rb +52 -17
- data/lib/aspera/cli/plugins/preview.rb +46 -32
- data/lib/aspera/cli/plugins/server.rb +57 -17
- data/lib/aspera/cli/plugins/shares.rb +34 -12
- data/lib/aspera/cli/sync_actions.rb +68 -0
- data/lib/aspera/cli/transfer_agent.rb +45 -55
- data/lib/aspera/cli/transfer_progress.rb +74 -0
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -1
- data/lib/aspera/command_line_builder.rb +14 -11
- data/lib/aspera/cos_node.rb +3 -2
- data/lib/aspera/environment.rb +17 -6
- data/lib/aspera/fasp/agent_aspera.rb +126 -0
- data/lib/aspera/fasp/agent_base.rb +31 -77
- data/lib/aspera/fasp/agent_connect.rb +21 -22
- data/lib/aspera/fasp/agent_direct.rb +88 -102
- data/lib/aspera/fasp/agent_httpgw.rb +196 -192
- data/lib/aspera/fasp/agent_node.rb +41 -34
- data/lib/aspera/fasp/agent_trsdk.rb +75 -34
- data/lib/aspera/fasp/error_info.rb +2 -2
- data/lib/aspera/fasp/faux_file.rb +52 -0
- data/lib/aspera/fasp/installation.rb +43 -184
- data/lib/aspera/fasp/management.rb +244 -0
- data/lib/aspera/fasp/parameters.rb +59 -26
- data/lib/aspera/fasp/parameters.yaml +75 -8
- data/lib/aspera/fasp/products.rb +162 -0
- data/lib/aspera/fasp/transfer_spec.rb +1 -1
- data/lib/aspera/fasp/uri.rb +4 -4
- data/lib/aspera/faspex_gw.rb +2 -2
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/hash_ext.rb +2 -2
- data/lib/aspera/json_rpc.rb +49 -0
- data/lib/aspera/line_logger.rb +23 -0
- data/lib/aspera/log.rb +57 -16
- data/lib/aspera/node.rb +97 -14
- data/lib/aspera/oauth.rb +36 -18
- data/lib/aspera/open_application.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/file_types.rb +4 -2
- data/lib/aspera/preview/generator.rb +22 -35
- data/lib/aspera/preview/options.rb +2 -0
- data/lib/aspera/preview/terminal.rb +24 -13
- data/lib/aspera/preview/utils.rb +19 -26
- data/lib/aspera/rest.rb +103 -72
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +15 -14
- data/lib/aspera/rest_errors_aspera.rb +37 -34
- data/lib/aspera/secret_hider.rb +14 -16
- data/lib/aspera/ssh.rb +4 -1
- data/lib/aspera/sync.rb +128 -122
- data/lib/aspera/temp_file_manager.rb +10 -3
- data/lib/aspera/web_auth.rb +10 -7
- data/lib/aspera/web_server_simple.rb +9 -4
- data.tar.gz.sig +0 -0
- metadata +33 -15
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/listener/line_dump.rb +0 -19
- data/lib/aspera/cli/listener/logger.rb +0 -22
- data/lib/aspera/cli/listener/progress.rb +0 -50
- data/lib/aspera/cli/listener/progress_multi.rb +0 -84
- data/lib/aspera/cli/plugins/sync.rb +0 -44
- data/lib/aspera/fasp/listener.rb +0 -13
@@ -1,17 +1,53 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/cli/
|
3
|
+
require 'aspera/cli/basic_auth_plugin'
|
4
|
+
require 'aspera/nagios'
|
4
5
|
require 'xmlsimple'
|
5
6
|
|
6
7
|
module Aspera
|
7
8
|
module Cli
|
8
9
|
module Plugins
|
9
10
|
class Orchestrator < Aspera::Cli::BasicAuthPlugin
|
11
|
+
class << self
|
12
|
+
STANDARD_PATH = '/aspera/orchestrator'
|
13
|
+
def detect(address_or_url)
|
14
|
+
address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
|
15
|
+
urls = [address_or_url]
|
16
|
+
urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
|
17
|
+
urls.each do |base_url|
|
18
|
+
next unless base_url.match?('https?://')
|
19
|
+
api = Rest.new(base_url: base_url)
|
20
|
+
test_endpoint = 'api/remote_node_ping'
|
21
|
+
result = api.read(test_endpoint, {format: :json})
|
22
|
+
next unless result[:data]['remote_orchestrator_info']
|
23
|
+
url = result[:http].uri.to_s
|
24
|
+
return {
|
25
|
+
version: result[:data]['remote_orchestrator_info']['orchestrator-version'],
|
26
|
+
url: url[0..url.index(test_endpoint) - 2]
|
27
|
+
}
|
28
|
+
rescue StandardError => e
|
29
|
+
Log.log.debug{"detect error: #{e}"}
|
30
|
+
end
|
31
|
+
return nil
|
32
|
+
end
|
33
|
+
|
34
|
+
def wizard(object:, private_key_path: nil, pub_key_pem: nil)
|
35
|
+
options = object.options
|
36
|
+
return {
|
37
|
+
preset_value: {
|
38
|
+
url: options.get_option(:url, mandatory: true),
|
39
|
+
username: options.get_option(:username, mandatory: true),
|
40
|
+
password: options.get_option(:password, mandatory: true)
|
41
|
+
},
|
42
|
+
test_args: 'workflow list'
|
43
|
+
}
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
10
47
|
def initialize(env)
|
11
48
|
super(env)
|
12
|
-
options.declare(:
|
13
|
-
options.declare(:
|
14
|
-
options.declare(:synchronous, 'Work step:parameter expected as result', values: :bool, default: :no)
|
49
|
+
options.declare(:result, "Specify result value as: 'work_step:parameter'")
|
50
|
+
options.declare(:synchronous, 'Wait for completion', values: :bool, default: :no)
|
15
51
|
options.declare(:ret_style, 'How return type is requested in api', values: %i[header arg ext], default: :arg)
|
16
52
|
options.declare(:auth_style, 'Authentication type', values: %i[arg_pass head_basic apikey], default: :head_basic)
|
17
53
|
options.parse_options!
|
@@ -64,7 +100,7 @@ module Aspera
|
|
64
100
|
end
|
65
101
|
result = @api_orch.call(call_args)
|
66
102
|
result[:data] = XmlSimple.xml_in(result[:http].body, opt[:xml_opt] || {'ForceArray' => true}) if format.eql?('xml')
|
67
|
-
Log.dump(:data, result[:data])
|
103
|
+
Log.log.debug{Log.dump(:data, result[:data])}
|
68
104
|
return result
|
69
105
|
end
|
70
106
|
|
@@ -74,7 +110,7 @@ module Aspera
|
|
74
110
|
when :arg_pass
|
75
111
|
rest_params[:auth] = {
|
76
112
|
type: :url,
|
77
|
-
|
113
|
+
url_query: {
|
78
114
|
'login' => options.get_option(:username, mandatory: true),
|
79
115
|
'password' => options.get_option(:password, mandatory: true) }}
|
80
116
|
when :head_basic
|
@@ -118,9 +154,9 @@ module Aspera
|
|
118
154
|
end
|
119
155
|
case command
|
120
156
|
when :status
|
121
|
-
|
122
|
-
|
123
|
-
result = call_ao('workflows_status',
|
157
|
+
call_opts = {}
|
158
|
+
call_opts[:id] = wf_id unless wf_id.eql?(ExtendedValue::ALL)
|
159
|
+
result = call_ao('workflows_status', call_opts)[:data]
|
124
160
|
return {type: :object_list, data: result['workflows']['workflow']}
|
125
161
|
when :list
|
126
162
|
result = call_ao('workflows_list', id: 0)[:data]
|
@@ -145,19 +181,18 @@ module Aspera
|
|
145
181
|
}
|
146
182
|
call_params = {format: :json}
|
147
183
|
override_accept = nil
|
148
|
-
#
|
149
|
-
|
150
|
-
self.options.get_option(:params, mandatory: true).each do |name, value|
|
184
|
+
# get external parameters if any
|
185
|
+
options.get_next_argument('external_parameters', mandatory: false, type: Hash, default: {}).each do |name, value|
|
151
186
|
call_params["external_parameters[#{name}]"] = value
|
152
187
|
end
|
153
188
|
# synchronous call ?
|
154
|
-
call_params['synchronous'] = true if
|
189
|
+
call_params['synchronous'] = true if options.get_option(:synchronous, mandatory: true)
|
155
190
|
# expected result for synchro call ?
|
156
|
-
|
157
|
-
unless
|
191
|
+
result_location = options.get_option(:result)
|
192
|
+
unless result_location.nil?
|
158
193
|
result[:type] = :status
|
159
|
-
fields =
|
160
|
-
raise "Expects: work_step:result_name
|
194
|
+
fields = result_location.split(':')
|
195
|
+
raise Cli::BadArgument, "Expects: work_step:result_name : #{result_location}" if fields.length != 2
|
161
196
|
call_params['explicit_output_step'] = fields[0]
|
162
197
|
call_params['explicit_output_variable'] = fields[1]
|
163
198
|
# implicitly, call is synchronous
|
@@ -1,17 +1,18 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# cspell:ignore trevents
|
3
4
|
require 'aspera/cli/basic_auth_plugin'
|
4
5
|
require 'aspera/preview/generator'
|
5
6
|
require 'aspera/preview/options'
|
6
7
|
require 'aspera/preview/utils'
|
7
8
|
require 'aspera/preview/file_types'
|
9
|
+
require 'aspera/preview/terminal'
|
8
10
|
require 'aspera/fasp/transfer_spec'
|
9
11
|
require 'aspera/persistency_action_once'
|
10
12
|
require 'aspera/node'
|
11
13
|
require 'aspera/hash_ext'
|
12
14
|
require 'aspera/timer_limiter'
|
13
15
|
require 'aspera/id_generator'
|
14
|
-
require 'date'
|
15
16
|
require 'securerandom'
|
16
17
|
|
17
18
|
module Aspera
|
@@ -26,8 +27,11 @@ module Aspera
|
|
26
27
|
PREVIEW_BASENAME = 'preview'
|
27
28
|
# subfolder in system tmp folder
|
28
29
|
TMP_DIR_PREFIX = 'prev_tmp'
|
30
|
+
# same value as in aspera.conf
|
29
31
|
DEFAULT_PREVIEWS_FOLDER = 'previews'
|
32
|
+
# mark that this is used by a particular access key
|
30
33
|
AK_MARKER_FILE = '.aspera_access_key'
|
34
|
+
# URL prefix for local storage
|
31
35
|
PVCL_LOCAL_STORAGE = 'file:///'
|
32
36
|
LOG_LIMITER_SEC = 30.0
|
33
37
|
private_constant :PREV_GEN_TAG,
|
@@ -53,6 +57,8 @@ module Aspera
|
|
53
57
|
@gen_options = Aspera::Preview::Options.new
|
54
58
|
# used to trigger periodic processing
|
55
59
|
@periodic = TimerLimiter.new(LOG_LIMITER_SEC)
|
60
|
+
# Proc
|
61
|
+
@filter_block = nil
|
56
62
|
# link CLI options to gen_info attributes
|
57
63
|
options.declare(
|
58
64
|
:skip_format, 'Skip this preview format (multiple possible)', values: Aspera::Preview::Generator::PREVIEW_FORMATS,
|
@@ -66,7 +72,7 @@ module Aspera
|
|
66
72
|
options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
|
67
73
|
options.declare(:temp_folder, 'Path to temp folder', default: Dir.tmpdir)
|
68
74
|
options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, default: [])
|
69
|
-
options.declare(:
|
75
|
+
options.declare(:base, 'Basename of output for for test')
|
70
76
|
options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
|
71
77
|
options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
|
72
78
|
options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', values: :bool, default: false)
|
@@ -232,6 +238,7 @@ module Aspera
|
|
232
238
|
def get_infos_remote(gen_infos, entry)
|
233
239
|
# store source directly here
|
234
240
|
local_original_filepath = File.join(@tmp_folder, entry['name'])
|
241
|
+
# require 'date'
|
235
242
|
# original_mtime=DateTime.parse(entry['modified_time'])
|
236
243
|
# out: where previews are generated
|
237
244
|
local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
|
@@ -255,9 +262,10 @@ module Aspera
|
|
255
262
|
"#{entry['id']}#{PREVIEW_FOLDER_SUFFIX}"
|
256
263
|
end
|
257
264
|
|
258
|
-
|
259
|
-
|
260
|
-
|
265
|
+
# Generate a file name based on basename and format (extension)
|
266
|
+
def preview_filename(preview_format, base_name=nil)
|
267
|
+
base_name ||= PREVIEW_BASENAME
|
268
|
+
return "#{base_name}.#{preview_format}"
|
261
269
|
end
|
262
270
|
|
263
271
|
# generate preview files for one folder entry (file) if necessary
|
@@ -290,10 +298,13 @@ module Aspera
|
|
290
298
|
next false if gen_info[:preview_newer_than_original]
|
291
299
|
end
|
292
300
|
end
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
301
|
+
begin
|
302
|
+
# need generator for further checks
|
303
|
+
gen_info[:generator] = Aspera::Preview::Generator.new(gen_info[:src], gen_info[:dst], @gen_options, @tmp_folder, entry['content_type'])
|
304
|
+
rescue
|
305
|
+
# no conversion supported
|
306
|
+
next false
|
307
|
+
end
|
297
308
|
# shall we skip it ?
|
298
309
|
next false if @skip_types.include?(gen_info[:generator].conversion_type)
|
299
310
|
# ok we need to generate
|
@@ -307,7 +318,7 @@ module Aspera
|
|
307
318
|
# download original file to temp folder
|
308
319
|
do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
|
309
320
|
end
|
310
|
-
Log.log.info{"source: #{entry['id']}: #{entry['path']}
|
321
|
+
Log.log.info{"source: #{entry['id']}: #{entry['path']}"}
|
311
322
|
gen_infos.each do |gen_info|
|
312
323
|
gen_info[:generator].generate rescue nil
|
313
324
|
end
|
@@ -328,24 +339,22 @@ module Aspera
|
|
328
339
|
end # generate_preview
|
329
340
|
|
330
341
|
# scan all files in provided folder entry
|
331
|
-
# @param
|
332
|
-
def scan_folder_files(top_entry,
|
333
|
-
|
342
|
+
# @param top_path subpath to start folder scan inside
|
343
|
+
def scan_folder_files(top_entry, top_path=nil)
|
344
|
+
unless top_path.nil?
|
334
345
|
# canonical path: start with / and ends with /
|
335
|
-
|
336
|
-
scan_start = "#{scan_start}/" # unless scan_start.end_with?('/')
|
346
|
+
top_path = '/' + top_path.split('/').reject(&:empty?).join('/') + '/'
|
337
347
|
end
|
338
|
-
|
339
|
-
Log.log.debug{"scan: #{top_entry} : #{scan_start}".green}
|
348
|
+
Log.log.debug{"scan: #{top_entry} : #{top_path}".green}
|
340
349
|
# don't use recursive call, use list instead
|
341
350
|
entries_to_process = [top_entry]
|
342
351
|
until entries_to_process.empty?
|
343
352
|
entry = entries_to_process.shift
|
344
|
-
# process this entry only if it is within the
|
353
|
+
# process this entry only if it is within the top_path
|
345
354
|
entry_path_with_slash = entry['path']
|
346
355
|
Log.log.info{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
|
347
356
|
entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
|
348
|
-
if !
|
357
|
+
if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
|
349
358
|
Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
|
350
359
|
next
|
351
360
|
end
|
@@ -353,7 +362,7 @@ module Aspera
|
|
353
362
|
begin
|
354
363
|
case entry['type']
|
355
364
|
when 'file'
|
356
|
-
if filter_block.call(entry)
|
365
|
+
if @filter_block.call(entry)
|
357
366
|
generate_preview(entry)
|
358
367
|
else
|
359
368
|
Log.log.debug('skip by filter')
|
@@ -386,11 +395,11 @@ module Aspera
|
|
386
395
|
end
|
387
396
|
end
|
388
397
|
|
389
|
-
ACTIONS = %i[scan events trevents check test].freeze
|
398
|
+
ACTIONS = %i[scan events trevents check test show].freeze
|
390
399
|
|
391
400
|
def execute_action
|
392
401
|
command = options.get_next_command(ACTIONS)
|
393
|
-
unless %i[check test].include?(command)
|
402
|
+
unless %i[check test show].include?(command)
|
394
403
|
# this will use node api
|
395
404
|
@api_node = Aspera::Node.new(params: basic_auth_params)
|
396
405
|
@transfer_server_address = URI.parse(@api_node.params[:base_url]).host
|
@@ -408,7 +417,7 @@ module Aspera
|
|
408
417
|
if @access_remote
|
409
418
|
# NOTE: the filter "name", it's why we take the first one
|
410
419
|
@previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
|
411
|
-
raise
|
420
|
+
raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
|
412
421
|
'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
|
413
422
|
else
|
414
423
|
raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
|
@@ -417,9 +426,9 @@ module Aspera
|
|
417
426
|
@local_storage_root = @local_storage_root[PVCL_LOCAL_STORAGE.length..-1] if @local_storage_root.start_with?(PVCL_LOCAL_STORAGE)
|
418
427
|
# TODO: windows could have "C:" ?
|
419
428
|
raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/')
|
420
|
-
raise
|
429
|
+
raise Cli::Error, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
|
421
430
|
@local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
|
422
|
-
raise
|
431
|
+
raise Cli::Error, "Folder #{@local_preview_folder} does not exist locally. " \
|
423
432
|
'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
|
424
433
|
# protection to avoid clash of file id for two different access keys
|
425
434
|
marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
|
@@ -450,9 +459,11 @@ module Aspera
|
|
450
459
|
else
|
451
460
|
@api_node.read("files/#{scan_id}")[:data]
|
452
461
|
end
|
462
|
+
@filter_block = Aspera::Node.file_matcher_from_argument(options)
|
453
463
|
scan_folder_files(folder_info, scan_path)
|
454
464
|
return Main.result_status('scan finished')
|
455
465
|
when :events, :trevents
|
466
|
+
@filter_block = Aspera::Node.file_matcher_from_argument(options)
|
456
467
|
iteration_persistency = nil
|
457
468
|
if options.get_option(:once_only, mandatory: true)
|
458
469
|
iteration_persistency = PersistencyActionOnce.new(
|
@@ -470,15 +481,18 @@ module Aspera
|
|
470
481
|
return Main.result_status("#{command} finished")
|
471
482
|
when :check
|
472
483
|
return Main.result_status('Tools validated')
|
473
|
-
when :test
|
474
|
-
format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS)
|
484
|
+
when :test, :show
|
475
485
|
source = options.get_next_argument('source file')
|
476
|
-
|
477
|
-
|
478
|
-
|
479
|
-
raise "out format #{format} not supported" unless g.supported?
|
486
|
+
format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS, default: :png)
|
487
|
+
generated_file_path = preview_filename(format, options.get_option(:base))
|
488
|
+
g = Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder, nil)
|
480
489
|
g.generate
|
481
|
-
|
490
|
+
if command.eql?(:show)
|
491
|
+
terminal_options = options.get_option(:query, default: {}).symbolize_keys
|
492
|
+
Log.log.debug{"preview: #{generated_file_path}"}
|
493
|
+
formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
|
494
|
+
end
|
495
|
+
return Main.result_status("generated: #{generated_file_path}")
|
482
496
|
else
|
483
497
|
raise 'error'
|
484
498
|
end
|
@@ -1,9 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
# cspell:ignore ascmd zmode zuid zgid fasping
|
3
4
|
require 'aspera/cli/basic_auth_plugin'
|
4
|
-
require 'aspera/cli/
|
5
|
-
require 'aspera/ascmd'
|
5
|
+
require 'aspera/cli/sync_actions'
|
6
6
|
require 'aspera/fasp/transfer_spec'
|
7
|
+
require 'aspera/ascmd'
|
7
8
|
require 'aspera/ssh'
|
8
9
|
require 'aspera/nagios'
|
9
10
|
require 'tempfile'
|
@@ -13,17 +14,8 @@ module Aspera
|
|
13
14
|
module Cli
|
14
15
|
module Plugins
|
15
16
|
# implement basic remote access with FASP/SSH
|
16
|
-
class SyncSpecServer
|
17
|
-
def initialize(transfer_spec)
|
18
|
-
@transfer_spec = transfer_spec
|
19
|
-
end
|
20
|
-
|
21
|
-
def transfer_spec(direction, local_path, remote_path)
|
22
|
-
return @transfer_spec
|
23
|
-
end
|
24
|
-
end
|
25
|
-
|
26
17
|
class Server < Aspera::Cli::BasicAuthPlugin
|
18
|
+
include SyncActions
|
27
19
|
SSH_SCHEME = 'ssh'
|
28
20
|
LOCAL_SCHEME = 'local'
|
29
21
|
HTTPS_SCHEME = 'https'
|
@@ -49,11 +41,59 @@ module Aspera
|
|
49
41
|
end
|
50
42
|
end
|
51
43
|
|
44
|
+
class << self
|
45
|
+
def application_name
|
46
|
+
'HSTS Fasp/SSH'
|
47
|
+
end
|
48
|
+
|
49
|
+
def detect(address_or_url)
|
50
|
+
urls = if address_or_url.match?(%r{^[a-z]{1,6}://})
|
51
|
+
[address_or_url]
|
52
|
+
else
|
53
|
+
[
|
54
|
+
"ssh://#{address_or_url}:33001",
|
55
|
+
"ssh://#{address_or_url}:22"
|
56
|
+
]
|
57
|
+
# wss not practical as it requires a token
|
58
|
+
end
|
59
|
+
|
60
|
+
urls.each do |base_url|
|
61
|
+
server_uri = URI.parse(base_url)
|
62
|
+
Log.log.debug{"URI=#{server_uri}, host=#{server_uri.hostname}, port=#{server_uri.port}, scheme=#{server_uri.scheme}"}
|
63
|
+
next unless server_uri.scheme.eql?(SSH_SCHEME)
|
64
|
+
begin
|
65
|
+
socket = TCPSocket.new(server_uri.hostname, server_uri.port)
|
66
|
+
socket.puts('SSH-2.0-Ascli_0.0')
|
67
|
+
version = socket.gets.chomp
|
68
|
+
if version.match?(/^SSH-2.0-/)
|
69
|
+
return {version: version.gsub(/^SSH-2.0-/, ''), url: base_url}
|
70
|
+
end
|
71
|
+
rescue StandardError => e
|
72
|
+
Log.log.debug{"detect error: #{e}"}
|
73
|
+
end
|
74
|
+
end
|
75
|
+
return nil
|
76
|
+
end
|
77
|
+
|
78
|
+
def wizard(object:, private_key_path: nil, pub_key_pem: nil)
|
79
|
+
options = object.options
|
80
|
+
return {
|
81
|
+
preset_value: {
|
82
|
+
url: options.get_option(:url, mandatory: true),
|
83
|
+
username: options.get_option(:username, mandatory: true),
|
84
|
+
password: options.get_option(:password, mandatory: true)
|
85
|
+
},
|
86
|
+
test_args: 'files br /'
|
87
|
+
}
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
52
91
|
def initialize(env)
|
53
92
|
super(env)
|
54
93
|
options.declare(:ssh_keys, 'SSH key path list (Array or single)')
|
55
94
|
options.declare(:passphrase, 'SSH private key passphrase')
|
56
95
|
options.declare(:ssh_options, 'SSH options', types: Hash, default: {})
|
96
|
+
SyncActions.declare_options(options)
|
57
97
|
options.parse_options!
|
58
98
|
@ssh_opts = options.get_option(:ssh_options).symbolize_keys
|
59
99
|
end
|
@@ -138,8 +178,8 @@ module Aspera
|
|
138
178
|
Fasp::TransferSpec.action_to_direction(transfer_spec, command)
|
139
179
|
return Main.result_transfer(transfer.start(transfer_spec))
|
140
180
|
when :sync
|
141
|
-
|
142
|
-
return
|
181
|
+
# lets ignore the arguments provided by execute_sync_action, we just give the transfer spec
|
182
|
+
return execute_sync_action {transfer_spec}
|
143
183
|
end
|
144
184
|
end
|
145
185
|
|
@@ -191,10 +231,10 @@ module Aspera
|
|
191
231
|
when *TRANSFER_COMMANDS
|
192
232
|
return execute_transfer(command, server_transfer_spec)
|
193
233
|
when *Aspera::AsCmd::OPERATIONS
|
194
|
-
|
234
|
+
command_arguments = options.get_next_argument('ascmd command arguments', expected: :multiple, mandatory: false)
|
195
235
|
ascmd = Aspera::AsCmd.new(ascmd_executor)
|
196
236
|
begin
|
197
|
-
result = ascmd.send(:execute_single, command,
|
237
|
+
result = ascmd.send(:execute_single, command, command_arguments)
|
198
238
|
case command
|
199
239
|
when :mkdir, :mv, :cp, :rm
|
200
240
|
return Main.result_success
|
@@ -206,7 +246,7 @@ module Aspera
|
|
206
246
|
return {type: :single_object, data: result.stringify_keys}
|
207
247
|
end
|
208
248
|
rescue Aspera::AsCmd::Error => e
|
209
|
-
raise
|
249
|
+
raise Cli::BadArgument, e.extended_message
|
210
250
|
end
|
211
251
|
else raise 'internal error: unexpected action'
|
212
252
|
end
|
@@ -7,20 +7,43 @@ module Aspera
|
|
7
7
|
module Plugins
|
8
8
|
# Plugin for Aspera Shares v1
|
9
9
|
class Shares < Aspera::Cli::BasicAuthPlugin
|
10
|
+
API_BASE = 'node_api'
|
10
11
|
class << self
|
11
|
-
def detect(
|
12
|
-
|
13
|
-
|
12
|
+
def detect(address_or_url)
|
13
|
+
address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
|
14
|
+
api = Rest.new(base_url: address_or_url, redirect_max: 1)
|
15
|
+
found = false
|
14
16
|
begin
|
15
17
|
# shall fail: shares requires auth, but we check error message
|
16
18
|
# TODO: use ping instead ?
|
17
|
-
api.read(
|
19
|
+
api.read("#{API_BASE}/app")
|
18
20
|
rescue RestCallError => e
|
19
21
|
if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
|
20
|
-
|
22
|
+
found = true
|
21
23
|
end
|
22
24
|
end
|
23
|
-
nil
|
25
|
+
return nil unless found
|
26
|
+
version = 'unknown'
|
27
|
+
test_page = api.call({ operation: 'GET', subpath: 'login' })
|
28
|
+
if (m = test_page[:http].body.match(/\(v(1\..*)\)/))
|
29
|
+
version = m[1]
|
30
|
+
end
|
31
|
+
return {
|
32
|
+
version: version,
|
33
|
+
url: address_or_url
|
34
|
+
}
|
35
|
+
end
|
36
|
+
|
37
|
+
def wizard(object:, private_key_path: nil, pub_key_pem: nil)
|
38
|
+
options = object.options
|
39
|
+
return {
|
40
|
+
preset_value: {
|
41
|
+
url: options.get_option(:url, mandatory: true),
|
42
|
+
username: options.get_option(:username, mandatory: true),
|
43
|
+
password: options.get_option(:password, mandatory: true)
|
44
|
+
},
|
45
|
+
test_args: 'files br /'
|
46
|
+
}
|
24
47
|
end
|
25
48
|
end
|
26
49
|
|
@@ -44,7 +67,7 @@ module Aspera
|
|
44
67
|
nagios = Nagios.new
|
45
68
|
begin
|
46
69
|
Rest
|
47
|
-
.new(base_url: options.get_option(:url, mandatory: true)
|
70
|
+
.new(base_url: "#{options.get_option(:url, mandatory: true)}/#{API_BASE}")
|
48
71
|
.call(
|
49
72
|
operation: 'GET',
|
50
73
|
subpath: 'ping',
|
@@ -56,9 +79,9 @@ module Aspera
|
|
56
79
|
end
|
57
80
|
return nagios.result
|
58
81
|
when :repository, :files
|
59
|
-
api_shares_node = basic_auth_api(
|
82
|
+
api_shares_node = basic_auth_api(API_BASE)
|
60
83
|
repo_command = options.get_next_command(Node::COMMANDS_SHARES)
|
61
|
-
return Node.new(@agents
|
84
|
+
return Node.new(@agents, api: api_shares_node).execute_action(repo_command)
|
62
85
|
when :admin
|
63
86
|
api_shares_admin = basic_auth_api('api/v1')
|
64
87
|
admin_command = options.get_next_command(%i[user group share node].freeze)
|
@@ -92,7 +115,7 @@ module Aspera
|
|
92
115
|
display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:any)
|
93
116
|
return entity_command(entity_verb, api_shares_admin, entities_path, display_fields: display_fields)
|
94
117
|
when :import
|
95
|
-
return do_bulk_operation(
|
118
|
+
return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
|
96
119
|
entity_parameters = entity_parameters.transform_keys{|k|k.gsub(/\s+/, '_').downcase}
|
97
120
|
raise 'expecting Hash' unless entity_parameters.is_a?(Hash)
|
98
121
|
SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if entity_parameters[p].nil?}
|
@@ -102,8 +125,7 @@ module Aspera
|
|
102
125
|
api_shares_admin.create("#{entities_path}/import", entity_parameters)[:data]
|
103
126
|
end
|
104
127
|
when :add
|
105
|
-
return do_bulk_operation(
|
106
|
-
raise "expecting string (name), have #{entity_name.class}" unless entity_name.is_a?(String)
|
128
|
+
return do_bulk_operation(command: entity_verb, descr: "#{entity_type} name", values: String) do |entity_name|
|
107
129
|
api_shares_admin.create(entities_path, {entity_type=>entity_name})[:data]
|
108
130
|
end
|
109
131
|
when *USR_GRP_SETTINGS
|
@@ -0,0 +1,68 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'aspera/sync'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
module Cli
|
7
|
+
# Module for sync actions
|
8
|
+
module SyncActions
|
9
|
+
SIMPLE_ARGUMENTS_SYNC = {
|
10
|
+
direction: Aspera::Sync::DIRECTIONS,
|
11
|
+
local_dir: String,
|
12
|
+
remote_dir: String
|
13
|
+
}.stringify_keys.freeze
|
14
|
+
|
15
|
+
class << self
|
16
|
+
def declare_options(options)
|
17
|
+
options.declare(:sync_info, 'Information for sync instance and sessions', types: Hash)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def execute_sync_action(&block)
|
22
|
+
# options = Aspera::Cli::Manager.new
|
23
|
+
raise 'Internal Error: No block given' unless block
|
24
|
+
command = options.get_next_command(%i[start admin])
|
25
|
+
# try to get 3 arguments as simple arguments
|
26
|
+
case command
|
27
|
+
when :start
|
28
|
+
simple_session_args = {}
|
29
|
+
SIMPLE_ARGUMENTS_SYNC.each do |arg, check|
|
30
|
+
value = options.get_next_argument(
|
31
|
+
arg,
|
32
|
+
type: check.is_a?(Class) ? check : nil,
|
33
|
+
expected: check.is_a?(Class) ? :single : check,
|
34
|
+
mandatory: false)
|
35
|
+
break if value.nil?
|
36
|
+
simple_session_args[arg] = value.to_s
|
37
|
+
end
|
38
|
+
async_params = nil
|
39
|
+
if simple_session_args.empty?
|
40
|
+
async_params = options.get_option(:sync_info, mandatory: true)
|
41
|
+
else
|
42
|
+
raise Cli::BadArgument,
|
43
|
+
"Provide zero or 3 arguments: #{SIMPLE_ARGUMENTS_SYNC.keys.join(',')}" unless simple_session_args.keys.sort == SIMPLE_ARGUMENTS_SYNC.keys.sort
|
44
|
+
async_params = options.get_option(
|
45
|
+
:sync_info,
|
46
|
+
mandatory: false,
|
47
|
+
default: {'sessions' => [{'name' => File.basename(simple_session_args['local_dir'])}]})
|
48
|
+
raise "sync_info shall be a Hash with key 'sessions' with Array of Hash: #{async_params}" unless async_params.is_a?(Hash) &&
|
49
|
+
async_params['sessions']&.is_a?(Array) &&
|
50
|
+
async_params['sessions'].first.is_a?(Hash)
|
51
|
+
async_params['sessions'].first.merge!(simple_session_args)
|
52
|
+
end
|
53
|
+
Log.log.debug{Log.dump('async_params', async_params)}
|
54
|
+
Aspera::Sync.start(async_params, &block)
|
55
|
+
return Main.result_success
|
56
|
+
when :admin
|
57
|
+
command2 = options.get_next_command([:status])
|
58
|
+
case command2
|
59
|
+
when :status
|
60
|
+
sync_session_name = options.get_next_argument('name of sync session', mandatory: false, type: String)
|
61
|
+
async_params = options.get_option(:sync_info, mandatory: true)
|
62
|
+
return {type: :single_object, data: Aspera::Sync.admin_status(async_params, sync_session_name)}
|
63
|
+
end # command2
|
64
|
+
end # command
|
65
|
+
end # execute_action
|
66
|
+
end # SyncActions
|
67
|
+
end # Cli
|
68
|
+
end # Aspera
|