aspera-cli 4.14.0 → 4.15.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +54 -3
  4. data/CONTRIBUTING.md +7 -7
  5. data/README.md +1457 -880
  6. data/bin/ascli +18 -9
  7. data/bin/asession +12 -14
  8. data/examples/proxy.pac +1 -1
  9. data/lib/aspera/aoc.rb +198 -127
  10. data/lib/aspera/ascmd.rb +24 -14
  11. data/lib/aspera/cli/basic_auth_plugin.rb +9 -6
  12. data/lib/aspera/cli/error.rb +17 -0
  13. data/lib/aspera/cli/extended_value.rb +47 -12
  14. data/lib/aspera/cli/formatter.rb +260 -171
  15. data/lib/aspera/cli/hints.rb +80 -0
  16. data/lib/aspera/cli/main.rb +101 -147
  17. data/lib/aspera/cli/manager.rb +160 -124
  18. data/lib/aspera/cli/plugin.rb +70 -59
  19. data/lib/aspera/cli/plugins/alee.rb +0 -1
  20. data/lib/aspera/cli/plugins/aoc.rb +239 -273
  21. data/lib/aspera/cli/plugins/ats.rb +8 -5
  22. data/lib/aspera/cli/plugins/bss.rb +2 -2
  23. data/lib/aspera/cli/plugins/config.rb +516 -375
  24. data/lib/aspera/cli/plugins/console.rb +40 -0
  25. data/lib/aspera/cli/plugins/cos.rb +4 -5
  26. data/lib/aspera/cli/plugins/faspex.rb +99 -84
  27. data/lib/aspera/cli/plugins/faspex5.rb +179 -148
  28. data/lib/aspera/cli/plugins/node.rb +219 -153
  29. data/lib/aspera/cli/plugins/orchestrator.rb +52 -17
  30. data/lib/aspera/cli/plugins/preview.rb +46 -32
  31. data/lib/aspera/cli/plugins/server.rb +57 -17
  32. data/lib/aspera/cli/plugins/shares.rb +34 -12
  33. data/lib/aspera/cli/sync_actions.rb +68 -0
  34. data/lib/aspera/cli/transfer_agent.rb +45 -55
  35. data/lib/aspera/cli/transfer_progress.rb +74 -0
  36. data/lib/aspera/cli/version.rb +1 -1
  37. data/lib/aspera/colors.rb +3 -1
  38. data/lib/aspera/command_line_builder.rb +14 -11
  39. data/lib/aspera/cos_node.rb +3 -2
  40. data/lib/aspera/environment.rb +17 -6
  41. data/lib/aspera/fasp/agent_aspera.rb +126 -0
  42. data/lib/aspera/fasp/agent_base.rb +31 -77
  43. data/lib/aspera/fasp/agent_connect.rb +21 -22
  44. data/lib/aspera/fasp/agent_direct.rb +88 -102
  45. data/lib/aspera/fasp/agent_httpgw.rb +196 -192
  46. data/lib/aspera/fasp/agent_node.rb +41 -34
  47. data/lib/aspera/fasp/agent_trsdk.rb +75 -34
  48. data/lib/aspera/fasp/error_info.rb +2 -2
  49. data/lib/aspera/fasp/faux_file.rb +52 -0
  50. data/lib/aspera/fasp/installation.rb +43 -184
  51. data/lib/aspera/fasp/management.rb +244 -0
  52. data/lib/aspera/fasp/parameters.rb +59 -26
  53. data/lib/aspera/fasp/parameters.yaml +75 -8
  54. data/lib/aspera/fasp/products.rb +162 -0
  55. data/lib/aspera/fasp/transfer_spec.rb +1 -1
  56. data/lib/aspera/fasp/uri.rb +4 -4
  57. data/lib/aspera/faspex_gw.rb +2 -2
  58. data/lib/aspera/faspex_postproc.rb +2 -2
  59. data/lib/aspera/hash_ext.rb +2 -2
  60. data/lib/aspera/json_rpc.rb +49 -0
  61. data/lib/aspera/line_logger.rb +23 -0
  62. data/lib/aspera/log.rb +57 -16
  63. data/lib/aspera/node.rb +97 -14
  64. data/lib/aspera/oauth.rb +36 -18
  65. data/lib/aspera/open_application.rb +4 -4
  66. data/lib/aspera/persistency_folder.rb +2 -2
  67. data/lib/aspera/preview/file_types.rb +4 -2
  68. data/lib/aspera/preview/generator.rb +22 -35
  69. data/lib/aspera/preview/options.rb +2 -0
  70. data/lib/aspera/preview/terminal.rb +24 -13
  71. data/lib/aspera/preview/utils.rb +19 -26
  72. data/lib/aspera/rest.rb +103 -72
  73. data/lib/aspera/rest_call_error.rb +1 -1
  74. data/lib/aspera/rest_error_analyzer.rb +15 -14
  75. data/lib/aspera/rest_errors_aspera.rb +37 -34
  76. data/lib/aspera/secret_hider.rb +14 -16
  77. data/lib/aspera/ssh.rb +4 -1
  78. data/lib/aspera/sync.rb +128 -122
  79. data/lib/aspera/temp_file_manager.rb +10 -3
  80. data/lib/aspera/web_auth.rb +10 -7
  81. data/lib/aspera/web_server_simple.rb +9 -4
  82. data.tar.gz.sig +0 -0
  83. metadata +33 -15
  84. metadata.gz.sig +0 -0
  85. data/lib/aspera/cli/listener/line_dump.rb +0 -19
  86. data/lib/aspera/cli/listener/logger.rb +0 -22
  87. data/lib/aspera/cli/listener/progress.rb +0 -50
  88. data/lib/aspera/cli/listener/progress_multi.rb +0 -84
  89. data/lib/aspera/cli/plugins/sync.rb +0 -44
  90. data/lib/aspera/fasp/listener.rb +0 -13
@@ -1,17 +1,53 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/cli/plugins/node'
3
+ require 'aspera/cli/basic_auth_plugin'
4
+ require 'aspera/nagios'
4
5
  require 'xmlsimple'
5
6
 
6
7
  module Aspera
7
8
  module Cli
8
9
  module Plugins
9
10
  class Orchestrator < Aspera::Cli::BasicAuthPlugin
11
+ class << self
12
+ STANDARD_PATH = '/aspera/orchestrator'
13
+ def detect(address_or_url)
14
+ address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
15
+ urls = [address_or_url]
16
+ urls.push("#{address_or_url}#{STANDARD_PATH}") unless address_or_url.end_with?(STANDARD_PATH)
17
+ urls.each do |base_url|
18
+ next unless base_url.match?('https?://')
19
+ api = Rest.new(base_url: base_url)
20
+ test_endpoint = 'api/remote_node_ping'
21
+ result = api.read(test_endpoint, {format: :json})
22
+ next unless result[:data]['remote_orchestrator_info']
23
+ url = result[:http].uri.to_s
24
+ return {
25
+ version: result[:data]['remote_orchestrator_info']['orchestrator-version'],
26
+ url: url[0..url.index(test_endpoint) - 2]
27
+ }
28
+ rescue StandardError => e
29
+ Log.log.debug{"detect error: #{e}"}
30
+ end
31
+ return nil
32
+ end
33
+
34
+ def wizard(object:, private_key_path: nil, pub_key_pem: nil)
35
+ options = object.options
36
+ return {
37
+ preset_value: {
38
+ url: options.get_option(:url, mandatory: true),
39
+ username: options.get_option(:username, mandatory: true),
40
+ password: options.get_option(:password, mandatory: true)
41
+ },
42
+ test_args: 'workflow list'
43
+ }
44
+ end
45
+ end
46
+
10
47
  def initialize(env)
11
48
  super(env)
12
- options.declare(:params, 'Start parameters', types: Hash, default: {})
13
- options.declare(:result, "Specify result value as: 'work step:parameter'")
14
- options.declare(:synchronous, 'Work step:parameter expected as result', values: :bool, default: :no)
49
+ options.declare(:result, "Specify result value as: 'work_step:parameter'")
50
+ options.declare(:synchronous, 'Wait for completion', values: :bool, default: :no)
15
51
  options.declare(:ret_style, 'How return type is requested in api', values: %i[header arg ext], default: :arg)
16
52
  options.declare(:auth_style, 'Authentication type', values: %i[arg_pass head_basic apikey], default: :head_basic)
17
53
  options.parse_options!
@@ -64,7 +100,7 @@ module Aspera
64
100
  end
65
101
  result = @api_orch.call(call_args)
66
102
  result[:data] = XmlSimple.xml_in(result[:http].body, opt[:xml_opt] || {'ForceArray' => true}) if format.eql?('xml')
67
- Log.dump(:data, result[:data])
103
+ Log.log.debug{Log.dump(:data, result[:data])}
68
104
  return result
69
105
  end
70
106
 
@@ -74,7 +110,7 @@ module Aspera
74
110
  when :arg_pass
75
111
  rest_params[:auth] = {
76
112
  type: :url,
77
- url_creds: {
113
+ url_query: {
78
114
  'login' => options.get_option(:username, mandatory: true),
79
115
  'password' => options.get_option(:password, mandatory: true) }}
80
116
  when :head_basic
@@ -118,9 +154,9 @@ module Aspera
118
154
  end
119
155
  case command
120
156
  when :status
121
- options = {}
122
- options[:id] = wf_id unless wf_id.eql?(VAL_ALL)
123
- result = call_ao('workflows_status', options)[:data]
157
+ call_opts = {}
158
+ call_opts[:id] = wf_id unless wf_id.eql?(ExtendedValue::ALL)
159
+ result = call_ao('workflows_status', call_opts)[:data]
124
160
  return {type: :object_list, data: result['workflows']['workflow']}
125
161
  when :list
126
162
  result = call_ao('workflows_list', id: 0)[:data]
@@ -145,19 +181,18 @@ module Aspera
145
181
  }
146
182
  call_params = {format: :json}
147
183
  override_accept = nil
148
- # set external parameters if any
149
- # TODO: make not an option, but a parameter
150
- self.options.get_option(:params, mandatory: true).each do |name, value|
184
+ # get external parameters if any
185
+ options.get_next_argument('external_parameters', mandatory: false, type: Hash, default: {}).each do |name, value|
151
186
  call_params["external_parameters[#{name}]"] = value
152
187
  end
153
188
  # synchronous call ?
154
- call_params['synchronous'] = true if self.options.get_option(:synchronous, mandatory: true)
189
+ call_params['synchronous'] = true if options.get_option(:synchronous, mandatory: true)
155
190
  # expected result for synchro call ?
156
- expected = self.options.get_option(:result)
157
- unless expected.nil?
191
+ result_location = options.get_option(:result)
192
+ unless result_location.nil?
158
193
  result[:type] = :status
159
- fields = expected.split(':')
160
- raise "Expects: work_step:result_name format, but got #{expected}" if fields.length != 2
194
+ fields = result_location.split(':')
195
+ raise Cli::BadArgument, "Expects: work_step:result_name : #{result_location}" if fields.length != 2
161
196
  call_params['explicit_output_step'] = fields[0]
162
197
  call_params['explicit_output_variable'] = fields[1]
163
198
  # implicitly, call is synchronous
@@ -1,17 +1,18 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # cspell:ignore trevents
3
4
  require 'aspera/cli/basic_auth_plugin'
4
5
  require 'aspera/preview/generator'
5
6
  require 'aspera/preview/options'
6
7
  require 'aspera/preview/utils'
7
8
  require 'aspera/preview/file_types'
9
+ require 'aspera/preview/terminal'
8
10
  require 'aspera/fasp/transfer_spec'
9
11
  require 'aspera/persistency_action_once'
10
12
  require 'aspera/node'
11
13
  require 'aspera/hash_ext'
12
14
  require 'aspera/timer_limiter'
13
15
  require 'aspera/id_generator'
14
- require 'date'
15
16
  require 'securerandom'
16
17
 
17
18
  module Aspera
@@ -26,8 +27,11 @@ module Aspera
26
27
  PREVIEW_BASENAME = 'preview'
27
28
  # subfolder in system tmp folder
28
29
  TMP_DIR_PREFIX = 'prev_tmp'
30
+ # same value as in aspera.conf
29
31
  DEFAULT_PREVIEWS_FOLDER = 'previews'
32
+ # mark that this is used by a particular access key
30
33
  AK_MARKER_FILE = '.aspera_access_key'
34
+ # URL prefix for local storage
31
35
  PVCL_LOCAL_STORAGE = 'file:///'
32
36
  LOG_LIMITER_SEC = 30.0
33
37
  private_constant :PREV_GEN_TAG,
@@ -53,6 +57,8 @@ module Aspera
53
57
  @gen_options = Aspera::Preview::Options.new
54
58
  # used to trigger periodic processing
55
59
  @periodic = TimerLimiter.new(LOG_LIMITER_SEC)
60
+ # Proc
61
+ @filter_block = nil
56
62
  # link CLI options to gen_info attributes
57
63
  options.declare(
58
64
  :skip_format, 'Skip this preview format (multiple possible)', values: Aspera::Preview::Generator::PREVIEW_FORMATS,
@@ -66,7 +72,7 @@ module Aspera
66
72
  options.declare(:previews_folder, 'Preview folder in storage root', handler: {o: self, m: :option_previews_folder}, default: DEFAULT_PREVIEWS_FOLDER)
67
73
  options.declare(:temp_folder, 'Path to temp folder', default: Dir.tmpdir)
68
74
  options.declare(:skip_folders, 'List of folder to skip', handler: {o: self, m: :option_skip_folders}, default: [])
69
- options.declare(:case, 'Basename of output for for test')
75
+ options.declare(:base, 'Basename of output for for test')
70
76
  options.declare(:scan_path, 'Subpath in folder id to start scan in (default=/)')
71
77
  options.declare(:scan_id, 'Folder id in storage to start scan in, default is access key main folder id')
72
78
  options.declare(:mimemagic, 'Use Mime type detection of gem mimemagic', values: :bool, default: false)
@@ -232,6 +238,7 @@ module Aspera
232
238
  def get_infos_remote(gen_infos, entry)
233
239
  # store source directly here
234
240
  local_original_filepath = File.join(@tmp_folder, entry['name'])
241
+ # require 'date'
235
242
  # original_mtime=DateTime.parse(entry['modified_time'])
236
243
  # out: where previews are generated
237
244
  local_entry_preview_dir = File.join(@tmp_folder, entry_preview_folder_name(entry))
@@ -255,9 +262,10 @@ module Aspera
255
262
  "#{entry['id']}#{PREVIEW_FOLDER_SUFFIX}"
256
263
  end
257
264
 
258
- def preview_filename(preview_format, filename=nil)
259
- filename ||= PREVIEW_BASENAME
260
- return "#{filename}.#{preview_format}"
265
+ # Generate a file name based on basename and format (extension)
266
+ def preview_filename(preview_format, base_name=nil)
267
+ base_name ||= PREVIEW_BASENAME
268
+ return "#{base_name}.#{preview_format}"
261
269
  end
262
270
 
263
271
  # generate preview files for one folder entry (file) if necessary
@@ -290,10 +298,13 @@ module Aspera
290
298
  next false if gen_info[:preview_newer_than_original]
291
299
  end
292
300
  end
293
- # need generator for further checks
294
- gen_info[:generator] = Aspera::Preview::Generator.new(@gen_options, gen_info[:src], gen_info[:dst], @tmp_folder, entry['content_type'])
295
- # get conversion_type (if known) and check if supported
296
- next false unless gen_info[:generator].supported?
301
+ begin
302
+ # need generator for further checks
303
+ gen_info[:generator] = Aspera::Preview::Generator.new(gen_info[:src], gen_info[:dst], @gen_options, @tmp_folder, entry['content_type'])
304
+ rescue
305
+ # no conversion supported
306
+ next false
307
+ end
297
308
  # shall we skip it ?
298
309
  next false if @skip_types.include?(gen_info[:generator].conversion_type)
299
310
  # ok we need to generate
@@ -307,7 +318,7 @@ module Aspera
307
318
  # download original file to temp folder
308
319
  do_transfer(Fasp::TransferSpec::DIRECTION_RECEIVE, entry['parent_file_id'], entry['name'], @tmp_folder)
309
320
  end
310
- Log.log.info{"source: #{entry['id']}: #{entry['path']})"}
321
+ Log.log.info{"source: #{entry['id']}: #{entry['path']}"}
311
322
  gen_infos.each do |gen_info|
312
323
  gen_info[:generator].generate rescue nil
313
324
  end
@@ -328,24 +339,22 @@ module Aspera
328
339
  end # generate_preview
329
340
 
330
341
  # scan all files in provided folder entry
331
- # @param scan_start subpath to start folder scan inside
332
- def scan_folder_files(top_entry, scan_start=nil)
333
- if !scan_start.nil?
342
+ # @param top_path subpath to start folder scan inside
343
+ def scan_folder_files(top_entry, top_path=nil)
344
+ unless top_path.nil?
334
345
  # canonical path: start with / and ends with /
335
- scan_start = '/' + scan_start.split('/').reject(&:empty?).join('/')
336
- scan_start = "#{scan_start}/" # unless scan_start.end_with?('/')
346
+ top_path = '/' + top_path.split('/').reject(&:empty?).join('/') + '/'
337
347
  end
338
- filter_block = Aspera::Node.file_matcher(value_or_query(allowed_types: String))
339
- Log.log.debug{"scan: #{top_entry} : #{scan_start}".green}
348
+ Log.log.debug{"scan: #{top_entry} : #{top_path}".green}
340
349
  # don't use recursive call, use list instead
341
350
  entries_to_process = [top_entry]
342
351
  until entries_to_process.empty?
343
352
  entry = entries_to_process.shift
344
- # process this entry only if it is within the scan_start
353
+ # process this entry only if it is within the top_path
345
354
  entry_path_with_slash = entry['path']
346
355
  Log.log.info{"processing entry #{entry_path_with_slash}"} if @periodic.trigger?
347
356
  entry_path_with_slash = "#{entry_path_with_slash}/" unless entry_path_with_slash.end_with?('/')
348
- if !scan_start.nil? && !scan_start.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(scan_start)
357
+ if !top_path.nil? && !top_path.start_with?(entry_path_with_slash) && !entry_path_with_slash.start_with?(top_path)
349
358
  Log.log.debug{"#{entry['path']} folder (skip start)".bg_red}
350
359
  next
351
360
  end
@@ -353,7 +362,7 @@ module Aspera
353
362
  begin
354
363
  case entry['type']
355
364
  when 'file'
356
- if filter_block.call(entry)
365
+ if @filter_block.call(entry)
357
366
  generate_preview(entry)
358
367
  else
359
368
  Log.log.debug('skip by filter')
@@ -386,11 +395,11 @@ module Aspera
386
395
  end
387
396
  end
388
397
 
389
- ACTIONS = %i[scan events trevents check test].freeze
398
+ ACTIONS = %i[scan events trevents check test show].freeze
390
399
 
391
400
  def execute_action
392
401
  command = options.get_next_command(ACTIONS)
393
- unless %i[check test].include?(command)
402
+ unless %i[check test show].include?(command)
394
403
  # this will use node api
395
404
  @api_node = Aspera::Node.new(params: basic_auth_params)
396
405
  @transfer_server_address = URI.parse(@api_node.params[:base_url]).host
@@ -408,7 +417,7 @@ module Aspera
408
417
  if @access_remote
409
418
  # NOTE: the filter "name", it's why we take the first one
410
419
  @previews_folder_entry = get_folder_entries(@access_key_self['root_file_id'], {name: @option_previews_folder}).first
411
- raise CliError, "Folder #{@option_previews_folder} does not exist on node. "\
420
+ raise Cli::Error, "Folder #{@option_previews_folder} does not exist on node. " \
412
421
  'Please create it in the storage root, or specify an alternate name.' if @previews_folder_entry.nil?
413
422
  else
414
423
  raise 'only local storage allowed in this mode' unless @access_key_self['storage']['type'].eql?('local')
@@ -417,9 +426,9 @@ module Aspera
417
426
  @local_storage_root = @local_storage_root[PVCL_LOCAL_STORAGE.length..-1] if @local_storage_root.start_with?(PVCL_LOCAL_STORAGE)
418
427
  # TODO: windows could have "C:" ?
419
428
  raise "not local storage: #{@local_storage_root}" unless @local_storage_root.start_with?('/')
420
- raise CliError, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
429
+ raise Cli::Error, "Local storage root folder #{@local_storage_root} does not exist." unless File.directory?(@local_storage_root)
421
430
  @local_preview_folder = File.join(@local_storage_root, @option_previews_folder)
422
- raise CliError, "Folder #{@local_preview_folder} does not exist locally. "\
431
+ raise Cli::Error, "Folder #{@local_preview_folder} does not exist locally. " \
423
432
  'Please create it, or specify an alternate name.' unless File.directory?(@local_preview_folder)
424
433
  # protection to avoid clash of file id for two different access keys
425
434
  marker_file = File.join(@local_preview_folder, AK_MARKER_FILE)
@@ -450,9 +459,11 @@ module Aspera
450
459
  else
451
460
  @api_node.read("files/#{scan_id}")[:data]
452
461
  end
462
+ @filter_block = Aspera::Node.file_matcher_from_argument(options)
453
463
  scan_folder_files(folder_info, scan_path)
454
464
  return Main.result_status('scan finished')
455
465
  when :events, :trevents
466
+ @filter_block = Aspera::Node.file_matcher_from_argument(options)
456
467
  iteration_persistency = nil
457
468
  if options.get_option(:once_only, mandatory: true)
458
469
  iteration_persistency = PersistencyActionOnce.new(
@@ -470,15 +481,18 @@ module Aspera
470
481
  return Main.result_status("#{command} finished")
471
482
  when :check
472
483
  return Main.result_status('Tools validated')
473
- when :test
474
- format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS)
484
+ when :test, :show
475
485
  source = options.get_next_argument('source file')
476
- dest = preview_filename(format, options.get_option(:case))
477
- g = Aspera::Preview::Generator.new(@gen_options, source, dest, @tmp_folder, nil)
478
- raise "cannot find file type for #{source}" if g.conversion_type.nil?
479
- raise "out format #{format} not supported" unless g.supported?
486
+ format = options.get_next_argument('format', expected: Aspera::Preview::Generator::PREVIEW_FORMATS, default: :png)
487
+ generated_file_path = preview_filename(format, options.get_option(:base))
488
+ g = Aspera::Preview::Generator.new(source, generated_file_path, @gen_options, @tmp_folder, nil)
480
489
  g.generate
481
- return Main.result_status("generated: #{dest}")
490
+ if command.eql?(:show)
491
+ terminal_options = options.get_option(:query, default: {}).symbolize_keys
492
+ Log.log.debug{"preview: #{generated_file_path}"}
493
+ formatter.display_status(Aspera::Preview::Terminal.build(File.read(generated_file_path), **terminal_options))
494
+ end
495
+ return Main.result_status("generated: #{generated_file_path}")
482
496
  else
483
497
  raise 'error'
484
498
  end
@@ -1,9 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
+ # cspell:ignore ascmd zmode zuid zgid fasping
3
4
  require 'aspera/cli/basic_auth_plugin'
4
- require 'aspera/cli/plugins/sync'
5
- require 'aspera/ascmd'
5
+ require 'aspera/cli/sync_actions'
6
6
  require 'aspera/fasp/transfer_spec'
7
+ require 'aspera/ascmd'
7
8
  require 'aspera/ssh'
8
9
  require 'aspera/nagios'
9
10
  require 'tempfile'
@@ -13,17 +14,8 @@ module Aspera
13
14
  module Cli
14
15
  module Plugins
15
16
  # implement basic remote access with FASP/SSH
16
- class SyncSpecServer
17
- def initialize(transfer_spec)
18
- @transfer_spec = transfer_spec
19
- end
20
-
21
- def transfer_spec(direction, local_path, remote_path)
22
- return @transfer_spec
23
- end
24
- end
25
-
26
17
  class Server < Aspera::Cli::BasicAuthPlugin
18
+ include SyncActions
27
19
  SSH_SCHEME = 'ssh'
28
20
  LOCAL_SCHEME = 'local'
29
21
  HTTPS_SCHEME = 'https'
@@ -49,11 +41,59 @@ module Aspera
49
41
  end
50
42
  end
51
43
 
44
+ class << self
45
+ def application_name
46
+ 'HSTS Fasp/SSH'
47
+ end
48
+
49
+ def detect(address_or_url)
50
+ urls = if address_or_url.match?(%r{^[a-z]{1,6}://})
51
+ [address_or_url]
52
+ else
53
+ [
54
+ "ssh://#{address_or_url}:33001",
55
+ "ssh://#{address_or_url}:22"
56
+ ]
57
+ # wss not practical as it requires a token
58
+ end
59
+
60
+ urls.each do |base_url|
61
+ server_uri = URI.parse(base_url)
62
+ Log.log.debug{"URI=#{server_uri}, host=#{server_uri.hostname}, port=#{server_uri.port}, scheme=#{server_uri.scheme}"}
63
+ next unless server_uri.scheme.eql?(SSH_SCHEME)
64
+ begin
65
+ socket = TCPSocket.new(server_uri.hostname, server_uri.port)
66
+ socket.puts('SSH-2.0-Ascli_0.0')
67
+ version = socket.gets.chomp
68
+ if version.match?(/^SSH-2.0-/)
69
+ return {version: version.gsub(/^SSH-2.0-/, ''), url: base_url}
70
+ end
71
+ rescue StandardError => e
72
+ Log.log.debug{"detect error: #{e}"}
73
+ end
74
+ end
75
+ return nil
76
+ end
77
+
78
+ def wizard(object:, private_key_path: nil, pub_key_pem: nil)
79
+ options = object.options
80
+ return {
81
+ preset_value: {
82
+ url: options.get_option(:url, mandatory: true),
83
+ username: options.get_option(:username, mandatory: true),
84
+ password: options.get_option(:password, mandatory: true)
85
+ },
86
+ test_args: 'files br /'
87
+ }
88
+ end
89
+ end
90
+
52
91
  def initialize(env)
53
92
  super(env)
54
93
  options.declare(:ssh_keys, 'SSH key path list (Array or single)')
55
94
  options.declare(:passphrase, 'SSH private key passphrase')
56
95
  options.declare(:ssh_options, 'SSH options', types: Hash, default: {})
96
+ SyncActions.declare_options(options)
57
97
  options.parse_options!
58
98
  @ssh_opts = options.get_option(:ssh_options).symbolize_keys
59
99
  end
@@ -138,8 +178,8 @@ module Aspera
138
178
  Fasp::TransferSpec.action_to_direction(transfer_spec, command)
139
179
  return Main.result_transfer(transfer.start(transfer_spec))
140
180
  when :sync
141
- sync_plugin = Plugins::Sync.new(@agents, sync_spec: SyncSpecServer.new(transfer_spec))
142
- return sync_plugin.execute_action
181
+ # lets ignore the arguments provided by execute_sync_action, we just give the transfer spec
182
+ return execute_sync_action {transfer_spec}
143
183
  end
144
184
  end
145
185
 
@@ -191,10 +231,10 @@ module Aspera
191
231
  when *TRANSFER_COMMANDS
192
232
  return execute_transfer(command, server_transfer_spec)
193
233
  when *Aspera::AsCmd::OPERATIONS
194
- args = options.get_next_argument('ascmd command arguments', expected: :multiple, mandatory: false)
234
+ command_arguments = options.get_next_argument('ascmd command arguments', expected: :multiple, mandatory: false)
195
235
  ascmd = Aspera::AsCmd.new(ascmd_executor)
196
236
  begin
197
- result = ascmd.send(:execute_single, command, args)
237
+ result = ascmd.send(:execute_single, command, command_arguments)
198
238
  case command
199
239
  when :mkdir, :mv, :cp, :rm
200
240
  return Main.result_success
@@ -206,7 +246,7 @@ module Aspera
206
246
  return {type: :single_object, data: result.stringify_keys}
207
247
  end
208
248
  rescue Aspera::AsCmd::Error => e
209
- raise CliBadArgument, e.extended_message
249
+ raise Cli::BadArgument, e.extended_message
210
250
  end
211
251
  else raise 'internal error: unexpected action'
212
252
  end
@@ -7,20 +7,43 @@ module Aspera
7
7
  module Plugins
8
8
  # Plugin for Aspera Shares v1
9
9
  class Shares < Aspera::Cli::BasicAuthPlugin
10
+ API_BASE = 'node_api'
10
11
  class << self
11
- def detect(base_url)
12
- api = Rest.new({base_url: base_url})
13
- # Shares
12
+ def detect(address_or_url)
13
+ address_or_url = "https://#{address_or_url}" unless address_or_url.match?(%r{^[a-z]{1,6}://})
14
+ api = Rest.new(base_url: address_or_url, redirect_max: 1)
15
+ found = false
14
16
  begin
15
17
  # shall fail: shares requires auth, but we check error message
16
18
  # TODO: use ping instead ?
17
- api.read('node_api/app')
19
+ api.read("#{API_BASE}/app")
18
20
  rescue RestCallError => e
19
21
  if e.response.code.to_s.eql?('401') && e.response.body.eql?('{"error":{"user_message":"API user authentication failed"}}')
20
- return {version: 'unknown'}
22
+ found = true
21
23
  end
22
24
  end
23
- nil
25
+ return nil unless found
26
+ version = 'unknown'
27
+ test_page = api.call({ operation: 'GET', subpath: 'login' })
28
+ if (m = test_page[:http].body.match(/\(v(1\..*)\)/))
29
+ version = m[1]
30
+ end
31
+ return {
32
+ version: version,
33
+ url: address_or_url
34
+ }
35
+ end
36
+
37
+ def wizard(object:, private_key_path: nil, pub_key_pem: nil)
38
+ options = object.options
39
+ return {
40
+ preset_value: {
41
+ url: options.get_option(:url, mandatory: true),
42
+ username: options.get_option(:username, mandatory: true),
43
+ password: options.get_option(:password, mandatory: true)
44
+ },
45
+ test_args: 'files br /'
46
+ }
24
47
  end
25
48
  end
26
49
 
@@ -44,7 +67,7 @@ module Aspera
44
67
  nagios = Nagios.new
45
68
  begin
46
69
  Rest
47
- .new(base_url: options.get_option(:url, mandatory: true) + '/node_api')
70
+ .new(base_url: "#{options.get_option(:url, mandatory: true)}/#{API_BASE}")
48
71
  .call(
49
72
  operation: 'GET',
50
73
  subpath: 'ping',
@@ -56,9 +79,9 @@ module Aspera
56
79
  end
57
80
  return nagios.result
58
81
  when :repository, :files
59
- api_shares_node = basic_auth_api('node_api')
82
+ api_shares_node = basic_auth_api(API_BASE)
60
83
  repo_command = options.get_next_command(Node::COMMANDS_SHARES)
61
- return Node.new(@agents.merge(skip_basic_auth_options: true, node_api: api_shares_node)).execute_action(repo_command)
84
+ return Node.new(@agents, api: api_shares_node).execute_action(repo_command)
62
85
  when :admin
63
86
  api_shares_admin = basic_auth_api('api/v1')
64
87
  admin_command = options.get_next_command(%i[user group share node].freeze)
@@ -92,7 +115,7 @@ module Aspera
92
115
  display_fields.push(:directory_user) if entity_type.eql?(:user) && entities_location.eql?(:any)
93
116
  return entity_command(entity_verb, api_shares_admin, entities_path, display_fields: display_fields)
94
117
  when :import
95
- return do_bulk_operation(value_create_modify(type: :bulk_hash), 'created') do |entity_parameters|
118
+ return do_bulk_operation(command: entity_verb, descr: 'user information') do |entity_parameters|
96
119
  entity_parameters = entity_parameters.transform_keys{|k|k.gsub(/\s+/, '_').downcase}
97
120
  raise 'expecting Hash' unless entity_parameters.is_a?(Hash)
98
121
  SAML_IMPORT_MANDATORY.each{|p|raise "missing mandatory field: #{p}" if entity_parameters[p].nil?}
@@ -102,8 +125,7 @@ module Aspera
102
125
  api_shares_admin.create("#{entities_path}/import", entity_parameters)[:data]
103
126
  end
104
127
  when :add
105
- return do_bulk_operation(value_create_modify(type: :bulk_hash), 'created') do |entity_name|
106
- raise "expecting string (name), have #{entity_name.class}" unless entity_name.is_a?(String)
128
+ return do_bulk_operation(command: entity_verb, descr: "#{entity_type} name", values: String) do |entity_name|
107
129
  api_shares_admin.create(entities_path, {entity_type=>entity_name})[:data]
108
130
  end
109
131
  when *USR_GRP_SETTINGS
@@ -0,0 +1,68 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'aspera/sync'
4
+
5
+ module Aspera
6
+ module Cli
7
+ # Module for sync actions
8
+ module SyncActions
9
+ SIMPLE_ARGUMENTS_SYNC = {
10
+ direction: Aspera::Sync::DIRECTIONS,
11
+ local_dir: String,
12
+ remote_dir: String
13
+ }.stringify_keys.freeze
14
+
15
+ class << self
16
+ def declare_options(options)
17
+ options.declare(:sync_info, 'Information for sync instance and sessions', types: Hash)
18
+ end
19
+ end
20
+
21
+ def execute_sync_action(&block)
22
+ # options = Aspera::Cli::Manager.new
23
+ raise 'Internal Error: No block given' unless block
24
+ command = options.get_next_command(%i[start admin])
25
+ # try to get 3 arguments as simple arguments
26
+ case command
27
+ when :start
28
+ simple_session_args = {}
29
+ SIMPLE_ARGUMENTS_SYNC.each do |arg, check|
30
+ value = options.get_next_argument(
31
+ arg,
32
+ type: check.is_a?(Class) ? check : nil,
33
+ expected: check.is_a?(Class) ? :single : check,
34
+ mandatory: false)
35
+ break if value.nil?
36
+ simple_session_args[arg] = value.to_s
37
+ end
38
+ async_params = nil
39
+ if simple_session_args.empty?
40
+ async_params = options.get_option(:sync_info, mandatory: true)
41
+ else
42
+ raise Cli::BadArgument,
43
+ "Provide zero or 3 arguments: #{SIMPLE_ARGUMENTS_SYNC.keys.join(',')}" unless simple_session_args.keys.sort == SIMPLE_ARGUMENTS_SYNC.keys.sort
44
+ async_params = options.get_option(
45
+ :sync_info,
46
+ mandatory: false,
47
+ default: {'sessions' => [{'name' => File.basename(simple_session_args['local_dir'])}]})
48
+ raise "sync_info shall be a Hash with key 'sessions' with Array of Hash: #{async_params}" unless async_params.is_a?(Hash) &&
49
+ async_params['sessions']&.is_a?(Array) &&
50
+ async_params['sessions'].first.is_a?(Hash)
51
+ async_params['sessions'].first.merge!(simple_session_args)
52
+ end
53
+ Log.log.debug{Log.dump('async_params', async_params)}
54
+ Aspera::Sync.start(async_params, &block)
55
+ return Main.result_success
56
+ when :admin
57
+ command2 = options.get_next_command([:status])
58
+ case command2
59
+ when :status
60
+ sync_session_name = options.get_next_argument('name of sync session', mandatory: false, type: String)
61
+ async_params = options.get_option(:sync_info, mandatory: true)
62
+ return {type: :single_object, data: Aspera::Sync.admin_status(async_params, sync_session_name)}
63
+ end # command2
64
+ end # command
65
+ end # execute_action
66
+ end # SyncActions
67
+ end # Cli
68
+ end # Aspera