aspera-cli 4.24.1 → 4.24.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (87) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +15 -2
  4. data/README.md +745 -436
  5. data/bin/ascli +20 -1
  6. data/bin/asession +23 -27
  7. data/lib/aspera/agent/base.rb +10 -21
  8. data/lib/aspera/agent/connect.rb +2 -3
  9. data/lib/aspera/agent/desktop.rb +2 -2
  10. data/lib/aspera/agent/direct.rb +49 -32
  11. data/lib/aspera/agent/factory.rb +31 -0
  12. data/lib/aspera/api/aoc.rb +79 -49
  13. data/lib/aspera/api/faspex.rb +212 -0
  14. data/lib/aspera/api/node.rb +99 -84
  15. data/lib/aspera/ascp/installation.rb +22 -21
  16. data/lib/aspera/ascp/management.rb +119 -23
  17. data/lib/aspera/assert.rb +14 -8
  18. data/lib/aspera/cli/extended_value.rb +15 -15
  19. data/lib/aspera/cli/formatter.rb +7 -5
  20. data/lib/aspera/cli/hints.rb +8 -0
  21. data/lib/aspera/cli/info.rb +4 -4
  22. data/lib/aspera/cli/main.rb +55 -70
  23. data/lib/aspera/cli/manager.rb +7 -4
  24. data/lib/aspera/cli/plugins/alee.rb +2 -1
  25. data/lib/aspera/cli/plugins/aoc.rb +110 -186
  26. data/lib/aspera/cli/plugins/ats.rb +4 -4
  27. data/lib/aspera/cli/plugins/base.rb +335 -0
  28. data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
  29. data/lib/aspera/cli/plugins/config.rb +249 -220
  30. data/lib/aspera/cli/plugins/console.rb +15 -15
  31. data/lib/aspera/cli/plugins/cos.rb +2 -2
  32. data/lib/aspera/cli/plugins/factory.rb +78 -0
  33. data/lib/aspera/cli/plugins/faspex.rb +17 -20
  34. data/lib/aspera/cli/plugins/faspex5.rb +79 -193
  35. data/lib/aspera/cli/plugins/faspio.rb +14 -13
  36. data/lib/aspera/cli/plugins/httpgw.rb +13 -12
  37. data/lib/aspera/cli/plugins/node.rb +34 -32
  38. data/lib/aspera/cli/plugins/oauth.rb +48 -0
  39. data/lib/aspera/cli/plugins/orchestrator.rb +15 -13
  40. data/lib/aspera/cli/plugins/preview.rb +4 -4
  41. data/lib/aspera/cli/plugins/server.rb +15 -13
  42. data/lib/aspera/cli/plugins/shares.rb +18 -15
  43. data/lib/aspera/cli/sync_actions.rb +1 -1
  44. data/lib/aspera/cli/transfer_agent.rb +24 -20
  45. data/lib/aspera/cli/transfer_progress.rb +6 -6
  46. data/lib/aspera/cli/version.rb +3 -3
  47. data/lib/aspera/cli/wizard.rb +65 -53
  48. data/lib/aspera/colors.rb +6 -0
  49. data/lib/aspera/command_line_builder.rb +45 -50
  50. data/lib/aspera/command_line_converter.rb +2 -1
  51. data/lib/aspera/coverage.rb +1 -1
  52. data/lib/aspera/data_repository.rb +1 -1
  53. data/lib/aspera/environment.rb +10 -7
  54. data/lib/aspera/faspex_gw.rb +6 -4
  55. data/lib/aspera/faspex_postproc.rb +1 -1
  56. data/lib/aspera/keychain/macos_security.rb +1 -1
  57. data/lib/aspera/log.rb +37 -9
  58. data/lib/aspera/nagios.rb +1 -1
  59. data/lib/aspera/oauth/base.rb +17 -10
  60. data/lib/aspera/oauth/factory.rb +8 -8
  61. data/lib/aspera/oauth/web.rb +2 -2
  62. data/lib/aspera/products/connect.rb +4 -3
  63. data/lib/aspera/products/desktop.rb +1 -4
  64. data/lib/aspera/products/other.rb +9 -1
  65. data/lib/aspera/products/transferd.rb +0 -1
  66. data/lib/aspera/rest.rb +126 -83
  67. data/lib/aspera/ssh.rb +3 -3
  68. data/lib/aspera/sync/args.schema.yaml +46 -3
  69. data/lib/aspera/sync/conf.schema.yaml +130 -94
  70. data/lib/aspera/sync/operations.rb +16 -16
  71. data/lib/aspera/temp_file_manager.rb +17 -5
  72. data/lib/aspera/transfer/error.rb +16 -7
  73. data/lib/aspera/transfer/parameters.rb +34 -20
  74. data/lib/aspera/transfer/resumer.rb +74 -0
  75. data/lib/aspera/transfer/spec.rb +4 -3
  76. data/lib/aspera/transfer/spec.schema.yaml +132 -51
  77. data/lib/aspera/transfer/spec_doc.rb +41 -35
  78. data/lib/aspera/uri_reader.rb +1 -1
  79. data/lib/aspera/web_auth.rb +6 -6
  80. data.tar.gz.sig +0 -0
  81. metadata +9 -7
  82. metadata.gz.sig +0 -0
  83. data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
  84. data/lib/aspera/cli/plugin.rb +0 -333
  85. data/lib/aspera/cli/plugin_factory.rb +0 -81
  86. data/lib/aspera/resumer.rb +0 -77
  87. data/lib/aspera/transfer/error_info.rb +0 -91
data/bin/ascli CHANGED
@@ -1,6 +1,8 @@
1
1
  #!/usr/bin/env ruby
2
2
  # frozen_string_literal: true
3
3
 
4
+ # This is the main script to execute the Aspera CLI
5
+
4
6
  old_verbose = $VERBOSE
5
7
  $VERBOSE = nil
6
8
  # internal representation of strings (ruby) is UTF-8
@@ -9,9 +11,26 @@ Encoding.default_internal = Encoding::UTF_8
9
11
  Encoding.default_external = Encoding::UTF_8
10
12
  $VERBOSE = old_verbose
11
13
 
14
+ require 'aspera/log'
15
+ require 'aspera/cli/info'
16
+
17
+ # Early debug for parser
18
+ # Note: does not accept short option names, nor extended values
19
+ Aspera::Log.instance.program_name = Aspera::Cli::Info::CMD_NAME
20
+ ARGV.each do |arg|
21
+ case arg
22
+ when '--' then break
23
+ when /^--log-level=(.*)/ then Aspera::Log.instance.level = Regexp.last_match(1).to_sym
24
+ when /^--log-format=(.*)/ then Aspera::Log.instance.formatter = Regexp.last_match(1) unless Regexp.last_match(1).start_with?('@ruby:')
25
+ when /^--logger=(.*)/ then Aspera::Log.instance.logger_type = Regexp.last_match(1).to_sym
26
+ end
27
+ rescue => e
28
+ $stderr.puts("Error: #{e}") # rubocop:disable Style/StderrPuts
29
+ Process.exit(1)
30
+ end
31
+
12
32
  require 'aspera/coverage'
13
33
  require 'aspera/environment'
14
34
  require 'aspera/cli/main'
15
- Aspera::Cli::Main.early_debug_setup(ARGV)
16
35
  Aspera::Environment.instance.fix_home
17
36
  Aspera::Cli::Main.new(ARGV).process_command_line
data/bin/asession CHANGED
@@ -7,16 +7,16 @@ require 'aspera/cli/extended_value'
7
7
  require 'aspera/products/transferd'
8
8
  require 'aspera/log'
9
9
  require 'json'
10
- # extended transfer spec parameter (only used in asession)
10
+ # Extended transfer spec parameter (only used in asession)
11
11
  PARAM_SPEC = 'spec'
12
- # log level
12
+ # Log level
13
13
  PARAM_LOG_LEVEL = 'loglevel'
14
- # transfer agent options
14
+ # Transfer agent options
15
15
  PARAM_AGENT = 'agent'
16
- # by default go to /tmp/username.filelist
16
+ # By default go to /tmp/username.filelist
17
17
  PARAM_TMP_FILE_LIST_FOLDER = 'file_list_folder'
18
18
  PARAM_SDK = 'sdk'
19
- # place transfer spec in that
19
+ # Place transfer spec in that
20
20
  SAMPLE_DEMO = '"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ssh_port":33001,"remote_password":"demoaspera"'
21
21
  SAMPLE_DEMO2 = '"direction":"receive","destination_root":"./test.dir"'
22
22
  def assert_usage(assertion, error_message)
@@ -52,27 +52,27 @@ def assert_usage(assertion, error_message)
52
52
  Process.exit(0)
53
53
  end
54
54
  parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
55
- # by default assume JSON input on stdin if no argument
55
+ # By default assume JSON input on stdin if no argument
56
56
  if ARGV.empty?
57
57
  ARGV.push('@json:@stdin')
58
58
  parameter_source_err_msg = ' (JSON on stdin)'
59
59
  end
60
- # anyway expect only one argument: session information
60
+ # Anyway expect only one argument: session information
61
61
  assert_usage(ARGV.length.eql?(1), 'exactly one argument is expected')
62
62
  assert_usage(!['-h', '--help'].include?(ARGV.first), nil)
63
- # parse transfer spec
63
+ # Parse transfer spec
64
64
  begin
65
65
  session_argument = ARGV.pop
66
66
  session_spec = Aspera::Cli::ExtendedValue.instance.evaluate(session_argument)
67
67
  rescue
68
68
  assert_usage(false, "Cannot parse argument: #{session_argument}")
69
69
  end
70
- # ensure right type
70
+ # Ensure right type for parameter
71
71
  assert_usage(session_spec.is_a?(Hash), "The value must be a Hash#{parameter_source_err_msg}")
72
72
  assert_usage(session_spec[PARAM_SPEC].is_a?(Hash), "The value must contain key #{PARAM_SPEC} with Hash value")
73
- # additional debug capability
74
- Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL] if session_spec.key?(PARAM_LOG_LEVEL)
75
- # possibly override temp folder
73
+ # Additional debug capability
74
+ Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL].to_sym if session_spec.key?(PARAM_LOG_LEVEL)
75
+ # Possibly override temp folder
76
76
  Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER] if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
77
77
  session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
78
78
  Aspera::Products::Transferd.sdk_directory = session_spec[PARAM_SDK]
@@ -80,25 +80,21 @@ session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
80
80
  agent_params = session_spec[PARAM_AGENT]
81
81
  agent_params['quiet'] = true
82
82
  agent_params['management_cb'] = ->(event) do
83
- puts JSON.generate(Aspera::Ascp::Management.enhanced_event_format(event))
83
+ puts JSON.generate(Aspera::Ascp::Management.event_native_to_snake(event))
84
84
  end
85
- # get local agent (ascp), disable ascp output on stdout to not mix with JSON events
85
+ # Get local agent (ascp), disable ascp output on stdout to not mix with JSON events
86
86
  client = Aspera::Agent::Direct.new(**agent_params.symbolize_keys)
87
- # start transfer (asynchronous)
88
- job_id = client.start_transfer(session_spec[PARAM_SPEC])
89
- # async commands
87
+ # Start transfer (asynchronous)
88
+ client.start_transfer(session_spec[PARAM_SPEC])
89
+ # commands to ascp on mgt port
90
90
  Thread.new do
91
- # we assume here a single session
92
- session_id = client.sessions_by_job(job_id).first
93
- begin
94
- loop do
95
- data = JSON.parse($stdin.gets)
96
- client.send_command(session_id, data)
97
- end
98
- rescue
99
- Process.exit(1)
91
+ loop do
92
+ data = JSON.parse($stdin.gets)
93
+ client.send_command(data)
100
94
  end
95
+ rescue
96
+ Process.exit(1)
101
97
  end
102
- # no exit code: status is success (0)
98
+ # No exit code: status is success (0)
103
99
  client.wait_for_transfers_completion
104
100
  client.shutdown
@@ -1,6 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'aspera/log'
4
3
  require 'aspera/assert'
5
4
  module Aspera
6
5
  module Agent
@@ -10,25 +9,6 @@ module Aspera
10
9
  # - `wait_for_transfers_completion` : waits for all transfer sessions to finish
11
10
  # - `notify_progress` : called back by transfer agent to notify transfer progress
12
11
  class Base
13
- RUBY_EXT = '.rb'
14
- private_constant :RUBY_EXT
15
- class << self
16
- def factory_create(agent, options)
17
- # Aspera.assert_values(agent, agent_list)
18
- require "aspera/agent/#{agent}"
19
- Aspera::Agent.const_get(agent.to_s.capitalize).new(**options)
20
- end
21
-
22
- # discover available agents
23
- # @return [Array] list of symbols of agents
24
- def agent_list
25
- base_class = File.basename(__FILE__)
26
- Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
27
- file.end_with?(RUBY_EXT) && !file.eql?(base_class)
28
- end.map{ |file| file[0..(-1 - RUBY_EXT.length)].to_sym}
29
- end
30
- end
31
-
32
12
  # Wait for all sessions to terminate and return the status of each session
33
13
  def wait_for_completion
34
14
  # list of: :success or "error message string"
@@ -48,8 +28,17 @@ module Aspera
48
28
  nil
49
29
  end
50
30
 
51
- def initialize(progress: nil)
31
+ attr_reader :config_dir
32
+
33
+ # Base transfer agent object
34
+ # @param progress [Object] Progress bar
35
+ # @param config_dir [String] Config folder
36
+ def initialize(
37
+ progress: nil,
38
+ config_dir: nil
39
+ )
52
40
  @progress = progress
41
+ @config_dir = config_dir
53
42
  end
54
43
 
55
44
  def notify_progress(*pos_args, **kw_args)
@@ -24,6 +24,7 @@ module Aspera
24
24
  raise Error, 'Using connect requires a graphical environment' unless Environment.instance.graphical?
25
25
  method_index = 0
26
26
  begin
27
+ # raise exception if connect not started and file does not exist
27
28
  connect_url = connect_api_url
28
29
  Log.log.debug{"found: #{connect_url}"}
29
30
  @connect_api = Rest.new(
@@ -135,9 +136,7 @@ module Aspera
135
136
 
136
137
  # @return the file path of local connect where API's URI can be read
137
138
  def connect_api_url
138
- connect_locations = Products::Other.find(Products::Connect.locations).first
139
- raise "Product: #{name} not found, please install." if connect_locations.nil?
140
- folder = File.join(connect_locations[:run_root], 'var', 'run')
139
+ folder = File.join(Products::Other.find(Products::Connect.locations).first[:run_root], 'var', 'run')
141
140
  ['', 's'].each do |ext|
142
141
  uri_file = File.join(folder, "http#{ext}.uri")
143
142
  Log.log.debug{"checking connect port file: #{uri_file}"}
@@ -102,7 +102,7 @@ module Aspera
102
102
 
103
103
  # @return [String] the url where transferd is listening
104
104
  def aspera_client_api_url
105
- log_file = Products::Desktop.log_file
105
+ log_file = File.join(Products::Other.find(Products::Desktop.locations).first[:log_root], Products::Desktop::LOG_FILENAME)
106
106
  url = 'http://127.0.0.1:33024'
107
107
  File.open(log_file, 'r') do |file|
108
108
  file.each_line do |line|
@@ -111,7 +111,7 @@ module Aspera
111
111
  url = "http://#{m[1]}"
112
112
  end
113
113
  end
114
- end
114
+ end if File.exist?(log_file)
115
115
  # raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
116
116
  return url
117
117
  end
@@ -6,7 +6,7 @@ require 'aspera/ascp/management'
6
6
  require 'aspera/transfer/parameters'
7
7
  require 'aspera/transfer/error'
8
8
  require 'aspera/transfer/spec'
9
- require 'aspera/resumer'
9
+ require 'aspera/transfer/resumer'
10
10
  require 'aspera/log'
11
11
  require 'aspera/assert'
12
12
  require 'socket'
@@ -24,18 +24,18 @@ module Aspera
24
24
  SELECT_AVAILABLE_PORT = 0
25
25
  private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
26
26
 
27
- # options for initialize (same as values in option transfer_info)
28
- # @param ascp_args [Array] additional arguments to ascp
29
- # @param wss [Boolean] true: if both SSH and wss in ts: prefer wss
30
- # @param quiet [Boolean] by default no native ascp progress bar
31
- # @param monitor [Boolean] set to false to eliminate management port
32
- # @param trusted_certs [Array,NilClass] list of files with trusted certificates (stores)
33
- # @param client_ssh_key [String] client ssh key option (from CLIENT_SSH_KEY_OPTIONS)
34
- # @param check_ignore_cb [Proc] callback with host,port
35
- # @param spawn_timeout_sec [Integer] timeout for ascp spawn
36
- # @param spawn_delay_sec [Integer] optional delay to start between sessions
37
- # @param multi_incr_udp [Boolean,NilClass] true: increment udp port for each session
38
- # @param resume [Hash,NilClass] resume policy
27
+ # Options: same as values in option `transfer_info`
28
+ # @param ascp_args [Array] (Params) Optional Additional arguments to ascp
29
+ # @param wss [Boolean] (Params) `true`: if both SSH and wss in ts: prefer wss
30
+ # @param quiet [Boolean] (Params) By default no native `ascp` progress bar
31
+ # @param monitor [Boolean] (Params) Set to `false` to eliminate management port
32
+ # @param trusted_certs [Array] (Params) Optional list of files with trusted certificates (stores)
33
+ # @param client_ssh_key [String] (Params) Client SSH key option (from CLIENT_SSH_KEY_OPTIONS)
34
+ # @param check_ignore_cb [Proc] (Params) Callback with host,port
35
+ # @param spawn_timeout_sec [Integer] Timeout for ascp spawn
36
+ # @param spawn_delay_sec [Integer] Optional delay to start between sessions
37
+ # @param multi_incr_udp [Boolean] Optional `true`: increment UDP port for each session
38
+ # @param resume [Hash] Optional Resume policy
39
39
  # @param management_cb [Proc] callback for management events
40
40
  # @param base_options [Hash] other options for base class
41
41
  def initialize(
@@ -54,7 +54,7 @@ module Aspera
54
54
  **base_options
55
55
  )
56
56
  super(**base_options)
57
- # special transfer parameters
57
+ # Special transfer parameters provided
58
58
  @tr_opts = {
59
59
  ascp_args: ascp_args,
60
60
  wss: wss,
@@ -69,19 +69,22 @@ module Aspera
69
69
  @multi_incr_udp = multi_incr_udp.nil? ? Environment.instance.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
70
70
  @monitor = monitor
71
71
  @management_cb = management_cb
72
- @resume_policy = Resumer.new(resume.nil? ? {} : resume.symbolize_keys)
72
+ resume = {} if resume.nil?
73
+ Aspera.assert_type(resume, Hash){'resume'}
74
+ @resume_policy = Transfer::Resumer.new(**resume.symbolize_keys)
73
75
  # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
74
76
  @sessions = []
75
77
  # mutex protects global data accessed by threads
76
78
  @mutex = Mutex.new
77
79
  @pre_calc_sent = false
78
80
  @pre_calc_last_size = nil
81
+ @command_file = File.join(config_dir || '.', "send_#{$PROCESS_ID}")
79
82
  end
80
83
 
81
- # start ascp transfer(s) (non blocking), single or multi-session
82
- # session information added to @sessions
83
- # @param transfer_spec [Hash] aspera transfer specification
84
- # @param token_regenerator [Object] object with method refreshed_transfer_token
84
+ # Start `ascp` transfer(s) (non blocking), single or multi-session
85
+ # Session information added to @sessions
86
+ # @param transfer_spec [Hash] Aspera transfer specification
87
+ # @param token_regenerator [Object] Object with method refreshed_transfer_token
85
88
  def start_transfer(transfer_spec, token_regenerator: nil)
86
89
  # clone transfer spec because we modify it (first level keys)
87
90
  transfer_spec = transfer_spec.clone
@@ -188,16 +191,23 @@ module Aspera
188
191
  @sessions.select{ |session| session[:job_id].eql?(job_id)}
189
192
  end
190
193
 
191
- # send command to management port of command (used in `asession)
192
- # @param job_id identified transfer process
193
- # @param session_index index of session (for multi session)
194
- # @param data command on mgt port, examples:
194
+ # Send command to management port of command (used in `asession).
195
+ # Examples:
195
196
  # {'type'=>'START','source'=>_path_,'destination'=>_path_}
196
197
  # {'type'=>'DONE'}
197
- def send_command(job_id, data)
198
- session = @sessions.find{ |session| session[:job_id].eql?(job_id)}
199
- Log.log.debug{"command: #{data}"}
200
- session[:io].puts(Ascp::Management.command_to_stream(data))
198
+ # @param data [Hash] Command on mgt port
199
+ # @param id [String] Optional identifier or transfer session
200
+ def send_command(data, id: nil)
201
+ Log.dump(:command, data)
202
+ sessions = id ? @sessions.select{ |session| session[:job_id].eql?(id)} : @sessions
203
+ if sessions.empty?
204
+ Log.log.warn('No transfer session')
205
+ return
206
+ end
207
+ message = Ascp::Management.command_to_stream(data)
208
+ sessions.each do |session|
209
+ session[:io].puts(message)
210
+ end
201
211
  end
202
212
 
203
213
  private
@@ -301,6 +311,14 @@ module Aspera
301
311
  session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
302
312
  @management_cb&.call(event)
303
313
  process_progress(event)
314
+ next unless File.exist?(@command_file)
315
+ begin
316
+ commands = JSON.parse(File.read(@command_file))
317
+ send_command(commands)
318
+ rescue => e
319
+ Log.log.error{e.to_s}
320
+ end
321
+ File.delete(@command_file)
304
322
  end
305
323
  Log.log.debug('management io closed')
306
324
  # check that last status was received before process exit
@@ -317,7 +335,7 @@ module Aspera
317
335
  Log.log.warn('Regenerating token for transfer')
318
336
  env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
319
337
  end
320
- raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
338
+ raise Transfer::Error.new(last_event['Description'], code: last_event['Code'].to_i)
321
339
  else Aspera.error_unexpected_value(last_event['Type'], :error){'last event type'}
322
340
  end
323
341
  rescue SystemCallError => e
@@ -347,7 +365,7 @@ module Aspera
347
365
  # status is nil if an exception occurred before starting command
348
366
  if !status&.success?
349
367
  message = "#{name} failed (#{status})"
350
- # raise error only if there was not already an exception (ERROR_INFO)
368
+ # raise error only if there was not already an exception (`$ERROR_INFO`)
351
369
  raise Transfer::Error, message unless $ERROR_INFO
352
370
  # else display this message also, as main exception is already here
353
371
  Log.log.error(message)
@@ -361,9 +379,8 @@ module Aspera
361
379
 
362
380
  attr_reader :sessions
363
381
 
364
- # notify progress to callback
365
- # @param event management port event
366
- # @param session sessin object
382
+ # Notify progress to callback
383
+ # @param event [Hash] management port event
367
384
  def process_progress(event)
368
385
  session_id = event['SessionId']
369
386
  case event['Type']
@@ -0,0 +1,31 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'singleton'
4
+ require 'aspera/log'
5
+ require 'aspera/environment'
6
+ module Aspera
7
+ module Agent
8
+ # Factory for Agents
9
+ class Factory
10
+ include Singleton
11
+
12
+ # Create new agent
13
+ def create(agent, options)
14
+ Log.dump(:options, options)
15
+ require "aspera/agent/#{agent}"
16
+ Aspera::Agent.const_get(agent.to_s.capitalize).new(**options)
17
+ end
18
+
19
+ # Discover available agents
20
+ # @return [Array] list of symbols of agents
21
+ def list
22
+ Dir.children(File.dirname(File.expand_path(__FILE__)))
23
+ .select{ |file| file.end_with?(Environment::RB_EXT)}
24
+ .map{ |file| File.basename(file, Environment::RB_EXT).to_sym}
25
+ .reject{ |item| IGNORED_ITEMS.include?(item)}
26
+ end
27
+ IGNORED_ITEMS = %i[factory base]
28
+ private_constant :IGNORED_ITEMS
29
+ end
30
+ end
31
+ end
@@ -8,11 +8,10 @@ require 'aspera/data_repository'
8
8
  require 'aspera/transfer/spec'
9
9
  require 'aspera/api/node'
10
10
  require 'base64'
11
- require 'cgi'
12
11
 
13
12
  module Aspera
14
13
  module Api
15
- class AoC < Aspera::Rest
14
+ class AoC < Rest
16
15
  PRODUCT_NAME = 'Aspera on Cloud'
17
16
  # use default workspace if it is set, else none
18
17
  DEFAULT_WORKSPACE = ''
@@ -140,6 +139,43 @@ module Aspera
140
139
  organization: org_domain[:organization]
141
140
  }
142
141
  end
142
+
143
+ # Call block with same query using paging and response information.
144
+ # Block must return a hash with :data and :http keys
145
+ # @return [Hash] {items: , total: }
146
+ def call_paging(query: {}, formatter: nil)
147
+ Aspera.assert_type(query, Hash){'query'}
148
+ Aspera.assert(block_given?)
149
+ # set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
150
+ query['per_page'] = 1000 unless query.key?('per_page')
151
+ max_items = query.delete(Rest::MAX_ITEMS)
152
+ max_pages = query.delete(Rest::MAX_PAGES)
153
+ item_list = []
154
+ total_count = nil
155
+ current_page = query['page']
156
+ current_page = 1 if current_page.nil?
157
+ page_count = 0
158
+ loop do
159
+ new_query = query.clone
160
+ new_query['page'] = current_page
161
+ result = yield(new_query)
162
+ Aspera.assert(result[:data])
163
+ Aspera.assert(result[:http])
164
+ total_count = result[:http]['X-Total-Count']
165
+ page_count += 1
166
+ current_page += 1
167
+ add_items = result[:data]
168
+ break if add_items.empty?
169
+ # append new items to full list
170
+ item_list += add_items
171
+ break if !max_items.nil? && item_list.count >= max_items
172
+ break if !max_pages.nil? && page_count >= max_pages
173
+ formatter&.long_operation_running("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
174
+ end
175
+ formatter&.long_operation_terminated
176
+ item_list = item_list[0..max_items - 1] if !max_items.nil? && item_list.count > max_items
177
+ return {items: item_list, total: total_count}
178
+ end
143
179
  end
144
180
 
145
181
  attr_reader :private_link
@@ -147,8 +183,8 @@ module Aspera
147
183
  def initialize(url:, auth:, subpath: API_V1, client_id: nil, client_secret: nil, scope: nil, redirect_uri: nil, private_key: nil, passphrase: nil, username: nil,
148
184
  password: nil, workspace: nil, secret_finder: nil)
149
185
  # test here because link may set url
150
- raise ArgumentError, 'Missing mandatory option: url' if url.nil?
151
- raise ArgumentError, 'Missing mandatory option: scope' if scope.nil?
186
+ raise ParameterError, 'Missing mandatory option: url' if url.nil?
187
+ raise ParameterError, 'Missing mandatory option: scope' if scope.nil?
152
188
  # default values for client id
153
189
  client_id, client_secret = self.class.get_client_info if client_id.nil?
154
190
  # access key secrets are provided out of band to get node api access
@@ -173,22 +209,21 @@ module Aspera
173
209
  auth_params[:grant_method] = if url_info.key?(:token)
174
210
  :url_json
175
211
  else
176
- raise ArgumentError, 'Missing mandatory option: auth' if auth.nil?
212
+ raise ParameterError, 'Missing mandatory option: auth' if auth.nil?
177
213
  auth
178
214
  end
179
215
  # this is the base API url
180
216
  api_url_base = self.class.api_base_url(api_domain: url_info[:instance_domain])
181
217
  # auth URL
182
218
  auth_params[:base_url] = "#{api_url_base}/#{OAUTH_API_SUBPATH}/#{url_info[:organization]}"
183
-
184
219
  # fill other auth parameters based on OAuth method
185
220
  case auth_params[:grant_method]
186
221
  when :web
187
- raise ArgumentError, 'Missing mandatory option: redirect_uri' if redirect_uri.nil?
222
+ raise ParameterError, 'Missing mandatory option: redirect_uri' if redirect_uri.nil?
188
223
  auth_params[:redirect_uri] = redirect_uri
189
224
  when :jwt
190
- raise ArgumentError, 'Missing mandatory option: private_key' if private_key.nil?
191
- raise ArgumentError, 'Missing mandatory option: username' if username.nil?
225
+ raise ParameterError, 'Missing mandatory option: private_key' if private_key.nil?
226
+ raise ParameterError, 'Missing mandatory option: username' if username.nil?
192
227
  auth_params[:private_key_obj] = OpenSSL::PKey::RSA.new(private_key, passphrase)
193
228
  auth_params[:payload] = {
194
229
  iss: auth_params[:client_id], # issuer
@@ -205,7 +240,7 @@ module Aspera
205
240
  auth_params[:json][:password] = password unless password.nil?
206
241
  # basic auth required for /token
207
242
  auth_params[:auth] = {type: :basic, username: auth_params[:client_id], password: auth_params[:client_secret]}
208
- else Aspera.error_unexpected_value(auth_params[:grant_method])
243
+ else Aspera.error_unexpected_value(auth_params[:grant_method]){'auth, use one of: :web, :jwt'}
209
244
  end
210
245
  super(
211
246
  base_url: "#{api_url_base}/#{subpath}",
@@ -213,12 +248,13 @@ module Aspera
213
248
  )
214
249
  end
215
250
 
216
- def public_link
217
- return unless auth_params[:grant_method].eql?(:url_json)
218
- return @cache_url_token_info unless @cache_url_token_info.nil?
219
- # TODO: can there be several in list ?
220
- @cache_url_token_info = read('url_tokens').first
221
- return @cache_url_token_info
251
+ # read using the query and paging
252
+ # @return [Hash] {items: , total: }
253
+ def read_with_paging(subpath, query: {}, formatter: nil)
254
+ return self.class.call_paging(query: query, formatter: formatter) do |paged_query|
255
+ # Use `call` instead of `read` to get headers
256
+ call(operation: 'GET', subpath: subpath, headers: {'Accept' => Rest::MIME_JSON}, query: paged_query)
257
+ end
222
258
  end
223
259
 
224
260
  def assert_public_link_types(expected)
@@ -230,7 +266,17 @@ module Aspera
230
266
  return [] # TODO : public_link['id'] ?
231
267
  end
232
268
 
233
- # cached user information
269
+ # Cached public link information
270
+ # @return [Hash, nil] token info if public link or nil
271
+ def public_link
272
+ return unless auth_params[:grant_method].eql?(:url_json)
273
+ return @cache_url_token_info unless @cache_url_token_info.nil?
274
+ # TODO: can there be several in list ?
275
+ @cache_url_token_info = read('url_tokens').first
276
+ return @cache_url_token_info
277
+ end
278
+
279
+ # Cached user information
234
280
  def current_user_info(exception: false)
235
281
  return @cache_user_info unless @cache_user_info.nil?
236
282
  # get our user's default information
@@ -246,21 +292,9 @@ module Aspera
246
292
  return @cache_user_info
247
293
  end
248
294
 
295
+ # Cached workspace information
249
296
  def workspace
250
- Aspera.assert(!@workspace_info.nil?){'AoC workspace context is not set'}
251
- @workspace_info
252
- end
253
-
254
- def home
255
- Aspera.assert(!@home_info.nil?){'AoC home context is not set'}
256
- @home_info
257
- end
258
-
259
- # Set the application context
260
- # @param application [Symbol,NilClass] :files or :packages
261
- # @return [Hash] current context information: workspace, and home node/file if app is "Files"
262
- def context=(application)
263
- Aspera.assert_values(application, %i[files packages])
297
+ return @workspace_info unless @workspace_info.nil?
264
298
  ws_id =
265
299
  if !public_link.nil?
266
300
  Log.log.debug('Using workspace of public link')
@@ -278,26 +312,21 @@ module Aspera
278
312
  else
279
313
  lookup_by_name('workspaces', @workspace_name)['id']
280
314
  end
281
- ws_info =
282
- if ws_id.nil?
283
- nil
284
- else
285
- read("workspaces/#{ws_id}")
286
- end
287
315
  @workspace_info =
288
- if ws_info.nil?
316
+ if ws_id.nil?
289
317
  {
290
- id: nil,
291
- name: "Shared #{application}"
318
+ name: 'Shared (no workspace)'
292
319
  }
293
320
  else
294
- {
295
- id: ws_info['id'],
296
- name: ws_info['name']
297
- }
321
+ read("workspaces/#{ws_id}").slice('id', 'name', 'home_node_id', 'home_file_id').symbolize_keys
298
322
  end
299
- Log.dump(:context, @workspace_info)
300
- return unless application.eql?(:files)
323
+ Log.dump(:workspace_info, @workspace_info)
324
+ @workspace_info
325
+ end
326
+
327
+ # Cached Home information for current user in Files app
328
+ def home
329
+ return @home_info unless @home_info.nil?
301
330
  @home_info =
302
331
  if !public_link.nil?
303
332
  assert_public_link_types(['view_shared_file'])
@@ -310,10 +339,10 @@ module Aspera
310
339
  node_id: private_link[:node_id],
311
340
  file_id: private_link[:file_id]
312
341
  }
313
- elsif ws_info
342
+ elsif workspace[:home_node_id] && workspace[:home_file_id]
314
343
  {
315
- node_id: ws_info['home_node_id'],
316
- file_id: ws_info['home_file_id']
344
+ node_id: workspace[:home_node_id],
345
+ file_id: workspace[:home_file_id]
317
346
  }
318
347
  else
319
348
  # not part of any workspace, but has some folder shared
@@ -325,6 +354,7 @@ module Aspera
325
354
  end
326
355
  raise "Cannot get user's home node id, check your default workspace or specify one" if @home_info[:node_id].to_s.empty?
327
356
  Log.dump(:context, @home_info)
357
+ @home_info
328
358
  end
329
359
 
330
360
  # @param node_id [String] identifier of node in AoC