aspera-cli 4.25.1 → 4.25.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (54) hide show
  1. checksums.yaml +4 -4
  2. checksums.yaml.gz.sig +0 -0
  3. data/CHANGELOG.md +456 -405
  4. data/CONTRIBUTING.md +22 -18
  5. data/README.md +33 -9741
  6. data/bin/asession +111 -88
  7. data/lib/aspera/agent/connect.rb +1 -1
  8. data/lib/aspera/agent/desktop.rb +1 -1
  9. data/lib/aspera/agent/direct.rb +19 -18
  10. data/lib/aspera/agent/node.rb +1 -1
  11. data/lib/aspera/api/aoc.rb +44 -20
  12. data/lib/aspera/api/faspex.rb +25 -6
  13. data/lib/aspera/api/node.rb +20 -16
  14. data/lib/aspera/ascp/installation.rb +32 -51
  15. data/lib/aspera/assert.rb +2 -2
  16. data/lib/aspera/cli/extended_value.rb +1 -0
  17. data/lib/aspera/cli/formatter.rb +0 -4
  18. data/lib/aspera/cli/hints.rb +18 -4
  19. data/lib/aspera/cli/main.rb +3 -6
  20. data/lib/aspera/cli/manager.rb +46 -30
  21. data/lib/aspera/cli/plugins/aoc.rb +155 -131
  22. data/lib/aspera/cli/plugins/base.rb +15 -18
  23. data/lib/aspera/cli/plugins/config.rb +50 -87
  24. data/lib/aspera/cli/plugins/factory.rb +2 -2
  25. data/lib/aspera/cli/plugins/faspex.rb +4 -4
  26. data/lib/aspera/cli/plugins/faspex5.rb +74 -76
  27. data/lib/aspera/cli/plugins/node.rb +3 -5
  28. data/lib/aspera/cli/plugins/oauth.rb +26 -25
  29. data/lib/aspera/cli/plugins/preview.rb +9 -14
  30. data/lib/aspera/cli/plugins/shares.rb +15 -7
  31. data/lib/aspera/cli/transfer_agent.rb +2 -2
  32. data/lib/aspera/cli/version.rb +1 -1
  33. data/lib/aspera/colors.rb +7 -0
  34. data/lib/aspera/environment.rb +30 -16
  35. data/lib/aspera/faspex_gw.rb +6 -6
  36. data/lib/aspera/faspex_postproc.rb +20 -14
  37. data/lib/aspera/hash_ext.rb +8 -0
  38. data/lib/aspera/log.rb +15 -15
  39. data/lib/aspera/markdown.rb +22 -0
  40. data/lib/aspera/node_simulator.rb +1 -1
  41. data/lib/aspera/oauth/base.rb +2 -2
  42. data/lib/aspera/oauth/url_json.rb +2 -2
  43. data/lib/aspera/oauth/web.rb +1 -1
  44. data/lib/aspera/preview/generator.rb +9 -9
  45. data/lib/aspera/rest.rb +44 -37
  46. data/lib/aspera/rest_call_error.rb +16 -8
  47. data/lib/aspera/rest_error_analyzer.rb +38 -36
  48. data/lib/aspera/rest_errors_aspera.rb +19 -18
  49. data/lib/aspera/transfer/resumer.rb +2 -2
  50. data/lib/aspera/yaml.rb +49 -0
  51. data.tar.gz.sig +0 -0
  52. metadata +17 -3
  53. metadata.gz.sig +0 -0
  54. data/release_notes.md +0 -8
data/bin/asession CHANGED
@@ -6,95 +6,118 @@ require 'aspera/agent/direct'
6
6
  require 'aspera/cli/extended_value'
7
7
  require 'aspera/products/transferd'
8
8
  require 'aspera/log'
9
+ require 'aspera/assert'
9
10
  require 'json'
10
- # Extended transfer spec parameter (only used in asession)
11
- PARAM_SPEC = 'spec'
12
- # Log level
13
- PARAM_LOG_LEVEL = 'loglevel'
14
- # Transfer agent options
15
- PARAM_AGENT = 'agent'
16
- # By default go to /tmp/username.filelist
17
- PARAM_TMP_FILE_LIST_FOLDER = 'file_list_folder'
18
- PARAM_SDK = 'sdk'
19
- # Place transfer spec in that
20
- SAMPLE_DEMO = '"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ssh_port":33001,"remote_password":"demoaspera"'
21
- SAMPLE_DEMO2 = '"direction":"receive","destination_root":"./test.dir"'
22
- def assert_usage(assertion, error_message)
23
- return if assertion
24
- # rubocop:disable Style/StderrPuts
25
- $stderr.puts('ERROR: '.red.blink + error_message) if error_message
26
- $stderr.puts('USAGE')
27
- $stderr.puts(' asession')
28
- $stderr.puts(' asession -h|--help')
29
- $stderr.puts(' asession [<session spec extended value>]')
30
- $stderr.puts(' ')
31
- $stderr.puts(' If no argument is provided, default will be used: @json:@stdin')
32
- $stderr.puts(' -h, --help display this message')
33
- $stderr.puts(' <session spec extended value> a dictionary (Hash)')
34
- $stderr.puts(' The value can be either:')
35
- $stderr.puts(" the JSON description itself, e.g. @json:'{\"xx\":\"yy\",...}'")
36
- $stderr.puts(' @json:@stdin, if the JSON is provided from stdin')
37
- $stderr.puts(' @json:@file:<path>, if the JSON is provided from a file')
38
- $stderr.puts(' The following keys are recognized in session spec:')
39
- $stderr.puts(" #{PARAM_SPEC} : mandatory, contains the transfer spec")
40
- $stderr.puts(" #{PARAM_LOG_LEVEL} : modify log level (to stderr)")
41
- $stderr.puts(" #{PARAM_AGENT} : modify transfer agent parameters, e.g. ascp_args")
42
- $stderr.puts(" #{PARAM_TMP_FILE_LIST_FOLDER} : location of temporary files")
43
- $stderr.puts(" #{PARAM_SDK} : location of SDK (ascp)")
44
- $stderr.puts(' Asynchronous commands can be provided on STDIN, examples:')
45
- $stderr.puts(' {"type":"START","source":"/aspera-test-dir-tiny/200KB.2"}')
46
- $stderr.puts(' {"type":"START","source":"xx","destination":"yy"}')
47
- $stderr.puts(' {"type":"DONE"}')
48
- $stderr.puts('EXAMPLES')
49
- $stderr.puts(%Q( asession @json:'{"#{PARAM_SPEC}":{#{SAMPLE_DEMO},#{SAMPLE_DEMO2},"paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}}'))
50
- $stderr.puts(%Q( echo '{"#{PARAM_SPEC}":{"remote_host":...}}'|asession @json:@stdin))
51
- # rubocop:enable Style/StderrPuts
52
- Process.exit(0)
53
- end
54
- parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
55
- # By default assume JSON input on stdin if no argument
56
- if ARGV.empty?
57
- ARGV.push('@json:@stdin')
58
- parameter_source_err_msg = ' (JSON on stdin)'
11
+
12
+ class UsageError < StandardError; end
13
+
14
+ class SimpleTransferManager
15
+ # First level parameters in session spec
16
+ # - Extended transfer spec parameter (only used in asession)
17
+ PARAM_SPEC = 'spec'
18
+ # - Log level
19
+ PARAM_LOG_LEVEL = 'loglevel'
20
+ # - Transfer agent options
21
+ PARAM_AGENT = 'agent'
22
+ # - By default go to /tmp/username.filelist
23
+ PARAM_TMP_FILE_LIST_FOLDER = 'file_list_folder'
24
+ # - SDK location
25
+ PARAM_SDK = 'sdk'
26
+ # Default SDK location
27
+ SDK_DEFAULT_DIR = File.join(Dir.home, '.aspera', 'sdk')
28
+ # Place transfer spec in that
29
+ SAMPLE_DEMO = '"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ssh_port":33001,"remote_password":"demoaspera"'
30
+ SAMPLE_DEMO2 = '"direction":"receive","destination_root":"./test.dir"'
31
+ STDIN_INPUT = '@json:@stdin:'
32
+ class << self
33
+ # Assert usage condition, otherwise display usage message and exit
34
+ # @param assertion [Boolean] Assertion to verify
35
+ # @param error_message [String] Error message to display
36
+ def usage
37
+ # rubocop:disable Style/StderrPuts
38
+ $stderr.puts('USAGE')
39
+ $stderr.puts(' asession')
40
+ $stderr.puts(' asession -h|--help')
41
+ $stderr.puts(' asession [<session spec extended value>]')
42
+ $stderr.puts(' ')
43
+ $stderr.puts(' If no argument is provided, default will be used: @json:@stdin')
44
+ $stderr.puts(' -h, --help display this message')
45
+ $stderr.puts(' <session spec extended value> a dictionary (Hash)')
46
+ $stderr.puts(' The value can be either:')
47
+ $stderr.puts(" the JSON description itself, e.g. @json:'{\"xx\":\"yy\",...}'")
48
+ $stderr.puts(' @json:@stdin, if the JSON is provided from stdin')
49
+ $stderr.puts(' @json:@file:<path>, if the JSON is provided from a file')
50
+ $stderr.puts(' The following keys are recognized in session spec:')
51
+ $stderr.puts(" #{PARAM_SPEC} : mandatory, contains the transfer spec")
52
+ $stderr.puts(" #{PARAM_LOG_LEVEL} : modify log level (to stderr)")
53
+ $stderr.puts(" #{PARAM_AGENT} : modify transfer agent parameters, e.g. ascp_args")
54
+ $stderr.puts(" #{PARAM_TMP_FILE_LIST_FOLDER} : location of temporary files")
55
+ $stderr.puts(" #{PARAM_SDK} : location of SDK (ascp)")
56
+ $stderr.puts(' Asynchronous commands can be provided on STDIN, examples:')
57
+ $stderr.puts(' {"type":"START","source":"/aspera-test-dir-tiny/200KB.2"}')
58
+ $stderr.puts(' {"type":"START","source":"xx","destination":"yy"}')
59
+ $stderr.puts(' {"type":"DONE"}')
60
+ $stderr.puts('EXAMPLES')
61
+ $stderr.puts(%Q( asession @json:'{"#{PARAM_SPEC}":{#{SAMPLE_DEMO},#{SAMPLE_DEMO2},"paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}}'))
62
+ $stderr.puts(%Q( echo '{"#{PARAM_SPEC}":{"remote_host":...}}'|asession #{STDIN_INPUT}))
63
+ # rubocop:enable Style/StderrPuts
64
+ end
65
+ end
66
+ def initialize(args)
67
+ parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
68
+ # By default assume JSON input on stdin if no argument
69
+ if args.empty?
70
+ args.push(STDIN_INPUT)
71
+ parameter_source_err_msg = ' (JSON on stdin)'
72
+ end
73
+ # Anyway expect only one argument: session information
74
+ Aspera.assert(args.length.eql?(1), 'exactly one argument is expected', type: UsageError)
75
+ if ['-h', '--help'].include?(args.first)
76
+ SimpleTransferManager.usage
77
+ exit(0)
78
+ end
79
+ # Parse transfer spec
80
+ @session_spec = Aspera::Cli::ExtendedValue.instance.evaluate(args.pop, context: 'asession parameter')
81
+ # Ensure right type for parameter
82
+ Aspera.assert(@session_spec.is_a?(Hash), "The value must be a Hash#{parameter_source_err_msg}", type: UsageError)
83
+ Aspera.assert(@session_spec[PARAM_SPEC].is_a?(Hash), "The value must contain key #{PARAM_SPEC} with Hash value", type: UsageError)
84
+ # Additional debug capability
85
+ Aspera::Log.instance.level = @session_spec[PARAM_LOG_LEVEL].to_sym if @session_spec.key?(PARAM_LOG_LEVEL)
86
+ # Possibly override temp folder
87
+ Aspera::Transfer::Parameters.file_list_folder = @session_spec[PARAM_TMP_FILE_LIST_FOLDER] if @session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
88
+ Aspera::Products::Transferd.sdk_directory = @session_spec[PARAM_SDK] || SDK_DEFAULT_DIR
89
+ agent_params = @session_spec[PARAM_AGENT] || {}
90
+ agent_params['quiet'] = true
91
+ agent_params['management_cb'] = ->(event) do
92
+ puts JSON.generate(Aspera::Ascp::Management.event_native_to_snake(event))
93
+ end
94
+ # Get local agent (ascp), disable ascp output on stdout to not mix with JSON events
95
+ @client = Aspera::Agent::Direct.new(**agent_params.symbolize_keys)
96
+ end
97
+
98
+ def start
99
+ # Start transfer (asynchronous)
100
+ @client.start_transfer(@session_spec[PARAM_SPEC])
101
+ # commands to ascp on mgt port
102
+ Thread.new do
103
+ loop do
104
+ data = JSON.parse($stdin.gets)
105
+ @client.send_command(data)
106
+ end
107
+ rescue => e
108
+ Aspera::Log.log.error("Error reading commands from stdin: #{e.message}")
109
+ Process.exit(1)
110
+ end
111
+ # No exit code: status is success (0)
112
+ @client.wait_for_transfers_completion
113
+ @client.shutdown
114
+ end
59
115
  end
60
- # Anyway expect only one argument: session information
61
- assert_usage(ARGV.length.eql?(1), 'exactly one argument is expected')
62
- assert_usage(!['-h', '--help'].include?(ARGV.first), nil)
63
- # Parse transfer spec
116
+
64
117
  begin
65
- session_argument = ARGV.pop
66
- session_spec = Aspera::Cli::ExtendedValue.instance.evaluate(session_argument)
67
- rescue
68
- assert_usage(false, "Cannot parse argument: #{session_argument}")
69
- end
70
- # Ensure right type for parameter
71
- assert_usage(session_spec.is_a?(Hash), "The value must be a Hash#{parameter_source_err_msg}")
72
- assert_usage(session_spec[PARAM_SPEC].is_a?(Hash), "The value must contain key #{PARAM_SPEC} with Hash value")
73
- # Additional debug capability
74
- Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL].to_sym if session_spec.key?(PARAM_LOG_LEVEL)
75
- # Possibly override temp folder
76
- Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER] if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
77
- session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
78
- Aspera::Products::Transferd.sdk_directory = session_spec[PARAM_SDK]
79
- session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
80
- agent_params = session_spec[PARAM_AGENT]
81
- agent_params['quiet'] = true
82
- agent_params['management_cb'] = ->(event) do
83
- puts JSON.generate(Aspera::Ascp::Management.event_native_to_snake(event))
84
- end
85
- # Get local agent (ascp), disable ascp output on stdout to not mix with JSON events
86
- client = Aspera::Agent::Direct.new(**agent_params.symbolize_keys)
87
- # Start transfer (asynchronous)
88
- client.start_transfer(session_spec[PARAM_SPEC])
89
- # commands to ascp on mgt port
90
- Thread.new do
91
- loop do
92
- data = JSON.parse($stdin.gets)
93
- client.send_command(data)
94
- end
95
- rescue
96
- Process.exit(1)
118
+ SimpleTransferManager.new(ARGV).start
119
+ rescue UsageError => e
120
+ Aspera::Log.log.error(e.message)
121
+ SimpleTransferManager.usage
122
+ exit(1)
97
123
  end
98
- # No exit code: status is success (0)
99
- client.wait_for_transfers_completion
100
- client.shutdown
@@ -32,7 +32,7 @@ module Aspera
32
32
  headers: {'Origin' => RestParameters.instance.user_agent}
33
33
  )
34
34
  connect_info = @connect_api.read('info/version')
35
- Log.log.info('Connect was reached') if method_index > 0
35
+ Log.log.debug('Connect was reached') if method_index > 0
36
36
  Log.dump(:connect_version, connect_info)
37
37
  rescue StandardError => e # Errno::ECONNREFUSED
38
38
  Log.log.debug{"Exception: #{e}"}
@@ -30,7 +30,7 @@ module Aspera
30
30
  @client_app_api = Aspera::JsonRpcClient.new(Aspera::Rest.new(base_url: aspera_client_api_url))
31
31
  client_info = @client_app_api.get_info
32
32
  Log.dump(:client_version, client_info)
33
- Log.log.info('Client was reached') if method_index > 0
33
+ Log.log.debug('Client was reached') if method_index > 0
34
34
  rescue Errno::ECONNREFUSED => e
35
35
  start_url = START_URIS[method_index]
36
36
  method_index += 1
@@ -16,9 +16,9 @@ require 'English'
16
16
 
17
17
  module Aspera
18
18
  module Agent
19
- # executes a local "ascp", create mgt port
19
+ # Execute a local `ascp` and use its management port to monitor progress
20
20
  class Direct < Base
21
- # ascp started locally, so listen local
21
+ # `ascp` started locally, so listen local
22
22
  LISTEN_LOCAL_ADDRESS = '127.0.0.1'
23
23
  # 0 means: use any available port
24
24
  SELECT_AVAILABLE_PORT = 0
@@ -48,13 +48,13 @@ module Aspera
48
48
  spawn_timeout_sec: 2,
49
49
  spawn_delay_sec: 2,
50
50
  multi_incr_udp: nil,
51
- resume: nil,
51
+ resume: {},
52
52
  monitor: true,
53
53
  management_cb: nil,
54
54
  **base_options
55
55
  )
56
56
  super(**base_options)
57
- # Special transfer parameters provided
57
+ # Options that have impact on `ascp` command line generated
58
58
  @tr_opts = {
59
59
  ascp_args: ascp_args,
60
60
  wss: wss,
@@ -69,7 +69,6 @@ module Aspera
69
69
  @multi_incr_udp = multi_incr_udp.nil? ? Environment.instance.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
70
70
  @monitor = monitor
71
71
  @management_cb = management_cb
72
- resume = {} if resume.nil?
73
72
  Aspera.assert_type(resume, Hash){'resume'}
74
73
  @resume_policy = Transfer::Resumer.new(**resume.symbolize_keys)
75
74
  # all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
@@ -78,6 +77,7 @@ module Aspera
78
77
  @mutex = Mutex.new
79
78
  @pre_calc_sent = false
80
79
  @pre_calc_last_size = nil
80
+ # Check on all management messages if that file exists, and if so, read commands from it
81
81
  @command_file = File.join(config_dir || '.', "send_#{$PROCESS_ID}")
82
82
  end
83
83
 
@@ -115,7 +115,7 @@ module Aspera
115
115
  Log.log.debug('multi_session count is zero: no multi session')
116
116
  multi_session_info = nil
117
117
  elsif @multi_incr_udp # multi_session_info[:count] > 0
118
- # if option not true: keep default udp port for all sessions
118
+ # if option not `true`: keep default UDP port for all sessions
119
119
  multi_session_info[:udp_base] = transfer_spec.key?('fasp_port') ? transfer_spec['fasp_port'] : Transfer::Spec::UDP_PORT
120
120
  # delete from original transfer spec, as we will increment values
121
121
  transfer_spec.delete('fasp_port')
@@ -225,9 +225,9 @@ module Aspera
225
225
  start_and_monitor_process(session: session, **session[:env_args])
226
226
  end
227
227
  Log.log.debug('transfer ok'.bg_green)
228
- rescue StandardError => e
228
+ rescue => e
229
229
  session[:error] = e
230
- Log.log.error{"Transfer thread error: #{e.class}:\n#{e.message}:\n#{e.backtrace.join("\n")}".red} if Log.instance.level.eql?(:debug)
230
+ raise if Log.log.debug? || !e.is_a?(Transfer::Error)
231
231
  end
232
232
  Log.log.debug{"EXIT (#{Thread.current[:name]})"}
233
233
  end
@@ -242,7 +242,7 @@ module Aspera
242
242
  # @param env [Hash] Environment variables (comes from ascp_args)
243
243
  # @param args [Array] Command line arguments (comes from ascp_args)
244
244
  # @return [nil] when process has exited
245
- # @throw FaspError on error
245
+ # @raise FaspError on error
246
246
  def start_and_monitor_process(
247
247
  session:,
248
248
  name:,
@@ -251,15 +251,15 @@ module Aspera
251
251
  )
252
252
  Aspera.assert_type(session, Hash)
253
253
  notify_progress(:sessions_init, info: 'starting')
254
+ # Do not use `capture_stderr`
255
+ capture_stderr = false
256
+ stderr_r, stderr_w = nil
257
+ spawn_args = {}
258
+ command_pid = nil
259
+ # Get location of command executable (ascp, async)
260
+ command_path = Ascp::Installation.instance.path(name)
261
+ command_arguments = [command_path]
254
262
  begin
255
- # do not use
256
- capture_stderr = false
257
- stderr_r, stderr_w = nil
258
- spawn_args = {}
259
- command_pid = nil
260
- # get location of command executable (ascp, async)
261
- command_path = Ascp::Installation.instance.path(name)
262
- command_arguments = [command_path]
263
263
  if @monitor
264
264
  # we use Socket directly, instead of TCPServer, as it gives access to lower level options
265
265
  socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
@@ -353,7 +353,7 @@ module Aspera
353
353
  Process.kill(:INT, command_pid) if @monitor && !Environment.instance.os.eql?(Environment::OS_WINDOWS)
354
354
  # collect process exit status or wait for termination
355
355
  _, status = Process.wait2(command_pid)
356
- if stderr_r
356
+ if capture_stderr
357
357
  # process stderr of ascp
358
358
  stderr_flag = false
359
359
  stderr_r.each_line do |line|
@@ -379,6 +379,7 @@ module Aspera
379
379
 
380
380
  private
381
381
 
382
+ # [Array<Hash>] List of sessions, one per `ascp` process
382
383
  attr_reader :sessions
383
384
 
384
385
  # Notify progress to callback
@@ -77,7 +77,7 @@ module Aspera
77
77
  # status is empty sometimes with status 200...
78
78
  transfer_data = node_api_.read("ops/transfers/#{@transfer_id}") || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
79
79
  case transfer_data['status']
80
- when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
80
+ when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)', 'waiting(api error)'
81
81
  notify_progress(:sessions_init, info: transfer_data['status'])
82
82
  when 'running'
83
83
  if !session_started
@@ -17,7 +17,7 @@ module Aspera
17
17
  DEFAULT_WORKSPACE = ''
18
18
  # Production domain of AoC
19
19
  SAAS_DOMAIN_PROD = 'ibmaspera.com' # cspell:disable-line
20
- # to avoid infinite loop in pub link redirection
20
+ # To avoid infinite loop in pub link redirection
21
21
  MAX_AOC_URL_REDIRECT = 10
22
22
  CLIENT_ID_PREFIX = 'aspera.'
23
23
  # Well-known AoC global client apps
@@ -47,7 +47,7 @@ module Aspera
47
47
  :PERMISSIONS_CREATED,
48
48
  :ID_AK_ADMIN
49
49
 
50
- # various API scopes supported
50
+ # Various API scopes supported
51
51
  module Scope
52
52
  SELF = 'self'
53
53
  USER = 'user:all'
@@ -142,11 +142,14 @@ module Aspera
142
142
  }
143
143
  end
144
144
 
145
- # Call block with same query using paging and response information.
146
- # Block must return an Array with data and http response
147
- # @return [Hash] {items: , total: }
148
- def call_paging(query: nil, formatter: nil)
149
- query = {} if query.nil?
145
+ # Call `block` with same query using paging and response information.
146
+ # Block must return a 2 element `Array` with data and http response
147
+ # @param query [Hash] Additionnal query parameters
148
+ # @param progress [nil, Object] Uses methods: `long_operation_running` and `long_operation_terminated`
149
+ # @return [Hash] Items and total number of items
150
+ # @option return [Array<Hash>] :items The list of items
151
+ # @option return [Integer] :total The total number of items
152
+ def call_paging(query: {}, progress: nil)
150
153
  Aspera.assert_type(query, Hash){'query'}
151
154
  Aspera.assert(block_given?)
152
155
  # set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
@@ -174,9 +177,9 @@ module Aspera
174
177
  break if !max_items.nil? && item_list.count >= max_items
175
178
  break if !max_pages.nil? && page_count >= max_pages
176
179
  break if total_count&.<=(item_list.count)
177
- formatter&.long_operation_running("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
180
+ progress&.long_operation_running("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
178
181
  end
179
- formatter&.long_operation_terminated
182
+ progress&.long_operation_terminated
180
183
  item_list = item_list[0..max_items - 1] if !max_items.nil? && item_list.count > max_items
181
184
  return {items: item_list, total: total_count}
182
185
  end
@@ -195,6 +198,23 @@ module Aspera
195
198
  def workspace_access?(permission)
196
199
  permission['access_id'].start_with?("#{ID_AK_ADMIN}_WS_")
197
200
  end
201
+
202
+ # Expand access levels to full list of levels.
203
+ # @param levels [nil, String, Array] Access levels
204
+ # @return [Array] Expanded access levels
205
+ def expand_access_levels(levels)
206
+ case levels
207
+ when nil, 'edit' then Node::ACCESS_LEVELS
208
+ when 'preview' then %w[list preview]
209
+ when 'download' then %w[list preview read]
210
+ when 'upload' then %w[mkdir write]
211
+ when Array
212
+ Aspera.assert_array_all(levels, String){'access_levels'}
213
+ levels.each{ |level| Aspera.assert_value(level, Node::ACCESS_LEVELS){'access_level'}}
214
+ levels
215
+ else Aspera.error_unexpected_value(levels){"access_levels must be a list of #{Node::ACCESS_LEVELS.join(', ')} or one of edit, preview, download, upload"}
216
+ end
217
+ end
198
218
  end
199
219
 
200
220
  attr_reader :private_link
@@ -212,7 +232,8 @@ module Aspera
212
232
  username: nil,
213
233
  password: nil,
214
234
  workspace: nil,
215
- secret_finder: nil
235
+ secret_finder: nil,
236
+ progress_disp: nil
216
237
  )
217
238
  # Test here because link may set url
218
239
  Aspera.assert(url, 'Missing mandatory option: url', type: ParameterError)
@@ -223,6 +244,7 @@ module Aspera
223
244
  # key: access key
224
245
  # value: associated secret
225
246
  @secret_finder = secret_finder
247
+ @progress_disp = progress_disp
226
248
  @workspace_name = workspace
227
249
  @cache_user_info = nil
228
250
  @cache_url_token_info = nil
@@ -282,10 +304,12 @@ module Aspera
282
304
  )
283
305
  end
284
306
 
285
- # read using the query and paging
307
+ # Read using the query and paging
308
+ # @param subpath [String] Entity path
309
+ # @param query [nil, Hash] Additional query
286
310
  # @return [Hash] {items: , total: }
287
- def read_with_paging(subpath, query = nil, formatter: nil)
288
- return self.class.call_paging(query: query, formatter: formatter) do |paged_query|
311
+ def read_with_paging(subpath, query = nil)
312
+ return self.class.call_paging(query: query, progress: @progress_disp) do |paged_query|
289
313
  read(subpath, query: paged_query, ret: :both)
290
314
  end
291
315
  end
@@ -316,9 +340,9 @@ module Aspera
316
340
  @cache_user_info =
317
341
  begin
318
342
  read('self')
319
- rescue StandardError => e
320
- raise e if exception
321
- Log.log.debug{"ignoring error: #{e}"}
343
+ rescue Aspera::RestCallError => e
344
+ raise if exception || e.message.include?('invalid_grant')
345
+ Log.log.debug{"Ignoring error: (#{e.class}) #{e}"}
322
346
  {}
323
347
  end
324
348
  USER_INFO_FIELDS_MIN.each{ |f| @cache_user_info[f] = nil if @cache_user_info[f].nil?}
@@ -394,10 +418,10 @@ module Aspera
394
418
  # @param node_id [String] identifier of node in AoC
395
419
  # @param workspace_id [String,nil] workspace identifier
396
420
  # @param workspace_name [String,nil] workspace name
397
- # @param scope [String,nil] e.g. Node::SCOPE_USER, or Node::SCOPE_ADMIN, or nil (requires secret)
421
+ # @param scope [String,nil] e.g. Node::Scope::USER, or Node::Scope::ADMIN, or nil (requires secret)
398
422
  # @param package_info [Hash,nil] created package information
399
423
  # @returns [Node] a node API for access key
400
- def node_api_from(node_id:, workspace_id: nil, workspace_name: nil, scope: Node::SCOPE_USER, package_info: nil)
424
+ def node_api_from(node_id:, workspace_id: nil, workspace_name: nil, scope: Node::Scope::USER, package_info: nil)
401
425
  Aspera.assert_type(node_id, String)
402
426
  node_info = read("nodes/#{node_id}")
403
427
  workspace_name = read("workspaces/#{workspace_id}")['name'] if workspace_name.nil? && !workspace_id.nil?
@@ -428,7 +452,7 @@ module Aspera
428
452
  node_params[:auth] = auth_params.clone
429
453
  node_params[:auth][:params] ||= {}
430
454
  node_params[:auth][:params][:scope] = Node.token_scope(node_info['access_key'], scope)
431
- node_params[:auth][:params][:owner_access] = true if scope.eql?(Node::SCOPE_ADMIN)
455
+ node_params[:auth][:params][:owner_access] = true if scope.eql?(Node::Scope::ADMIN)
432
456
  # Special header required for bearer token only
433
457
  node_params[:headers] = {Node::HEADER_X_ASPERA_ACCESS_KEY => node_info['access_key']}
434
458
  end
@@ -693,7 +717,7 @@ module Aspera
693
717
  # (optional). The name of the folder to be displayed to the destination user.
694
718
  # Use it if its value is different from the "share_as" field.
695
719
  event_creation['link_name'] = app_info[:opt_link_name] unless app_info[:opt_link_name].nil?
696
- create('events', event_creation)
720
+ create('events', event_creation, query: {admin: true})
697
721
  end
698
722
  end
699
723
  end
@@ -60,19 +60,32 @@ module Aspera
60
60
  PATH_AUTH = 'auth'
61
61
  PATH_API_V5 = 'api/v5'
62
62
  PATH_HEALTH = 'configuration/ping'
63
- private_constant :PATH_API_V5,
64
- :PATH_HEALTH,
65
- :PATH_AUTH
63
+ private_constant :PATH_AUTH,
64
+ :PATH_API_V5,
65
+ :PATH_HEALTH
66
66
  RECIPIENT_TYPES = %w[user workgroup external_user distribution_list shared_inbox].freeze
67
67
  PACKAGE_TERMINATED = %w[completed failed].freeze
68
68
  # list of supported mailbox types (to list packages)
69
- API_LIST_MAILBOX_TYPES = %w[inbox inbox_history inbox_all inbox_all_history outbox outbox_history pending pending_history all].freeze
69
+ SENT_MAILBOX_TYPES = %w[outbox outbox_history].freeze
70
+ API_LIST_MAILBOX_TYPES = (%w[inbox inbox_history inbox_all inbox_all_history pending pending_history all] + SENT_MAILBOX_TYPES).freeze
70
71
  # PACKAGE_SEND_FROM_REMOTE_SOURCE = 'remote_source'
71
72
  # Faspex API v5: get transfer spec for connect
72
73
  TRANSFER_CONNECT = 'connect'
73
74
  ADMIN_RESOURCES = %i[
74
- accounts distribution_lists contacts jobs workgroups shared_inboxes nodes oauth_clients registrations saml_configs
75
- metadata_profiles email_notifications alternate_addresses webhooks
75
+ accounts
76
+ distribution_lists
77
+ contacts
78
+ jobs
79
+ workgroups
80
+ shared_inboxes
81
+ nodes
82
+ oauth_clients
83
+ registrations
84
+ saml_configs
85
+ metadata_profiles
86
+ email_notifications
87
+ alternate_addresses
88
+ webhooks
76
89
  ].freeze
77
90
  # states for jobs not in final state
78
91
  JOB_RUNNING = %w[queued working].freeze
@@ -114,6 +127,12 @@ module Aspera
114
127
  def public_link?(url)
115
128
  url.include?('?context=')
116
129
  end
130
+
131
+ # Depending on box, the package files are either: `received` or `sent`
132
+ # @return [:sent, :received] the type of mailbox
133
+ def box_type(box)
134
+ SENT_MAILBOX_TYPES.include?(box) || box == 'ALL' ? :sent : :received
135
+ end
117
136
  end
118
137
  attr_reader :pub_link_context
119
138
 
@@ -16,9 +16,15 @@ module Aspera
16
16
  module Api
17
17
  # Provides additional functions using node API with gen4 extensions (access keys)
18
18
  class Node < Aspera::Rest
19
- # Separator between node.AK and user:all
20
- SCOPE_SEPARATOR = ':'
21
- SCOPE_NODE_PREFIX = 'node.'
19
+ # Format of node scope : node.<access key>:<scope>
20
+ module Scope
21
+ # Node sub-scopes
22
+ USER = 'user:all'
23
+ ADMIN = 'admin:all'
24
+ # Separator between node.AK and user:all
25
+ SEPARATOR = ':'
26
+ NODE_PREFIX = 'node.'
27
+ end
22
28
  # Accepted types in `file_matcher`
23
29
  MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
24
30
  # Delimiter in decoded node token
@@ -29,20 +35,16 @@ module Aspera
29
35
  REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
30
36
  # Methods of @app_info[:api]
31
37
  REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
32
- private_constant :SCOPE_SEPARATOR, :SCOPE_NODE_PREFIX, :MATCH_TYPES,
38
+ private_constant :MATCH_TYPES,
33
39
  :SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
34
40
  :REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
35
-
36
- # Node API permissions
41
+ # Node API permissions: delete list mkdir preview read rename write
37
42
  ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
38
43
  # Special HTTP Headers
39
44
  HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
40
45
  HEADER_X_TOTAL_COUNT = 'X-Total-Count'
41
46
  HEADER_X_CACHE_CONTROL = 'X-Aspera-Cache-Control'
42
47
  HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
43
- # Node sub-scopes
44
- SCOPE_USER = 'user:all'
45
- SCOPE_ADMIN = 'admin:all'
46
48
  # / in cloud
47
49
  PATH_SEPARATOR = '/'
48
50
 
@@ -63,7 +65,7 @@ module Aspera
63
65
  # Adds cache control header, as globally specified to read request
64
66
  # Use like this: read(...,**cache_control)
65
67
  def cache_control
66
- headers = {'Accept' => Rest::MIME_JSON}
68
+ headers = {'Accept' => Mime::JSON}
67
69
  headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless use_node_cache
68
70
  {headers: headers}
69
71
  end
@@ -126,16 +128,16 @@ module Aspera
126
128
  # Node API scopes
127
129
  # @return [String] node scope
128
130
  def token_scope(access_key, scope)
129
- return [SCOPE_NODE_PREFIX, access_key, SCOPE_SEPARATOR, scope].join('')
131
+ return [Scope::NODE_PREFIX, access_key, Scope::SEPARATOR, scope].join('')
130
132
  end
131
133
 
132
134
  # Decode node scope into access key and scope
133
135
  # @return [Hash]
134
136
  def decode_scope(scope)
135
- items = scope.split(SCOPE_SEPARATOR, 2)
137
+ items = scope.split(Scope::SEPARATOR, 2)
136
138
  Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
137
- Aspera.assert(items[0].start_with?(SCOPE_NODE_PREFIX)){"invalid scope: #{scope}"}
138
- return {access_key: items[0][SCOPE_NODE_PREFIX.length..-1], scope: items[1]}
139
+ Aspera.assert(items[0].start_with?(Scope::NODE_PREFIX)){"invalid scope: #{scope}"}
140
+ return {access_key: items[0][Scope::NODE_PREFIX.length..-1], scope: items[1]}
139
141
  end
140
142
 
141
143
  # Create an Aspera Node bearer token
@@ -151,7 +153,7 @@ module Aspera
151
153
  # Manage convenience parameters
152
154
  expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
153
155
  payload.delete('_validity')
154
- scope = payload['_scope'] || SCOPE_USER
156
+ scope = payload['_scope'] || Scope::USER
155
157
  payload.delete('_scope')
156
158
  payload['scope'] ||= token_scope(access_key, scope)
157
159
  payload['auth_type'] ||= 'access_key'
@@ -306,7 +308,9 @@ module Aspera
306
308
  # @param top_file_id [String] id initial file id
307
309
  # @param path [String] file or folder path (end with "/" is like setting process_last_link)
308
310
  # @param process_last_link [Boolean] if true, follow the last link
309
- # @return [Hash] {.api,.file_id}
311
+ # @return [Hash] Result data
312
+ # @option return [Aspera::Rest] :api REST client instance
313
+ # @option return [String] :file_id File identifier
310
314
  def resolve_api_fid(top_file_id, path, process_last_link = false)
311
315
  Aspera.assert_type(top_file_id, String)
312
316
  Aspera.assert_type(path, String)