aspera-cli 4.25.1 → 4.25.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +456 -405
- data/CONTRIBUTING.md +22 -18
- data/README.md +33 -9741
- data/bin/asession +111 -88
- data/lib/aspera/agent/connect.rb +1 -1
- data/lib/aspera/agent/desktop.rb +1 -1
- data/lib/aspera/agent/direct.rb +19 -18
- data/lib/aspera/agent/node.rb +1 -1
- data/lib/aspera/api/aoc.rb +44 -20
- data/lib/aspera/api/faspex.rb +25 -6
- data/lib/aspera/api/node.rb +20 -16
- data/lib/aspera/ascp/installation.rb +32 -51
- data/lib/aspera/assert.rb +2 -2
- data/lib/aspera/cli/extended_value.rb +1 -0
- data/lib/aspera/cli/formatter.rb +0 -4
- data/lib/aspera/cli/hints.rb +18 -4
- data/lib/aspera/cli/main.rb +3 -6
- data/lib/aspera/cli/manager.rb +46 -30
- data/lib/aspera/cli/plugins/aoc.rb +155 -131
- data/lib/aspera/cli/plugins/base.rb +15 -18
- data/lib/aspera/cli/plugins/config.rb +50 -87
- data/lib/aspera/cli/plugins/factory.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +4 -4
- data/lib/aspera/cli/plugins/faspex5.rb +74 -76
- data/lib/aspera/cli/plugins/node.rb +3 -5
- data/lib/aspera/cli/plugins/oauth.rb +26 -25
- data/lib/aspera/cli/plugins/preview.rb +9 -14
- data/lib/aspera/cli/plugins/shares.rb +15 -7
- data/lib/aspera/cli/transfer_agent.rb +2 -2
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +7 -0
- data/lib/aspera/environment.rb +30 -16
- data/lib/aspera/faspex_gw.rb +6 -6
- data/lib/aspera/faspex_postproc.rb +20 -14
- data/lib/aspera/hash_ext.rb +8 -0
- data/lib/aspera/log.rb +15 -15
- data/lib/aspera/markdown.rb +22 -0
- data/lib/aspera/node_simulator.rb +1 -1
- data/lib/aspera/oauth/base.rb +2 -2
- data/lib/aspera/oauth/url_json.rb +2 -2
- data/lib/aspera/oauth/web.rb +1 -1
- data/lib/aspera/preview/generator.rb +9 -9
- data/lib/aspera/rest.rb +44 -37
- data/lib/aspera/rest_call_error.rb +16 -8
- data/lib/aspera/rest_error_analyzer.rb +38 -36
- data/lib/aspera/rest_errors_aspera.rb +19 -18
- data/lib/aspera/transfer/resumer.rb +2 -2
- data/lib/aspera/yaml.rb +49 -0
- data.tar.gz.sig +0 -0
- metadata +17 -3
- metadata.gz.sig +0 -0
- data/release_notes.md +0 -8
data/bin/asession
CHANGED
|
@@ -6,95 +6,118 @@ require 'aspera/agent/direct'
|
|
|
6
6
|
require 'aspera/cli/extended_value'
|
|
7
7
|
require 'aspera/products/transferd'
|
|
8
8
|
require 'aspera/log'
|
|
9
|
+
require 'aspera/assert'
|
|
9
10
|
require 'json'
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
#
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
#
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
11
|
+
|
|
12
|
+
class UsageError < StandardError; end
|
|
13
|
+
|
|
14
|
+
class SimpleTransferManager
|
|
15
|
+
# First level parameters in session spec
|
|
16
|
+
# - Extended transfer spec parameter (only used in asession)
|
|
17
|
+
PARAM_SPEC = 'spec'
|
|
18
|
+
# - Log level
|
|
19
|
+
PARAM_LOG_LEVEL = 'loglevel'
|
|
20
|
+
# - Transfer agent options
|
|
21
|
+
PARAM_AGENT = 'agent'
|
|
22
|
+
# - By default go to /tmp/username.filelist
|
|
23
|
+
PARAM_TMP_FILE_LIST_FOLDER = 'file_list_folder'
|
|
24
|
+
# - SDK location
|
|
25
|
+
PARAM_SDK = 'sdk'
|
|
26
|
+
# Default SDK location
|
|
27
|
+
SDK_DEFAULT_DIR = File.join(Dir.home, '.aspera', 'sdk')
|
|
28
|
+
# Place transfer spec in that
|
|
29
|
+
SAMPLE_DEMO = '"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ssh_port":33001,"remote_password":"demoaspera"'
|
|
30
|
+
SAMPLE_DEMO2 = '"direction":"receive","destination_root":"./test.dir"'
|
|
31
|
+
STDIN_INPUT = '@json:@stdin:'
|
|
32
|
+
class << self
|
|
33
|
+
# Assert usage condition, otherwise display usage message and exit
|
|
34
|
+
# @param assertion [Boolean] Assertion to verify
|
|
35
|
+
# @param error_message [String] Error message to display
|
|
36
|
+
def usage
|
|
37
|
+
# rubocop:disable Style/StderrPuts
|
|
38
|
+
$stderr.puts('USAGE')
|
|
39
|
+
$stderr.puts(' asession')
|
|
40
|
+
$stderr.puts(' asession -h|--help')
|
|
41
|
+
$stderr.puts(' asession [<session spec extended value>]')
|
|
42
|
+
$stderr.puts(' ')
|
|
43
|
+
$stderr.puts(' If no argument is provided, default will be used: @json:@stdin')
|
|
44
|
+
$stderr.puts(' -h, --help display this message')
|
|
45
|
+
$stderr.puts(' <session spec extended value> a dictionary (Hash)')
|
|
46
|
+
$stderr.puts(' The value can be either:')
|
|
47
|
+
$stderr.puts(" the JSON description itself, e.g. @json:'{\"xx\":\"yy\",...}'")
|
|
48
|
+
$stderr.puts(' @json:@stdin, if the JSON is provided from stdin')
|
|
49
|
+
$stderr.puts(' @json:@file:<path>, if the JSON is provided from a file')
|
|
50
|
+
$stderr.puts(' The following keys are recognized in session spec:')
|
|
51
|
+
$stderr.puts(" #{PARAM_SPEC} : mandatory, contains the transfer spec")
|
|
52
|
+
$stderr.puts(" #{PARAM_LOG_LEVEL} : modify log level (to stderr)")
|
|
53
|
+
$stderr.puts(" #{PARAM_AGENT} : modify transfer agent parameters, e.g. ascp_args")
|
|
54
|
+
$stderr.puts(" #{PARAM_TMP_FILE_LIST_FOLDER} : location of temporary files")
|
|
55
|
+
$stderr.puts(" #{PARAM_SDK} : location of SDK (ascp)")
|
|
56
|
+
$stderr.puts(' Asynchronous commands can be provided on STDIN, examples:')
|
|
57
|
+
$stderr.puts(' {"type":"START","source":"/aspera-test-dir-tiny/200KB.2"}')
|
|
58
|
+
$stderr.puts(' {"type":"START","source":"xx","destination":"yy"}')
|
|
59
|
+
$stderr.puts(' {"type":"DONE"}')
|
|
60
|
+
$stderr.puts('EXAMPLES')
|
|
61
|
+
$stderr.puts(%Q( asession @json:'{"#{PARAM_SPEC}":{#{SAMPLE_DEMO},#{SAMPLE_DEMO2},"paths":[{"source":"/aspera-test-dir-tiny/200KB.1"}]}}'))
|
|
62
|
+
$stderr.puts(%Q( echo '{"#{PARAM_SPEC}":{"remote_host":...}}'|asession #{STDIN_INPUT}))
|
|
63
|
+
# rubocop:enable Style/StderrPuts
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
def initialize(args)
|
|
67
|
+
parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
|
|
68
|
+
# By default assume JSON input on stdin if no argument
|
|
69
|
+
if args.empty?
|
|
70
|
+
args.push(STDIN_INPUT)
|
|
71
|
+
parameter_source_err_msg = ' (JSON on stdin)'
|
|
72
|
+
end
|
|
73
|
+
# Anyway expect only one argument: session information
|
|
74
|
+
Aspera.assert(args.length.eql?(1), 'exactly one argument is expected', type: UsageError)
|
|
75
|
+
if ['-h', '--help'].include?(args.first)
|
|
76
|
+
SimpleTransferManager.usage
|
|
77
|
+
exit(0)
|
|
78
|
+
end
|
|
79
|
+
# Parse transfer spec
|
|
80
|
+
@session_spec = Aspera::Cli::ExtendedValue.instance.evaluate(args.pop, context: 'asession parameter')
|
|
81
|
+
# Ensure right type for parameter
|
|
82
|
+
Aspera.assert(@session_spec.is_a?(Hash), "The value must be a Hash#{parameter_source_err_msg}", type: UsageError)
|
|
83
|
+
Aspera.assert(@session_spec[PARAM_SPEC].is_a?(Hash), "The value must contain key #{PARAM_SPEC} with Hash value", type: UsageError)
|
|
84
|
+
# Additional debug capability
|
|
85
|
+
Aspera::Log.instance.level = @session_spec[PARAM_LOG_LEVEL].to_sym if @session_spec.key?(PARAM_LOG_LEVEL)
|
|
86
|
+
# Possibly override temp folder
|
|
87
|
+
Aspera::Transfer::Parameters.file_list_folder = @session_spec[PARAM_TMP_FILE_LIST_FOLDER] if @session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
|
|
88
|
+
Aspera::Products::Transferd.sdk_directory = @session_spec[PARAM_SDK] || SDK_DEFAULT_DIR
|
|
89
|
+
agent_params = @session_spec[PARAM_AGENT] || {}
|
|
90
|
+
agent_params['quiet'] = true
|
|
91
|
+
agent_params['management_cb'] = ->(event) do
|
|
92
|
+
puts JSON.generate(Aspera::Ascp::Management.event_native_to_snake(event))
|
|
93
|
+
end
|
|
94
|
+
# Get local agent (ascp), disable ascp output on stdout to not mix with JSON events
|
|
95
|
+
@client = Aspera::Agent::Direct.new(**agent_params.symbolize_keys)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def start
|
|
99
|
+
# Start transfer (asynchronous)
|
|
100
|
+
@client.start_transfer(@session_spec[PARAM_SPEC])
|
|
101
|
+
# commands to ascp on mgt port
|
|
102
|
+
Thread.new do
|
|
103
|
+
loop do
|
|
104
|
+
data = JSON.parse($stdin.gets)
|
|
105
|
+
@client.send_command(data)
|
|
106
|
+
end
|
|
107
|
+
rescue => e
|
|
108
|
+
Aspera::Log.log.error("Error reading commands from stdin: #{e.message}")
|
|
109
|
+
Process.exit(1)
|
|
110
|
+
end
|
|
111
|
+
# No exit code: status is success (0)
|
|
112
|
+
@client.wait_for_transfers_completion
|
|
113
|
+
@client.shutdown
|
|
114
|
+
end
|
|
59
115
|
end
|
|
60
|
-
|
|
61
|
-
assert_usage(ARGV.length.eql?(1), 'exactly one argument is expected')
|
|
62
|
-
assert_usage(!['-h', '--help'].include?(ARGV.first), nil)
|
|
63
|
-
# Parse transfer spec
|
|
116
|
+
|
|
64
117
|
begin
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
# Ensure right type for parameter
|
|
71
|
-
assert_usage(session_spec.is_a?(Hash), "The value must be a Hash#{parameter_source_err_msg}")
|
|
72
|
-
assert_usage(session_spec[PARAM_SPEC].is_a?(Hash), "The value must contain key #{PARAM_SPEC} with Hash value")
|
|
73
|
-
# Additional debug capability
|
|
74
|
-
Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL].to_sym if session_spec.key?(PARAM_LOG_LEVEL)
|
|
75
|
-
# Possibly override temp folder
|
|
76
|
-
Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER] if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
|
|
77
|
-
session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
|
|
78
|
-
Aspera::Products::Transferd.sdk_directory = session_spec[PARAM_SDK]
|
|
79
|
-
session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
|
|
80
|
-
agent_params = session_spec[PARAM_AGENT]
|
|
81
|
-
agent_params['quiet'] = true
|
|
82
|
-
agent_params['management_cb'] = ->(event) do
|
|
83
|
-
puts JSON.generate(Aspera::Ascp::Management.event_native_to_snake(event))
|
|
84
|
-
end
|
|
85
|
-
# Get local agent (ascp), disable ascp output on stdout to not mix with JSON events
|
|
86
|
-
client = Aspera::Agent::Direct.new(**agent_params.symbolize_keys)
|
|
87
|
-
# Start transfer (asynchronous)
|
|
88
|
-
client.start_transfer(session_spec[PARAM_SPEC])
|
|
89
|
-
# commands to ascp on mgt port
|
|
90
|
-
Thread.new do
|
|
91
|
-
loop do
|
|
92
|
-
data = JSON.parse($stdin.gets)
|
|
93
|
-
client.send_command(data)
|
|
94
|
-
end
|
|
95
|
-
rescue
|
|
96
|
-
Process.exit(1)
|
|
118
|
+
SimpleTransferManager.new(ARGV).start
|
|
119
|
+
rescue UsageError => e
|
|
120
|
+
Aspera::Log.log.error(e.message)
|
|
121
|
+
SimpleTransferManager.usage
|
|
122
|
+
exit(1)
|
|
97
123
|
end
|
|
98
|
-
# No exit code: status is success (0)
|
|
99
|
-
client.wait_for_transfers_completion
|
|
100
|
-
client.shutdown
|
data/lib/aspera/agent/connect.rb
CHANGED
|
@@ -32,7 +32,7 @@ module Aspera
|
|
|
32
32
|
headers: {'Origin' => RestParameters.instance.user_agent}
|
|
33
33
|
)
|
|
34
34
|
connect_info = @connect_api.read('info/version')
|
|
35
|
-
Log.log.
|
|
35
|
+
Log.log.debug('Connect was reached') if method_index > 0
|
|
36
36
|
Log.dump(:connect_version, connect_info)
|
|
37
37
|
rescue StandardError => e # Errno::ECONNREFUSED
|
|
38
38
|
Log.log.debug{"Exception: #{e}"}
|
data/lib/aspera/agent/desktop.rb
CHANGED
|
@@ -30,7 +30,7 @@ module Aspera
|
|
|
30
30
|
@client_app_api = Aspera::JsonRpcClient.new(Aspera::Rest.new(base_url: aspera_client_api_url))
|
|
31
31
|
client_info = @client_app_api.get_info
|
|
32
32
|
Log.dump(:client_version, client_info)
|
|
33
|
-
Log.log.
|
|
33
|
+
Log.log.debug('Client was reached') if method_index > 0
|
|
34
34
|
rescue Errno::ECONNREFUSED => e
|
|
35
35
|
start_url = START_URIS[method_index]
|
|
36
36
|
method_index += 1
|
data/lib/aspera/agent/direct.rb
CHANGED
|
@@ -16,9 +16,9 @@ require 'English'
|
|
|
16
16
|
|
|
17
17
|
module Aspera
|
|
18
18
|
module Agent
|
|
19
|
-
#
|
|
19
|
+
# Execute a local `ascp` and use its management port to monitor progress
|
|
20
20
|
class Direct < Base
|
|
21
|
-
# ascp started locally, so listen local
|
|
21
|
+
# `ascp` started locally, so listen local
|
|
22
22
|
LISTEN_LOCAL_ADDRESS = '127.0.0.1'
|
|
23
23
|
# 0 means: use any available port
|
|
24
24
|
SELECT_AVAILABLE_PORT = 0
|
|
@@ -48,13 +48,13 @@ module Aspera
|
|
|
48
48
|
spawn_timeout_sec: 2,
|
|
49
49
|
spawn_delay_sec: 2,
|
|
50
50
|
multi_incr_udp: nil,
|
|
51
|
-
resume:
|
|
51
|
+
resume: {},
|
|
52
52
|
monitor: true,
|
|
53
53
|
management_cb: nil,
|
|
54
54
|
**base_options
|
|
55
55
|
)
|
|
56
56
|
super(**base_options)
|
|
57
|
-
#
|
|
57
|
+
# Options that have impact on `ascp` command line generated
|
|
58
58
|
@tr_opts = {
|
|
59
59
|
ascp_args: ascp_args,
|
|
60
60
|
wss: wss,
|
|
@@ -69,7 +69,6 @@ module Aspera
|
|
|
69
69
|
@multi_incr_udp = multi_incr_udp.nil? ? Environment.instance.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
|
70
70
|
@monitor = monitor
|
|
71
71
|
@management_cb = management_cb
|
|
72
|
-
resume = {} if resume.nil?
|
|
73
72
|
Aspera.assert_type(resume, Hash){'resume'}
|
|
74
73
|
@resume_policy = Transfer::Resumer.new(**resume.symbolize_keys)
|
|
75
74
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
|
@@ -78,6 +77,7 @@ module Aspera
|
|
|
78
77
|
@mutex = Mutex.new
|
|
79
78
|
@pre_calc_sent = false
|
|
80
79
|
@pre_calc_last_size = nil
|
|
80
|
+
# Check on all management messages if that file exists, and if so, read commands from it
|
|
81
81
|
@command_file = File.join(config_dir || '.', "send_#{$PROCESS_ID}")
|
|
82
82
|
end
|
|
83
83
|
|
|
@@ -115,7 +115,7 @@ module Aspera
|
|
|
115
115
|
Log.log.debug('multi_session count is zero: no multi session')
|
|
116
116
|
multi_session_info = nil
|
|
117
117
|
elsif @multi_incr_udp # multi_session_info[:count] > 0
|
|
118
|
-
# if option not true
|
|
118
|
+
# if option not `true`: keep default UDP port for all sessions
|
|
119
119
|
multi_session_info[:udp_base] = transfer_spec.key?('fasp_port') ? transfer_spec['fasp_port'] : Transfer::Spec::UDP_PORT
|
|
120
120
|
# delete from original transfer spec, as we will increment values
|
|
121
121
|
transfer_spec.delete('fasp_port')
|
|
@@ -225,9 +225,9 @@ module Aspera
|
|
|
225
225
|
start_and_monitor_process(session: session, **session[:env_args])
|
|
226
226
|
end
|
|
227
227
|
Log.log.debug('transfer ok'.bg_green)
|
|
228
|
-
rescue
|
|
228
|
+
rescue => e
|
|
229
229
|
session[:error] = e
|
|
230
|
-
Log.log.
|
|
230
|
+
raise if Log.log.debug? || !e.is_a?(Transfer::Error)
|
|
231
231
|
end
|
|
232
232
|
Log.log.debug{"EXIT (#{Thread.current[:name]})"}
|
|
233
233
|
end
|
|
@@ -242,7 +242,7 @@ module Aspera
|
|
|
242
242
|
# @param env [Hash] Environment variables (comes from ascp_args)
|
|
243
243
|
# @param args [Array] Command line arguments (comes from ascp_args)
|
|
244
244
|
# @return [nil] when process has exited
|
|
245
|
-
# @
|
|
245
|
+
# @raise FaspError on error
|
|
246
246
|
def start_and_monitor_process(
|
|
247
247
|
session:,
|
|
248
248
|
name:,
|
|
@@ -251,15 +251,15 @@ module Aspera
|
|
|
251
251
|
)
|
|
252
252
|
Aspera.assert_type(session, Hash)
|
|
253
253
|
notify_progress(:sessions_init, info: 'starting')
|
|
254
|
+
# Do not use `capture_stderr`
|
|
255
|
+
capture_stderr = false
|
|
256
|
+
stderr_r, stderr_w = nil
|
|
257
|
+
spawn_args = {}
|
|
258
|
+
command_pid = nil
|
|
259
|
+
# Get location of command executable (ascp, async)
|
|
260
|
+
command_path = Ascp::Installation.instance.path(name)
|
|
261
|
+
command_arguments = [command_path]
|
|
254
262
|
begin
|
|
255
|
-
# do not use
|
|
256
|
-
capture_stderr = false
|
|
257
|
-
stderr_r, stderr_w = nil
|
|
258
|
-
spawn_args = {}
|
|
259
|
-
command_pid = nil
|
|
260
|
-
# get location of command executable (ascp, async)
|
|
261
|
-
command_path = Ascp::Installation.instance.path(name)
|
|
262
|
-
command_arguments = [command_path]
|
|
263
263
|
if @monitor
|
|
264
264
|
# we use Socket directly, instead of TCPServer, as it gives access to lower level options
|
|
265
265
|
socket_class = defined?(JRUBY_VERSION) ? ServerSocket : Socket
|
|
@@ -353,7 +353,7 @@ module Aspera
|
|
|
353
353
|
Process.kill(:INT, command_pid) if @monitor && !Environment.instance.os.eql?(Environment::OS_WINDOWS)
|
|
354
354
|
# collect process exit status or wait for termination
|
|
355
355
|
_, status = Process.wait2(command_pid)
|
|
356
|
-
if
|
|
356
|
+
if capture_stderr
|
|
357
357
|
# process stderr of ascp
|
|
358
358
|
stderr_flag = false
|
|
359
359
|
stderr_r.each_line do |line|
|
|
@@ -379,6 +379,7 @@ module Aspera
|
|
|
379
379
|
|
|
380
380
|
private
|
|
381
381
|
|
|
382
|
+
# [Array<Hash>] List of sessions, one per `ascp` process
|
|
382
383
|
attr_reader :sessions
|
|
383
384
|
|
|
384
385
|
# Notify progress to callback
|
data/lib/aspera/agent/node.rb
CHANGED
|
@@ -77,7 +77,7 @@ module Aspera
|
|
|
77
77
|
# status is empty sometimes with status 200...
|
|
78
78
|
transfer_data = node_api_.read("ops/transfers/#{@transfer_id}") || {'status' => 'unknown'} rescue {'status' => 'waiting(api error)'}
|
|
79
79
|
case transfer_data['status']
|
|
80
|
-
when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)'
|
|
80
|
+
when 'waiting', 'partially_completed', 'unknown', 'waiting(read error)', 'waiting(api error)'
|
|
81
81
|
notify_progress(:sessions_init, info: transfer_data['status'])
|
|
82
82
|
when 'running'
|
|
83
83
|
if !session_started
|
data/lib/aspera/api/aoc.rb
CHANGED
|
@@ -17,7 +17,7 @@ module Aspera
|
|
|
17
17
|
DEFAULT_WORKSPACE = ''
|
|
18
18
|
# Production domain of AoC
|
|
19
19
|
SAAS_DOMAIN_PROD = 'ibmaspera.com' # cspell:disable-line
|
|
20
|
-
#
|
|
20
|
+
# To avoid infinite loop in pub link redirection
|
|
21
21
|
MAX_AOC_URL_REDIRECT = 10
|
|
22
22
|
CLIENT_ID_PREFIX = 'aspera.'
|
|
23
23
|
# Well-known AoC global client apps
|
|
@@ -47,7 +47,7 @@ module Aspera
|
|
|
47
47
|
:PERMISSIONS_CREATED,
|
|
48
48
|
:ID_AK_ADMIN
|
|
49
49
|
|
|
50
|
-
#
|
|
50
|
+
# Various API scopes supported
|
|
51
51
|
module Scope
|
|
52
52
|
SELF = 'self'
|
|
53
53
|
USER = 'user:all'
|
|
@@ -142,11 +142,14 @@ module Aspera
|
|
|
142
142
|
}
|
|
143
143
|
end
|
|
144
144
|
|
|
145
|
-
# Call block with same query using paging and response information.
|
|
146
|
-
# Block must return
|
|
147
|
-
# @
|
|
148
|
-
|
|
149
|
-
|
|
145
|
+
# Call `block` with same query using paging and response information.
|
|
146
|
+
# Block must return a 2 element `Array` with data and http response
|
|
147
|
+
# @param query [Hash] Additionnal query parameters
|
|
148
|
+
# @param progress [nil, Object] Uses methods: `long_operation_running` and `long_operation_terminated`
|
|
149
|
+
# @return [Hash] Items and total number of items
|
|
150
|
+
# @option return [Array<Hash>] :items The list of items
|
|
151
|
+
# @option return [Integer] :total The total number of items
|
|
152
|
+
def call_paging(query: {}, progress: nil)
|
|
150
153
|
Aspera.assert_type(query, Hash){'query'}
|
|
151
154
|
Aspera.assert(block_given?)
|
|
152
155
|
# set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
|
|
@@ -174,9 +177,9 @@ module Aspera
|
|
|
174
177
|
break if !max_items.nil? && item_list.count >= max_items
|
|
175
178
|
break if !max_pages.nil? && page_count >= max_pages
|
|
176
179
|
break if total_count&.<=(item_list.count)
|
|
177
|
-
|
|
180
|
+
progress&.long_operation_running("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
|
|
178
181
|
end
|
|
179
|
-
|
|
182
|
+
progress&.long_operation_terminated
|
|
180
183
|
item_list = item_list[0..max_items - 1] if !max_items.nil? && item_list.count > max_items
|
|
181
184
|
return {items: item_list, total: total_count}
|
|
182
185
|
end
|
|
@@ -195,6 +198,23 @@ module Aspera
|
|
|
195
198
|
def workspace_access?(permission)
|
|
196
199
|
permission['access_id'].start_with?("#{ID_AK_ADMIN}_WS_")
|
|
197
200
|
end
|
|
201
|
+
|
|
202
|
+
# Expand access levels to full list of levels.
|
|
203
|
+
# @param levels [nil, String, Array] Access levels
|
|
204
|
+
# @return [Array] Expanded access levels
|
|
205
|
+
def expand_access_levels(levels)
|
|
206
|
+
case levels
|
|
207
|
+
when nil, 'edit' then Node::ACCESS_LEVELS
|
|
208
|
+
when 'preview' then %w[list preview]
|
|
209
|
+
when 'download' then %w[list preview read]
|
|
210
|
+
when 'upload' then %w[mkdir write]
|
|
211
|
+
when Array
|
|
212
|
+
Aspera.assert_array_all(levels, String){'access_levels'}
|
|
213
|
+
levels.each{ |level| Aspera.assert_value(level, Node::ACCESS_LEVELS){'access_level'}}
|
|
214
|
+
levels
|
|
215
|
+
else Aspera.error_unexpected_value(levels){"access_levels must be a list of #{Node::ACCESS_LEVELS.join(', ')} or one of edit, preview, download, upload"}
|
|
216
|
+
end
|
|
217
|
+
end
|
|
198
218
|
end
|
|
199
219
|
|
|
200
220
|
attr_reader :private_link
|
|
@@ -212,7 +232,8 @@ module Aspera
|
|
|
212
232
|
username: nil,
|
|
213
233
|
password: nil,
|
|
214
234
|
workspace: nil,
|
|
215
|
-
secret_finder: nil
|
|
235
|
+
secret_finder: nil,
|
|
236
|
+
progress_disp: nil
|
|
216
237
|
)
|
|
217
238
|
# Test here because link may set url
|
|
218
239
|
Aspera.assert(url, 'Missing mandatory option: url', type: ParameterError)
|
|
@@ -223,6 +244,7 @@ module Aspera
|
|
|
223
244
|
# key: access key
|
|
224
245
|
# value: associated secret
|
|
225
246
|
@secret_finder = secret_finder
|
|
247
|
+
@progress_disp = progress_disp
|
|
226
248
|
@workspace_name = workspace
|
|
227
249
|
@cache_user_info = nil
|
|
228
250
|
@cache_url_token_info = nil
|
|
@@ -282,10 +304,12 @@ module Aspera
|
|
|
282
304
|
)
|
|
283
305
|
end
|
|
284
306
|
|
|
285
|
-
#
|
|
307
|
+
# Read using the query and paging
|
|
308
|
+
# @param subpath [String] Entity path
|
|
309
|
+
# @param query [nil, Hash] Additional query
|
|
286
310
|
# @return [Hash] {items: , total: }
|
|
287
|
-
def read_with_paging(subpath, query = nil
|
|
288
|
-
return self.class.call_paging(query: query,
|
|
311
|
+
def read_with_paging(subpath, query = nil)
|
|
312
|
+
return self.class.call_paging(query: query, progress: @progress_disp) do |paged_query|
|
|
289
313
|
read(subpath, query: paged_query, ret: :both)
|
|
290
314
|
end
|
|
291
315
|
end
|
|
@@ -316,9 +340,9 @@ module Aspera
|
|
|
316
340
|
@cache_user_info =
|
|
317
341
|
begin
|
|
318
342
|
read('self')
|
|
319
|
-
rescue
|
|
320
|
-
raise
|
|
321
|
-
Log.log.debug{"
|
|
343
|
+
rescue Aspera::RestCallError => e
|
|
344
|
+
raise if exception || e.message.include?('invalid_grant')
|
|
345
|
+
Log.log.debug{"Ignoring error: (#{e.class}) #{e}"}
|
|
322
346
|
{}
|
|
323
347
|
end
|
|
324
348
|
USER_INFO_FIELDS_MIN.each{ |f| @cache_user_info[f] = nil if @cache_user_info[f].nil?}
|
|
@@ -394,10 +418,10 @@ module Aspera
|
|
|
394
418
|
# @param node_id [String] identifier of node in AoC
|
|
395
419
|
# @param workspace_id [String,nil] workspace identifier
|
|
396
420
|
# @param workspace_name [String,nil] workspace name
|
|
397
|
-
# @param scope [String,nil] e.g. Node::
|
|
421
|
+
# @param scope [String,nil] e.g. Node::Scope::USER, or Node::Scope::ADMIN, or nil (requires secret)
|
|
398
422
|
# @param package_info [Hash,nil] created package information
|
|
399
423
|
# @returns [Node] a node API for access key
|
|
400
|
-
def node_api_from(node_id:, workspace_id: nil, workspace_name: nil, scope: Node::
|
|
424
|
+
def node_api_from(node_id:, workspace_id: nil, workspace_name: nil, scope: Node::Scope::USER, package_info: nil)
|
|
401
425
|
Aspera.assert_type(node_id, String)
|
|
402
426
|
node_info = read("nodes/#{node_id}")
|
|
403
427
|
workspace_name = read("workspaces/#{workspace_id}")['name'] if workspace_name.nil? && !workspace_id.nil?
|
|
@@ -428,7 +452,7 @@ module Aspera
|
|
|
428
452
|
node_params[:auth] = auth_params.clone
|
|
429
453
|
node_params[:auth][:params] ||= {}
|
|
430
454
|
node_params[:auth][:params][:scope] = Node.token_scope(node_info['access_key'], scope)
|
|
431
|
-
node_params[:auth][:params][:owner_access] = true if scope.eql?(Node::
|
|
455
|
+
node_params[:auth][:params][:owner_access] = true if scope.eql?(Node::Scope::ADMIN)
|
|
432
456
|
# Special header required for bearer token only
|
|
433
457
|
node_params[:headers] = {Node::HEADER_X_ASPERA_ACCESS_KEY => node_info['access_key']}
|
|
434
458
|
end
|
|
@@ -693,7 +717,7 @@ module Aspera
|
|
|
693
717
|
# (optional). The name of the folder to be displayed to the destination user.
|
|
694
718
|
# Use it if its value is different from the "share_as" field.
|
|
695
719
|
event_creation['link_name'] = app_info[:opt_link_name] unless app_info[:opt_link_name].nil?
|
|
696
|
-
create('events', event_creation)
|
|
720
|
+
create('events', event_creation, query: {admin: true})
|
|
697
721
|
end
|
|
698
722
|
end
|
|
699
723
|
end
|
data/lib/aspera/api/faspex.rb
CHANGED
|
@@ -60,19 +60,32 @@ module Aspera
|
|
|
60
60
|
PATH_AUTH = 'auth'
|
|
61
61
|
PATH_API_V5 = 'api/v5'
|
|
62
62
|
PATH_HEALTH = 'configuration/ping'
|
|
63
|
-
private_constant :
|
|
64
|
-
:
|
|
65
|
-
:
|
|
63
|
+
private_constant :PATH_AUTH,
|
|
64
|
+
:PATH_API_V5,
|
|
65
|
+
:PATH_HEALTH
|
|
66
66
|
RECIPIENT_TYPES = %w[user workgroup external_user distribution_list shared_inbox].freeze
|
|
67
67
|
PACKAGE_TERMINATED = %w[completed failed].freeze
|
|
68
68
|
# list of supported mailbox types (to list packages)
|
|
69
|
-
|
|
69
|
+
SENT_MAILBOX_TYPES = %w[outbox outbox_history].freeze
|
|
70
|
+
API_LIST_MAILBOX_TYPES = (%w[inbox inbox_history inbox_all inbox_all_history pending pending_history all] + SENT_MAILBOX_TYPES).freeze
|
|
70
71
|
# PACKAGE_SEND_FROM_REMOTE_SOURCE = 'remote_source'
|
|
71
72
|
# Faspex API v5: get transfer spec for connect
|
|
72
73
|
TRANSFER_CONNECT = 'connect'
|
|
73
74
|
ADMIN_RESOURCES = %i[
|
|
74
|
-
accounts
|
|
75
|
-
|
|
75
|
+
accounts
|
|
76
|
+
distribution_lists
|
|
77
|
+
contacts
|
|
78
|
+
jobs
|
|
79
|
+
workgroups
|
|
80
|
+
shared_inboxes
|
|
81
|
+
nodes
|
|
82
|
+
oauth_clients
|
|
83
|
+
registrations
|
|
84
|
+
saml_configs
|
|
85
|
+
metadata_profiles
|
|
86
|
+
email_notifications
|
|
87
|
+
alternate_addresses
|
|
88
|
+
webhooks
|
|
76
89
|
].freeze
|
|
77
90
|
# states for jobs not in final state
|
|
78
91
|
JOB_RUNNING = %w[queued working].freeze
|
|
@@ -114,6 +127,12 @@ module Aspera
|
|
|
114
127
|
def public_link?(url)
|
|
115
128
|
url.include?('?context=')
|
|
116
129
|
end
|
|
130
|
+
|
|
131
|
+
# Depending on box, the package files are either: `received` or `sent`
|
|
132
|
+
# @return [:sent, :received] the type of mailbox
|
|
133
|
+
def box_type(box)
|
|
134
|
+
SENT_MAILBOX_TYPES.include?(box) || box == 'ALL' ? :sent : :received
|
|
135
|
+
end
|
|
117
136
|
end
|
|
118
137
|
attr_reader :pub_link_context
|
|
119
138
|
|
data/lib/aspera/api/node.rb
CHANGED
|
@@ -16,9 +16,15 @@ module Aspera
|
|
|
16
16
|
module Api
|
|
17
17
|
# Provides additional functions using node API with gen4 extensions (access keys)
|
|
18
18
|
class Node < Aspera::Rest
|
|
19
|
-
#
|
|
20
|
-
|
|
21
|
-
|
|
19
|
+
# Format of node scope : node.<access key>:<scope>
|
|
20
|
+
module Scope
|
|
21
|
+
# Node sub-scopes
|
|
22
|
+
USER = 'user:all'
|
|
23
|
+
ADMIN = 'admin:all'
|
|
24
|
+
# Separator between node.AK and user:all
|
|
25
|
+
SEPARATOR = ':'
|
|
26
|
+
NODE_PREFIX = 'node.'
|
|
27
|
+
end
|
|
22
28
|
# Accepted types in `file_matcher`
|
|
23
29
|
MATCH_TYPES = [String, Proc, Regexp, NilClass].freeze
|
|
24
30
|
# Delimiter in decoded node token
|
|
@@ -29,20 +35,16 @@ module Aspera
|
|
|
29
35
|
REQUIRED_APP_INFO_FIELDS = %i[api app node_info workspace_id workspace_name].freeze
|
|
30
36
|
# Methods of @app_info[:api]
|
|
31
37
|
REQUIRED_APP_API_METHODS = %i[node_api_from add_ts_tags].freeze
|
|
32
|
-
private_constant :
|
|
38
|
+
private_constant :MATCH_TYPES,
|
|
33
39
|
:SIGNATURE_DELIMITER, :BEARER_TOKEN_VALIDITY_DEFAULT,
|
|
34
40
|
:REQUIRED_APP_INFO_FIELDS, :REQUIRED_APP_API_METHODS
|
|
35
|
-
|
|
36
|
-
# Node API permissions
|
|
41
|
+
# Node API permissions: delete list mkdir preview read rename write
|
|
37
42
|
ACCESS_LEVELS = %w[delete list mkdir preview read rename write].freeze
|
|
38
43
|
# Special HTTP Headers
|
|
39
44
|
HEADER_X_ASPERA_ACCESS_KEY = 'X-Aspera-AccessKey'
|
|
40
45
|
HEADER_X_TOTAL_COUNT = 'X-Total-Count'
|
|
41
46
|
HEADER_X_CACHE_CONTROL = 'X-Aspera-Cache-Control'
|
|
42
47
|
HEADER_X_NEXT_ITER_TOKEN = 'X-Aspera-Next-Iteration-Token'
|
|
43
|
-
# Node sub-scopes
|
|
44
|
-
SCOPE_USER = 'user:all'
|
|
45
|
-
SCOPE_ADMIN = 'admin:all'
|
|
46
48
|
# / in cloud
|
|
47
49
|
PATH_SEPARATOR = '/'
|
|
48
50
|
|
|
@@ -63,7 +65,7 @@ module Aspera
|
|
|
63
65
|
# Adds cache control header, as globally specified to read request
|
|
64
66
|
# Use like this: read(...,**cache_control)
|
|
65
67
|
def cache_control
|
|
66
|
-
headers = {'Accept' =>
|
|
68
|
+
headers = {'Accept' => Mime::JSON}
|
|
67
69
|
headers[HEADER_X_CACHE_CONTROL] = 'no-cache' unless use_node_cache
|
|
68
70
|
{headers: headers}
|
|
69
71
|
end
|
|
@@ -126,16 +128,16 @@ module Aspera
|
|
|
126
128
|
# Node API scopes
|
|
127
129
|
# @return [String] node scope
|
|
128
130
|
def token_scope(access_key, scope)
|
|
129
|
-
return [
|
|
131
|
+
return [Scope::NODE_PREFIX, access_key, Scope::SEPARATOR, scope].join('')
|
|
130
132
|
end
|
|
131
133
|
|
|
132
134
|
# Decode node scope into access key and scope
|
|
133
135
|
# @return [Hash]
|
|
134
136
|
def decode_scope(scope)
|
|
135
|
-
items = scope.split(
|
|
137
|
+
items = scope.split(Scope::SEPARATOR, 2)
|
|
136
138
|
Aspera.assert(items.length.eql?(2)){"invalid scope: #{scope}"}
|
|
137
|
-
Aspera.assert(items[0].start_with?(
|
|
138
|
-
return {access_key: items[0][
|
|
139
|
+
Aspera.assert(items[0].start_with?(Scope::NODE_PREFIX)){"invalid scope: #{scope}"}
|
|
140
|
+
return {access_key: items[0][Scope::NODE_PREFIX.length..-1], scope: items[1]}
|
|
139
141
|
end
|
|
140
142
|
|
|
141
143
|
# Create an Aspera Node bearer token
|
|
@@ -151,7 +153,7 @@ module Aspera
|
|
|
151
153
|
# Manage convenience parameters
|
|
152
154
|
expiration_sec = payload['_validity'] || BEARER_TOKEN_VALIDITY_DEFAULT
|
|
153
155
|
payload.delete('_validity')
|
|
154
|
-
scope = payload['_scope'] ||
|
|
156
|
+
scope = payload['_scope'] || Scope::USER
|
|
155
157
|
payload.delete('_scope')
|
|
156
158
|
payload['scope'] ||= token_scope(access_key, scope)
|
|
157
159
|
payload['auth_type'] ||= 'access_key'
|
|
@@ -306,7 +308,9 @@ module Aspera
|
|
|
306
308
|
# @param top_file_id [String] id initial file id
|
|
307
309
|
# @param path [String] file or folder path (end with "/" is like setting process_last_link)
|
|
308
310
|
# @param process_last_link [Boolean] if true, follow the last link
|
|
309
|
-
# @return [Hash]
|
|
311
|
+
# @return [Hash] Result data
|
|
312
|
+
# @option return [Aspera::Rest] :api REST client instance
|
|
313
|
+
# @option return [String] :file_id File identifier
|
|
310
314
|
def resolve_api_fid(top_file_id, path, process_last_link = false)
|
|
311
315
|
Aspera.assert_type(top_file_id, String)
|
|
312
316
|
Aspera.assert_type(path, String)
|