aspera-cli 4.24.1 → 4.24.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +15 -2
- data/README.md +745 -436
- data/bin/ascli +20 -1
- data/bin/asession +23 -27
- data/lib/aspera/agent/base.rb +10 -21
- data/lib/aspera/agent/connect.rb +2 -3
- data/lib/aspera/agent/desktop.rb +2 -2
- data/lib/aspera/agent/direct.rb +49 -32
- data/lib/aspera/agent/factory.rb +31 -0
- data/lib/aspera/api/aoc.rb +79 -49
- data/lib/aspera/api/faspex.rb +212 -0
- data/lib/aspera/api/node.rb +99 -84
- data/lib/aspera/ascp/installation.rb +22 -21
- data/lib/aspera/ascp/management.rb +119 -23
- data/lib/aspera/assert.rb +14 -8
- data/lib/aspera/cli/extended_value.rb +15 -15
- data/lib/aspera/cli/formatter.rb +7 -5
- data/lib/aspera/cli/hints.rb +8 -0
- data/lib/aspera/cli/info.rb +4 -4
- data/lib/aspera/cli/main.rb +55 -70
- data/lib/aspera/cli/manager.rb +7 -4
- data/lib/aspera/cli/plugins/alee.rb +2 -1
- data/lib/aspera/cli/plugins/aoc.rb +110 -186
- data/lib/aspera/cli/plugins/ats.rb +4 -4
- data/lib/aspera/cli/plugins/base.rb +335 -0
- data/lib/aspera/cli/plugins/basic_auth.rb +45 -0
- data/lib/aspera/cli/plugins/config.rb +249 -220
- data/lib/aspera/cli/plugins/console.rb +15 -15
- data/lib/aspera/cli/plugins/cos.rb +2 -2
- data/lib/aspera/cli/plugins/factory.rb +78 -0
- data/lib/aspera/cli/plugins/faspex.rb +17 -20
- data/lib/aspera/cli/plugins/faspex5.rb +79 -193
- data/lib/aspera/cli/plugins/faspio.rb +14 -13
- data/lib/aspera/cli/plugins/httpgw.rb +13 -12
- data/lib/aspera/cli/plugins/node.rb +34 -32
- data/lib/aspera/cli/plugins/oauth.rb +48 -0
- data/lib/aspera/cli/plugins/orchestrator.rb +15 -13
- data/lib/aspera/cli/plugins/preview.rb +4 -4
- data/lib/aspera/cli/plugins/server.rb +15 -13
- data/lib/aspera/cli/plugins/shares.rb +18 -15
- data/lib/aspera/cli/sync_actions.rb +1 -1
- data/lib/aspera/cli/transfer_agent.rb +24 -20
- data/lib/aspera/cli/transfer_progress.rb +6 -6
- data/lib/aspera/cli/version.rb +3 -3
- data/lib/aspera/cli/wizard.rb +65 -53
- data/lib/aspera/colors.rb +6 -0
- data/lib/aspera/command_line_builder.rb +45 -50
- data/lib/aspera/command_line_converter.rb +2 -1
- data/lib/aspera/coverage.rb +1 -1
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +10 -7
- data/lib/aspera/faspex_gw.rb +6 -4
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/keychain/macos_security.rb +1 -1
- data/lib/aspera/log.rb +37 -9
- data/lib/aspera/nagios.rb +1 -1
- data/lib/aspera/oauth/base.rb +17 -10
- data/lib/aspera/oauth/factory.rb +8 -8
- data/lib/aspera/oauth/web.rb +2 -2
- data/lib/aspera/products/connect.rb +4 -3
- data/lib/aspera/products/desktop.rb +1 -4
- data/lib/aspera/products/other.rb +9 -1
- data/lib/aspera/products/transferd.rb +0 -1
- data/lib/aspera/rest.rb +126 -83
- data/lib/aspera/ssh.rb +3 -3
- data/lib/aspera/sync/args.schema.yaml +46 -3
- data/lib/aspera/sync/conf.schema.yaml +130 -94
- data/lib/aspera/sync/operations.rb +16 -16
- data/lib/aspera/temp_file_manager.rb +17 -5
- data/lib/aspera/transfer/error.rb +16 -7
- data/lib/aspera/transfer/parameters.rb +34 -20
- data/lib/aspera/transfer/resumer.rb +74 -0
- data/lib/aspera/transfer/spec.rb +4 -3
- data/lib/aspera/transfer/spec.schema.yaml +132 -51
- data/lib/aspera/transfer/spec_doc.rb +41 -35
- data/lib/aspera/uri_reader.rb +1 -1
- data/lib/aspera/web_auth.rb +6 -6
- data.tar.gz.sig +0 -0
- metadata +9 -7
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +0 -43
- data/lib/aspera/cli/plugin.rb +0 -333
- data/lib/aspera/cli/plugin_factory.rb +0 -81
- data/lib/aspera/resumer.rb +0 -77
- data/lib/aspera/transfer/error_info.rb +0 -91
data/bin/ascli
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
#!/usr/bin/env ruby
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
# This is the main script to execute the Aspera CLI
|
|
5
|
+
|
|
4
6
|
old_verbose = $VERBOSE
|
|
5
7
|
$VERBOSE = nil
|
|
6
8
|
# internal representation of strings (ruby) is UTF-8
|
|
@@ -9,9 +11,26 @@ Encoding.default_internal = Encoding::UTF_8
|
|
|
9
11
|
Encoding.default_external = Encoding::UTF_8
|
|
10
12
|
$VERBOSE = old_verbose
|
|
11
13
|
|
|
14
|
+
require 'aspera/log'
|
|
15
|
+
require 'aspera/cli/info'
|
|
16
|
+
|
|
17
|
+
# Early debug for parser
|
|
18
|
+
# Note: does not accept short option names, nor extended values
|
|
19
|
+
Aspera::Log.instance.program_name = Aspera::Cli::Info::CMD_NAME
|
|
20
|
+
ARGV.each do |arg|
|
|
21
|
+
case arg
|
|
22
|
+
when '--' then break
|
|
23
|
+
when /^--log-level=(.*)/ then Aspera::Log.instance.level = Regexp.last_match(1).to_sym
|
|
24
|
+
when /^--log-format=(.*)/ then Aspera::Log.instance.formatter = Regexp.last_match(1) unless Regexp.last_match(1).start_with?('@ruby:')
|
|
25
|
+
when /^--logger=(.*)/ then Aspera::Log.instance.logger_type = Regexp.last_match(1).to_sym
|
|
26
|
+
end
|
|
27
|
+
rescue => e
|
|
28
|
+
$stderr.puts("Error: #{e}") # rubocop:disable Style/StderrPuts
|
|
29
|
+
Process.exit(1)
|
|
30
|
+
end
|
|
31
|
+
|
|
12
32
|
require 'aspera/coverage'
|
|
13
33
|
require 'aspera/environment'
|
|
14
34
|
require 'aspera/cli/main'
|
|
15
|
-
Aspera::Cli::Main.early_debug_setup(ARGV)
|
|
16
35
|
Aspera::Environment.instance.fix_home
|
|
17
36
|
Aspera::Cli::Main.new(ARGV).process_command_line
|
data/bin/asession
CHANGED
|
@@ -7,16 +7,16 @@ require 'aspera/cli/extended_value'
|
|
|
7
7
|
require 'aspera/products/transferd'
|
|
8
8
|
require 'aspera/log'
|
|
9
9
|
require 'json'
|
|
10
|
-
#
|
|
10
|
+
# Extended transfer spec parameter (only used in asession)
|
|
11
11
|
PARAM_SPEC = 'spec'
|
|
12
|
-
#
|
|
12
|
+
# Log level
|
|
13
13
|
PARAM_LOG_LEVEL = 'loglevel'
|
|
14
|
-
#
|
|
14
|
+
# Transfer agent options
|
|
15
15
|
PARAM_AGENT = 'agent'
|
|
16
|
-
#
|
|
16
|
+
# By default go to /tmp/username.filelist
|
|
17
17
|
PARAM_TMP_FILE_LIST_FOLDER = 'file_list_folder'
|
|
18
18
|
PARAM_SDK = 'sdk'
|
|
19
|
-
#
|
|
19
|
+
# Place transfer spec in that
|
|
20
20
|
SAMPLE_DEMO = '"remote_host":"demo.asperasoft.com","remote_user":"asperaweb","ssh_port":33001,"remote_password":"demoaspera"'
|
|
21
21
|
SAMPLE_DEMO2 = '"direction":"receive","destination_root":"./test.dir"'
|
|
22
22
|
def assert_usage(assertion, error_message)
|
|
@@ -52,27 +52,27 @@ def assert_usage(assertion, error_message)
|
|
|
52
52
|
Process.exit(0)
|
|
53
53
|
end
|
|
54
54
|
parameter_source_err_msg = ' (argument), did you specify: "@json:" ?'
|
|
55
|
-
#
|
|
55
|
+
# By default assume JSON input on stdin if no argument
|
|
56
56
|
if ARGV.empty?
|
|
57
57
|
ARGV.push('@json:@stdin')
|
|
58
58
|
parameter_source_err_msg = ' (JSON on stdin)'
|
|
59
59
|
end
|
|
60
|
-
#
|
|
60
|
+
# Anyway expect only one argument: session information
|
|
61
61
|
assert_usage(ARGV.length.eql?(1), 'exactly one argument is expected')
|
|
62
62
|
assert_usage(!['-h', '--help'].include?(ARGV.first), nil)
|
|
63
|
-
#
|
|
63
|
+
# Parse transfer spec
|
|
64
64
|
begin
|
|
65
65
|
session_argument = ARGV.pop
|
|
66
66
|
session_spec = Aspera::Cli::ExtendedValue.instance.evaluate(session_argument)
|
|
67
67
|
rescue
|
|
68
68
|
assert_usage(false, "Cannot parse argument: #{session_argument}")
|
|
69
69
|
end
|
|
70
|
-
#
|
|
70
|
+
# Ensure right type for parameter
|
|
71
71
|
assert_usage(session_spec.is_a?(Hash), "The value must be a Hash#{parameter_source_err_msg}")
|
|
72
72
|
assert_usage(session_spec[PARAM_SPEC].is_a?(Hash), "The value must contain key #{PARAM_SPEC} with Hash value")
|
|
73
|
-
#
|
|
74
|
-
Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL] if session_spec.key?(PARAM_LOG_LEVEL)
|
|
75
|
-
#
|
|
73
|
+
# Additional debug capability
|
|
74
|
+
Aspera::Log.instance.level = session_spec[PARAM_LOG_LEVEL].to_sym if session_spec.key?(PARAM_LOG_LEVEL)
|
|
75
|
+
# Possibly override temp folder
|
|
76
76
|
Aspera::Transfer::Parameters.file_list_folder = session_spec[PARAM_TMP_FILE_LIST_FOLDER] if session_spec.key?(PARAM_TMP_FILE_LIST_FOLDER)
|
|
77
77
|
session_spec[PARAM_SDK] = File.join(Dir.home, '.aspera', 'sdk') unless session_spec.key?(PARAM_SDK)
|
|
78
78
|
Aspera::Products::Transferd.sdk_directory = session_spec[PARAM_SDK]
|
|
@@ -80,25 +80,21 @@ session_spec[PARAM_AGENT] = {} unless session_spec.key?(PARAM_AGENT)
|
|
|
80
80
|
agent_params = session_spec[PARAM_AGENT]
|
|
81
81
|
agent_params['quiet'] = true
|
|
82
82
|
agent_params['management_cb'] = ->(event) do
|
|
83
|
-
puts JSON.generate(Aspera::Ascp::Management.
|
|
83
|
+
puts JSON.generate(Aspera::Ascp::Management.event_native_to_snake(event))
|
|
84
84
|
end
|
|
85
|
-
#
|
|
85
|
+
# Get local agent (ascp), disable ascp output on stdout to not mix with JSON events
|
|
86
86
|
client = Aspera::Agent::Direct.new(**agent_params.symbolize_keys)
|
|
87
|
-
#
|
|
88
|
-
|
|
89
|
-
#
|
|
87
|
+
# Start transfer (asynchronous)
|
|
88
|
+
client.start_transfer(session_spec[PARAM_SPEC])
|
|
89
|
+
# commands to ascp on mgt port
|
|
90
90
|
Thread.new do
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
loop do
|
|
95
|
-
data = JSON.parse($stdin.gets)
|
|
96
|
-
client.send_command(session_id, data)
|
|
97
|
-
end
|
|
98
|
-
rescue
|
|
99
|
-
Process.exit(1)
|
|
91
|
+
loop do
|
|
92
|
+
data = JSON.parse($stdin.gets)
|
|
93
|
+
client.send_command(data)
|
|
100
94
|
end
|
|
95
|
+
rescue
|
|
96
|
+
Process.exit(1)
|
|
101
97
|
end
|
|
102
|
-
#
|
|
98
|
+
# No exit code: status is success (0)
|
|
103
99
|
client.wait_for_transfers_completion
|
|
104
100
|
client.shutdown
|
data/lib/aspera/agent/base.rb
CHANGED
|
@@ -1,6 +1,5 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/log'
|
|
4
3
|
require 'aspera/assert'
|
|
5
4
|
module Aspera
|
|
6
5
|
module Agent
|
|
@@ -10,25 +9,6 @@ module Aspera
|
|
|
10
9
|
# - `wait_for_transfers_completion` : waits for all transfer sessions to finish
|
|
11
10
|
# - `notify_progress` : called back by transfer agent to notify transfer progress
|
|
12
11
|
class Base
|
|
13
|
-
RUBY_EXT = '.rb'
|
|
14
|
-
private_constant :RUBY_EXT
|
|
15
|
-
class << self
|
|
16
|
-
def factory_create(agent, options)
|
|
17
|
-
# Aspera.assert_values(agent, agent_list)
|
|
18
|
-
require "aspera/agent/#{agent}"
|
|
19
|
-
Aspera::Agent.const_get(agent.to_s.capitalize).new(**options)
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
# discover available agents
|
|
23
|
-
# @return [Array] list of symbols of agents
|
|
24
|
-
def agent_list
|
|
25
|
-
base_class = File.basename(__FILE__)
|
|
26
|
-
Dir.entries(File.dirname(File.expand_path(__FILE__))).select do |file|
|
|
27
|
-
file.end_with?(RUBY_EXT) && !file.eql?(base_class)
|
|
28
|
-
end.map{ |file| file[0..(-1 - RUBY_EXT.length)].to_sym}
|
|
29
|
-
end
|
|
30
|
-
end
|
|
31
|
-
|
|
32
12
|
# Wait for all sessions to terminate and return the status of each session
|
|
33
13
|
def wait_for_completion
|
|
34
14
|
# list of: :success or "error message string"
|
|
@@ -48,8 +28,17 @@ module Aspera
|
|
|
48
28
|
nil
|
|
49
29
|
end
|
|
50
30
|
|
|
51
|
-
|
|
31
|
+
attr_reader :config_dir
|
|
32
|
+
|
|
33
|
+
# Base transfer agent object
|
|
34
|
+
# @param progress [Object] Progress bar
|
|
35
|
+
# @param config_dir [String] Config folder
|
|
36
|
+
def initialize(
|
|
37
|
+
progress: nil,
|
|
38
|
+
config_dir: nil
|
|
39
|
+
)
|
|
52
40
|
@progress = progress
|
|
41
|
+
@config_dir = config_dir
|
|
53
42
|
end
|
|
54
43
|
|
|
55
44
|
def notify_progress(*pos_args, **kw_args)
|
data/lib/aspera/agent/connect.rb
CHANGED
|
@@ -24,6 +24,7 @@ module Aspera
|
|
|
24
24
|
raise Error, 'Using connect requires a graphical environment' unless Environment.instance.graphical?
|
|
25
25
|
method_index = 0
|
|
26
26
|
begin
|
|
27
|
+
# raise exception if connect not started and file does not exist
|
|
27
28
|
connect_url = connect_api_url
|
|
28
29
|
Log.log.debug{"found: #{connect_url}"}
|
|
29
30
|
@connect_api = Rest.new(
|
|
@@ -135,9 +136,7 @@ module Aspera
|
|
|
135
136
|
|
|
136
137
|
# @return the file path of local connect where API's URI can be read
|
|
137
138
|
def connect_api_url
|
|
138
|
-
|
|
139
|
-
raise "Product: #{name} not found, please install." if connect_locations.nil?
|
|
140
|
-
folder = File.join(connect_locations[:run_root], 'var', 'run')
|
|
139
|
+
folder = File.join(Products::Other.find(Products::Connect.locations).first[:run_root], 'var', 'run')
|
|
141
140
|
['', 's'].each do |ext|
|
|
142
141
|
uri_file = File.join(folder, "http#{ext}.uri")
|
|
143
142
|
Log.log.debug{"checking connect port file: #{uri_file}"}
|
data/lib/aspera/agent/desktop.rb
CHANGED
|
@@ -102,7 +102,7 @@ module Aspera
|
|
|
102
102
|
|
|
103
103
|
# @return [String] the url where transferd is listening
|
|
104
104
|
def aspera_client_api_url
|
|
105
|
-
log_file = Products::Desktop.
|
|
105
|
+
log_file = File.join(Products::Other.find(Products::Desktop.locations).first[:log_root], Products::Desktop::LOG_FILENAME)
|
|
106
106
|
url = 'http://127.0.0.1:33024'
|
|
107
107
|
File.open(log_file, 'r') do |file|
|
|
108
108
|
file.each_line do |line|
|
|
@@ -111,7 +111,7 @@ module Aspera
|
|
|
111
111
|
url = "http://#{m[1]}"
|
|
112
112
|
end
|
|
113
113
|
end
|
|
114
|
-
end
|
|
114
|
+
end if File.exist?(log_file)
|
|
115
115
|
# raise StandardError, "Unable to find the JSON-RPC server URL in #{log_file}" if url.nil?
|
|
116
116
|
return url
|
|
117
117
|
end
|
data/lib/aspera/agent/direct.rb
CHANGED
|
@@ -6,7 +6,7 @@ require 'aspera/ascp/management'
|
|
|
6
6
|
require 'aspera/transfer/parameters'
|
|
7
7
|
require 'aspera/transfer/error'
|
|
8
8
|
require 'aspera/transfer/spec'
|
|
9
|
-
require 'aspera/resumer'
|
|
9
|
+
require 'aspera/transfer/resumer'
|
|
10
10
|
require 'aspera/log'
|
|
11
11
|
require 'aspera/assert'
|
|
12
12
|
require 'socket'
|
|
@@ -24,18 +24,18 @@ module Aspera
|
|
|
24
24
|
SELECT_AVAILABLE_PORT = 0
|
|
25
25
|
private_constant :LISTEN_LOCAL_ADDRESS, :SELECT_AVAILABLE_PORT
|
|
26
26
|
|
|
27
|
-
#
|
|
28
|
-
# @param ascp_args [Array]
|
|
29
|
-
# @param wss [Boolean] true
|
|
30
|
-
# @param quiet [Boolean]
|
|
31
|
-
# @param monitor [Boolean]
|
|
32
|
-
# @param trusted_certs [Array
|
|
33
|
-
# @param client_ssh_key [String]
|
|
34
|
-
# @param check_ignore_cb [Proc]
|
|
35
|
-
# @param spawn_timeout_sec [Integer]
|
|
36
|
-
# @param spawn_delay_sec [Integer]
|
|
37
|
-
# @param multi_incr_udp [Boolean
|
|
38
|
-
# @param resume [Hash
|
|
27
|
+
# Options: same as values in option `transfer_info`
|
|
28
|
+
# @param ascp_args [Array] (Params) Optional Additional arguments to ascp
|
|
29
|
+
# @param wss [Boolean] (Params) `true`: if both SSH and wss in ts: prefer wss
|
|
30
|
+
# @param quiet [Boolean] (Params) By default no native `ascp` progress bar
|
|
31
|
+
# @param monitor [Boolean] (Params) Set to `false` to eliminate management port
|
|
32
|
+
# @param trusted_certs [Array] (Params) Optional list of files with trusted certificates (stores)
|
|
33
|
+
# @param client_ssh_key [String] (Params) Client SSH key option (from CLIENT_SSH_KEY_OPTIONS)
|
|
34
|
+
# @param check_ignore_cb [Proc] (Params) Callback with host,port
|
|
35
|
+
# @param spawn_timeout_sec [Integer] Timeout for ascp spawn
|
|
36
|
+
# @param spawn_delay_sec [Integer] Optional delay to start between sessions
|
|
37
|
+
# @param multi_incr_udp [Boolean] Optional `true`: increment UDP port for each session
|
|
38
|
+
# @param resume [Hash] Optional Resume policy
|
|
39
39
|
# @param management_cb [Proc] callback for management events
|
|
40
40
|
# @param base_options [Hash] other options for base class
|
|
41
41
|
def initialize(
|
|
@@ -54,7 +54,7 @@ module Aspera
|
|
|
54
54
|
**base_options
|
|
55
55
|
)
|
|
56
56
|
super(**base_options)
|
|
57
|
-
#
|
|
57
|
+
# Special transfer parameters provided
|
|
58
58
|
@tr_opts = {
|
|
59
59
|
ascp_args: ascp_args,
|
|
60
60
|
wss: wss,
|
|
@@ -69,19 +69,22 @@ module Aspera
|
|
|
69
69
|
@multi_incr_udp = multi_incr_udp.nil? ? Environment.instance.os.eql?(Environment::OS_WINDOWS) : multi_incr_udp
|
|
70
70
|
@monitor = monitor
|
|
71
71
|
@management_cb = management_cb
|
|
72
|
-
|
|
72
|
+
resume = {} if resume.nil?
|
|
73
|
+
Aspera.assert_type(resume, Hash){'resume'}
|
|
74
|
+
@resume_policy = Transfer::Resumer.new(**resume.symbolize_keys)
|
|
73
75
|
# all transfer jobs, key = SecureRandom.uuid, protected by mutex, cond var on change
|
|
74
76
|
@sessions = []
|
|
75
77
|
# mutex protects global data accessed by threads
|
|
76
78
|
@mutex = Mutex.new
|
|
77
79
|
@pre_calc_sent = false
|
|
78
80
|
@pre_calc_last_size = nil
|
|
81
|
+
@command_file = File.join(config_dir || '.', "send_#{$PROCESS_ID}")
|
|
79
82
|
end
|
|
80
83
|
|
|
81
|
-
#
|
|
82
|
-
#
|
|
83
|
-
# @param transfer_spec
|
|
84
|
-
# @param token_regenerator [Object]
|
|
84
|
+
# Start `ascp` transfer(s) (non blocking), single or multi-session
|
|
85
|
+
# Session information added to @sessions
|
|
86
|
+
# @param transfer_spec [Hash] Aspera transfer specification
|
|
87
|
+
# @param token_regenerator [Object] Object with method refreshed_transfer_token
|
|
85
88
|
def start_transfer(transfer_spec, token_regenerator: nil)
|
|
86
89
|
# clone transfer spec because we modify it (first level keys)
|
|
87
90
|
transfer_spec = transfer_spec.clone
|
|
@@ -188,16 +191,23 @@ module Aspera
|
|
|
188
191
|
@sessions.select{ |session| session[:job_id].eql?(job_id)}
|
|
189
192
|
end
|
|
190
193
|
|
|
191
|
-
#
|
|
192
|
-
#
|
|
193
|
-
# @param session_index index of session (for multi session)
|
|
194
|
-
# @param data command on mgt port, examples:
|
|
194
|
+
# Send command to management port of command (used in `asession).
|
|
195
|
+
# Examples:
|
|
195
196
|
# {'type'=>'START','source'=>_path_,'destination'=>_path_}
|
|
196
197
|
# {'type'=>'DONE'}
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
198
|
+
# @param data [Hash] Command on mgt port
|
|
199
|
+
# @param id [String] Optional identifier or transfer session
|
|
200
|
+
def send_command(data, id: nil)
|
|
201
|
+
Log.dump(:command, data)
|
|
202
|
+
sessions = id ? @sessions.select{ |session| session[:job_id].eql?(id)} : @sessions
|
|
203
|
+
if sessions.empty?
|
|
204
|
+
Log.log.warn('No transfer session')
|
|
205
|
+
return
|
|
206
|
+
end
|
|
207
|
+
message = Ascp::Management.command_to_stream(data)
|
|
208
|
+
sessions.each do |session|
|
|
209
|
+
session[:io].puts(message)
|
|
210
|
+
end
|
|
201
211
|
end
|
|
202
212
|
|
|
203
213
|
private
|
|
@@ -301,6 +311,14 @@ module Aspera
|
|
|
301
311
|
session[:id] = event['SessionId'] if event['Type'].eql?('INIT')
|
|
302
312
|
@management_cb&.call(event)
|
|
303
313
|
process_progress(event)
|
|
314
|
+
next unless File.exist?(@command_file)
|
|
315
|
+
begin
|
|
316
|
+
commands = JSON.parse(File.read(@command_file))
|
|
317
|
+
send_command(commands)
|
|
318
|
+
rescue => e
|
|
319
|
+
Log.log.error{e.to_s}
|
|
320
|
+
end
|
|
321
|
+
File.delete(@command_file)
|
|
304
322
|
end
|
|
305
323
|
Log.log.debug('management io closed')
|
|
306
324
|
# check that last status was received before process exit
|
|
@@ -317,7 +335,7 @@ module Aspera
|
|
|
317
335
|
Log.log.warn('Regenerating token for transfer')
|
|
318
336
|
env['ASPERA_SCP_TOKEN'] = session[:token_regenerator].refreshed_transfer_token
|
|
319
337
|
end
|
|
320
|
-
raise Transfer::Error.new(last_event['Description'], last_event['Code'].to_i)
|
|
338
|
+
raise Transfer::Error.new(last_event['Description'], code: last_event['Code'].to_i)
|
|
321
339
|
else Aspera.error_unexpected_value(last_event['Type'], :error){'last event type'}
|
|
322
340
|
end
|
|
323
341
|
rescue SystemCallError => e
|
|
@@ -347,7 +365,7 @@ module Aspera
|
|
|
347
365
|
# status is nil if an exception occurred before starting command
|
|
348
366
|
if !status&.success?
|
|
349
367
|
message = "#{name} failed (#{status})"
|
|
350
|
-
# raise error only if there was not already an exception (ERROR_INFO)
|
|
368
|
+
# raise error only if there was not already an exception (`$ERROR_INFO`)
|
|
351
369
|
raise Transfer::Error, message unless $ERROR_INFO
|
|
352
370
|
# else display this message also, as main exception is already here
|
|
353
371
|
Log.log.error(message)
|
|
@@ -361,9 +379,8 @@ module Aspera
|
|
|
361
379
|
|
|
362
380
|
attr_reader :sessions
|
|
363
381
|
|
|
364
|
-
#
|
|
365
|
-
# @param event management port event
|
|
366
|
-
# @param session sessin object
|
|
382
|
+
# Notify progress to callback
|
|
383
|
+
# @param event [Hash] management port event
|
|
367
384
|
def process_progress(event)
|
|
368
385
|
session_id = event['SessionId']
|
|
369
386
|
case event['Type']
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'singleton'
|
|
4
|
+
require 'aspera/log'
|
|
5
|
+
require 'aspera/environment'
|
|
6
|
+
module Aspera
|
|
7
|
+
module Agent
|
|
8
|
+
# Factory for Agents
|
|
9
|
+
class Factory
|
|
10
|
+
include Singleton
|
|
11
|
+
|
|
12
|
+
# Create new agent
|
|
13
|
+
def create(agent, options)
|
|
14
|
+
Log.dump(:options, options)
|
|
15
|
+
require "aspera/agent/#{agent}"
|
|
16
|
+
Aspera::Agent.const_get(agent.to_s.capitalize).new(**options)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Discover available agents
|
|
20
|
+
# @return [Array] list of symbols of agents
|
|
21
|
+
def list
|
|
22
|
+
Dir.children(File.dirname(File.expand_path(__FILE__)))
|
|
23
|
+
.select{ |file| file.end_with?(Environment::RB_EXT)}
|
|
24
|
+
.map{ |file| File.basename(file, Environment::RB_EXT).to_sym}
|
|
25
|
+
.reject{ |item| IGNORED_ITEMS.include?(item)}
|
|
26
|
+
end
|
|
27
|
+
IGNORED_ITEMS = %i[factory base]
|
|
28
|
+
private_constant :IGNORED_ITEMS
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
data/lib/aspera/api/aoc.rb
CHANGED
|
@@ -8,11 +8,10 @@ require 'aspera/data_repository'
|
|
|
8
8
|
require 'aspera/transfer/spec'
|
|
9
9
|
require 'aspera/api/node'
|
|
10
10
|
require 'base64'
|
|
11
|
-
require 'cgi'
|
|
12
11
|
|
|
13
12
|
module Aspera
|
|
14
13
|
module Api
|
|
15
|
-
class AoC <
|
|
14
|
+
class AoC < Rest
|
|
16
15
|
PRODUCT_NAME = 'Aspera on Cloud'
|
|
17
16
|
# use default workspace if it is set, else none
|
|
18
17
|
DEFAULT_WORKSPACE = ''
|
|
@@ -140,6 +139,43 @@ module Aspera
|
|
|
140
139
|
organization: org_domain[:organization]
|
|
141
140
|
}
|
|
142
141
|
end
|
|
142
|
+
|
|
143
|
+
# Call block with same query using paging and response information.
|
|
144
|
+
# Block must return a hash with :data and :http keys
|
|
145
|
+
# @return [Hash] {items: , total: }
|
|
146
|
+
def call_paging(query: {}, formatter: nil)
|
|
147
|
+
Aspera.assert_type(query, Hash){'query'}
|
|
148
|
+
Aspera.assert(block_given?)
|
|
149
|
+
# set default large page if user does not specify own parameters. AoC Caps to 1000 anyway
|
|
150
|
+
query['per_page'] = 1000 unless query.key?('per_page')
|
|
151
|
+
max_items = query.delete(Rest::MAX_ITEMS)
|
|
152
|
+
max_pages = query.delete(Rest::MAX_PAGES)
|
|
153
|
+
item_list = []
|
|
154
|
+
total_count = nil
|
|
155
|
+
current_page = query['page']
|
|
156
|
+
current_page = 1 if current_page.nil?
|
|
157
|
+
page_count = 0
|
|
158
|
+
loop do
|
|
159
|
+
new_query = query.clone
|
|
160
|
+
new_query['page'] = current_page
|
|
161
|
+
result = yield(new_query)
|
|
162
|
+
Aspera.assert(result[:data])
|
|
163
|
+
Aspera.assert(result[:http])
|
|
164
|
+
total_count = result[:http]['X-Total-Count']
|
|
165
|
+
page_count += 1
|
|
166
|
+
current_page += 1
|
|
167
|
+
add_items = result[:data]
|
|
168
|
+
break if add_items.empty?
|
|
169
|
+
# append new items to full list
|
|
170
|
+
item_list += add_items
|
|
171
|
+
break if !max_items.nil? && item_list.count >= max_items
|
|
172
|
+
break if !max_pages.nil? && page_count >= max_pages
|
|
173
|
+
formatter&.long_operation_running("#{item_list.count} / #{total_count}") unless total_count.eql?(item_list.count.to_s)
|
|
174
|
+
end
|
|
175
|
+
formatter&.long_operation_terminated
|
|
176
|
+
item_list = item_list[0..max_items - 1] if !max_items.nil? && item_list.count > max_items
|
|
177
|
+
return {items: item_list, total: total_count}
|
|
178
|
+
end
|
|
143
179
|
end
|
|
144
180
|
|
|
145
181
|
attr_reader :private_link
|
|
@@ -147,8 +183,8 @@ module Aspera
|
|
|
147
183
|
def initialize(url:, auth:, subpath: API_V1, client_id: nil, client_secret: nil, scope: nil, redirect_uri: nil, private_key: nil, passphrase: nil, username: nil,
|
|
148
184
|
password: nil, workspace: nil, secret_finder: nil)
|
|
149
185
|
# test here because link may set url
|
|
150
|
-
raise
|
|
151
|
-
raise
|
|
186
|
+
raise ParameterError, 'Missing mandatory option: url' if url.nil?
|
|
187
|
+
raise ParameterError, 'Missing mandatory option: scope' if scope.nil?
|
|
152
188
|
# default values for client id
|
|
153
189
|
client_id, client_secret = self.class.get_client_info if client_id.nil?
|
|
154
190
|
# access key secrets are provided out of band to get node api access
|
|
@@ -173,22 +209,21 @@ module Aspera
|
|
|
173
209
|
auth_params[:grant_method] = if url_info.key?(:token)
|
|
174
210
|
:url_json
|
|
175
211
|
else
|
|
176
|
-
raise
|
|
212
|
+
raise ParameterError, 'Missing mandatory option: auth' if auth.nil?
|
|
177
213
|
auth
|
|
178
214
|
end
|
|
179
215
|
# this is the base API url
|
|
180
216
|
api_url_base = self.class.api_base_url(api_domain: url_info[:instance_domain])
|
|
181
217
|
# auth URL
|
|
182
218
|
auth_params[:base_url] = "#{api_url_base}/#{OAUTH_API_SUBPATH}/#{url_info[:organization]}"
|
|
183
|
-
|
|
184
219
|
# fill other auth parameters based on OAuth method
|
|
185
220
|
case auth_params[:grant_method]
|
|
186
221
|
when :web
|
|
187
|
-
raise
|
|
222
|
+
raise ParameterError, 'Missing mandatory option: redirect_uri' if redirect_uri.nil?
|
|
188
223
|
auth_params[:redirect_uri] = redirect_uri
|
|
189
224
|
when :jwt
|
|
190
|
-
raise
|
|
191
|
-
raise
|
|
225
|
+
raise ParameterError, 'Missing mandatory option: private_key' if private_key.nil?
|
|
226
|
+
raise ParameterError, 'Missing mandatory option: username' if username.nil?
|
|
192
227
|
auth_params[:private_key_obj] = OpenSSL::PKey::RSA.new(private_key, passphrase)
|
|
193
228
|
auth_params[:payload] = {
|
|
194
229
|
iss: auth_params[:client_id], # issuer
|
|
@@ -205,7 +240,7 @@ module Aspera
|
|
|
205
240
|
auth_params[:json][:password] = password unless password.nil?
|
|
206
241
|
# basic auth required for /token
|
|
207
242
|
auth_params[:auth] = {type: :basic, username: auth_params[:client_id], password: auth_params[:client_secret]}
|
|
208
|
-
else Aspera.error_unexpected_value(auth_params[:grant_method])
|
|
243
|
+
else Aspera.error_unexpected_value(auth_params[:grant_method]){'auth, use one of: :web, :jwt'}
|
|
209
244
|
end
|
|
210
245
|
super(
|
|
211
246
|
base_url: "#{api_url_base}/#{subpath}",
|
|
@@ -213,12 +248,13 @@ module Aspera
|
|
|
213
248
|
)
|
|
214
249
|
end
|
|
215
250
|
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
251
|
+
# read using the query and paging
|
|
252
|
+
# @return [Hash] {items: , total: }
|
|
253
|
+
def read_with_paging(subpath, query: {}, formatter: nil)
|
|
254
|
+
return self.class.call_paging(query: query, formatter: formatter) do |paged_query|
|
|
255
|
+
# Use `call` instead of `read` to get headers
|
|
256
|
+
call(operation: 'GET', subpath: subpath, headers: {'Accept' => Rest::MIME_JSON}, query: paged_query)
|
|
257
|
+
end
|
|
222
258
|
end
|
|
223
259
|
|
|
224
260
|
def assert_public_link_types(expected)
|
|
@@ -230,7 +266,17 @@ module Aspera
|
|
|
230
266
|
return [] # TODO : public_link['id'] ?
|
|
231
267
|
end
|
|
232
268
|
|
|
233
|
-
#
|
|
269
|
+
# Cached public link information
|
|
270
|
+
# @return [Hash, nil] token info if public link or nil
|
|
271
|
+
def public_link
|
|
272
|
+
return unless auth_params[:grant_method].eql?(:url_json)
|
|
273
|
+
return @cache_url_token_info unless @cache_url_token_info.nil?
|
|
274
|
+
# TODO: can there be several in list ?
|
|
275
|
+
@cache_url_token_info = read('url_tokens').first
|
|
276
|
+
return @cache_url_token_info
|
|
277
|
+
end
|
|
278
|
+
|
|
279
|
+
# Cached user information
|
|
234
280
|
def current_user_info(exception: false)
|
|
235
281
|
return @cache_user_info unless @cache_user_info.nil?
|
|
236
282
|
# get our user's default information
|
|
@@ -246,21 +292,9 @@ module Aspera
|
|
|
246
292
|
return @cache_user_info
|
|
247
293
|
end
|
|
248
294
|
|
|
295
|
+
# Cached workspace information
|
|
249
296
|
def workspace
|
|
250
|
-
|
|
251
|
-
@workspace_info
|
|
252
|
-
end
|
|
253
|
-
|
|
254
|
-
def home
|
|
255
|
-
Aspera.assert(!@home_info.nil?){'AoC home context is not set'}
|
|
256
|
-
@home_info
|
|
257
|
-
end
|
|
258
|
-
|
|
259
|
-
# Set the application context
|
|
260
|
-
# @param application [Symbol,NilClass] :files or :packages
|
|
261
|
-
# @return [Hash] current context information: workspace, and home node/file if app is "Files"
|
|
262
|
-
def context=(application)
|
|
263
|
-
Aspera.assert_values(application, %i[files packages])
|
|
297
|
+
return @workspace_info unless @workspace_info.nil?
|
|
264
298
|
ws_id =
|
|
265
299
|
if !public_link.nil?
|
|
266
300
|
Log.log.debug('Using workspace of public link')
|
|
@@ -278,26 +312,21 @@ module Aspera
|
|
|
278
312
|
else
|
|
279
313
|
lookup_by_name('workspaces', @workspace_name)['id']
|
|
280
314
|
end
|
|
281
|
-
ws_info =
|
|
282
|
-
if ws_id.nil?
|
|
283
|
-
nil
|
|
284
|
-
else
|
|
285
|
-
read("workspaces/#{ws_id}")
|
|
286
|
-
end
|
|
287
315
|
@workspace_info =
|
|
288
|
-
if
|
|
316
|
+
if ws_id.nil?
|
|
289
317
|
{
|
|
290
|
-
|
|
291
|
-
name: "Shared #{application}"
|
|
318
|
+
name: 'Shared (no workspace)'
|
|
292
319
|
}
|
|
293
320
|
else
|
|
294
|
-
{
|
|
295
|
-
id: ws_info['id'],
|
|
296
|
-
name: ws_info['name']
|
|
297
|
-
}
|
|
321
|
+
read("workspaces/#{ws_id}").slice('id', 'name', 'home_node_id', 'home_file_id').symbolize_keys
|
|
298
322
|
end
|
|
299
|
-
Log.dump(:
|
|
300
|
-
|
|
323
|
+
Log.dump(:workspace_info, @workspace_info)
|
|
324
|
+
@workspace_info
|
|
325
|
+
end
|
|
326
|
+
|
|
327
|
+
# Cached Home information for current user in Files app
|
|
328
|
+
def home
|
|
329
|
+
return @home_info unless @home_info.nil?
|
|
301
330
|
@home_info =
|
|
302
331
|
if !public_link.nil?
|
|
303
332
|
assert_public_link_types(['view_shared_file'])
|
|
@@ -310,10 +339,10 @@ module Aspera
|
|
|
310
339
|
node_id: private_link[:node_id],
|
|
311
340
|
file_id: private_link[:file_id]
|
|
312
341
|
}
|
|
313
|
-
elsif
|
|
342
|
+
elsif workspace[:home_node_id] && workspace[:home_file_id]
|
|
314
343
|
{
|
|
315
|
-
node_id:
|
|
316
|
-
file_id:
|
|
344
|
+
node_id: workspace[:home_node_id],
|
|
345
|
+
file_id: workspace[:home_file_id]
|
|
317
346
|
}
|
|
318
347
|
else
|
|
319
348
|
# not part of any workspace, but has some folder shared
|
|
@@ -325,6 +354,7 @@ module Aspera
|
|
|
325
354
|
end
|
|
326
355
|
raise "Cannot get user's home node id, check your default workspace or specify one" if @home_info[:node_id].to_s.empty?
|
|
327
356
|
Log.dump(:context, @home_info)
|
|
357
|
+
@home_info
|
|
328
358
|
end
|
|
329
359
|
|
|
330
360
|
# @param node_id [String] identifier of node in AoC
|