aspera-cli 4.9.0 → 4.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +20 -0
- data/CHANGELOG.md +509 -0
- data/CONTRIBUTING.md +118 -0
- data/README.md +1241 -916
- data/bin/ascli +4 -4
- data/bin/asession +11 -11
- data/docs/test_env.conf +32 -21
- data/examples/aoc.rb +4 -4
- data/examples/dascli +16 -9
- data/examples/faspex4.rb +8 -8
- data/examples/node.rb +12 -12
- data/examples/server.rb +10 -10
- data/lib/aspera/aoc.rb +273 -266
- data/lib/aspera/ascmd.rb +56 -54
- data/lib/aspera/ats_api.rb +4 -4
- data/lib/aspera/cli/basic_auth_plugin.rb +15 -12
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formater.rb +64 -64
- data/lib/aspera/cli/info.rb +2 -2
- data/lib/aspera/cli/listener/line_dump.rb +1 -1
- data/lib/aspera/cli/listener/logger.rb +1 -1
- data/lib/aspera/cli/listener/progress.rb +5 -6
- data/lib/aspera/cli/listener/progress_multi.rb +14 -19
- data/lib/aspera/cli/main.rb +66 -67
- data/lib/aspera/cli/manager.rb +112 -110
- data/lib/aspera/cli/plugin.rb +57 -36
- data/lib/aspera/cli/plugins/alee.rb +4 -4
- data/lib/aspera/cli/plugins/aoc.rb +309 -670
- data/lib/aspera/cli/plugins/ats.rb +44 -46
- data/lib/aspera/cli/plugins/bss.rb +10 -10
- data/lib/aspera/cli/plugins/config.rb +497 -378
- data/lib/aspera/cli/plugins/console.rb +12 -12
- data/lib/aspera/cli/plugins/cos.rb +18 -20
- data/lib/aspera/cli/plugins/faspex.rb +112 -114
- data/lib/aspera/cli/plugins/faspex5.rb +71 -46
- data/lib/aspera/cli/plugins/node.rb +379 -283
- data/lib/aspera/cli/plugins/orchestrator.rb +46 -46
- data/lib/aspera/cli/plugins/preview.rb +122 -114
- data/lib/aspera/cli/plugins/server.rb +137 -83
- data/lib/aspera/cli/plugins/shares.rb +30 -29
- data/lib/aspera/cli/plugins/sync.rb +13 -33
- data/lib/aspera/cli/transfer_agent.rb +60 -59
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +3 -3
- data/lib/aspera/command_line_builder.rb +27 -27
- data/lib/aspera/cos_node.rb +22 -20
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +35 -15
- data/lib/aspera/fasp/agent_base.rb +15 -15
- data/lib/aspera/fasp/agent_connect.rb +23 -21
- data/lib/aspera/fasp/agent_direct.rb +66 -64
- data/lib/aspera/fasp/agent_httpgw.rb +141 -78
- data/lib/aspera/fasp/agent_node.rb +23 -21
- data/lib/aspera/fasp/agent_trsdk.rb +20 -20
- data/lib/aspera/fasp/error.rb +3 -2
- data/lib/aspera/fasp/error_info.rb +11 -8
- data/lib/aspera/fasp/installation.rb +79 -79
- data/lib/aspera/fasp/listener.rb +1 -1
- data/lib/aspera/fasp/parameters.rb +86 -71
- data/lib/aspera/fasp/parameters.yaml +7 -4
- data/lib/aspera/fasp/resume_policy.rb +8 -8
- data/lib/aspera/fasp/transfer_spec.rb +35 -2
- data/lib/aspera/fasp/uri.rb +7 -7
- data/lib/aspera/faspex_gw.rb +7 -5
- data/lib/aspera/hash_ext.rb +3 -3
- data/lib/aspera/id_generator.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +38 -105
- data/lib/aspera/keychain/macos_security.rb +128 -57
- data/lib/aspera/log.rb +7 -7
- data/lib/aspera/nagios.rb +19 -18
- data/lib/aspera/node.rb +209 -35
- data/lib/aspera/oauth.rb +37 -36
- data/lib/aspera/open_application.rb +19 -11
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +16 -15
- data/lib/aspera/preview/file_types.rb +8 -8
- data/lib/aspera/preview/generator.rb +67 -67
- data/lib/aspera/preview/utils.rb +27 -27
- data/lib/aspera/proxy_auto_config.js +41 -41
- data/lib/aspera/proxy_auto_config.rb +21 -14
- data/lib/aspera/rest.rb +72 -67
- data/lib/aspera/rest_call_error.rb +2 -1
- data/lib/aspera/rest_error_analyzer.rb +18 -17
- data/lib/aspera/rest_errors_aspera.rb +16 -16
- data/lib/aspera/secret_hider.rb +15 -13
- data/lib/aspera/ssh.rb +11 -10
- data/lib/aspera/sync.rb +158 -44
- data/lib/aspera/temp_file_manager.rb +2 -2
- data/lib/aspera/uri_reader.rb +4 -4
- data/lib/aspera/web_auth.rb +14 -13
- data.tar.gz.sig +0 -0
- metadata +11 -36
- metadata.gz.sig +0 -0
@@ -9,7 +9,7 @@ require 'securerandom'
|
|
9
9
|
module Aspera
|
10
10
|
module Cli
|
11
11
|
module Plugins
|
12
|
-
class Faspex5 < BasicAuthPlugin
|
12
|
+
class Faspex5 < Aspera::Cli::BasicAuthPlugin
|
13
13
|
class << self
|
14
14
|
def detect(base_url)
|
15
15
|
api = Rest.new(base_url: base_url, redirect_max: 1)
|
@@ -21,32 +21,31 @@ module Aspera
|
|
21
21
|
end
|
22
22
|
end
|
23
23
|
|
24
|
-
VAL_ALL = 'ALL'
|
25
24
|
TRANSFER_CONNECT = 'connect'
|
26
|
-
private_constant :
|
25
|
+
private_constant :TRANSFER_CONNECT
|
27
26
|
|
28
27
|
def initialize(env)
|
29
28
|
super(env)
|
30
|
-
options.add_opt_simple(:client_id,'OAuth client identifier')
|
31
|
-
options.add_opt_simple(:client_secret,'OAuth client secret')
|
32
|
-
options.add_opt_simple(:redirect_uri,'OAuth redirect URI for web authentication')
|
33
|
-
options.add_opt_list(:auth,[Oauth::STD_AUTH_TYPES
|
34
|
-
options.add_opt_simple(:private_key,'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
|
35
|
-
options.add_opt_simple(:passphrase,'RSA private key passphrase')
|
36
|
-
options.set_option(:auth
|
29
|
+
options.add_opt_simple(:client_id, 'OAuth client identifier')
|
30
|
+
options.add_opt_simple(:client_secret, 'OAuth client secret')
|
31
|
+
options.add_opt_simple(:redirect_uri, 'OAuth redirect URI for web authentication')
|
32
|
+
options.add_opt_list(:auth, [:boot].concat(Oauth::STD_AUTH_TYPES), 'OAuth type of authentication')
|
33
|
+
options.add_opt_simple(:private_key, 'OAuth JWT RSA private key PEM value (prefix file path with @file:)')
|
34
|
+
options.add_opt_simple(:passphrase, 'RSA private key passphrase')
|
35
|
+
options.set_option(:auth, :jwt)
|
37
36
|
options.parse_options!
|
38
37
|
end
|
39
38
|
|
40
39
|
def set_api
|
41
|
-
@faxpex5_api_base_url = options.get_option(:url,is_type: :mandatory).gsub(%r{/+$},'')
|
40
|
+
@faxpex5_api_base_url = options.get_option(:url, is_type: :mandatory).gsub(%r{/+$}, '')
|
42
41
|
@faxpex5_api_auth_url = "#{@faxpex5_api_base_url}/auth"
|
43
42
|
faxpex5_api_v5_url = "#{@faxpex5_api_base_url}/api/v5"
|
44
|
-
case options.get_option(:auth,is_type: :mandatory)
|
43
|
+
case options.get_option(:auth, is_type: :mandatory)
|
45
44
|
when :boot
|
46
45
|
# the password here is the token copied directly from browser in developer mode
|
47
46
|
@api_v5 = Rest.new({
|
48
47
|
base_url: faxpex5_api_v5_url,
|
49
|
-
headers: {'Authorization' => options.get_option(:password,is_type: :mandatory)}
|
48
|
+
headers: {'Authorization' => options.get_option(:password, is_type: :mandatory)}
|
50
49
|
})
|
51
50
|
when :web
|
52
51
|
# opens a browser and ask user to auth using web
|
@@ -56,11 +55,11 @@ module Aspera
|
|
56
55
|
type: :oauth2,
|
57
56
|
base_url: @faxpex5_api_auth_url,
|
58
57
|
crtype: :web,
|
59
|
-
client_id: options.get_option(:client_id,is_type: :mandatory),
|
60
|
-
web: {redirect_uri: options.get_option(:redirect_uri,is_type: :mandatory)}
|
58
|
+
client_id: options.get_option(:client_id, is_type: :mandatory),
|
59
|
+
web: {redirect_uri: options.get_option(:redirect_uri, is_type: :mandatory)}
|
61
60
|
}})
|
62
61
|
when :jwt
|
63
|
-
app_client_id = options.get_option(:client_id,is_type: :mandatory)
|
62
|
+
app_client_id = options.get_option(:client_id, is_type: :mandatory)
|
64
63
|
@api_v5 = Rest.new({
|
65
64
|
base_url: faxpex5_api_v5_url,
|
66
65
|
auth: {
|
@@ -72,31 +71,33 @@ module Aspera
|
|
72
71
|
payload: {
|
73
72
|
iss: app_client_id, # issuer
|
74
73
|
aud: app_client_id, # audience TODO: ???
|
75
|
-
sub: "user:#{options.get_option(:username,is_type: :mandatory)}" # subject also "client:#{app_client_id}" + auth user/pass
|
74
|
+
sub: "user:#{options.get_option(:username, is_type: :mandatory)}" # subject also "client:#{app_client_id}" + auth user/pass
|
76
75
|
},
|
77
|
-
#auth: {type: :basic, options.get_option(:username,is_type: :mandatory), options.get_option(:password,is_type: :mandatory),
|
78
|
-
private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key,is_type: :mandatory),options.get_option(:passphrase)),
|
76
|
+
# auth: {type: :basic, options.get_option(:username,is_type: :mandatory), options.get_option(:password,is_type: :mandatory),
|
77
|
+
private_key_obj: OpenSSL::PKey::RSA.new(options.get_option(:private_key, is_type: :mandatory), options.get_option(:passphrase)),
|
79
78
|
headers: {typ: 'JWT'}
|
80
79
|
}
|
81
80
|
}})
|
82
81
|
end
|
83
82
|
end
|
84
83
|
|
85
|
-
ACTIONS = %i[health package admin
|
84
|
+
ACTIONS = %i[health version user bearer_token package admin].freeze
|
86
85
|
|
87
86
|
def execute_action
|
88
87
|
set_api
|
89
88
|
command = options.get_next_command(ACTIONS)
|
90
89
|
case command
|
90
|
+
when :version
|
91
|
+
return { type: :single_object, data: @api_v5.read('version')[:data] }
|
91
92
|
when :health
|
92
93
|
nagios = Nagios.new
|
93
94
|
begin
|
94
|
-
result=Rest.new(base_url: @faxpex5_api_base_url).read('health')[:data]
|
95
|
-
result.each do |k,v|
|
96
|
-
nagios.add_ok(k,v.to_s)
|
95
|
+
result = Rest.new(base_url: @faxpex5_api_base_url).read('health')[:data]
|
96
|
+
result.each do |k, v|
|
97
|
+
nagios.add_ok(k, v.to_s)
|
97
98
|
end
|
98
99
|
rescue StandardError => e
|
99
|
-
nagios.add_critical('faspex api',e.to_s)
|
100
|
+
nagios.add_critical('faspex api', e.to_s)
|
100
101
|
end
|
101
102
|
return nagios.result
|
102
103
|
when :user
|
@@ -106,10 +107,12 @@ module Aspera
|
|
106
107
|
when :show
|
107
108
|
return { type: :single_object, data: @api_v5.read('account/preferences')[:data] }
|
108
109
|
when :modify
|
109
|
-
@api_v5.update('account/preferences',options.get_next_argument('modified parameters
|
110
|
+
@api_v5.update('account/preferences', options.get_next_argument('modified parameters', type: Hash))
|
110
111
|
return Main.result_status('modified')
|
111
112
|
end
|
112
113
|
end
|
114
|
+
when :bearer_token
|
115
|
+
return {type: :text, data: @api_v5.oauth_token}
|
113
116
|
when :package
|
114
117
|
command = options.get_next_command(%i[list show send receive])
|
115
118
|
case command
|
@@ -117,48 +120,70 @@ module Aspera
|
|
117
120
|
parameters = options.get_option(:value)
|
118
121
|
return {
|
119
122
|
type: :object_list,
|
120
|
-
data: @api_v5.read('packages',parameters)[:data]['packages'],
|
123
|
+
data: @api_v5.read('packages', parameters)[:data]['packages'],
|
121
124
|
fields: %w[id title release_date total_bytes total_files created_time state]
|
122
125
|
}
|
123
126
|
when :show
|
124
127
|
id = instance_identifier
|
125
128
|
return {type: :single_object, data: @api_v5.read("packages/#{id}")[:data]}
|
126
129
|
when :send
|
127
|
-
parameters = options.get_option(:value,is_type: :mandatory)
|
128
|
-
raise CliBadArgument,'value must be hash, refer to API' unless parameters.is_a?(Hash)
|
129
|
-
package = @api_v5.create('packages',parameters)[:data]
|
130
|
-
# TODO: option to send from remote source
|
131
|
-
transfer_spec = @api_v5.
|
130
|
+
parameters = options.get_option(:value, is_type: :mandatory)
|
131
|
+
raise CliBadArgument, 'value must be hash, refer to API' unless parameters.is_a?(Hash)
|
132
|
+
package = @api_v5.create('packages', parameters)[:data]
|
133
|
+
# TODO: option to send from remote source or httpgw
|
134
|
+
transfer_spec = @api_v5.call(
|
135
|
+
operation: 'POST',
|
136
|
+
subpath: "packages/#{package['id']}/transfer_spec/upload",
|
137
|
+
headers: {'Accept' => 'application/json'},
|
138
|
+
url_params: {transfer_type: TRANSFER_CONNECT},
|
139
|
+
json_params: {paths: transfer.ts_source_paths.map{|i|i['source']}}
|
140
|
+
)[:data]
|
132
141
|
transfer_spec.delete('authentication')
|
133
|
-
return Main.result_transfer(transfer.start(transfer_spec
|
142
|
+
return Main.result_transfer(transfer.start(transfer_spec))
|
134
143
|
when :receive
|
135
144
|
pkg_type = 'received'
|
136
145
|
pack_id = instance_identifier
|
137
146
|
package_ids = [pack_id]
|
138
147
|
skip_ids_data = []
|
139
148
|
skip_ids_persistency = nil
|
140
|
-
if options.get_option(:once_only,is_type: :mandatory)
|
149
|
+
if options.get_option(:once_only, is_type: :mandatory)
|
141
150
|
# read ids from persistency
|
142
151
|
skip_ids_persistency = PersistencyActionOnce.new(
|
143
152
|
manager: @agents[:persistency],
|
144
153
|
data: skip_ids_data,
|
145
|
-
id: IdGenerator.from_list([
|
154
|
+
id: IdGenerator.from_list([
|
155
|
+
'faspex_recv',
|
156
|
+
options.get_option(:url, is_type: :mandatory),
|
157
|
+
options.get_option(:username, is_type: :mandatory),
|
158
|
+
pkg_type]))
|
146
159
|
end
|
147
|
-
if
|
160
|
+
if VAL_ALL.eql?(pack_id)
|
148
161
|
# TODO: if packages have same name, they will overwrite
|
149
162
|
parameters = options.get_option(:value)
|
150
|
-
parameters ||= {'type' => 'received','subtype' => 'mypackages','limit' => 1000}
|
151
|
-
raise CliBadArgument,'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
|
152
|
-
package_ids = @api_v5.read('packages',parameters)[:data]['packages'].map{|p|p['id']}
|
163
|
+
parameters ||= {'type' => 'received', 'subtype' => 'mypackages', 'limit' => 1000}
|
164
|
+
raise CliBadArgument, 'value filter must be Hash (API GET)' unless parameters.is_a?(Hash)
|
165
|
+
package_ids = @api_v5.read('packages', parameters)[:data]['packages'].map{|p|p['id']}
|
153
166
|
package_ids.reject!{|i|skip_ids_data.include?(i)}
|
154
167
|
end
|
155
168
|
result_transfer = []
|
156
169
|
package_ids.each do |pkgid|
|
170
|
+
param_file_list = {}
|
171
|
+
begin
|
172
|
+
param_file_list['paths'] = transfer.ts_source_paths.map{|i|i['source']}
|
173
|
+
rescue Aspera::Cli::CliBadArgument
|
174
|
+
# paths is optional
|
175
|
+
end
|
157
176
|
# TODO: allow from sent as well ?
|
158
|
-
transfer_spec = @api_v5.
|
177
|
+
transfer_spec = @api_v5.call(
|
178
|
+
operation: 'POST',
|
179
|
+
subpath: "packages/#{pkgid}/transfer_spec/download",
|
180
|
+
headers: {'Accept' => 'application/json'},
|
181
|
+
url_params: {transfer_type: TRANSFER_CONNECT, type: pkg_type},
|
182
|
+
json_params: param_file_list
|
183
|
+
)[:data]
|
159
184
|
transfer_spec.delete('authentication')
|
160
|
-
statuses = transfer.start(transfer_spec
|
161
|
-
result_transfer.push({'package' => pkgid,Main::STATUS_FIELD => statuses})
|
185
|
+
statuses = transfer.start(transfer_spec)
|
186
|
+
result_transfer.push({'package' => pkgid, Main::STATUS_FIELD => statuses})
|
162
187
|
# skip only if all sessions completed
|
163
188
|
skip_ids_data.push(pkgid) if TransferAgent.session_status(statuses).eql?(:success)
|
164
189
|
end
|
@@ -172,19 +197,19 @@ module Aspera
|
|
172
197
|
res_path = list_key = res_type.to_s
|
173
198
|
case res_type
|
174
199
|
when :metadata_profiles
|
175
|
-
res_path='configuration/metadata_profiles'
|
176
|
-
list_key='profiles'
|
200
|
+
res_path = 'configuration/metadata_profiles'
|
201
|
+
list_key = 'profiles'
|
177
202
|
end
|
178
203
|
display_fields =
|
179
204
|
case res_type
|
180
|
-
when :accounts then [:all_but,'user_profile_data_attributes']
|
181
|
-
when :oauth_clients then [:all_but,'public_key']
|
205
|
+
when :accounts then [:all_but, 'user_profile_data_attributes']
|
206
|
+
when :oauth_clients then [:all_but, 'public_key']
|
182
207
|
end
|
183
208
|
adm_api = @api_v5
|
184
209
|
if res_type.eql?(:oauth_clients)
|
185
210
|
adm_api = Rest.new(@api_v5.params.merge({base_url: @faxpex5_api_auth_url}))
|
186
211
|
end
|
187
|
-
return entity_action(adm_api,res_path,item_list_key: list_key, display_fields: display_fields)
|
212
|
+
return entity_action(adm_api, res_path, item_list_key: list_key, display_fields: display_fields)
|
188
213
|
end
|
189
214
|
end # case command
|
190
215
|
end # action
|