aspera-cli 4.6.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
data/lib/aspera/aoc.rb
CHANGED
@@ -1,28 +1,34 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/log'
|
2
3
|
require 'aspera/rest'
|
3
4
|
require 'aspera/hash_ext'
|
4
5
|
require 'aspera/data_repository'
|
5
|
-
require 'aspera/fasp/
|
6
|
+
require 'aspera/fasp/transfer_spec'
|
6
7
|
require 'base64'
|
7
8
|
|
9
|
+
Aspera::Oauth.register_token_creator(:aoc_pub_link,lambda{|o|
|
10
|
+
o.token_auth_api.call({
|
11
|
+
operation: 'POST',
|
12
|
+
subpath: o.params[:path_token],
|
13
|
+
headers: {'Accept'=>'application/json'},
|
14
|
+
json_params: o.params[:aoc_pub_link][:json],
|
15
|
+
url_params: o.params[:aoc_pub_link][:url].merge(scope: o.params[:scope]) # scope is here because it changes over time (node)
|
16
|
+
})
|
17
|
+
})
|
18
|
+
|
8
19
|
module Aspera
|
9
20
|
class AoC < Rest
|
10
|
-
private
|
11
|
-
@@use_standard_ports = true
|
12
|
-
|
13
|
-
API_V1='api/v1'
|
14
|
-
|
15
21
|
PRODUCT_NAME='Aspera on Cloud'
|
16
22
|
# Production domain of AoC
|
17
23
|
PROD_DOMAIN='ibmaspera.com'
|
18
24
|
# to avoid infinite loop in pub link redirection
|
19
25
|
MAX_REDIRECT=10
|
20
|
-
|
26
|
+
# Well-known AoC globals client apps
|
27
|
+
GLOBAL_CLIENT_APPS=['aspera.global-cli-client','aspera.drive']
|
21
28
|
# index offset in data repository of client app
|
22
29
|
DATA_REPO_INDEX_START = 4
|
23
30
|
# cookie prefix so that console can decode identity
|
24
31
|
COOKIE_PREFIX='aspera.aoc'
|
25
|
-
|
26
32
|
# path in URL of public links
|
27
33
|
PUBLIC_LINK_PATHS=['/packages/public/receive','/packages/public/send','/files/public']
|
28
34
|
JWT_AUDIENCE='https://api.asperafiles.com/api/v1/oauth2/token'
|
@@ -30,10 +36,8 @@ module Aspera
|
|
30
36
|
# minimum fields for user info if retrieval fails
|
31
37
|
USER_INFO_FIELDS_MIN=['name','email','id','default_workspace_id','organization_id']
|
32
38
|
|
33
|
-
private_constant :
|
34
|
-
:OAUTH_API_SUBPATH,:COOKIE_PREFIX,:USER_INFO_FIELDS_MIN
|
39
|
+
private_constant :MAX_REDIRECT,:GLOBAL_CLIENT_APPS,:DATA_REPO_INDEX_START,:COOKIE_PREFIX,:PUBLIC_LINK_PATHS,:JWT_AUDIENCE,:OAUTH_API_SUBPATH,:USER_INFO_FIELDS_MIN
|
35
40
|
|
36
|
-
public
|
37
41
|
# various API scopes supported
|
38
42
|
SCOPE_FILES_SELF='self'
|
39
43
|
SCOPE_FILES_USER='user:all'
|
@@ -45,14 +49,19 @@ module Aspera
|
|
45
49
|
PATH_SEPARATOR='/'
|
46
50
|
FILES_APP='files'
|
47
51
|
PACKAGES_APP='packages'
|
52
|
+
API_V1='api/v1'
|
53
|
+
|
54
|
+
# class instance variable, access with accessors on class
|
55
|
+
@use_standard_ports = true
|
48
56
|
|
49
57
|
# class static methods
|
50
58
|
class << self
|
59
|
+
attr_accessor :use_standard_ports
|
51
60
|
# strings /Applications/Aspera\ Drive.app/Contents/MacOS/AsperaDrive|grep -E '.{100}==$'|base64 --decode
|
52
|
-
def get_client_info(client_name=
|
53
|
-
client_index=
|
61
|
+
def get_client_info(client_name=GLOBAL_CLIENT_APPS.first)
|
62
|
+
client_index=GLOBAL_CLIENT_APPS.index(client_name)
|
54
63
|
raise "no such pre-defined client: #{client_name}" if client_index.nil?
|
55
|
-
return client_name,Base64.urlsafe_encode64(DataRepository.instance.
|
64
|
+
return client_name,Base64.urlsafe_encode64(DataRepository.instance.data(DATA_REPO_INDEX_START+client_index))
|
56
65
|
end
|
57
66
|
|
58
67
|
# @param url of AoC instance
|
@@ -76,55 +85,50 @@ module Aspera
|
|
76
85
|
|
77
86
|
def metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
|
78
87
|
return Rest.new({
|
79
|
-
:
|
80
|
-
:
|
88
|
+
base_url: "#{api_base_url(api_domain)}/metering/v1",
|
89
|
+
headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
|
81
90
|
})
|
82
91
|
end
|
83
92
|
|
84
93
|
# node API scopes
|
85
94
|
def node_scope(access_key,scope)
|
86
|
-
return
|
87
|
-
end
|
88
|
-
|
89
|
-
def set_use_default_ports(val)
|
90
|
-
@@use_standard_ports=val
|
95
|
+
return "node.#{access_key}:#{scope}"
|
91
96
|
end
|
92
97
|
|
93
98
|
# check option "link"
|
94
99
|
# if present try to get token value (resolve redirection if short links used)
|
95
100
|
# then set options url/token/auth
|
96
|
-
def resolve_pub_link(
|
101
|
+
def resolve_pub_link(a_auth,a_opt)
|
102
|
+
public_link_url=a_opt[:link]
|
97
103
|
return if public_link_url.nil?
|
98
|
-
|
99
|
-
url_param_token_pair=nil
|
104
|
+
raise 'do not use both link and url options' unless a_opt[:url].nil?
|
100
105
|
redirect_count=0
|
101
|
-
|
106
|
+
while redirect_count <= MAX_REDIRECT
|
102
107
|
uri=URI.parse(public_link_url)
|
108
|
+
# detect if it's an expected format
|
103
109
|
if PUBLIC_LINK_PATHS.include?(uri.path)
|
104
|
-
url_param_token_pair=URI
|
105
|
-
if url_param_token_pair.nil?
|
106
|
-
raise ArgumentError,"link option must be URL with 'token' parameter"
|
107
|
-
end
|
110
|
+
url_param_token_pair=URI.decode_www_form(uri.query).select{|e|e.first.eql?('token')}.first
|
111
|
+
raise ArgumentError,'link option must be URL with "token" parameter' if url_param_token_pair.nil?
|
108
112
|
# ok we get it !
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
113
|
+
a_opt[:url]='https://'+uri.host
|
114
|
+
a_auth[:crtype]=:aoc_pub_link
|
115
|
+
a_auth[:aoc_pub_link]={
|
116
|
+
url: {grant_type: 'url_token'}, # URL args
|
117
|
+
json: {url_token: url_param_token_pair.last} # JSON body
|
118
|
+
}
|
119
|
+
# password protection of link
|
120
|
+
a_auth[:aoc_pub_link][:json][:password]=a_opt[:password] unless a_opt[:password].nil?
|
121
|
+
return # SUCCESS
|
113
122
|
end
|
114
123
|
Log.log.debug("no expected format: #{public_link_url}")
|
115
|
-
raise "exceeded max redirection: #{MAX_REDIRECT}" if redirect_count > MAX_REDIRECT
|
116
124
|
r = Net::HTTP.get_response(uri)
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
|
121
|
-
|
122
|
-
# not a redirection
|
123
|
-
raise ArgumentError,'link option must be redirect or have token parameter'
|
124
|
-
end
|
125
|
+
# not a redirection
|
126
|
+
raise ArgumentError,'link option must be redirect or have token parameter' unless r.code.start_with?('3')
|
127
|
+
public_link_url = r['location']
|
128
|
+
raise 'no location in redirection' if public_link_url.nil?
|
129
|
+
Log.log.debug("redirect to: #{public_link_url}")
|
125
130
|
end # loop
|
126
|
-
|
127
|
-
raise RuntimeError,'too many redirections'
|
131
|
+
raise "exceeded max redirection: #{MAX_REDIRECT}"
|
128
132
|
end
|
129
133
|
|
130
134
|
# additional transfer spec (tags) for package information
|
@@ -139,9 +143,10 @@ module Aspera
|
|
139
143
|
# add details to show in analytics
|
140
144
|
def analytics_ts(app,direction,ws_id,ws_name)
|
141
145
|
# translate transfer to operation
|
142
|
-
operation=
|
143
|
-
|
144
|
-
when
|
146
|
+
operation=
|
147
|
+
case direction
|
148
|
+
when Fasp::TransferSpec::DIRECTION_SEND then 'upload'
|
149
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE then 'download'
|
145
150
|
else raise "ERROR: unexpected value: #{direction}"
|
146
151
|
end
|
147
152
|
|
@@ -151,8 +156,8 @@ module Aspera
|
|
151
156
|
'usage_id' => "aspera.files.workspace.#{ws_id}", # activity tracking
|
152
157
|
'files' => {
|
153
158
|
'files_transfer_action' => "#{operation}_#{app.gsub(/s$/,'')}",
|
154
|
-
'workspace_name' => ws_name,
|
155
|
-
'workspace_id' => ws_id
|
159
|
+
'workspace_name' => ws_name, # activity tracking
|
160
|
+
'workspace_id' => ws_id
|
156
161
|
}
|
157
162
|
}
|
158
163
|
}
|
@@ -162,6 +167,8 @@ module Aspera
|
|
162
167
|
|
163
168
|
# @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath,:password (for pub link)
|
164
169
|
def initialize(opt)
|
170
|
+
raise ArgumentError,'Missing mandatory option: scope' if opt[:scope].nil?
|
171
|
+
|
165
172
|
# access key secrets are provided out of band to get node api access
|
166
173
|
# key: access key
|
167
174
|
# value: associated secret
|
@@ -169,21 +176,15 @@ module Aspera
|
|
169
176
|
@user_info=nil
|
170
177
|
|
171
178
|
# init rest params
|
172
|
-
aoc_rest_p={:
|
179
|
+
aoc_rest_p={auth: {type: :oauth2}}
|
173
180
|
# shortcut to auth section
|
174
181
|
aoc_auth_p=aoc_rest_p[:auth]
|
175
182
|
|
176
|
-
# sets [:
|
177
|
-
self.class.resolve_pub_link(
|
183
|
+
# sets opt[:url], aoc_rest_p[:auth][:crtype], [:auth][:aoc_pub_link] if there is a link
|
184
|
+
self.class.resolve_pub_link(aoc_auth_p,opt)
|
178
185
|
|
179
|
-
|
180
|
-
|
181
|
-
opt[:url] = aoc_rest_p[:org_url]
|
182
|
-
aoc_rest_p.delete(:org_url)
|
183
|
-
else
|
184
|
-
# else url is mandatory
|
185
|
-
raise ArgumentError,"Missing mandatory option: url" if opt[:url].nil?
|
186
|
-
end
|
186
|
+
# test here because link may set url
|
187
|
+
raise ArgumentError,'Missing mandatory option: url' if opt[:url].nil?
|
187
188
|
|
188
189
|
# get org name and domain from url
|
189
190
|
organization,instance_domain=self.class.parse_url(opt[:url])
|
@@ -195,56 +196,67 @@ module Aspera
|
|
195
196
|
aoc_auth_p[:base_url] = "#{api_url_base}/#{OAUTH_API_SUBPATH}/#{organization}"
|
196
197
|
aoc_auth_p[:client_id]=opt[:client_id]
|
197
198
|
aoc_auth_p[:client_secret] = opt[:client_secret]
|
199
|
+
aoc_auth_p[:scope] = opt[:scope]
|
198
200
|
|
199
|
-
if
|
200
|
-
|
201
|
-
|
201
|
+
# filled if pub link
|
202
|
+
if !aoc_auth_p.has_key?(:crtype)
|
203
|
+
raise ArgumentError,'Missing mandatory option: auth' if opt[:auth].nil?
|
204
|
+
aoc_auth_p[:crtype] = opt[:auth]
|
202
205
|
end
|
203
206
|
|
204
207
|
if aoc_auth_p[:client_id].nil?
|
205
208
|
aoc_auth_p[:client_id],aoc_auth_p[:client_secret] = self.class.get_client_info()
|
206
209
|
end
|
207
210
|
|
208
|
-
raise ArgumentError,"Missing mandatory option: scope" if opt[:scope].nil?
|
209
|
-
aoc_auth_p[:scope] = opt[:scope]
|
210
|
-
|
211
211
|
# fill other auth parameters based on Oauth method
|
212
|
-
case aoc_auth_p[:
|
212
|
+
case aoc_auth_p[:crtype]
|
213
213
|
when :web
|
214
|
-
raise ArgumentError,
|
215
|
-
aoc_auth_p[:
|
214
|
+
raise ArgumentError,'Missing mandatory option: redirect_uri' if opt[:redirect_uri].nil?
|
215
|
+
aoc_auth_p[:web]={redirect_uri: opt[:redirect_uri]}
|
216
216
|
when :jwt
|
217
|
+
raise ArgumentError,'Missing mandatory option: private_key' if opt[:private_key].nil?
|
218
|
+
raise ArgumentError,'Missing mandatory option: username' if opt[:username].nil?
|
219
|
+
aoc_auth_p[:jwt]={
|
220
|
+
private_key_obj: OpenSSL::PKey::RSA.new(opt[:private_key]),
|
221
|
+
payload: {
|
222
|
+
iss: aoc_auth_p[:client_id], # issuer
|
223
|
+
sub: opt[:username], # subject
|
224
|
+
aud: JWT_AUDIENCE
|
225
|
+
}
|
226
|
+
}
|
217
227
|
# add jwt payload for global ids
|
218
|
-
if
|
219
|
-
|
220
|
-
|
221
|
-
|
222
|
-
|
223
|
-
private_key_PEM_string=opt[:private_key]
|
224
|
-
aoc_auth_p[:jwt_audience] = JWT_AUDIENCE
|
225
|
-
aoc_auth_p[:jwt_subject] = opt[:username]
|
226
|
-
aoc_auth_p[:jwt_private_key_obj] = OpenSSL::PKey::RSA.new(private_key_PEM_string)
|
227
|
-
when :url_token
|
228
|
-
aoc_auth_p[:password]=opt[:password] unless opt[:password].nil?
|
229
|
-
# nothing more
|
230
|
-
else raise "ERROR: unsupported auth method: #{aoc_auth_p[:grant]}"
|
228
|
+
aoc_auth_p[:jwt][:payload][:org]=organization if GLOBAL_CLIENT_APPS.include?(aoc_auth_p[:client_id])
|
229
|
+
when :aoc_pub_link
|
230
|
+
# basic auth required for /token
|
231
|
+
aoc_auth_p[:auth]={type: :basic, username: aoc_auth_p[:client_id],password: aoc_auth_p[:client_secret]}
|
232
|
+
else raise "ERROR: unsupported auth method: #{aoc_auth_p[:crtype]}"
|
231
233
|
end
|
232
234
|
super(aoc_rest_p)
|
233
235
|
end
|
234
236
|
|
237
|
+
def url_token_data
|
238
|
+
return nil unless params[:auth][:crtype].eql?(:aoc_pub_link)
|
239
|
+
# TODO: can there be several in list ?
|
240
|
+
return read('url_tokens')[:data].first
|
241
|
+
end
|
242
|
+
|
235
243
|
def key_chain=(keychain)
|
236
|
-
raise
|
237
|
-
raise
|
244
|
+
raise 'keychain already set' unless @key_chain.nil?
|
245
|
+
raise 'keychain must have get_secret' unless keychain.respond_to?(:get_secret)
|
238
246
|
@key_chain=keychain
|
239
|
-
nil
|
240
247
|
end
|
241
248
|
|
242
249
|
# cached user information
|
243
250
|
def user_info
|
244
251
|
if @user_info.nil?
|
245
252
|
# get our user's default information
|
246
|
-
@user_info=
|
247
|
-
|
253
|
+
@user_info=
|
254
|
+
begin
|
255
|
+
read('self')[:data]
|
256
|
+
rescue StandardError => e
|
257
|
+
Log.log.debug("ignoring error: #{e}")
|
258
|
+
{}
|
259
|
+
end
|
248
260
|
USER_INFO_FIELDS_MIN.each{|f|@user_info[f]='unknown' if @user_info[f].nil?}
|
249
261
|
end
|
250
262
|
return @user_info
|
@@ -262,8 +274,10 @@ module Aspera
|
|
262
274
|
# - transfer spec for aspera on cloud, based on node information and file id
|
263
275
|
# - source and token regeneration method
|
264
276
|
def tr_spec(app,direction,node_file,ts_add)
|
265
|
-
#
|
266
|
-
|
277
|
+
# get node api
|
278
|
+
node_api=get_node_api(node_file[:node_info])
|
279
|
+
# this lambda returns the bearer token for node, if
|
280
|
+
token_generation_lambda=lambda{|do_refresh|node_api.oauth_token(force_refresh: do_refresh)}
|
267
281
|
# prepare transfer specification
|
268
282
|
# note xfer_id and xfer_retry are set by the transfer agent itself
|
269
283
|
transfer_spec={
|
@@ -273,7 +287,7 @@ module Aspera
|
|
273
287
|
'aspera' => {
|
274
288
|
'app' => app,
|
275
289
|
'files' => {
|
276
|
-
'node_id' => node_file[:node_info]['id']
|
290
|
+
'node_id' => node_file[:node_info]['id']
|
277
291
|
}, # files
|
278
292
|
'node' => {
|
279
293
|
'access_key' => node_file[:node_info]['access_key'],
|
@@ -284,42 +298,40 @@ module Aspera
|
|
284
298
|
} # tags
|
285
299
|
}
|
286
300
|
# add remote host info
|
287
|
-
if
|
301
|
+
if self.class.use_standard_ports
|
288
302
|
# get default TCP/UDP ports and transfer user
|
289
|
-
transfer_spec.merge!(Fasp::
|
303
|
+
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
290
304
|
# by default: same address as node API
|
291
305
|
transfer_spec['remote_host']=node_file[:node_info]['host']
|
292
306
|
# 30 it's necessarily https scheme: webui does not allow anything else
|
293
|
-
if node_file[:node_info]['transfer_url'].is_a?(String)
|
307
|
+
if node_file[:node_info]['transfer_url'].is_a?(String) && !node_file[:node_info]['transfer_url'].empty?
|
294
308
|
transfer_spec['remote_host']=URI.parse(node_file[:node_info]['transfer_url']).host
|
295
309
|
end
|
296
310
|
else
|
297
311
|
# retrieve values from API
|
298
|
-
std_t_spec=
|
312
|
+
std_t_spec=node_api.create('files/download_setup',{transfer_requests: [{ transfer_request: {paths: [{'source'=>'/'}] } }] })[:data]['transfer_specs'].first['transfer_spec']
|
299
313
|
['remote_host','remote_user','ssh_port','fasp_port'].each {|i| transfer_spec[i]=std_t_spec[i]}
|
300
314
|
end
|
301
315
|
# add caller provided transfer spec
|
302
316
|
transfer_spec.deep_merge!(ts_add)
|
303
317
|
# additional information for transfer agent
|
304
318
|
source_and_token_generator={
|
305
|
-
|
306
|
-
:
|
319
|
+
src: :node_gen4,
|
320
|
+
regenerate_token: token_generation_lambda
|
307
321
|
}
|
308
322
|
return transfer_spec,source_and_token_generator
|
309
323
|
end
|
310
324
|
|
311
325
|
# returns a node API for access key
|
326
|
+
# @param node_info [Hash] with 'url' and 'access_key'
|
312
327
|
# @param scope e.g. SCOPE_NODE_USER
|
313
328
|
# no scope: requires secret
|
314
329
|
# if secret provided beforehand: use it
|
315
|
-
def get_node_api(node_info,
|
316
|
-
raise
|
317
|
-
|
318
|
-
|
319
|
-
|
320
|
-
if ak_secret.nil? and !options.has_key?(:scope)
|
321
|
-
raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}"
|
322
|
-
end
|
330
|
+
def get_node_api(node_info, scope: SCOPE_NODE_USER, use_secret: true)
|
331
|
+
raise 'internal error' unless node_info.is_a?(Hash) && node_info.has_key?('url') && node_info.has_key?('access_key')
|
332
|
+
# get optional secret unless :use_secret is false
|
333
|
+
ak_secret=@key_chain.get_secret(url: node_info['url'], username: node_info['access_key'], mandatory: false) if use_secret && !@key_chain.nil?
|
334
|
+
raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}" if ak_secret.nil? && scope.nil?
|
323
335
|
node_rest_params={base_url: node_info['url']}
|
324
336
|
# if secret is available
|
325
337
|
if !ak_secret.nil?
|
@@ -331,8 +343,8 @@ module Aspera
|
|
331
343
|
else
|
332
344
|
# X-Aspera-AccessKey required for bearer token only
|
333
345
|
node_rest_params[:headers]= {'X-Aspera-AccessKey'=>node_info['access_key']}
|
334
|
-
node_rest_params[:auth]=
|
335
|
-
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],
|
346
|
+
node_rest_params[:auth]=params[:auth].clone
|
347
|
+
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],scope)
|
336
348
|
end
|
337
349
|
return Node.new(node_rest_params)
|
338
350
|
end
|
@@ -341,7 +353,7 @@ module Aspera
|
|
341
353
|
# @return split values
|
342
354
|
def check_get_node_file(node_file)
|
343
355
|
raise "node_file must be Hash (got #{node_file.class})" unless node_file.is_a?(Hash)
|
344
|
-
raise
|
356
|
+
raise 'node_file must have 2 keys: :file_id and :node_info' unless node_file.keys.sort.eql?([:file_id,:node_info])
|
345
357
|
node_info=node_file[:node_info]
|
346
358
|
file_id=node_file[:file_id]
|
347
359
|
raise "node_info must be Hash (got #{node_info.class}: #{node_info})" unless node_info.is_a?(Hash)
|
@@ -357,28 +369,28 @@ module Aspera
|
|
357
369
|
@find_state[:found].push(entry.merge({'path'=>path})) if @find_state[:test_block].call(entry)
|
358
370
|
# process link
|
359
371
|
if entry[:type].eql?('link')
|
360
|
-
sub_node_info=
|
372
|
+
sub_node_info=read("nodes/#{entry['target_node_id']}")[:data]
|
361
373
|
sub_opt={method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
|
362
|
-
get_node_api(sub_node_info
|
374
|
+
get_node_api(sub_node_info).crawl(self,sub_opt)
|
363
375
|
end
|
364
|
-
rescue => e
|
376
|
+
rescue StandardError => e
|
365
377
|
Log.log.error("#{path}: #{e.message}")
|
366
378
|
end
|
367
379
|
# process all folders
|
368
380
|
return true
|
369
381
|
end
|
370
382
|
|
371
|
-
def find_files(
|
383
|
+
def find_files(top_node_file, test_block)
|
372
384
|
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
373
385
|
Log.log.debug("find_files: node_info=#{top_node_info}, fileid=#{top_file_id}")
|
374
386
|
@find_state={found: [], test_block: test_block}
|
375
|
-
get_node_api(top_node_info
|
387
|
+
get_node_api(top_node_info).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
|
376
388
|
result=@find_state[:found]
|
377
389
|
@find_state=nil
|
378
390
|
return result
|
379
391
|
end
|
380
392
|
|
381
|
-
def process_resolve_node_file(entry,
|
393
|
+
def process_resolve_node_file(entry,_path)
|
382
394
|
# stop digging here if not in right path
|
383
395
|
return false unless entry['name'].eql?(@resolve_state[:path].first)
|
384
396
|
# ok it matches, so we remove the match
|
@@ -389,11 +401,11 @@ module Aspera
|
|
389
401
|
raise "#{entry['name']} is a file, expecting folder to find: #{@resolve_state[:path]}" unless @resolve_state[:path].empty?
|
390
402
|
@resolve_state[:result][:file_id]=entry['id']
|
391
403
|
when 'link'
|
392
|
-
@resolve_state[:result][:node_info]=
|
404
|
+
@resolve_state[:result][:node_info]=read("nodes/#{entry['target_node_id']}")[:data]
|
393
405
|
if @resolve_state[:path].empty?
|
394
406
|
@resolve_state[:result][:file_id]=entry['target_id']
|
395
407
|
else
|
396
|
-
get_node_api(@resolve_state[:result][:node_info]
|
408
|
+
get_node_api(@resolve_state[:result][:node_info]).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
|
397
409
|
end
|
398
410
|
when 'folder'
|
399
411
|
if @resolve_state[:path].empty?
|
@@ -412,15 +424,15 @@ module Aspera
|
|
412
424
|
# @param top_node_file Array [root node,file id]
|
413
425
|
# @param element_path_string String path of element
|
414
426
|
# supports links to secondary nodes
|
415
|
-
def resolve_node_file(
|
427
|
+
def resolve_node_file(top_node_file, element_path_string)
|
416
428
|
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
417
|
-
path_elements=element_path_string.split(PATH_SEPARATOR).
|
429
|
+
path_elements=element_path_string.split(PATH_SEPARATOR).reject(&:empty?)
|
418
430
|
result={node_info: top_node_info, file_id: nil}
|
419
431
|
if path_elements.empty?
|
420
432
|
result[:file_id]=top_file_id
|
421
433
|
else
|
422
434
|
@resolve_state={path: path_elements, result: result}
|
423
|
-
get_node_api(top_node_info
|
435
|
+
get_node_api(top_node_info).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
|
424
436
|
not_found=@resolve_state[:path]
|
425
437
|
@resolve_state=nil
|
426
438
|
raise "entry not found: #{not_found}" if result[:file_id].nil?
|
@@ -435,19 +447,18 @@ module Aspera
|
|
435
447
|
# returns entities whose name contains value (case insensitive)
|
436
448
|
matching_items=read(entity_type,options.merge({'q'=>entity_name}))[:data]
|
437
449
|
case matching_items.length
|
438
|
-
when 1
|
439
|
-
when 0
|
450
|
+
when 1 then return matching_items.first
|
451
|
+
when 0 then raise 'not found'
|
440
452
|
else
|
441
453
|
# multiple case insensitive partial matches, try case insensitive full match
|
442
454
|
# (anyway AoC does not allow creation of 2 entities with same case insensitive name)
|
443
455
|
icase_matches=matching_items.select{|i|i['name'].casecmp?(entity_name)}
|
444
456
|
case icase_matches.length
|
445
|
-
when 1
|
446
|
-
when 0
|
447
|
-
else
|
457
|
+
when 1 then return icase_matches.first
|
458
|
+
when 0 then raise %Q(#{entity_type}: multiple case insensitive partial match for: "#{entity_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
|
459
|
+
else raise "Two entities cannot have the same case insensitive name: #{icase_matches.map{|i|i['name']}}"
|
448
460
|
end
|
449
461
|
end
|
450
462
|
end
|
451
|
-
|
452
463
|
end # AoC
|
453
464
|
end # Aspera
|