aspera-cli 4.6.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
data/lib/aspera/aoc.rb
CHANGED
|
@@ -1,28 +1,34 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/log'
|
|
2
3
|
require 'aspera/rest'
|
|
3
4
|
require 'aspera/hash_ext'
|
|
4
5
|
require 'aspera/data_repository'
|
|
5
|
-
require 'aspera/fasp/
|
|
6
|
+
require 'aspera/fasp/transfer_spec'
|
|
6
7
|
require 'base64'
|
|
7
8
|
|
|
9
|
+
Aspera::Oauth.register_token_creator(:aoc_pub_link,lambda{|o|
|
|
10
|
+
o.token_auth_api.call({
|
|
11
|
+
operation: 'POST',
|
|
12
|
+
subpath: o.params[:path_token],
|
|
13
|
+
headers: {'Accept'=>'application/json'},
|
|
14
|
+
json_params: o.params[:aoc_pub_link][:json],
|
|
15
|
+
url_params: o.params[:aoc_pub_link][:url].merge(scope: o.params[:scope]) # scope is here because it changes over time (node)
|
|
16
|
+
})
|
|
17
|
+
})
|
|
18
|
+
|
|
8
19
|
module Aspera
|
|
9
20
|
class AoC < Rest
|
|
10
|
-
private
|
|
11
|
-
@@use_standard_ports = true
|
|
12
|
-
|
|
13
|
-
API_V1='api/v1'
|
|
14
|
-
|
|
15
21
|
PRODUCT_NAME='Aspera on Cloud'
|
|
16
22
|
# Production domain of AoC
|
|
17
23
|
PROD_DOMAIN='ibmaspera.com'
|
|
18
24
|
# to avoid infinite loop in pub link redirection
|
|
19
25
|
MAX_REDIRECT=10
|
|
20
|
-
|
|
26
|
+
# Well-known AoC globals client apps
|
|
27
|
+
GLOBAL_CLIENT_APPS=['aspera.global-cli-client','aspera.drive']
|
|
21
28
|
# index offset in data repository of client app
|
|
22
29
|
DATA_REPO_INDEX_START = 4
|
|
23
30
|
# cookie prefix so that console can decode identity
|
|
24
31
|
COOKIE_PREFIX='aspera.aoc'
|
|
25
|
-
|
|
26
32
|
# path in URL of public links
|
|
27
33
|
PUBLIC_LINK_PATHS=['/packages/public/receive','/packages/public/send','/files/public']
|
|
28
34
|
JWT_AUDIENCE='https://api.asperafiles.com/api/v1/oauth2/token'
|
|
@@ -30,10 +36,8 @@ module Aspera
|
|
|
30
36
|
# minimum fields for user info if retrieval fails
|
|
31
37
|
USER_INFO_FIELDS_MIN=['name','email','id','default_workspace_id','organization_id']
|
|
32
38
|
|
|
33
|
-
private_constant :
|
|
34
|
-
:OAUTH_API_SUBPATH,:COOKIE_PREFIX,:USER_INFO_FIELDS_MIN
|
|
39
|
+
private_constant :MAX_REDIRECT,:GLOBAL_CLIENT_APPS,:DATA_REPO_INDEX_START,:COOKIE_PREFIX,:PUBLIC_LINK_PATHS,:JWT_AUDIENCE,:OAUTH_API_SUBPATH,:USER_INFO_FIELDS_MIN
|
|
35
40
|
|
|
36
|
-
public
|
|
37
41
|
# various API scopes supported
|
|
38
42
|
SCOPE_FILES_SELF='self'
|
|
39
43
|
SCOPE_FILES_USER='user:all'
|
|
@@ -45,14 +49,19 @@ module Aspera
|
|
|
45
49
|
PATH_SEPARATOR='/'
|
|
46
50
|
FILES_APP='files'
|
|
47
51
|
PACKAGES_APP='packages'
|
|
52
|
+
API_V1='api/v1'
|
|
53
|
+
|
|
54
|
+
# class instance variable, access with accessors on class
|
|
55
|
+
@use_standard_ports = true
|
|
48
56
|
|
|
49
57
|
# class static methods
|
|
50
58
|
class << self
|
|
59
|
+
attr_accessor :use_standard_ports
|
|
51
60
|
# strings /Applications/Aspera\ Drive.app/Contents/MacOS/AsperaDrive|grep -E '.{100}==$'|base64 --decode
|
|
52
|
-
def get_client_info(client_name=
|
|
53
|
-
client_index=
|
|
61
|
+
def get_client_info(client_name=GLOBAL_CLIENT_APPS.first)
|
|
62
|
+
client_index=GLOBAL_CLIENT_APPS.index(client_name)
|
|
54
63
|
raise "no such pre-defined client: #{client_name}" if client_index.nil?
|
|
55
|
-
return client_name,Base64.urlsafe_encode64(DataRepository.instance.
|
|
64
|
+
return client_name,Base64.urlsafe_encode64(DataRepository.instance.data(DATA_REPO_INDEX_START+client_index))
|
|
56
65
|
end
|
|
57
66
|
|
|
58
67
|
# @param url of AoC instance
|
|
@@ -76,55 +85,50 @@ module Aspera
|
|
|
76
85
|
|
|
77
86
|
def metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
|
|
78
87
|
return Rest.new({
|
|
79
|
-
:
|
|
80
|
-
:
|
|
88
|
+
base_url: "#{api_base_url(api_domain)}/metering/v1",
|
|
89
|
+
headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
|
|
81
90
|
})
|
|
82
91
|
end
|
|
83
92
|
|
|
84
93
|
# node API scopes
|
|
85
94
|
def node_scope(access_key,scope)
|
|
86
|
-
return
|
|
87
|
-
end
|
|
88
|
-
|
|
89
|
-
def set_use_default_ports(val)
|
|
90
|
-
@@use_standard_ports=val
|
|
95
|
+
return "node.#{access_key}:#{scope}"
|
|
91
96
|
end
|
|
92
97
|
|
|
93
98
|
# check option "link"
|
|
94
99
|
# if present try to get token value (resolve redirection if short links used)
|
|
95
100
|
# then set options url/token/auth
|
|
96
|
-
def resolve_pub_link(
|
|
101
|
+
def resolve_pub_link(a_auth,a_opt)
|
|
102
|
+
public_link_url=a_opt[:link]
|
|
97
103
|
return if public_link_url.nil?
|
|
98
|
-
|
|
99
|
-
url_param_token_pair=nil
|
|
104
|
+
raise 'do not use both link and url options' unless a_opt[:url].nil?
|
|
100
105
|
redirect_count=0
|
|
101
|
-
|
|
106
|
+
while redirect_count <= MAX_REDIRECT
|
|
102
107
|
uri=URI.parse(public_link_url)
|
|
108
|
+
# detect if it's an expected format
|
|
103
109
|
if PUBLIC_LINK_PATHS.include?(uri.path)
|
|
104
|
-
url_param_token_pair=URI
|
|
105
|
-
if url_param_token_pair.nil?
|
|
106
|
-
raise ArgumentError,"link option must be URL with 'token' parameter"
|
|
107
|
-
end
|
|
110
|
+
url_param_token_pair=URI.decode_www_form(uri.query).select{|e|e.first.eql?('token')}.first
|
|
111
|
+
raise ArgumentError,'link option must be URL with "token" parameter' if url_param_token_pair.nil?
|
|
108
112
|
# ok we get it !
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
+
a_opt[:url]='https://'+uri.host
|
|
114
|
+
a_auth[:crtype]=:aoc_pub_link
|
|
115
|
+
a_auth[:aoc_pub_link]={
|
|
116
|
+
url: {grant_type: 'url_token'}, # URL args
|
|
117
|
+
json: {url_token: url_param_token_pair.last} # JSON body
|
|
118
|
+
}
|
|
119
|
+
# password protection of link
|
|
120
|
+
a_auth[:aoc_pub_link][:json][:password]=a_opt[:password] unless a_opt[:password].nil?
|
|
121
|
+
return # SUCCESS
|
|
113
122
|
end
|
|
114
123
|
Log.log.debug("no expected format: #{public_link_url}")
|
|
115
|
-
raise "exceeded max redirection: #{MAX_REDIRECT}" if redirect_count > MAX_REDIRECT
|
|
116
124
|
r = Net::HTTP.get_response(uri)
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
# not a redirection
|
|
123
|
-
raise ArgumentError,'link option must be redirect or have token parameter'
|
|
124
|
-
end
|
|
125
|
+
# not a redirection
|
|
126
|
+
raise ArgumentError,'link option must be redirect or have token parameter' unless r.code.start_with?('3')
|
|
127
|
+
public_link_url = r['location']
|
|
128
|
+
raise 'no location in redirection' if public_link_url.nil?
|
|
129
|
+
Log.log.debug("redirect to: #{public_link_url}")
|
|
125
130
|
end # loop
|
|
126
|
-
|
|
127
|
-
raise RuntimeError,'too many redirections'
|
|
131
|
+
raise "exceeded max redirection: #{MAX_REDIRECT}"
|
|
128
132
|
end
|
|
129
133
|
|
|
130
134
|
# additional transfer spec (tags) for package information
|
|
@@ -139,9 +143,10 @@ module Aspera
|
|
|
139
143
|
# add details to show in analytics
|
|
140
144
|
def analytics_ts(app,direction,ws_id,ws_name)
|
|
141
145
|
# translate transfer to operation
|
|
142
|
-
operation=
|
|
143
|
-
|
|
144
|
-
when
|
|
146
|
+
operation=
|
|
147
|
+
case direction
|
|
148
|
+
when Fasp::TransferSpec::DIRECTION_SEND then 'upload'
|
|
149
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE then 'download'
|
|
145
150
|
else raise "ERROR: unexpected value: #{direction}"
|
|
146
151
|
end
|
|
147
152
|
|
|
@@ -151,8 +156,8 @@ module Aspera
|
|
|
151
156
|
'usage_id' => "aspera.files.workspace.#{ws_id}", # activity tracking
|
|
152
157
|
'files' => {
|
|
153
158
|
'files_transfer_action' => "#{operation}_#{app.gsub(/s$/,'')}",
|
|
154
|
-
'workspace_name' => ws_name,
|
|
155
|
-
'workspace_id' => ws_id
|
|
159
|
+
'workspace_name' => ws_name, # activity tracking
|
|
160
|
+
'workspace_id' => ws_id
|
|
156
161
|
}
|
|
157
162
|
}
|
|
158
163
|
}
|
|
@@ -162,6 +167,8 @@ module Aspera
|
|
|
162
167
|
|
|
163
168
|
# @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath,:password (for pub link)
|
|
164
169
|
def initialize(opt)
|
|
170
|
+
raise ArgumentError,'Missing mandatory option: scope' if opt[:scope].nil?
|
|
171
|
+
|
|
165
172
|
# access key secrets are provided out of band to get node api access
|
|
166
173
|
# key: access key
|
|
167
174
|
# value: associated secret
|
|
@@ -169,21 +176,15 @@ module Aspera
|
|
|
169
176
|
@user_info=nil
|
|
170
177
|
|
|
171
178
|
# init rest params
|
|
172
|
-
aoc_rest_p={:
|
|
179
|
+
aoc_rest_p={auth: {type: :oauth2}}
|
|
173
180
|
# shortcut to auth section
|
|
174
181
|
aoc_auth_p=aoc_rest_p[:auth]
|
|
175
182
|
|
|
176
|
-
# sets [:
|
|
177
|
-
self.class.resolve_pub_link(
|
|
183
|
+
# sets opt[:url], aoc_rest_p[:auth][:crtype], [:auth][:aoc_pub_link] if there is a link
|
|
184
|
+
self.class.resolve_pub_link(aoc_auth_p,opt)
|
|
178
185
|
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
opt[:url] = aoc_rest_p[:org_url]
|
|
182
|
-
aoc_rest_p.delete(:org_url)
|
|
183
|
-
else
|
|
184
|
-
# else url is mandatory
|
|
185
|
-
raise ArgumentError,"Missing mandatory option: url" if opt[:url].nil?
|
|
186
|
-
end
|
|
186
|
+
# test here because link may set url
|
|
187
|
+
raise ArgumentError,'Missing mandatory option: url' if opt[:url].nil?
|
|
187
188
|
|
|
188
189
|
# get org name and domain from url
|
|
189
190
|
organization,instance_domain=self.class.parse_url(opt[:url])
|
|
@@ -195,56 +196,67 @@ module Aspera
|
|
|
195
196
|
aoc_auth_p[:base_url] = "#{api_url_base}/#{OAUTH_API_SUBPATH}/#{organization}"
|
|
196
197
|
aoc_auth_p[:client_id]=opt[:client_id]
|
|
197
198
|
aoc_auth_p[:client_secret] = opt[:client_secret]
|
|
199
|
+
aoc_auth_p[:scope] = opt[:scope]
|
|
198
200
|
|
|
199
|
-
if
|
|
200
|
-
|
|
201
|
-
|
|
201
|
+
# filled if pub link
|
|
202
|
+
if !aoc_auth_p.has_key?(:crtype)
|
|
203
|
+
raise ArgumentError,'Missing mandatory option: auth' if opt[:auth].nil?
|
|
204
|
+
aoc_auth_p[:crtype] = opt[:auth]
|
|
202
205
|
end
|
|
203
206
|
|
|
204
207
|
if aoc_auth_p[:client_id].nil?
|
|
205
208
|
aoc_auth_p[:client_id],aoc_auth_p[:client_secret] = self.class.get_client_info()
|
|
206
209
|
end
|
|
207
210
|
|
|
208
|
-
raise ArgumentError,"Missing mandatory option: scope" if opt[:scope].nil?
|
|
209
|
-
aoc_auth_p[:scope] = opt[:scope]
|
|
210
|
-
|
|
211
211
|
# fill other auth parameters based on Oauth method
|
|
212
|
-
case aoc_auth_p[:
|
|
212
|
+
case aoc_auth_p[:crtype]
|
|
213
213
|
when :web
|
|
214
|
-
raise ArgumentError,
|
|
215
|
-
aoc_auth_p[:
|
|
214
|
+
raise ArgumentError,'Missing mandatory option: redirect_uri' if opt[:redirect_uri].nil?
|
|
215
|
+
aoc_auth_p[:web]={redirect_uri: opt[:redirect_uri]}
|
|
216
216
|
when :jwt
|
|
217
|
+
raise ArgumentError,'Missing mandatory option: private_key' if opt[:private_key].nil?
|
|
218
|
+
raise ArgumentError,'Missing mandatory option: username' if opt[:username].nil?
|
|
219
|
+
aoc_auth_p[:jwt]={
|
|
220
|
+
private_key_obj: OpenSSL::PKey::RSA.new(opt[:private_key]),
|
|
221
|
+
payload: {
|
|
222
|
+
iss: aoc_auth_p[:client_id], # issuer
|
|
223
|
+
sub: opt[:username], # subject
|
|
224
|
+
aud: JWT_AUDIENCE
|
|
225
|
+
}
|
|
226
|
+
}
|
|
217
227
|
# add jwt payload for global ids
|
|
218
|
-
if
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
private_key_PEM_string=opt[:private_key]
|
|
224
|
-
aoc_auth_p[:jwt_audience] = JWT_AUDIENCE
|
|
225
|
-
aoc_auth_p[:jwt_subject] = opt[:username]
|
|
226
|
-
aoc_auth_p[:jwt_private_key_obj] = OpenSSL::PKey::RSA.new(private_key_PEM_string)
|
|
227
|
-
when :url_token
|
|
228
|
-
aoc_auth_p[:password]=opt[:password] unless opt[:password].nil?
|
|
229
|
-
# nothing more
|
|
230
|
-
else raise "ERROR: unsupported auth method: #{aoc_auth_p[:grant]}"
|
|
228
|
+
aoc_auth_p[:jwt][:payload][:org]=organization if GLOBAL_CLIENT_APPS.include?(aoc_auth_p[:client_id])
|
|
229
|
+
when :aoc_pub_link
|
|
230
|
+
# basic auth required for /token
|
|
231
|
+
aoc_auth_p[:auth]={type: :basic, username: aoc_auth_p[:client_id],password: aoc_auth_p[:client_secret]}
|
|
232
|
+
else raise "ERROR: unsupported auth method: #{aoc_auth_p[:crtype]}"
|
|
231
233
|
end
|
|
232
234
|
super(aoc_rest_p)
|
|
233
235
|
end
|
|
234
236
|
|
|
237
|
+
def url_token_data
|
|
238
|
+
return nil unless params[:auth][:crtype].eql?(:aoc_pub_link)
|
|
239
|
+
# TODO: can there be several in list ?
|
|
240
|
+
return read('url_tokens')[:data].first
|
|
241
|
+
end
|
|
242
|
+
|
|
235
243
|
def key_chain=(keychain)
|
|
236
|
-
raise
|
|
237
|
-
raise
|
|
244
|
+
raise 'keychain already set' unless @key_chain.nil?
|
|
245
|
+
raise 'keychain must have get_secret' unless keychain.respond_to?(:get_secret)
|
|
238
246
|
@key_chain=keychain
|
|
239
|
-
nil
|
|
240
247
|
end
|
|
241
248
|
|
|
242
249
|
# cached user information
|
|
243
250
|
def user_info
|
|
244
251
|
if @user_info.nil?
|
|
245
252
|
# get our user's default information
|
|
246
|
-
@user_info=
|
|
247
|
-
|
|
253
|
+
@user_info=
|
|
254
|
+
begin
|
|
255
|
+
read('self')[:data]
|
|
256
|
+
rescue StandardError => e
|
|
257
|
+
Log.log.debug("ignoring error: #{e}")
|
|
258
|
+
{}
|
|
259
|
+
end
|
|
248
260
|
USER_INFO_FIELDS_MIN.each{|f|@user_info[f]='unknown' if @user_info[f].nil?}
|
|
249
261
|
end
|
|
250
262
|
return @user_info
|
|
@@ -262,8 +274,10 @@ module Aspera
|
|
|
262
274
|
# - transfer spec for aspera on cloud, based on node information and file id
|
|
263
275
|
# - source and token regeneration method
|
|
264
276
|
def tr_spec(app,direction,node_file,ts_add)
|
|
265
|
-
#
|
|
266
|
-
|
|
277
|
+
# get node api
|
|
278
|
+
node_api=get_node_api(node_file[:node_info])
|
|
279
|
+
# this lambda returns the bearer token for node, if
|
|
280
|
+
token_generation_lambda=lambda{|do_refresh|node_api.oauth_token(force_refresh: do_refresh)}
|
|
267
281
|
# prepare transfer specification
|
|
268
282
|
# note xfer_id and xfer_retry are set by the transfer agent itself
|
|
269
283
|
transfer_spec={
|
|
@@ -273,7 +287,7 @@ module Aspera
|
|
|
273
287
|
'aspera' => {
|
|
274
288
|
'app' => app,
|
|
275
289
|
'files' => {
|
|
276
|
-
'node_id' => node_file[:node_info]['id']
|
|
290
|
+
'node_id' => node_file[:node_info]['id']
|
|
277
291
|
}, # files
|
|
278
292
|
'node' => {
|
|
279
293
|
'access_key' => node_file[:node_info]['access_key'],
|
|
@@ -284,42 +298,40 @@ module Aspera
|
|
|
284
298
|
} # tags
|
|
285
299
|
}
|
|
286
300
|
# add remote host info
|
|
287
|
-
if
|
|
301
|
+
if self.class.use_standard_ports
|
|
288
302
|
# get default TCP/UDP ports and transfer user
|
|
289
|
-
transfer_spec.merge!(Fasp::
|
|
303
|
+
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
|
290
304
|
# by default: same address as node API
|
|
291
305
|
transfer_spec['remote_host']=node_file[:node_info]['host']
|
|
292
306
|
# 30 it's necessarily https scheme: webui does not allow anything else
|
|
293
|
-
if node_file[:node_info]['transfer_url'].is_a?(String)
|
|
307
|
+
if node_file[:node_info]['transfer_url'].is_a?(String) && !node_file[:node_info]['transfer_url'].empty?
|
|
294
308
|
transfer_spec['remote_host']=URI.parse(node_file[:node_info]['transfer_url']).host
|
|
295
309
|
end
|
|
296
310
|
else
|
|
297
311
|
# retrieve values from API
|
|
298
|
-
std_t_spec=
|
|
312
|
+
std_t_spec=node_api.create('files/download_setup',{transfer_requests: [{ transfer_request: {paths: [{'source'=>'/'}] } }] })[:data]['transfer_specs'].first['transfer_spec']
|
|
299
313
|
['remote_host','remote_user','ssh_port','fasp_port'].each {|i| transfer_spec[i]=std_t_spec[i]}
|
|
300
314
|
end
|
|
301
315
|
# add caller provided transfer spec
|
|
302
316
|
transfer_spec.deep_merge!(ts_add)
|
|
303
317
|
# additional information for transfer agent
|
|
304
318
|
source_and_token_generator={
|
|
305
|
-
|
|
306
|
-
:
|
|
319
|
+
src: :node_gen4,
|
|
320
|
+
regenerate_token: token_generation_lambda
|
|
307
321
|
}
|
|
308
322
|
return transfer_spec,source_and_token_generator
|
|
309
323
|
end
|
|
310
324
|
|
|
311
325
|
# returns a node API for access key
|
|
326
|
+
# @param node_info [Hash] with 'url' and 'access_key'
|
|
312
327
|
# @param scope e.g. SCOPE_NODE_USER
|
|
313
328
|
# no scope: requires secret
|
|
314
329
|
# if secret provided beforehand: use it
|
|
315
|
-
def get_node_api(node_info,
|
|
316
|
-
raise
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
if ak_secret.nil? and !options.has_key?(:scope)
|
|
321
|
-
raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}"
|
|
322
|
-
end
|
|
330
|
+
def get_node_api(node_info, scope: SCOPE_NODE_USER, use_secret: true)
|
|
331
|
+
raise 'internal error' unless node_info.is_a?(Hash) && node_info.has_key?('url') && node_info.has_key?('access_key')
|
|
332
|
+
# get optional secret unless :use_secret is false
|
|
333
|
+
ak_secret=@key_chain.get_secret(url: node_info['url'], username: node_info['access_key'], mandatory: false) if use_secret && !@key_chain.nil?
|
|
334
|
+
raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}" if ak_secret.nil? && scope.nil?
|
|
323
335
|
node_rest_params={base_url: node_info['url']}
|
|
324
336
|
# if secret is available
|
|
325
337
|
if !ak_secret.nil?
|
|
@@ -331,8 +343,8 @@ module Aspera
|
|
|
331
343
|
else
|
|
332
344
|
# X-Aspera-AccessKey required for bearer token only
|
|
333
345
|
node_rest_params[:headers]= {'X-Aspera-AccessKey'=>node_info['access_key']}
|
|
334
|
-
node_rest_params[:auth]=
|
|
335
|
-
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],
|
|
346
|
+
node_rest_params[:auth]=params[:auth].clone
|
|
347
|
+
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],scope)
|
|
336
348
|
end
|
|
337
349
|
return Node.new(node_rest_params)
|
|
338
350
|
end
|
|
@@ -341,7 +353,7 @@ module Aspera
|
|
|
341
353
|
# @return split values
|
|
342
354
|
def check_get_node_file(node_file)
|
|
343
355
|
raise "node_file must be Hash (got #{node_file.class})" unless node_file.is_a?(Hash)
|
|
344
|
-
raise
|
|
356
|
+
raise 'node_file must have 2 keys: :file_id and :node_info' unless node_file.keys.sort.eql?([:file_id,:node_info])
|
|
345
357
|
node_info=node_file[:node_info]
|
|
346
358
|
file_id=node_file[:file_id]
|
|
347
359
|
raise "node_info must be Hash (got #{node_info.class}: #{node_info})" unless node_info.is_a?(Hash)
|
|
@@ -357,28 +369,28 @@ module Aspera
|
|
|
357
369
|
@find_state[:found].push(entry.merge({'path'=>path})) if @find_state[:test_block].call(entry)
|
|
358
370
|
# process link
|
|
359
371
|
if entry[:type].eql?('link')
|
|
360
|
-
sub_node_info=
|
|
372
|
+
sub_node_info=read("nodes/#{entry['target_node_id']}")[:data]
|
|
361
373
|
sub_opt={method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
|
|
362
|
-
get_node_api(sub_node_info
|
|
374
|
+
get_node_api(sub_node_info).crawl(self,sub_opt)
|
|
363
375
|
end
|
|
364
|
-
rescue => e
|
|
376
|
+
rescue StandardError => e
|
|
365
377
|
Log.log.error("#{path}: #{e.message}")
|
|
366
378
|
end
|
|
367
379
|
# process all folders
|
|
368
380
|
return true
|
|
369
381
|
end
|
|
370
382
|
|
|
371
|
-
def find_files(
|
|
383
|
+
def find_files(top_node_file, test_block)
|
|
372
384
|
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
|
373
385
|
Log.log.debug("find_files: node_info=#{top_node_info}, fileid=#{top_file_id}")
|
|
374
386
|
@find_state={found: [], test_block: test_block}
|
|
375
|
-
get_node_api(top_node_info
|
|
387
|
+
get_node_api(top_node_info).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
|
|
376
388
|
result=@find_state[:found]
|
|
377
389
|
@find_state=nil
|
|
378
390
|
return result
|
|
379
391
|
end
|
|
380
392
|
|
|
381
|
-
def process_resolve_node_file(entry,
|
|
393
|
+
def process_resolve_node_file(entry,_path)
|
|
382
394
|
# stop digging here if not in right path
|
|
383
395
|
return false unless entry['name'].eql?(@resolve_state[:path].first)
|
|
384
396
|
# ok it matches, so we remove the match
|
|
@@ -389,11 +401,11 @@ module Aspera
|
|
|
389
401
|
raise "#{entry['name']} is a file, expecting folder to find: #{@resolve_state[:path]}" unless @resolve_state[:path].empty?
|
|
390
402
|
@resolve_state[:result][:file_id]=entry['id']
|
|
391
403
|
when 'link'
|
|
392
|
-
@resolve_state[:result][:node_info]=
|
|
404
|
+
@resolve_state[:result][:node_info]=read("nodes/#{entry['target_node_id']}")[:data]
|
|
393
405
|
if @resolve_state[:path].empty?
|
|
394
406
|
@resolve_state[:result][:file_id]=entry['target_id']
|
|
395
407
|
else
|
|
396
|
-
get_node_api(@resolve_state[:result][:node_info]
|
|
408
|
+
get_node_api(@resolve_state[:result][:node_info]).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
|
|
397
409
|
end
|
|
398
410
|
when 'folder'
|
|
399
411
|
if @resolve_state[:path].empty?
|
|
@@ -412,15 +424,15 @@ module Aspera
|
|
|
412
424
|
# @param top_node_file Array [root node,file id]
|
|
413
425
|
# @param element_path_string String path of element
|
|
414
426
|
# supports links to secondary nodes
|
|
415
|
-
def resolve_node_file(
|
|
427
|
+
def resolve_node_file(top_node_file, element_path_string)
|
|
416
428
|
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
|
417
|
-
path_elements=element_path_string.split(PATH_SEPARATOR).
|
|
429
|
+
path_elements=element_path_string.split(PATH_SEPARATOR).reject(&:empty?)
|
|
418
430
|
result={node_info: top_node_info, file_id: nil}
|
|
419
431
|
if path_elements.empty?
|
|
420
432
|
result[:file_id]=top_file_id
|
|
421
433
|
else
|
|
422
434
|
@resolve_state={path: path_elements, result: result}
|
|
423
|
-
get_node_api(top_node_info
|
|
435
|
+
get_node_api(top_node_info).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
|
|
424
436
|
not_found=@resolve_state[:path]
|
|
425
437
|
@resolve_state=nil
|
|
426
438
|
raise "entry not found: #{not_found}" if result[:file_id].nil?
|
|
@@ -435,19 +447,18 @@ module Aspera
|
|
|
435
447
|
# returns entities whose name contains value (case insensitive)
|
|
436
448
|
matching_items=read(entity_type,options.merge({'q'=>entity_name}))[:data]
|
|
437
449
|
case matching_items.length
|
|
438
|
-
when 1
|
|
439
|
-
when 0
|
|
450
|
+
when 1 then return matching_items.first
|
|
451
|
+
when 0 then raise 'not found'
|
|
440
452
|
else
|
|
441
453
|
# multiple case insensitive partial matches, try case insensitive full match
|
|
442
454
|
# (anyway AoC does not allow creation of 2 entities with same case insensitive name)
|
|
443
455
|
icase_matches=matching_items.select{|i|i['name'].casecmp?(entity_name)}
|
|
444
456
|
case icase_matches.length
|
|
445
|
-
when 1
|
|
446
|
-
when 0
|
|
447
|
-
else
|
|
457
|
+
when 1 then return icase_matches.first
|
|
458
|
+
when 0 then raise %Q(#{entity_type}: multiple case insensitive partial match for: "#{entity_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
|
|
459
|
+
else raise "Two entities cannot have the same case insensitive name: #{icase_matches.map{|i|i['name']}}"
|
|
448
460
|
end
|
|
449
461
|
end
|
|
450
462
|
end
|
|
451
|
-
|
|
452
463
|
end # AoC
|
|
453
464
|
end # Aspera
|