aspera-cli 4.7.0 → 4.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/README.md +1267 -999
- data/bin/ascli +20 -1
- data/bin/asession +37 -34
- data/docs/test_env.conf +7 -3
- data/examples/aoc.rb +13 -12
- data/examples/dascli +23 -0
- data/examples/faspex4.rb +34 -29
- data/examples/{transfer.rb → node.rb} +31 -59
- data/examples/server.rb +93 -0
- data/lib/aspera/aoc.rb +153 -143
- data/lib/aspera/ascmd.rb +56 -45
- data/lib/aspera/ats_api.rb +9 -6
- data/lib/aspera/cli/basic_auth_plugin.rb +18 -16
- data/lib/aspera/cli/extended_value.rb +33 -30
- data/lib/aspera/cli/formater.rb +105 -111
- data/lib/aspera/cli/info.rb +3 -2
- data/lib/aspera/cli/listener/line_dump.rb +1 -0
- data/lib/aspera/cli/listener/logger.rb +1 -0
- data/lib/aspera/cli/listener/progress.rb +13 -12
- data/lib/aspera/cli/listener/progress_multi.rb +21 -20
- data/lib/aspera/cli/main.rb +110 -90
- data/lib/aspera/cli/manager.rb +99 -88
- data/lib/aspera/cli/plugin.rb +98 -39
- data/lib/aspera/cli/plugins/alee.rb +6 -5
- data/lib/aspera/cli/plugins/aoc.rb +581 -450
- data/lib/aspera/cli/plugins/ats.rb +84 -83
- data/lib/aspera/cli/plugins/bss.rb +30 -27
- data/lib/aspera/cli/plugins/config.rb +488 -397
- data/lib/aspera/cli/plugins/console.rb +17 -15
- data/lib/aspera/cli/plugins/cos.rb +26 -35
- data/lib/aspera/cli/plugins/faspex.rb +206 -172
- data/lib/aspera/cli/plugins/faspex5.rb +109 -74
- data/lib/aspera/cli/plugins/node.rb +379 -189
- data/lib/aspera/cli/plugins/orchestrator.rb +71 -65
- data/lib/aspera/cli/plugins/preview.rb +131 -122
- data/lib/aspera/cli/plugins/server.rb +50 -150
- data/lib/aspera/cli/plugins/shares.rb +61 -27
- data/lib/aspera/cli/plugins/sync.rb +15 -14
- data/lib/aspera/cli/transfer_agent.rb +75 -64
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +29 -28
- data/lib/aspera/command_line_builder.rb +50 -43
- data/lib/aspera/cos_node.rb +64 -38
- data/lib/aspera/data_repository.rb +1 -0
- data/lib/aspera/environment.rb +33 -10
- data/lib/aspera/fasp/agent_base.rb +35 -30
- data/lib/aspera/fasp/agent_connect.rb +35 -30
- data/lib/aspera/fasp/agent_direct.rb +68 -60
- data/lib/aspera/fasp/agent_httpgw.rb +71 -64
- data/lib/aspera/fasp/agent_node.rb +24 -23
- data/lib/aspera/fasp/agent_trsdk.rb +19 -20
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +79 -68
- data/lib/aspera/fasp/installation.rb +130 -126
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +71 -60
- data/lib/aspera/fasp/parameters.yaml +69 -17
- data/lib/aspera/fasp/resume_policy.rb +14 -11
- data/lib/aspera/fasp/transfer_spec.rb +6 -5
- data/lib/aspera/fasp/uri.rb +25 -24
- data/lib/aspera/faspex_gw.rb +83 -72
- data/lib/aspera/hash_ext.rb +23 -13
- data/lib/aspera/id_generator.rb +16 -13
- data/lib/aspera/keychain/encrypted_hash.rb +61 -46
- data/lib/aspera/keychain/macos_security.rb +26 -24
- data/lib/aspera/log.rb +35 -39
- data/lib/aspera/nagios.rb +36 -28
- data/lib/aspera/node.rb +19 -19
- data/lib/aspera/oauth.rb +120 -100
- data/lib/aspera/open_application.rb +25 -22
- data/lib/aspera/persistency_action_once.rb +9 -8
- data/lib/aspera/persistency_folder.rb +13 -9
- data/lib/aspera/preview/file_types.rb +261 -266
- data/lib/aspera/preview/generator.rb +74 -73
- data/lib/aspera/preview/image_error.png +0 -0
- data/lib/aspera/preview/options.rb +7 -6
- data/lib/aspera/preview/utils.rb +30 -33
- data/lib/aspera/preview/video_error.png +0 -0
- data/lib/aspera/proxy_auto_config.rb +27 -23
- data/lib/aspera/rest.rb +73 -74
- data/lib/aspera/rest_call_error.rb +1 -0
- data/lib/aspera/rest_error_analyzer.rb +23 -19
- data/lib/aspera/rest_errors_aspera.rb +43 -40
- data/lib/aspera/secret_hider.rb +74 -0
- data/lib/aspera/ssh.rb +13 -10
- data/lib/aspera/sync.rb +49 -47
- data/lib/aspera/temp_file_manager.rb +7 -5
- data/lib/aspera/timer_limiter.rb +9 -8
- data/lib/aspera/uri_reader.rb +17 -18
- data/lib/aspera/web_auth.rb +17 -15
- data.tar.gz.sig +5 -0
- metadata +119 -35
- metadata.gz.sig +0 -0
- data/bin/dascli +0 -13
data/lib/aspera/aoc.rb
CHANGED
@@ -1,55 +1,62 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
+
|
2
3
|
require 'aspera/log'
|
3
4
|
require 'aspera/rest'
|
4
5
|
require 'aspera/hash_ext'
|
5
6
|
require 'aspera/data_repository'
|
6
7
|
require 'aspera/fasp/transfer_spec'
|
7
8
|
require 'base64'
|
9
|
+
require 'cgi'
|
8
10
|
|
9
11
|
Aspera::Oauth.register_token_creator(:aoc_pub_link,lambda{|o|
|
10
|
-
o.
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
12
|
+
o.api.call({
|
13
|
+
operation: 'POST',
|
14
|
+
subpath: o.gparams[:path_token],
|
15
|
+
headers: {'Accept' => 'application/json'},
|
16
|
+
json_params: o.sparams[:json],
|
17
|
+
url_params: o.sparams[:url].merge(scope: o.gparams[:scope]) # scope is here because it changes over time (node)
|
16
18
|
})
|
19
|
+
},lambda { |oauth|
|
20
|
+
return [oauth.sparams.dig(:json,:url_token)]
|
17
21
|
})
|
18
22
|
|
19
23
|
module Aspera
|
20
24
|
class AoC < Rest
|
21
|
-
PRODUCT_NAME='Aspera on Cloud'
|
25
|
+
PRODUCT_NAME = 'Aspera on Cloud'
|
22
26
|
# Production domain of AoC
|
23
|
-
PROD_DOMAIN='ibmaspera.com'
|
27
|
+
PROD_DOMAIN = 'ibmaspera.com'
|
24
28
|
# to avoid infinite loop in pub link redirection
|
25
|
-
MAX_REDIRECT=10
|
29
|
+
MAX_REDIRECT = 10
|
26
30
|
# Well-known AoC globals client apps
|
27
|
-
GLOBAL_CLIENT_APPS=[
|
31
|
+
GLOBAL_CLIENT_APPS = %w[aspera.global-cli-client aspera.drive].freeze
|
28
32
|
# index offset in data repository of client app
|
29
33
|
DATA_REPO_INDEX_START = 4
|
30
34
|
# cookie prefix so that console can decode identity
|
31
|
-
COOKIE_PREFIX='aspera.aoc'
|
35
|
+
COOKIE_PREFIX = 'aspera.aoc'
|
32
36
|
# path in URL of public links
|
33
|
-
PUBLIC_LINK_PATHS=[
|
34
|
-
JWT_AUDIENCE='https://api.asperafiles.com/api/v1/oauth2/token'
|
35
|
-
OAUTH_API_SUBPATH='api/v1/oauth2'
|
37
|
+
PUBLIC_LINK_PATHS = %w[/packages/public/receive /packages/public/send /files/public].freeze
|
38
|
+
JWT_AUDIENCE = 'https://api.asperafiles.com/api/v1/oauth2/token'
|
39
|
+
OAUTH_API_SUBPATH = 'api/v1/oauth2'
|
36
40
|
# minimum fields for user info if retrieval fails
|
37
|
-
USER_INFO_FIELDS_MIN=[
|
41
|
+
USER_INFO_FIELDS_MIN = %w[name email id default_workspace_id organization_id].freeze
|
38
42
|
|
39
|
-
private_constant :MAX_REDIRECT,:GLOBAL_CLIENT_APPS,:DATA_REPO_INDEX_START,:COOKIE_PREFIX,:PUBLIC_LINK_PATHS,:JWT_AUDIENCE
|
43
|
+
private_constant :MAX_REDIRECT,:GLOBAL_CLIENT_APPS,:DATA_REPO_INDEX_START,:COOKIE_PREFIX,:PUBLIC_LINK_PATHS,:JWT_AUDIENCE,
|
44
|
+
:OAUTH_API_SUBPATH,:USER_INFO_FIELDS_MIN
|
40
45
|
|
41
46
|
# various API scopes supported
|
42
|
-
SCOPE_FILES_SELF='self'
|
43
|
-
SCOPE_FILES_USER='user:all'
|
44
|
-
SCOPE_FILES_ADMIN='admin:all'
|
45
|
-
SCOPE_FILES_ADMIN_USER='admin-user:all'
|
46
|
-
SCOPE_FILES_ADMIN_USER_USER=SCOPE_FILES_ADMIN_USER+'+'+SCOPE_FILES_USER
|
47
|
-
SCOPE_NODE_USER='user:all'
|
48
|
-
SCOPE_NODE_ADMIN='admin:all'
|
49
|
-
PATH_SEPARATOR='/'
|
50
|
-
FILES_APP='files'
|
51
|
-
PACKAGES_APP='packages'
|
52
|
-
API_V1='api/v1'
|
47
|
+
SCOPE_FILES_SELF = 'self'
|
48
|
+
SCOPE_FILES_USER = 'user:all'
|
49
|
+
SCOPE_FILES_ADMIN = 'admin:all'
|
50
|
+
SCOPE_FILES_ADMIN_USER = 'admin-user:all'
|
51
|
+
SCOPE_FILES_ADMIN_USER_USER = SCOPE_FILES_ADMIN_USER + '+' + SCOPE_FILES_USER
|
52
|
+
SCOPE_NODE_USER = 'user:all'
|
53
|
+
SCOPE_NODE_ADMIN = 'admin:all'
|
54
|
+
PATH_SEPARATOR = '/'
|
55
|
+
FILES_APP = 'files'
|
56
|
+
PACKAGES_APP = 'packages'
|
57
|
+
API_V1 = 'api/v1'
|
58
|
+
# error message when entity not found
|
59
|
+
ENTITY_NOT_FOUND = 'No such'
|
53
60
|
|
54
61
|
# class instance variable, access with accessors on class
|
55
62
|
@use_standard_ports = true
|
@@ -59,19 +66,19 @@ module Aspera
|
|
59
66
|
attr_accessor :use_standard_ports
|
60
67
|
# strings /Applications/Aspera\ Drive.app/Contents/MacOS/AsperaDrive|grep -E '.{100}==$'|base64 --decode
|
61
68
|
def get_client_info(client_name=GLOBAL_CLIENT_APPS.first)
|
62
|
-
client_index=GLOBAL_CLIENT_APPS.index(client_name)
|
69
|
+
client_index = GLOBAL_CLIENT_APPS.index(client_name)
|
63
70
|
raise "no such pre-defined client: #{client_name}" if client_index.nil?
|
64
|
-
return client_name,Base64.urlsafe_encode64(DataRepository.instance.data(DATA_REPO_INDEX_START+client_index))
|
71
|
+
return client_name,Base64.urlsafe_encode64(DataRepository.instance.data(DATA_REPO_INDEX_START + client_index))
|
65
72
|
end
|
66
73
|
|
67
74
|
# @param url of AoC instance
|
68
75
|
# @return organization id in url and AoC domain: ibmaspera.com, asperafiles.com or qa.asperafiles.com, etc...
|
69
76
|
def parse_url(aoc_org_url)
|
70
|
-
uri=URI.parse(aoc_org_url.gsub(/\/+$/,''))
|
71
|
-
instance_fqdn=uri.host
|
77
|
+
uri = URI.parse(aoc_org_url.gsub(/\/+$/,''))
|
78
|
+
instance_fqdn = uri.host
|
72
79
|
Log.log.debug("instance_fqdn=#{instance_fqdn}")
|
73
80
|
raise "No host found in URL.Please check URL format: https://myorg.#{PROD_DOMAIN}" if instance_fqdn.nil?
|
74
|
-
organization,instance_domain=instance_fqdn.split('.',2)
|
81
|
+
organization,instance_domain = instance_fqdn.split('.',2)
|
75
82
|
Log.log.debug("instance_domain=#{instance_domain}")
|
76
83
|
Log.log.debug("organization=#{organization}")
|
77
84
|
raise "expecting a public FQDN for #{PRODUCT_NAME}" if instance_domain.nil?
|
@@ -79,14 +86,14 @@ module Aspera
|
|
79
86
|
end
|
80
87
|
|
81
88
|
# base API url depends on domain, which could be "qa.xxx"
|
82
|
-
def api_base_url(api_domain
|
83
|
-
return "https
|
89
|
+
def api_base_url(organization: 'api', api_domain: PROD_DOMAIN)
|
90
|
+
return "https://#{organization}.#{api_domain}"
|
84
91
|
end
|
85
92
|
|
86
93
|
def metering_api(entitlement_id,customer_id,api_domain=PROD_DOMAIN)
|
87
94
|
return Rest.new({
|
88
|
-
base_url:
|
89
|
-
headers:
|
95
|
+
base_url: "#{api_base_url(api_domain: api_domain)}/metering/v1",
|
96
|
+
headers: {'X-Aspera-Entitlement-Authorization' => Rest.basic_creds(entitlement_id,customer_id)}
|
90
97
|
})
|
91
98
|
end
|
92
99
|
|
@@ -99,25 +106,25 @@ module Aspera
|
|
99
106
|
# if present try to get token value (resolve redirection if short links used)
|
100
107
|
# then set options url/token/auth
|
101
108
|
def resolve_pub_link(a_auth,a_opt)
|
102
|
-
public_link_url=a_opt[:link]
|
109
|
+
public_link_url = a_opt[:link]
|
103
110
|
return if public_link_url.nil?
|
104
111
|
raise 'do not use both link and url options' unless a_opt[:url].nil?
|
105
|
-
redirect_count=0
|
112
|
+
redirect_count = 0
|
106
113
|
while redirect_count <= MAX_REDIRECT
|
107
|
-
uri=URI.parse(public_link_url)
|
114
|
+
uri = URI.parse(public_link_url)
|
108
115
|
# detect if it's an expected format
|
109
116
|
if PUBLIC_LINK_PATHS.include?(uri.path)
|
110
|
-
url_param_token_pair=URI.decode_www_form(uri.query).
|
117
|
+
url_param_token_pair = URI.decode_www_form(uri.query).find{|e|e.first.eql?('token')}
|
111
118
|
raise ArgumentError,'link option must be URL with "token" parameter' if url_param_token_pair.nil?
|
112
119
|
# ok we get it !
|
113
|
-
a_opt[:url]='https://'+uri.host
|
114
|
-
a_auth[:crtype]
|
115
|
-
a_auth[:aoc_pub_link]={
|
120
|
+
a_opt[:url] = 'https://' + uri.host
|
121
|
+
a_auth[:crtype] = :aoc_pub_link
|
122
|
+
a_auth[:aoc_pub_link] = {
|
116
123
|
url: {grant_type: 'url_token'}, # URL args
|
117
124
|
json: {url_token: url_param_token_pair.last} # JSON body
|
118
125
|
}
|
119
126
|
# password protection of link
|
120
|
-
a_auth[:aoc_pub_link][:json][:password]=a_opt[:password] unless a_opt[:password].nil?
|
127
|
+
a_auth[:aoc_pub_link][:json][:password] = a_opt[:password] unless a_opt[:password].nil?
|
121
128
|
return # SUCCESS
|
122
129
|
end
|
123
130
|
Log.log.debug("no expected format: #{public_link_url}")
|
@@ -133,52 +140,52 @@ module Aspera
|
|
133
140
|
|
134
141
|
# additional transfer spec (tags) for package information
|
135
142
|
def package_tags(package_info,operation)
|
136
|
-
return {'tags'=>{'aspera'=>{'files'=>{
|
143
|
+
return {'tags' => {'aspera' => {'files' => {
|
137
144
|
'package_id' => package_info['id'],
|
138
145
|
'package_name' => package_info['name'],
|
139
146
|
'package_operation' => operation
|
140
|
-
|
147
|
+
}}}}
|
141
148
|
end
|
142
149
|
|
143
150
|
# add details to show in analytics
|
144
151
|
def analytics_ts(app,direction,ws_id,ws_name)
|
145
152
|
# translate transfer to operation
|
146
|
-
operation=
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
153
|
+
operation =
|
154
|
+
case direction
|
155
|
+
when Fasp::TransferSpec::DIRECTION_SEND then 'upload'
|
156
|
+
when Fasp::TransferSpec::DIRECTION_RECEIVE then 'download'
|
157
|
+
else raise "ERROR: unexpected value: #{direction}"
|
158
|
+
end
|
152
159
|
|
153
160
|
return {
|
154
|
-
'tags'
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
161
|
+
'tags' => {
|
162
|
+
'aspera' => {
|
163
|
+
'usage_id' => "aspera.files.workspace.#{ws_id}", # activity tracking
|
164
|
+
'files' => {
|
165
|
+
'files_transfer_action' => "#{operation}_#{app.gsub(/s$/,'')}",
|
166
|
+
'workspace_name' => ws_name, # activity tracking
|
167
|
+
'workspace_id' => ws_id
|
168
|
+
}
|
169
|
+
}
|
163
170
|
}
|
164
171
|
}
|
165
172
|
end
|
166
173
|
end # static methods
|
167
174
|
|
168
|
-
# @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:username,:subpath,:password (for pub link)
|
175
|
+
# @param :link,:url,:auth,:client_id,:client_secret,:scope,:redirect_uri,:private_key,:passphrase,:username,:subpath,:password (for pub link)
|
169
176
|
def initialize(opt)
|
170
177
|
raise ArgumentError,'Missing mandatory option: scope' if opt[:scope].nil?
|
171
178
|
|
172
179
|
# access key secrets are provided out of band to get node api access
|
173
180
|
# key: access key
|
174
181
|
# value: associated secret
|
175
|
-
@key_chain=nil
|
176
|
-
@user_info=nil
|
182
|
+
@key_chain = nil
|
183
|
+
@user_info = nil
|
177
184
|
|
178
185
|
# init rest params
|
179
|
-
aoc_rest_p={auth: {type: :oauth2}}
|
186
|
+
aoc_rest_p = {auth: {type: :oauth2}}
|
180
187
|
# shortcut to auth section
|
181
|
-
aoc_auth_p=aoc_rest_p[:auth]
|
188
|
+
aoc_auth_p = aoc_rest_p[:auth]
|
182
189
|
|
183
190
|
# sets opt[:url], aoc_rest_p[:auth][:crtype], [:auth][:aoc_pub_link] if there is a link
|
184
191
|
self.class.resolve_pub_link(aoc_auth_p,opt)
|
@@ -187,14 +194,14 @@ module Aspera
|
|
187
194
|
raise ArgumentError,'Missing mandatory option: url' if opt[:url].nil?
|
188
195
|
|
189
196
|
# get org name and domain from url
|
190
|
-
organization,instance_domain=self.class.parse_url(opt[:url])
|
197
|
+
organization,instance_domain = self.class.parse_url(opt[:url])
|
191
198
|
# this is the base API url
|
192
|
-
api_url_base=self.class.api_base_url(instance_domain)
|
199
|
+
api_url_base = self.class.api_base_url(api_domain: instance_domain)
|
193
200
|
# API URL, including subpath (version ...)
|
194
|
-
aoc_rest_p[:base_url]="#{api_url_base}/#{opt[:subpath]}"
|
201
|
+
aoc_rest_p[:base_url] = "#{api_url_base}/#{opt[:subpath]}"
|
195
202
|
# base auth URL
|
196
203
|
aoc_auth_p[:base_url] = "#{api_url_base}/#{OAUTH_API_SUBPATH}/#{organization}"
|
197
|
-
aoc_auth_p[:client_id]=opt[:client_id]
|
204
|
+
aoc_auth_p[:client_id] = opt[:client_id]
|
198
205
|
aoc_auth_p[:client_secret] = opt[:client_secret]
|
199
206
|
aoc_auth_p[:scope] = opt[:scope]
|
200
207
|
|
@@ -205,30 +212,30 @@ module Aspera
|
|
205
212
|
end
|
206
213
|
|
207
214
|
if aoc_auth_p[:client_id].nil?
|
208
|
-
aoc_auth_p[:client_id],aoc_auth_p[:client_secret] = self.class.get_client_info
|
215
|
+
aoc_auth_p[:client_id],aoc_auth_p[:client_secret] = self.class.get_client_info
|
209
216
|
end
|
210
217
|
|
211
218
|
# fill other auth parameters based on Oauth method
|
212
219
|
case aoc_auth_p[:crtype]
|
213
220
|
when :web
|
214
221
|
raise ArgumentError,'Missing mandatory option: redirect_uri' if opt[:redirect_uri].nil?
|
215
|
-
aoc_auth_p[:web]={redirect_uri: opt[:redirect_uri]}
|
222
|
+
aoc_auth_p[:web] = {redirect_uri: opt[:redirect_uri]}
|
216
223
|
when :jwt
|
217
224
|
raise ArgumentError,'Missing mandatory option: private_key' if opt[:private_key].nil?
|
218
225
|
raise ArgumentError,'Missing mandatory option: username' if opt[:username].nil?
|
219
|
-
aoc_auth_p[:jwt]={
|
220
|
-
private_key_obj: OpenSSL::PKey::RSA.new(opt[:private_key]),
|
221
|
-
payload:
|
226
|
+
aoc_auth_p[:jwt] = {
|
227
|
+
private_key_obj: OpenSSL::PKey::RSA.new(opt[:private_key],opt[:passphrase]),
|
228
|
+
payload: {
|
222
229
|
iss: aoc_auth_p[:client_id], # issuer
|
223
230
|
sub: opt[:username], # subject
|
224
231
|
aud: JWT_AUDIENCE
|
225
232
|
}
|
226
233
|
}
|
227
234
|
# add jwt payload for global ids
|
228
|
-
aoc_auth_p[:jwt][:payload][:org]=organization if GLOBAL_CLIENT_APPS.include?(aoc_auth_p[:client_id])
|
235
|
+
aoc_auth_p[:jwt][:payload][:org] = organization if GLOBAL_CLIENT_APPS.include?(aoc_auth_p[:client_id])
|
229
236
|
when :aoc_pub_link
|
230
237
|
# basic auth required for /token
|
231
|
-
aoc_auth_p[:auth]={type: :basic, username: aoc_auth_p[:client_id],password: aoc_auth_p[:client_secret]}
|
238
|
+
aoc_auth_p[:auth] = {type: :basic, username: aoc_auth_p[:client_id],password: aoc_auth_p[:client_secret]}
|
232
239
|
else raise "ERROR: unsupported auth method: #{aoc_auth_p[:crtype]}"
|
233
240
|
end
|
234
241
|
super(aoc_rest_p)
|
@@ -243,21 +250,21 @@ module Aspera
|
|
243
250
|
def key_chain=(keychain)
|
244
251
|
raise 'keychain already set' unless @key_chain.nil?
|
245
252
|
raise 'keychain must have get_secret' unless keychain.respond_to?(:get_secret)
|
246
|
-
@key_chain=keychain
|
253
|
+
@key_chain = keychain
|
247
254
|
end
|
248
255
|
|
249
256
|
# cached user information
|
250
257
|
def user_info
|
251
258
|
if @user_info.nil?
|
252
259
|
# get our user's default information
|
253
|
-
@user_info=
|
254
|
-
|
255
|
-
|
256
|
-
|
257
|
-
|
258
|
-
|
259
|
-
|
260
|
-
USER_INFO_FIELDS_MIN.each{|f|@user_info[f]='unknown' if @user_info[f].nil?}
|
260
|
+
@user_info =
|
261
|
+
begin
|
262
|
+
read('self')[:data]
|
263
|
+
rescue StandardError => e
|
264
|
+
Log.log.debug("ignoring error: #{e}")
|
265
|
+
{}
|
266
|
+
end
|
267
|
+
USER_INFO_FIELDS_MIN.each{|f|@user_info[f] = 'unknown' if @user_info[f].nil?}
|
261
268
|
end
|
262
269
|
return @user_info
|
263
270
|
end
|
@@ -265,9 +272,9 @@ module Aspera
|
|
265
272
|
# build ts addon for IBM Aspera Console (cookie)
|
266
273
|
def console_ts(app)
|
267
274
|
# we are sure that fields are not nil
|
268
|
-
elements=[app,user_info['name'],user_info['email']].map{|e|Base64.strict_encode64(e)}
|
275
|
+
elements = [app,user_info['name'],user_info['email']].map{|e|Base64.strict_encode64(e)}
|
269
276
|
elements.unshift(COOKIE_PREFIX)
|
270
|
-
return {'cookie'=>elements.join(':')}
|
277
|
+
return {'cookie' => elements.join(':')}
|
271
278
|
end
|
272
279
|
|
273
280
|
# build "transfer info", 2 elements array with:
|
@@ -275,26 +282,26 @@ module Aspera
|
|
275
282
|
# - source and token regeneration method
|
276
283
|
def tr_spec(app,direction,node_file,ts_add)
|
277
284
|
# get node api
|
278
|
-
node_api=get_node_api(node_file[:node_info])
|
285
|
+
node_api = get_node_api(node_file[:node_info])
|
279
286
|
# this lambda returns the bearer token for node, if
|
280
|
-
token_generation_lambda=lambda{|do_refresh|node_api.oauth_token(force_refresh: do_refresh)}
|
287
|
+
token_generation_lambda = lambda{|do_refresh|node_api.oauth_token(force_refresh: do_refresh)}
|
281
288
|
# prepare transfer specification
|
282
289
|
# note xfer_id and xfer_retry are set by the transfer agent itself
|
283
|
-
transfer_spec={
|
284
|
-
'direction'
|
285
|
-
'token'
|
286
|
-
'tags'
|
287
|
-
|
288
|
-
|
289
|
-
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
|
295
|
-
|
296
|
-
|
297
|
-
|
290
|
+
transfer_spec = {
|
291
|
+
'direction' => direction,
|
292
|
+
'token' => token_generation_lambda.call(false), # first time, use cache
|
293
|
+
'tags' => {
|
294
|
+
'aspera' => {
|
295
|
+
'app' => app,
|
296
|
+
'files' => {
|
297
|
+
'node_id' => node_file[:node_info]['id']
|
298
|
+
}, # files
|
299
|
+
'node' => {
|
300
|
+
'access_key' => node_file[:node_info]['access_key'],
|
301
|
+
#'file_id' => ts_add['source_root_id']
|
302
|
+
'file_id' => node_file[:file_id]
|
303
|
+
} # node
|
304
|
+
} # aspera
|
298
305
|
} # tags
|
299
306
|
}
|
300
307
|
# add remote host info
|
@@ -302,22 +309,25 @@ module Aspera
|
|
302
309
|
# get default TCP/UDP ports and transfer user
|
303
310
|
transfer_spec.merge!(Fasp::TransferSpec::AK_TSPEC_BASE)
|
304
311
|
# by default: same address as node API
|
305
|
-
transfer_spec['remote_host']=node_file[:node_info]['host']
|
312
|
+
transfer_spec['remote_host'] = node_file[:node_info]['host']
|
306
313
|
# 30 it's necessarily https scheme: webui does not allow anything else
|
307
314
|
if node_file[:node_info]['transfer_url'].is_a?(String) && !node_file[:node_info]['transfer_url'].empty?
|
308
|
-
transfer_spec['remote_host']=URI.parse(node_file[:node_info]['transfer_url']).host
|
315
|
+
transfer_spec['remote_host'] = URI.parse(node_file[:node_info]['transfer_url']).host
|
309
316
|
end
|
310
317
|
else
|
311
318
|
# retrieve values from API
|
312
|
-
std_t_spec=node_api.create(
|
313
|
-
|
319
|
+
std_t_spec = node_api.create(
|
320
|
+
'files/download_setup',
|
321
|
+
{transfer_requests: [{ transfer_request: {paths: [{'source' => '/'}] } }] }
|
322
|
+
)[:data]['transfer_specs'].first['transfer_spec']
|
323
|
+
%w[remote_host remote_user ssh_port fasp_port].each {|i| transfer_spec[i] = std_t_spec[i]}
|
314
324
|
end
|
315
325
|
# add caller provided transfer spec
|
316
326
|
transfer_spec.deep_merge!(ts_add)
|
317
327
|
# additional information for transfer agent
|
318
|
-
source_and_token_generator={
|
319
|
-
src:
|
320
|
-
regenerate_token:
|
328
|
+
source_and_token_generator = {
|
329
|
+
src: :node_gen4,
|
330
|
+
regenerate_token: token_generation_lambda
|
321
331
|
}
|
322
332
|
return transfer_spec,source_and_token_generator
|
323
333
|
end
|
@@ -330,21 +340,21 @@ module Aspera
|
|
330
340
|
def get_node_api(node_info, scope: SCOPE_NODE_USER, use_secret: true)
|
331
341
|
raise 'internal error' unless node_info.is_a?(Hash) && node_info.has_key?('url') && node_info.has_key?('access_key')
|
332
342
|
# get optional secret unless :use_secret is false
|
333
|
-
ak_secret
|
343
|
+
ak_secret = @key_chain.get_secret(url: node_info['url'], username: node_info['access_key'], mandatory: false) if use_secret && !@key_chain.nil?
|
334
344
|
raise "There must be at least one of: 'secret' or 'scope' for access key #{node_info['access_key']}" if ak_secret.nil? && scope.nil?
|
335
|
-
node_rest_params={base_url: node_info['url']}
|
345
|
+
node_rest_params = {base_url: node_info['url']}
|
336
346
|
# if secret is available
|
337
347
|
if !ak_secret.nil?
|
338
|
-
node_rest_params[:auth]={
|
348
|
+
node_rest_params[:auth] = {
|
339
349
|
type: :basic,
|
340
350
|
username: node_info['access_key'],
|
341
351
|
password: ak_secret
|
342
352
|
}
|
343
353
|
else
|
344
354
|
# X-Aspera-AccessKey required for bearer token only
|
345
|
-
node_rest_params[:headers]= {'X-Aspera-AccessKey'=>node_info['access_key']}
|
346
|
-
node_rest_params[:auth]=params[:auth].clone
|
347
|
-
node_rest_params[:auth][:scope]=self.class.node_scope(node_info['access_key'],scope)
|
355
|
+
node_rest_params[:headers] = {'X-Aspera-AccessKey' => node_info['access_key']}
|
356
|
+
node_rest_params[:auth] = params[:auth].clone
|
357
|
+
node_rest_params[:auth][:scope] = self.class.node_scope(node_info['access_key'],scope)
|
348
358
|
end
|
349
359
|
return Node.new(node_rest_params)
|
350
360
|
end
|
@@ -353,9 +363,9 @@ module Aspera
|
|
353
363
|
# @return split values
|
354
364
|
def check_get_node_file(node_file)
|
355
365
|
raise "node_file must be Hash (got #{node_file.class})" unless node_file.is_a?(Hash)
|
356
|
-
raise 'node_file must have 2 keys: :file_id and :node_info' unless node_file.keys.sort.eql?([
|
357
|
-
node_info=node_file[:node_info]
|
358
|
-
file_id=node_file[:file_id]
|
366
|
+
raise 'node_file must have 2 keys: :file_id and :node_info' unless node_file.keys.sort.eql?(%i[file_id node_info])
|
367
|
+
node_info = node_file[:node_info]
|
368
|
+
file_id = node_file[:file_id]
|
359
369
|
raise "node_info must be Hash (got #{node_info.class}: #{node_info})" unless node_info.is_a?(Hash)
|
360
370
|
raise 'node_info must have id' unless node_info.has_key?('id')
|
361
371
|
raise 'file_id is empty' if file_id.to_s.empty?
|
@@ -366,11 +376,11 @@ module Aspera
|
|
366
376
|
def process_find_files(entry,path)
|
367
377
|
begin
|
368
378
|
# add to result if match filter
|
369
|
-
@find_state[:found].push(entry.merge({'path'=>path})) if @find_state[:test_block].call(entry)
|
379
|
+
@find_state[:found].push(entry.merge({'path' => path})) if @find_state[:test_block].call(entry)
|
370
380
|
# process link
|
371
381
|
if entry[:type].eql?('link')
|
372
|
-
sub_node_info=read("nodes/#{entry['target_node_id']}")[:data]
|
373
|
-
sub_opt={method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
|
382
|
+
sub_node_info = read("nodes/#{entry['target_node_id']}")[:data]
|
383
|
+
sub_opt = {method: process_find_files, top_file_id: entry['target_id'], top_file_path: path}
|
374
384
|
get_node_api(sub_node_info).crawl(self,sub_opt)
|
375
385
|
end
|
376
386
|
rescue StandardError => e
|
@@ -381,12 +391,12 @@ module Aspera
|
|
381
391
|
end
|
382
392
|
|
383
393
|
def find_files(top_node_file, test_block)
|
384
|
-
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
394
|
+
top_node_info,top_file_id = check_get_node_file(top_node_file)
|
385
395
|
Log.log.debug("find_files: node_info=#{top_node_info}, fileid=#{top_file_id}")
|
386
|
-
@find_state={found: [], test_block: test_block}
|
396
|
+
@find_state = {found: [], test_block: test_block}
|
387
397
|
get_node_api(top_node_info).crawl(self,{method: :process_find_files, top_file_id: top_file_id})
|
388
|
-
result
|
389
|
-
@find_state=nil
|
398
|
+
result = @find_state[:found]
|
399
|
+
@find_state = nil
|
390
400
|
return result
|
391
401
|
end
|
392
402
|
|
@@ -399,18 +409,18 @@ module Aspera
|
|
399
409
|
when 'file'
|
400
410
|
# file must be terminal
|
401
411
|
raise "#{entry['name']} is a file, expecting folder to find: #{@resolve_state[:path]}" unless @resolve_state[:path].empty?
|
402
|
-
@resolve_state[:result][:file_id]=entry['id']
|
412
|
+
@resolve_state[:result][:file_id] = entry['id']
|
403
413
|
when 'link'
|
404
|
-
@resolve_state[:result][:node_info]=read("nodes/#{entry['target_node_id']}")[:data]
|
414
|
+
@resolve_state[:result][:node_info] = read("nodes/#{entry['target_node_id']}")[:data]
|
405
415
|
if @resolve_state[:path].empty?
|
406
|
-
@resolve_state[:result][:file_id]=entry['target_id']
|
416
|
+
@resolve_state[:result][:file_id] = entry['target_id']
|
407
417
|
else
|
408
418
|
get_node_api(@resolve_state[:result][:node_info]).crawl(self,{method: :process_resolve_node_file, top_file_id: entry['target_id']})
|
409
419
|
end
|
410
420
|
when 'folder'
|
411
421
|
if @resolve_state[:path].empty?
|
412
422
|
# found: store
|
413
|
-
@resolve_state[:result][:file_id]=entry['id']
|
423
|
+
@resolve_state[:result][:file_id] = entry['id']
|
414
424
|
return false
|
415
425
|
end
|
416
426
|
else
|
@@ -425,16 +435,16 @@ module Aspera
|
|
425
435
|
# @param element_path_string String path of element
|
426
436
|
# supports links to secondary nodes
|
427
437
|
def resolve_node_file(top_node_file, element_path_string)
|
428
|
-
top_node_info,top_file_id=check_get_node_file(top_node_file)
|
429
|
-
path_elements=element_path_string.split(PATH_SEPARATOR).reject(&:empty?)
|
430
|
-
result={node_info: top_node_info, file_id: nil}
|
438
|
+
top_node_info,top_file_id = check_get_node_file(top_node_file)
|
439
|
+
path_elements = element_path_string.split(PATH_SEPARATOR).reject(&:empty?)
|
440
|
+
result = {node_info: top_node_info, file_id: nil}
|
431
441
|
if path_elements.empty?
|
432
|
-
result[:file_id]=top_file_id
|
442
|
+
result[:file_id] = top_file_id
|
433
443
|
else
|
434
|
-
@resolve_state={path: path_elements, result: result}
|
444
|
+
@resolve_state = {path: path_elements, result: result}
|
435
445
|
get_node_api(top_node_info).crawl(self,{method: :process_resolve_node_file, top_file_id: top_file_id})
|
436
|
-
not_found
|
437
|
-
@resolve_state=nil
|
446
|
+
not_found = @resolve_state[:path]
|
447
|
+
@resolve_state = nil
|
438
448
|
raise "entry not found: #{not_found}" if result[:file_id].nil?
|
439
449
|
end
|
440
450
|
return result
|
@@ -445,17 +455,17 @@ module Aspera
|
|
445
455
|
# @param options additional search options
|
446
456
|
def lookup_entity_by_name(entity_type,entity_name,options={})
|
447
457
|
# returns entities whose name contains value (case insensitive)
|
448
|
-
matching_items=read(entity_type,options.merge({'q'=>entity_name}))[:data]
|
458
|
+
matching_items = read(entity_type,options.merge({'q' => CGI.escape(entity_name)}))[:data]
|
449
459
|
case matching_items.length
|
450
460
|
when 1 then return matching_items.first
|
451
|
-
when 0 then raise
|
461
|
+
when 0 then raise %Q{#{ENTITY_NOT_FOUND} #{entity_type}: "#{entity_name}"}
|
452
462
|
else
|
453
463
|
# multiple case insensitive partial matches, try case insensitive full match
|
454
464
|
# (anyway AoC does not allow creation of 2 entities with same case insensitive name)
|
455
|
-
icase_matches=matching_items.select{|i|i['name'].casecmp?(entity_name)}
|
465
|
+
icase_matches = matching_items.select{|i|i['name'].casecmp?(entity_name)}
|
456
466
|
case icase_matches.length
|
457
467
|
when 1 then return icase_matches.first
|
458
|
-
when 0 then raise %Q(#{entity_type}: multiple case insensitive partial match for: "#{entity_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.)
|
468
|
+
when 0 then raise %Q(#{entity_type}: multiple case insensitive partial match for: "#{entity_name}": #{matching_items.map{|i|i['name']}} but no case insensitive full match. Please be more specific or give exact name.) # rubocop:disable Layout/LineLength
|
459
469
|
else raise "Two entities cannot have the same case insensitive name: #{icase_matches.map{|i|i['name']}}"
|
460
470
|
end
|
461
471
|
end
|