aspera-cli 4.17.0 → 4.18.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +2 -4
- data/CHANGELOG.md +23 -0
- data/CONTRIBUTING.md +15 -1
- data/README.md +620 -378
- data/bin/ascli +5 -0
- data/bin/asession +2 -2
- data/lib/aspera/agent/alpha.rb +6 -4
- data/lib/aspera/agent/base.rb +9 -6
- data/lib/aspera/agent/connect.rb +4 -4
- data/lib/aspera/agent/direct.rb +56 -37
- data/lib/aspera/agent/httpgw.rb +23 -324
- data/lib/aspera/agent/node.rb +19 -20
- data/lib/aspera/agent/trsdk.rb +19 -20
- data/lib/aspera/api/aoc.rb +17 -14
- data/lib/aspera/api/cos_node.rb +4 -4
- data/lib/aspera/api/httpgw.rb +339 -0
- data/lib/aspera/api/node.rb +34 -21
- data/lib/aspera/ascmd.rb +4 -3
- data/lib/aspera/ascp/installation.rb +15 -7
- data/lib/aspera/ascp/management.rb +2 -2
- data/lib/aspera/cli/basic_auth_plugin.rb +5 -9
- data/lib/aspera/cli/extended_value.rb +12 -6
- data/lib/aspera/cli/formatter.rb +155 -65
- data/lib/aspera/cli/hints.rb +18 -0
- data/lib/aspera/cli/main.rb +22 -29
- data/lib/aspera/cli/manager.rb +53 -36
- data/lib/aspera/cli/plugin.rb +26 -17
- data/lib/aspera/cli/plugin_factory.rb +31 -20
- data/lib/aspera/cli/plugins/alee.rb +14 -2
- data/lib/aspera/cli/plugins/aoc.rb +141 -131
- data/lib/aspera/cli/plugins/ats.rb +1 -1
- data/lib/aspera/cli/plugins/config.rb +52 -46
- data/lib/aspera/cli/plugins/console.rb +8 -5
- data/lib/aspera/cli/plugins/faspex.rb +27 -19
- data/lib/aspera/cli/plugins/faspex5.rb +222 -149
- data/lib/aspera/cli/plugins/faspio.rb +85 -0
- data/lib/aspera/cli/plugins/httpgw.rb +55 -0
- data/lib/aspera/cli/plugins/node.rb +86 -29
- data/lib/aspera/cli/plugins/orchestrator.rb +31 -29
- data/lib/aspera/cli/plugins/preview.rb +6 -2
- data/lib/aspera/cli/plugins/server.rb +5 -5
- data/lib/aspera/cli/plugins/shares.rb +16 -14
- data/lib/aspera/cli/sync_actions.rb +6 -6
- data/lib/aspera/cli/transfer_agent.rb +5 -4
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +7 -6
- data/lib/aspera/faspex_gw.rb +5 -4
- data/lib/aspera/faspex_postproc.rb +2 -2
- data/lib/aspera/log.rb +6 -3
- data/lib/aspera/node_simulator.rb +2 -2
- data/lib/aspera/oauth/base.rb +31 -19
- data/lib/aspera/oauth/factory.rb +12 -13
- data/lib/aspera/oauth/generic.rb +1 -0
- data/lib/aspera/oauth/jwt.rb +18 -15
- data/lib/aspera/oauth/url_json.rb +8 -6
- data/lib/aspera/open_application.rb +5 -7
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/preview/options.rb +3 -3
- data/lib/aspera/preview/terminal.rb +4 -4
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +5 -1
- data/lib/aspera/rest.rb +60 -74
- data/lib/aspera/rest_call_error.rb +1 -1
- data/lib/aspera/rest_error_analyzer.rb +2 -2
- data/lib/aspera/rest_errors_aspera.rb +1 -1
- data/lib/aspera/resumer.rb +1 -1
- data/lib/aspera/secret_hider.rb +2 -4
- data/lib/aspera/ssh.rb +1 -1
- data/lib/aspera/transfer/parameters.rb +39 -36
- data/lib/aspera/transfer/spec.rb +2 -0
- data/lib/aspera/transfer/sync.rb +2 -1
- data/lib/aspera/transfer/uri.rb +1 -1
- data/lib/aspera/uri_reader.rb +5 -4
- data/lib/aspera/web_auth.rb +1 -1
- data/lib/aspera/web_server_simple.rb +4 -3
- data.tar.gz.sig +0 -0
- metadata +5 -3
- metadata.gz.sig +0 -0
- data/lib/aspera/cli/plugins/bss.rb +0 -71
data/lib/aspera/rest.rb
CHANGED
@@ -1,11 +1,11 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
+
require 'aspera/rest_errors_aspera'
|
4
|
+
require 'aspera/rest_error_analyzer'
|
3
5
|
require 'aspera/log'
|
4
6
|
require 'aspera/assert'
|
5
7
|
require 'aspera/oauth'
|
6
|
-
require 'aspera/rest_error_analyzer'
|
7
8
|
require 'aspera/hash_ext'
|
8
|
-
require 'aspera/rest_errors_aspera'
|
9
9
|
require 'net/http'
|
10
10
|
require 'net/https'
|
11
11
|
require 'json'
|
@@ -153,47 +153,6 @@ module Aspera
|
|
153
153
|
return @http_session
|
154
154
|
end
|
155
155
|
|
156
|
-
def build_request(
|
157
|
-
operation:,
|
158
|
-
subpath:,
|
159
|
-
url_params:,
|
160
|
-
json_params:,
|
161
|
-
www_body_params:,
|
162
|
-
text_body_params:,
|
163
|
-
headers:
|
164
|
-
)
|
165
|
-
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
166
|
-
# URI.escape()
|
167
|
-
separator = !['', '/'].include?(subpath) || @base_url.end_with?('/') ? '/' : ''
|
168
|
-
uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", url_params)
|
169
|
-
Log.log.debug{"URI=#{uri}"}
|
170
|
-
begin
|
171
|
-
# instantiate request object based on string name
|
172
|
-
req = Net::HTTP.const_get(operation.capitalize).new(uri)
|
173
|
-
rescue NameError
|
174
|
-
raise "unsupported operation : #{operation}"
|
175
|
-
end
|
176
|
-
if !json_params.nil?
|
177
|
-
req.body = JSON.generate(json_params) # , ascii_only: true
|
178
|
-
req['Content-Type'] = 'application/json'
|
179
|
-
end
|
180
|
-
if !www_body_params.nil?
|
181
|
-
req.body = URI.encode_www_form(www_body_params)
|
182
|
-
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
183
|
-
end
|
184
|
-
if !text_body_params.nil?
|
185
|
-
req.body = text_body_params
|
186
|
-
end
|
187
|
-
# set headers
|
188
|
-
headers.each do |key, value|
|
189
|
-
req[key] = value
|
190
|
-
end
|
191
|
-
# :type = :basic
|
192
|
-
req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
|
193
|
-
Log.log.debug{Log.dump(:req_body, req.body)}
|
194
|
-
return req
|
195
|
-
end
|
196
|
-
|
197
156
|
public
|
198
157
|
|
199
158
|
attr_reader :auth_params
|
@@ -217,7 +176,7 @@ module Aspera
|
|
217
176
|
# :url_query [:url] a hash
|
218
177
|
# :* [:oauth2] see OAuth::Factory class
|
219
178
|
# @param not_auth_codes [Array] codes that trigger a refresh/regeneration of bearer token
|
220
|
-
# @param redirect_max [int] max
|
179
|
+
# @param redirect_max [int] max redirection allowed
|
221
180
|
def initialize(
|
222
181
|
base_url:,
|
223
182
|
auth: nil,
|
@@ -255,21 +214,21 @@ module Aspera
|
|
255
214
|
return @oauth
|
256
215
|
end
|
257
216
|
|
258
|
-
def oauth_token(force_refresh: false)
|
259
|
-
Aspera.assert_values(force_refresh, [true, false])
|
260
|
-
return oauth.get_authorization(use_refresh_token: force_refresh)
|
261
|
-
end
|
262
|
-
|
263
217
|
# HTTP/S REST call
|
218
|
+
# @param operation [String] HTTP operation (GET, POST, PUT, DELETE)
|
219
|
+
# @param subpath [String] subpath of REST API
|
220
|
+
# @param query [Hash] URL parameters
|
221
|
+
# @param body [Hash, String] body parameters
|
222
|
+
# @param body_type [Symbol] type of body parameters (:json, :www, :text, nil)
|
264
223
|
# @param save_to_file (filepath)
|
265
224
|
# @param return_error (bool)
|
225
|
+
# @param headers [Hash] additional headers
|
266
226
|
def call(
|
267
227
|
operation:,
|
268
228
|
subpath: nil,
|
269
|
-
|
270
|
-
|
271
|
-
|
272
|
-
text_body_params: nil,
|
229
|
+
query: nil,
|
230
|
+
body: nil,
|
231
|
+
body_type: nil,
|
273
232
|
save_to_file: nil,
|
274
233
|
return_error: false,
|
275
234
|
headers: nil
|
@@ -293,20 +252,49 @@ module Aspera
|
|
293
252
|
Log.log.debug('using Basic auth')
|
294
253
|
# done in build_req
|
295
254
|
when :oauth2
|
296
|
-
headers['Authorization'] =
|
255
|
+
headers['Authorization'] = oauth.token unless headers.key?('Authorization')
|
297
256
|
when :url
|
298
|
-
|
257
|
+
query ||= {}
|
299
258
|
@auth_params[:url_query].each do |key, value|
|
300
|
-
|
259
|
+
query[key] = value
|
301
260
|
end
|
302
261
|
else Aspera.error_unexpected_value(@auth_params[:type])
|
303
262
|
end
|
304
263
|
result = {http: nil}
|
305
|
-
# start a block to be able to retry the actual HTTP request
|
264
|
+
# start a block to be able to retry the actual HTTP request in case of OAuth token expiration
|
306
265
|
begin
|
307
|
-
|
308
|
-
|
309
|
-
|
266
|
+
# TODO: shall we percent encode subpath (spaces) test with access key delete with space in id
|
267
|
+
# URI.escape()
|
268
|
+
separator = !['', '/'].include?(subpath) || @base_url.end_with?('/') ? '/' : ''
|
269
|
+
uri = self.class.build_uri("#{@base_url}#{separator}#{subpath}", query)
|
270
|
+
Log.log.debug{"URI=#{uri}"}
|
271
|
+
begin
|
272
|
+
# instantiate request object based on string name
|
273
|
+
req = Net::HTTP.const_get(operation.capitalize).new(uri)
|
274
|
+
rescue NameError
|
275
|
+
raise "unsupported operation : #{operation}"
|
276
|
+
end
|
277
|
+
case body_type
|
278
|
+
when :json
|
279
|
+
req.body = JSON.generate(body) # , ascii_only: true
|
280
|
+
req['Content-Type'] = 'application/json'
|
281
|
+
when :www
|
282
|
+
req.body = URI.encode_www_form(body)
|
283
|
+
req['Content-Type'] = 'application/x-www-form-urlencoded'
|
284
|
+
when :text
|
285
|
+
req.body = body
|
286
|
+
req['Content-Type'] = 'text/plain'
|
287
|
+
when nil
|
288
|
+
else
|
289
|
+
raise "unsupported body type : #{body_type}"
|
290
|
+
end
|
291
|
+
# set headers
|
292
|
+
headers.each do |key, value|
|
293
|
+
req[key] = value
|
294
|
+
end
|
295
|
+
# :type = :basic
|
296
|
+
req.basic_auth(@auth_params[:username], @auth_params[:password]) if @auth_params[:type].eql?(:basic)
|
297
|
+
Log.log.debug{Log.dump(:req_body, req.body)}
|
310
298
|
# we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
|
311
299
|
oauth_tries ||= 2
|
312
300
|
# initialize with number of initial retries allowed, nil gives zero
|
@@ -347,7 +335,7 @@ module Aspera
|
|
347
335
|
# rename at the end
|
348
336
|
File.rename(target_file_tmp, target_file)
|
349
337
|
file_saved = true
|
350
|
-
end
|
338
|
+
end
|
351
339
|
end
|
352
340
|
# sometimes there is a UTF8 char (e.g. (c) ), TODO : related to mime type encoding ?
|
353
341
|
# result[:http].body.force_encoding('UTF-8') if result[:http].body.is_a?(String)
|
@@ -367,16 +355,16 @@ module Aspera
|
|
367
355
|
if @not_auth_codes.include?(result[:http].code.to_s) && @auth_params[:type].eql?(:oauth2)
|
368
356
|
begin
|
369
357
|
# try to use refresh token
|
370
|
-
req['Authorization'] =
|
358
|
+
req['Authorization'] = oauth.token(refresh: true)
|
371
359
|
rescue RestCallError => e_tok
|
372
360
|
e = e_tok
|
373
361
|
Log.log.error('refresh failed'.bg_red)
|
374
362
|
# regenerate a brand new token
|
375
|
-
req['Authorization'] =
|
363
|
+
req['Authorization'] = oauth.token(refresh: true)
|
376
364
|
end
|
377
365
|
Log.log.debug{"using new token=#{headers['Authorization']}"}
|
378
366
|
retry if (oauth_tries -= 1).nonzero?
|
379
|
-
end
|
367
|
+
end
|
380
368
|
# redirect ? (any code beginning with 3)
|
381
369
|
if e.response.is_a?(Net::HTTPRedirection) && tries_remain_redirect.positive?
|
382
370
|
tries_remain_redirect -= 1
|
@@ -390,13 +378,12 @@ module Aspera
|
|
390
378
|
end
|
391
379
|
# forwards the request to the new location
|
392
380
|
return self.class.new(base_url: new_url, redirect_max: tries_remain_redirect).call(
|
393
|
-
operation: operation,
|
394
|
-
url_params: url_params, www_body_params: www_body_params, text_body_params: text_body_params,
|
381
|
+
operation: operation, query: query, body: body, body_type: body_type,
|
395
382
|
save_to_file: save_to_file, return_error: return_error, headers: headers)
|
396
383
|
end
|
397
384
|
# raise exception if could not retry and not return error in result
|
398
385
|
raise e unless return_error
|
399
|
-
end
|
386
|
+
end
|
400
387
|
Log.log.debug{"result=#{result}"}
|
401
388
|
return result
|
402
389
|
end
|
@@ -405,21 +392,20 @@ module Aspera
|
|
405
392
|
# CRUD methods here
|
406
393
|
#
|
407
394
|
|
408
|
-
|
409
|
-
|
410
|
-
return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, encoding => params)
|
395
|
+
def create(subpath, params)
|
396
|
+
return call(operation: 'POST', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)
|
411
397
|
end
|
412
398
|
|
413
399
|
def read(subpath, query=nil)
|
414
|
-
return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'},
|
400
|
+
return call(operation: 'GET', subpath: subpath, headers: {'Accept' => 'application/json'}, query: query)
|
415
401
|
end
|
416
402
|
|
417
403
|
def update(subpath, params)
|
418
|
-
return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'},
|
404
|
+
return call(operation: 'PUT', subpath: subpath, headers: {'Accept' => 'application/json'}, body: params, body_type: :json)
|
419
405
|
end
|
420
406
|
|
421
407
|
def delete(subpath, params=nil)
|
422
|
-
return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'},
|
408
|
+
return call(operation: 'DELETE', subpath: subpath, headers: {'Accept' => 'application/json'}, query: params)
|
423
409
|
end
|
424
410
|
|
425
411
|
def cancel(subpath)
|
@@ -453,4 +439,4 @@ module Aspera
|
|
453
439
|
end
|
454
440
|
end
|
455
441
|
end
|
456
|
-
end
|
442
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/log'
|
4
3
|
require 'aspera/rest_call_error'
|
4
|
+
require 'aspera/log'
|
5
5
|
require 'singleton'
|
6
6
|
|
7
7
|
module Aspera
|
@@ -80,7 +80,7 @@ module Aspera
|
|
80
80
|
end
|
81
81
|
end
|
82
82
|
end
|
83
|
-
end
|
83
|
+
end
|
84
84
|
|
85
85
|
class << self
|
86
86
|
# used by handler to add an error description to list of errors
|
data/lib/aspera/resumer.rb
CHANGED
data/lib/aspera/secret_hider.rb
CHANGED
@@ -25,10 +25,8 @@ module Aspera
|
|
25
25
|
/(?<begin> (?:#{ASCP_ENV_SECRETS.join('|')})=)(\\.|[^ ])*(?<end> )/,
|
26
26
|
# rendered JSON or Ruby
|
27
27
|
/(?<begin>(?:(?<quote>["'])|:)[^"':=]*(?:#{ALL_SECRETS.join('|')})[^"':=]*\k<quote>?(?:=>|:) *")[^"]+(?<end>")/,
|
28
|
-
#
|
29
|
-
/(?<begin>
|
30
|
-
# option "secrets"
|
31
|
-
/(?<begin>(secrets)={)[^}]+(?<end>})/,
|
28
|
+
# logged data
|
29
|
+
/(?<begin>(?:#{ALL_SECRETS.join('|')})[ =:]+).*(?<end>$)/,
|
32
30
|
# private key values
|
33
31
|
/(?<begin>--+BEGIN [^-]+ KEY--+)[[:ascii:]]+?(?<end>--+?END [^-]+ KEY--+)/,
|
34
32
|
# cred in http dump
|
data/lib/aspera/ssh.rb
CHANGED
@@ -22,12 +22,10 @@ module Aspera
|
|
22
22
|
# Agents shown in manual for parameters (sub list)
|
23
23
|
SUPPORTED_AGENTS = %i[direct node connect trsdk httpgw].freeze
|
24
24
|
# Short names of columns in manual
|
25
|
-
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|
|
25
|
+
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|agent_sym|agent_sym.to_s[0].to_sym}
|
26
26
|
FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
|
27
|
-
# options that can be provided to the constructor, and then in @options
|
28
|
-
SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore_cb quiet trusted_certs].freeze
|
29
27
|
|
30
|
-
private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
|
28
|
+
private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
|
31
29
|
|
32
30
|
class << self
|
33
31
|
# Temp folder for file lists, must contain only file lists
|
@@ -35,18 +33,18 @@ module Aspera
|
|
35
33
|
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
36
34
|
@file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists') # cspell:disable-line
|
37
35
|
|
38
|
-
# @param
|
36
|
+
# @param formatter [Cli::Formatter] formatter to use
|
39
37
|
# @return a table suitable to display in manual
|
40
|
-
def man_table
|
38
|
+
def man_table(formatter)
|
41
39
|
result = []
|
42
40
|
Spec::DESCRIPTION.each do |name, options|
|
43
41
|
param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
|
44
42
|
# add flags for supported agents in doc
|
45
|
-
SUPPORTED_AGENTS.each do |
|
46
|
-
param[
|
43
|
+
SUPPORTED_AGENTS.each do |agent_sym|
|
44
|
+
param[agent_sym.to_s[0].to_sym] = Cli::Formatter.tick(options[:agents].nil? || options[:agents].include?(agent_sym))
|
47
45
|
end
|
48
46
|
# only keep lines that are usable in supported agents
|
49
|
-
next if SUPPORTED_AGENTS_SHORT.inject(true){|
|
47
|
+
next if SUPPORTED_AGENTS_SHORT.inject(true){|memory, agent_short_sym|memory && param[agent_short_sym].empty?}
|
50
48
|
param[:cli] =
|
51
49
|
case options[:cli][:type]
|
52
50
|
when :envvar then 'env:' + options[:cli][:variable]
|
@@ -63,8 +61,8 @@ module Aspera
|
|
63
61
|
end.map{|n|"{#{n}}"}.join('|')
|
64
62
|
conversion_tag = options[:cli].key?(:convert) ? '(conversion)' : ''
|
65
63
|
"#{options[:cli][:switch]} #{conversion_tag}#{values}"
|
66
|
-
when :special then
|
67
|
-
when :ignore then
|
64
|
+
when :special then formatter.special_format('special')
|
65
|
+
when :ignore then formatter.special_format('ignored')
|
68
66
|
else
|
69
67
|
param[:d].eql?(tick_yes) ? '' : 'n/a'
|
70
68
|
end
|
@@ -75,17 +73,17 @@ module Aspera
|
|
75
73
|
param[:description] = param[:description].gsub('/', '\\')
|
76
74
|
result.push(param)
|
77
75
|
end
|
78
|
-
return result.sort_by { |
|
76
|
+
return result.sort_by { |parameter_info| parameter_info[:name] }
|
79
77
|
end
|
80
78
|
|
81
79
|
# special encoding methods used in YAML (key: :convert)
|
82
|
-
def convert_remove_hyphen(
|
80
|
+
def convert_remove_hyphen(value); value.tr('-', ''); end
|
83
81
|
|
84
82
|
# special encoding methods used in YAML (key: :convert)
|
85
|
-
def convert_json64(
|
83
|
+
def convert_json64(value); Base64.strict_encode64(JSON.generate(value)); end
|
86
84
|
|
87
85
|
# special encoding methods used in YAML (key: :convert)
|
88
|
-
def convert_base64(
|
86
|
+
def convert_base64(value); Base64.strict_encode64(value); end
|
89
87
|
|
90
88
|
# file list is provided directly with ascp arguments
|
91
89
|
# @param ascp_args [Array,NilClass] ascp arguments
|
@@ -94,8 +92,8 @@ module Aspera
|
|
94
92
|
end
|
95
93
|
|
96
94
|
# temp file list files are created here
|
97
|
-
def file_list_folder=(
|
98
|
-
@file_list_folder =
|
95
|
+
def file_list_folder=(value)
|
96
|
+
@file_list_folder = value
|
99
97
|
return if @file_list_folder.nil?
|
100
98
|
FileUtils.mkdir_p(@file_list_folder)
|
101
99
|
TempFileManager.instance.cleanup_expired(@file_list_folder)
|
@@ -103,28 +101,33 @@ module Aspera
|
|
103
101
|
|
104
102
|
# static methods
|
105
103
|
attr_reader :file_list_folder
|
106
|
-
end
|
104
|
+
end
|
107
105
|
|
108
106
|
# @param options [Hash] key: :wss: bool, :ascp_args: array of strings
|
109
|
-
def initialize(
|
110
|
-
|
111
|
-
|
107
|
+
def initialize(
|
108
|
+
job_spec,
|
109
|
+
ascp_args:,
|
110
|
+
wss:,
|
111
|
+
quiet:,
|
112
|
+
trusted_certs:,
|
113
|
+
check_ignore_cb:
|
114
|
+
)
|
112
115
|
@job_spec = job_spec
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
@
|
117
|
-
|
118
|
-
|
119
|
-
Aspera.assert_type(@
|
120
|
-
Aspera.assert(@
|
116
|
+
@ascp_args = ascp_args
|
117
|
+
@wss = wss
|
118
|
+
@quiet = quiet
|
119
|
+
@trusted_certs = trusted_certs
|
120
|
+
@check_ignore_cb = check_ignore_cb
|
121
|
+
Aspera.assert_type(job_spec, Hash)
|
122
|
+
Aspera.assert_type(@ascp_args, Array){'ascp_args'}
|
123
|
+
Aspera.assert(@ascp_args.all?(String)){'ascp arguments must be Strings'}
|
121
124
|
@builder = CommandLineBuilder.new(@job_spec, Spec::DESCRIPTION)
|
122
125
|
end
|
123
126
|
|
124
127
|
# either place source files on command line, or add file list file
|
125
128
|
def process_file_list
|
126
129
|
# is the file list provided through ascp parameters?
|
127
|
-
ascp_file_list_provided = self.class.ascp_args_file_list?(@
|
130
|
+
ascp_file_list_provided = self.class.ascp_args_file_list?(@ascp_args)
|
128
131
|
# set if paths is mandatory in ts
|
129
132
|
@builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided # cspell:words keepalive
|
130
133
|
# get paths in transfer spec (after setting if it is mandatory)
|
@@ -162,7 +165,7 @@ module Aspera
|
|
162
165
|
def remote_certificates
|
163
166
|
certificates_to_use = []
|
164
167
|
# use web socket secure for session ?
|
165
|
-
if @builder.read_param('wss_enabled') && (@
|
168
|
+
if @builder.read_param('wss_enabled') && (@wss || !@job_spec.key?('fasp_port'))
|
166
169
|
# by default use web socket session if available, unless removed by user
|
167
170
|
@builder.add_command_line_options(['--ws-connect'])
|
168
171
|
# TODO: option to give order ssh,ws (legacy http is implied by ssh)
|
@@ -171,9 +174,9 @@ module Aspera
|
|
171
174
|
@job_spec.delete('fasp_port')
|
172
175
|
@job_spec.delete('sshfp')
|
173
176
|
# set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
174
|
-
certificates_to_use.concat(@
|
177
|
+
certificates_to_use.concat(@trusted_certs) if @trusted_certs.is_a?(Array)
|
175
178
|
# ignore cert for wss ?
|
176
|
-
if @
|
179
|
+
if @check_ignore_cb&.call(@job_spec['remote_host'], @job_spec['wss_port'])
|
177
180
|
wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
|
178
181
|
wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}"
|
179
182
|
File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
|
@@ -228,7 +231,7 @@ module Aspera
|
|
228
231
|
# destination will be base64 encoded, put this before source path arguments
|
229
232
|
@builder.add_command_line_options(['--dest64']) if base64_destination
|
230
233
|
# optional arguments, at the end to override previous ones (to allow override)
|
231
|
-
@builder.add_command_line_options(@
|
234
|
+
@builder.add_command_line_options(@ascp_args)
|
232
235
|
# get list of source files to transfer and build arg for ascp
|
233
236
|
process_file_list
|
234
237
|
# process destination folder
|
@@ -238,7 +241,7 @@ module Aspera
|
|
238
241
|
# destination MUST be last command line argument to ascp
|
239
242
|
@builder.add_command_line_options([destination_folder])
|
240
243
|
@builder.add_env_args(env_args)
|
241
|
-
env_args[:args].unshift('-q') if @
|
244
|
+
env_args[:args].unshift('-q') if @quiet
|
242
245
|
# add fallback cert and key as arguments if needed
|
243
246
|
if ['1', 1, true, 'force'].include?(@job_spec['http_fallback'])
|
244
247
|
env_args[:args].unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
|
@@ -247,6 +250,6 @@ module Aspera
|
|
247
250
|
Log.log.debug{"ascp args: #{env_args}"}
|
248
251
|
return env_args
|
249
252
|
end
|
250
|
-
end
|
253
|
+
end
|
251
254
|
end
|
252
255
|
end
|
data/lib/aspera/transfer/spec.rb
CHANGED
@@ -17,6 +17,8 @@ module Aspera
|
|
17
17
|
'ssh_port' => SSH_PORT,
|
18
18
|
'fasp_port' => UDP_PORT
|
19
19
|
}.freeze
|
20
|
+
# fields for transport
|
21
|
+
TRANSPORT_FIELDS = %w[remote_host remote_user ssh_port fasp_port wss_enabled wss_port].freeze
|
20
22
|
# reserved tag for Aspera
|
21
23
|
TAG_RESERVED = 'aspera'
|
22
24
|
class << self
|
data/lib/aspera/transfer/sync.rb
CHANGED
@@ -48,7 +48,8 @@ module Aspera
|
|
48
48
|
'cooloff' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
49
49
|
'pending_max' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
50
50
|
'scan_intensity' => { cli: { type: :opt_with_arg}, accepted_types: :string},
|
51
|
-
'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Transfer::Parameters.convert_remove_hyphen'},
|
51
|
+
'cipher' => { cli: { type: :opt_with_arg, convert: 'Aspera::Transfer::Parameters.convert_remove_hyphen'},
|
52
|
+
accepted_types: :string, ts: true},
|
52
53
|
'transfer_threads' => { cli: { type: :opt_with_arg}, accepted_types: :int},
|
53
54
|
'preserve_time' => { cli: { type: :opt_without_arg}, ts: :preserve_times},
|
54
55
|
'preserve_access_time' => { cli: { type: :opt_without_arg}, ts: nil},
|
data/lib/aspera/transfer/uri.rb
CHANGED
@@ -24,7 +24,7 @@ module Aspera
|
|
24
24
|
result_ts['ssh_port'] = @fasp_uri.port
|
25
25
|
result_ts['paths'] = [{'source' => URI.decode_www_form_component(@fasp_uri.path)}]
|
26
26
|
# faspex 4 does not encode trailing base64 padding, fix that to be able to decode properly
|
27
|
-
fixed_query = @fasp_uri.query.gsub(/(=+)$/){|
|
27
|
+
fixed_query = @fasp_uri.query.gsub(/(=+)$/){|trail_equals|'%3D' * trail_equals.length}
|
28
28
|
|
29
29
|
Rest.decode_query(fixed_query).each do |name, value|
|
30
30
|
case name
|
data/lib/aspera/uri_reader.rb
CHANGED
@@ -4,21 +4,22 @@ require 'uri'
|
|
4
4
|
require 'aspera/rest'
|
5
5
|
|
6
6
|
module Aspera
|
7
|
+
# read some content from some URI, support file: , http: and https: schemes
|
7
8
|
module UriReader
|
8
9
|
class << self
|
9
10
|
# read some content from some URI, support file: , http: and https: schemes
|
10
11
|
def read(uri_to_read)
|
11
|
-
|
12
|
-
case
|
12
|
+
uri = URI.parse(uri_to_read)
|
13
|
+
case uri.scheme
|
13
14
|
when 'http', 'https'
|
14
15
|
return Rest.new(base_url: uri_to_read, redirect_max: 5).call(operation: 'GET', subpath: '', headers: {'Accept' => 'text/plain'})[:data]
|
15
16
|
when 'file', NilClass
|
16
|
-
local_file_path =
|
17
|
+
local_file_path = uri.path
|
17
18
|
raise 'URL shall have a path, check syntax' if local_file_path.nil?
|
18
19
|
local_file_path = File.expand_path(local_file_path.gsub(%r{^/}, '')) if %r{^/(~|.|..)/}.match?(local_file_path)
|
19
20
|
return File.read(local_file_path)
|
20
21
|
else
|
21
|
-
raise "unknown scheme: [#{
|
22
|
+
raise "unknown scheme: [#{uri.scheme}] for [#{uri_to_read}]"
|
22
23
|
end
|
23
24
|
end
|
24
25
|
end
|
data/lib/aspera/web_auth.rb
CHANGED
@@ -28,7 +28,7 @@ module Aspera
|
|
28
28
|
response.body = '<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>'
|
29
29
|
return nil
|
30
30
|
end
|
31
|
-
end
|
31
|
+
end
|
32
32
|
|
33
33
|
# start a local web server, then start a browser that will callback the local server upon authentication
|
34
34
|
class WebAuth < WebServerSimple
|
@@ -7,6 +7,7 @@ require 'aspera/assert'
|
|
7
7
|
require 'openssl'
|
8
8
|
|
9
9
|
module Aspera
|
10
|
+
# Simple WEBrick server with HTTPS support
|
10
11
|
class WebServerSimple < WEBrick::HTTPServer
|
11
12
|
CERT_PARAMETERS = %i[key cert chain].freeze
|
12
13
|
GENERIC_ISSUER = '/C=FR/O=Test/OU=Test/CN=Test'
|
@@ -18,8 +19,8 @@ module Aspera
|
|
18
19
|
# generates and adds self signed cert to provided webrick options
|
19
20
|
def fill_self_signed_cert(cert, key, digest = 'SHA256')
|
20
21
|
cert.subject = cert.issuer = OpenSSL::X509::Name.parse(GENERIC_ISSUER)
|
21
|
-
cert.not_before = Time.now
|
22
|
-
cert.not_after
|
22
|
+
cert.not_before = cert.not_after = Time.now
|
23
|
+
cert.not_after += ONE_YEAR_SECONDS
|
23
24
|
cert.public_key = key.public_key
|
24
25
|
cert.serial = 0x0
|
25
26
|
cert.version = 2
|
@@ -56,7 +57,7 @@ module Aspera
|
|
56
57
|
else
|
57
58
|
Aspera.assert_type(certificate, Hash)
|
58
59
|
certificate = certificate.symbolize_keys
|
59
|
-
raise "unexpected key in certificate config: only: #{CERT_PARAMETERS.join(', ')}" if certificate.keys.any?{|
|
60
|
+
raise "unexpected key in certificate config: only: #{CERT_PARAMETERS.join(', ')}" if certificate.keys.any?{|key|!CERT_PARAMETERS.include?(key)}
|
60
61
|
webrick_options[:SSLPrivateKey] = if certificate.key?(:key)
|
61
62
|
OpenSSL::PKey::RSA.new(File.read(certificate[:key]))
|
62
63
|
else
|
data.tar.gz.sig
CHANGED
Binary file
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: aspera-cli
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.18.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Laurent Martin
|
@@ -37,7 +37,7 @@ cert_chain:
|
|
37
37
|
eTf9kxhVM40wGQOECVNA8UsEEZHD48eF+csUYZtAJOF5oxTI8UyV9T/o6CgO0c9/
|
38
38
|
Gzz+Qm5ULOUcPiJLjSpaiTrkiIVYiDGnqNSr6R1Hb1c=
|
39
39
|
-----END CERTIFICATE-----
|
40
|
-
date: 2024-
|
40
|
+
date: 2024-07-10 00:00:00.000000000 Z
|
41
41
|
dependencies:
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: blankslate
|
@@ -403,6 +403,7 @@ files:
|
|
403
403
|
- lib/aspera/api/aoc.rb
|
404
404
|
- lib/aspera/api/ats.rb
|
405
405
|
- lib/aspera/api/cos_node.rb
|
406
|
+
- lib/aspera/api/httpgw.rb
|
406
407
|
- lib/aspera/api/node.rb
|
407
408
|
- lib/aspera/ascmd.rb
|
408
409
|
- lib/aspera/ascp/installation.rb
|
@@ -422,12 +423,13 @@ files:
|
|
422
423
|
- lib/aspera/cli/plugins/alee.rb
|
423
424
|
- lib/aspera/cli/plugins/aoc.rb
|
424
425
|
- lib/aspera/cli/plugins/ats.rb
|
425
|
-
- lib/aspera/cli/plugins/bss.rb
|
426
426
|
- lib/aspera/cli/plugins/config.rb
|
427
427
|
- lib/aspera/cli/plugins/console.rb
|
428
428
|
- lib/aspera/cli/plugins/cos.rb
|
429
429
|
- lib/aspera/cli/plugins/faspex.rb
|
430
430
|
- lib/aspera/cli/plugins/faspex5.rb
|
431
|
+
- lib/aspera/cli/plugins/faspio.rb
|
432
|
+
- lib/aspera/cli/plugins/httpgw.rb
|
431
433
|
- lib/aspera/cli/plugins/node.rb
|
432
434
|
- lib/aspera/cli/plugins/orchestrator.rb
|
433
435
|
- lib/aspera/cli/plugins/preview.rb
|
metadata.gz.sig
CHANGED
Binary file
|