aspera-cli 4.16.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +50 -19
- data/CONTRIBUTING.md +3 -1
- data/README.md +965 -793
- data/bin/asession +29 -21
- data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
- data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
- data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
- data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
- data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
- data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
- data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
- data/lib/aspera/api/aoc.rb +586 -0
- data/lib/aspera/api/ats.rb +46 -0
- data/lib/aspera/api/cos_node.rb +95 -0
- data/lib/aspera/api/node.rb +344 -0
- data/lib/aspera/ascmd.rb +46 -10
- data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
- data/lib/aspera/{fasp → ascp}/management.rb +3 -8
- data/lib/aspera/{fasp → ascp}/products.rb +1 -1
- data/lib/aspera/assert.rb +30 -30
- data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
- data/lib/aspera/cli/extended_value.rb +1 -1
- data/lib/aspera/cli/formatter.rb +13 -13
- data/lib/aspera/cli/hints.rb +5 -5
- data/lib/aspera/cli/main.rb +35 -28
- data/lib/aspera/cli/manager.rb +25 -24
- data/lib/aspera/cli/plugin.rb +22 -15
- data/lib/aspera/cli/plugin_factory.rb +61 -0
- data/lib/aspera/cli/plugins/alee.rb +7 -7
- data/lib/aspera/cli/plugins/aoc.rb +83 -77
- data/lib/aspera/cli/plugins/ats.rb +32 -33
- data/lib/aspera/cli/plugins/bss.rb +3 -4
- data/lib/aspera/cli/plugins/config.rb +169 -186
- data/lib/aspera/cli/plugins/console.rb +8 -6
- data/lib/aspera/cli/plugins/cos.rb +19 -18
- data/lib/aspera/cli/plugins/faspex.rb +61 -54
- data/lib/aspera/cli/plugins/faspex5.rb +150 -103
- data/lib/aspera/cli/plugins/node.rb +68 -73
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
- data/lib/aspera/cli/plugins/preview.rb +31 -31
- data/lib/aspera/cli/plugins/server.rb +31 -33
- data/lib/aspera/cli/plugins/shares.rb +13 -11
- data/lib/aspera/cli/sync_actions.rb +8 -8
- data/lib/aspera/cli/transfer_agent.rb +32 -19
- data/lib/aspera/cli/transfer_progress.rb +1 -1
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -0
- data/lib/aspera/command_line_builder.rb +14 -14
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +2 -3
- data/lib/aspera/faspex_gw.rb +5 -6
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/id_generator.rb +2 -2
- data/lib/aspera/json_rpc.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +6 -6
- data/lib/aspera/keychain/macos_security.rb +27 -22
- data/lib/aspera/log.rb +2 -2
- data/lib/aspera/nagios.rb +3 -3
- data/lib/aspera/node_simulator.rb +5 -6
- data/lib/aspera/oauth/base.rb +143 -0
- data/lib/aspera/oauth/factory.rb +124 -0
- data/lib/aspera/oauth/generic.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +51 -0
- data/lib/aspera/oauth/url_json.rb +31 -0
- data/lib/aspera/oauth/web.rb +50 -0
- data/lib/aspera/oauth.rb +5 -331
- data/lib/aspera/open_application.rb +7 -7
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +5 -5
- data/lib/aspera/preview/terminal.rb +3 -2
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +4 -4
- data/lib/aspera/rest.rb +175 -144
- data/lib/aspera/rest_errors_aspera.rb +3 -3
- data/lib/aspera/resumer.rb +77 -0
- data/lib/aspera/ssh.rb +6 -1
- data/lib/aspera/{fasp → transfer}/error.rb +3 -3
- data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
- data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
- data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
- data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
- data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
- data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
- data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
- data/lib/aspera/web_server_simple.rb +11 -3
- data.tar.gz.sig +0 -0
- metadata +36 -63
- metadata.gz.sig +0 -0
- data/lib/aspera/aoc.rb +0 -601
- data/lib/aspera/ats_api.rb +0 -47
- data/lib/aspera/cos_node.rb +0 -94
- data/lib/aspera/fasp/resume_policy.rb +0 -79
- data/lib/aspera/node.rb +0 -339
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'singleton'
|
|
4
|
+
require 'aspera/log'
|
|
5
|
+
require 'aspera/assert'
|
|
6
|
+
|
|
7
|
+
module Aspera
|
|
8
|
+
# implements a simple resume policy
|
|
9
|
+
class Resumer
|
|
10
|
+
# list of supported parameters and default values
|
|
11
|
+
DEFAULTS = {
|
|
12
|
+
iter_max: 7,
|
|
13
|
+
sleep_initial: 2,
|
|
14
|
+
sleep_factor: 2,
|
|
15
|
+
sleep_max: 60
|
|
16
|
+
}.freeze
|
|
17
|
+
|
|
18
|
+
# @param params see DEFAULTS
|
|
19
|
+
def initialize(params=nil)
|
|
20
|
+
@parameters = DEFAULTS.dup
|
|
21
|
+
if !params.nil?
|
|
22
|
+
Aspera.assert_type(params, Hash)
|
|
23
|
+
params.each do |k, v|
|
|
24
|
+
Aspera.assert_values(k, DEFAULTS.keys){'resume parameter'}
|
|
25
|
+
Aspera.assert_type(v, Integer){k}
|
|
26
|
+
@parameters[k] = v
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
Log.log.debug{"resume params=#{@parameters}"}
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# calls block a number of times (resumes) until success or limit reached
|
|
33
|
+
# this is re-entrant, one resumer can handle multiple transfers in //
|
|
34
|
+
def execute_with_resume
|
|
35
|
+
Aspera.assert(block_given?)
|
|
36
|
+
# maximum of retry
|
|
37
|
+
remaining_resumes = @parameters[:iter_max]
|
|
38
|
+
sleep_seconds = @parameters[:sleep_initial]
|
|
39
|
+
Log.log.debug{"retries=#{remaining_resumes}"}
|
|
40
|
+
# try to send the file until ascp is successful
|
|
41
|
+
loop do
|
|
42
|
+
Log.log.debug('transfer starting')
|
|
43
|
+
begin
|
|
44
|
+
# call provided block
|
|
45
|
+
yield
|
|
46
|
+
# exit retry loop if success
|
|
47
|
+
break
|
|
48
|
+
rescue Transfer::Error => e
|
|
49
|
+
Log.log.warn{"An error occurred during transfer: #{e.message}"}
|
|
50
|
+
# failure in ascp
|
|
51
|
+
if e.retryable?
|
|
52
|
+
# exit if we exceed the max number of retry
|
|
53
|
+
raise Transfer::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
|
|
54
|
+
else
|
|
55
|
+
# give one chance only to non retryable errors
|
|
56
|
+
unless remaining_resumes.eql?(@parameters[:iter_max])
|
|
57
|
+
Log.log.error('non-retryable error'.red.blink)
|
|
58
|
+
raise e
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# take this retry in account
|
|
64
|
+
remaining_resumes -= 1
|
|
65
|
+
Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
|
|
66
|
+
|
|
67
|
+
# wait a bit before retrying, maybe network condition will be better
|
|
68
|
+
sleep(sleep_seconds)
|
|
69
|
+
|
|
70
|
+
# increase retry period
|
|
71
|
+
sleep_seconds *= @parameters[:sleep_factor]
|
|
72
|
+
# cap value
|
|
73
|
+
sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
|
|
74
|
+
end # loop
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
data/lib/aspera/ssh.rb
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
require 'net/ssh'
|
|
4
4
|
|
|
5
5
|
if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
|
|
6
|
-
# HACK: deactivate ed25519 and ecdsa private keys from
|
|
6
|
+
# HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
|
|
7
7
|
old_verbose = $VERBOSE
|
|
8
8
|
$VERBOSE = nil
|
|
9
9
|
begin
|
|
@@ -14,6 +14,11 @@ if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
|
|
|
14
14
|
$VERBOSE = old_verbose
|
|
15
15
|
end
|
|
16
16
|
|
|
17
|
+
if RUBY_ENGINE == 'jruby' && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
|
|
18
|
+
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } }
|
|
19
|
+
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ }
|
|
20
|
+
end
|
|
21
|
+
|
|
17
22
|
module Aspera
|
|
18
23
|
# A simple wrapper around Net::SSH
|
|
19
24
|
# executes one command and get its result from stdout
|
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/
|
|
3
|
+
require 'aspera/transfer/error_info'
|
|
4
4
|
|
|
5
5
|
module Aspera
|
|
6
|
-
module
|
|
6
|
+
module Transfer
|
|
7
7
|
# error raised if transfer fails
|
|
8
8
|
class Error < StandardError
|
|
9
9
|
attr_reader :err_code
|
|
@@ -14,7 +14,7 @@ module Aspera
|
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def info
|
|
17
|
-
r =
|
|
17
|
+
r = ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
|
|
18
18
|
return r.merge({i: @err_code})
|
|
19
19
|
end
|
|
20
20
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# cspell:words mnemo PROTO RCVR NOLIC PMTU BRTT VLINK BWMEAS SSEAR FIPS
|
|
4
4
|
|
|
5
5
|
module Aspera
|
|
6
|
-
module
|
|
6
|
+
module Transfer
|
|
7
7
|
# from https://www.google.com/search?q=FASP+error+codes
|
|
8
8
|
# Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
|
|
9
9
|
# rubocop:disable Layout/MultilineHashKeyLineBreaks
|
|
@@ -4,19 +4,19 @@ require 'aspera/log'
|
|
|
4
4
|
require 'aspera/assert'
|
|
5
5
|
require 'aspera/command_line_builder'
|
|
6
6
|
require 'aspera/temp_file_manager'
|
|
7
|
-
require 'aspera/
|
|
8
|
-
require 'aspera/
|
|
7
|
+
require 'aspera/transfer/error'
|
|
8
|
+
require 'aspera/transfer/spec'
|
|
9
|
+
require 'aspera/ascp/installation'
|
|
9
10
|
require 'aspera/cli/formatter'
|
|
10
11
|
require 'aspera/rest'
|
|
11
12
|
require 'securerandom'
|
|
12
13
|
require 'base64'
|
|
13
14
|
require 'json'
|
|
14
|
-
require 'yaml'
|
|
15
15
|
require 'fileutils'
|
|
16
16
|
require 'openssl'
|
|
17
17
|
|
|
18
18
|
module Aspera
|
|
19
|
-
module
|
|
19
|
+
module Transfer
|
|
20
20
|
# translate transfer specification to ascp parameter list
|
|
21
21
|
class Parameters
|
|
22
22
|
# Agents shown in manual for parameters (sub list)
|
|
@@ -25,7 +25,7 @@ module Aspera
|
|
|
25
25
|
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
|
26
26
|
FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
|
|
27
27
|
# options that can be provided to the constructor, and then in @options
|
|
28
|
-
SUPPORTED_OPTIONS = %i[ascp_args wss
|
|
28
|
+
SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore_cb quiet trusted_certs].freeze
|
|
29
29
|
|
|
30
30
|
private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS, :SUPPORTED_OPTIONS
|
|
31
31
|
|
|
@@ -34,22 +34,12 @@ module Aspera
|
|
|
34
34
|
# because of garbage collection takes any file there
|
|
35
35
|
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
|
36
36
|
@file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists') # cspell:disable-line
|
|
37
|
-
@param_description_cache = nil
|
|
38
|
-
# @return normalized description of transfer spec parameters, direct from yaml
|
|
39
|
-
def description
|
|
40
|
-
if @param_description_cache.nil?
|
|
41
|
-
# config file in same folder with same name as this source
|
|
42
|
-
description_from_yaml = YAML.load_file("#{__FILE__[0..-3]}yaml")
|
|
43
|
-
@param_description_cache = Aspera::CommandLineBuilder.normalize_description(description_from_yaml)
|
|
44
|
-
end
|
|
45
|
-
return @param_description_cache
|
|
46
|
-
end
|
|
47
37
|
|
|
48
38
|
# @param to_text [bool] replace HTML entities with text equivalent
|
|
49
39
|
# @return a table suitable to display in manual
|
|
50
40
|
def man_table
|
|
51
41
|
result = []
|
|
52
|
-
|
|
42
|
+
Spec::DESCRIPTION.each do |name, options|
|
|
53
43
|
param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
|
|
54
44
|
# add flags for supported agents in doc
|
|
55
45
|
SUPPORTED_AGENTS.each do |a|
|
|
@@ -85,13 +75,7 @@ module Aspera
|
|
|
85
75
|
param[:description] = param[:description].gsub('/', '\\')
|
|
86
76
|
result.push(param)
|
|
87
77
|
end
|
|
88
|
-
return result.
|
|
89
|
-
if a[:name].start_with?('EX_').eql?(b[:name].start_with?('EX_'))
|
|
90
|
-
a[:name] <=> b[:name]
|
|
91
|
-
else
|
|
92
|
-
b[:name] <=> a[:name]
|
|
93
|
-
end
|
|
94
|
-
end
|
|
78
|
+
return result.sort_by { |a| a[:name] }
|
|
95
79
|
end
|
|
96
80
|
|
|
97
81
|
# special encoding methods used in YAML (key: :convert)
|
|
@@ -104,13 +88,9 @@ module Aspera
|
|
|
104
88
|
def convert_base64(v); Base64.strict_encode64(v); end
|
|
105
89
|
|
|
106
90
|
# file list is provided directly with ascp arguments
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
ascp_args
|
|
110
|
-
(ts['EX_ascp_args'].is_a?(Array) && ts['EX_ascp_args'].any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
|
|
111
|
-
(ascp_args.is_a?(Array) && ascp_args.any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
|
|
112
|
-
ts.key?('EX_file_list') ||
|
|
113
|
-
ts.key?('EX_file_pair_list')
|
|
91
|
+
# @param ascp_args [Array,NilClass] ascp arguments
|
|
92
|
+
def ascp_args_file_list?(ascp_args)
|
|
93
|
+
ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
|
|
114
94
|
end
|
|
115
95
|
|
|
116
96
|
# temp file list files are created here
|
|
@@ -127,67 +107,56 @@ module Aspera
|
|
|
127
107
|
|
|
128
108
|
# @param options [Hash] key: :wss: bool, :ascp_args: array of strings
|
|
129
109
|
def initialize(job_spec, options)
|
|
130
|
-
assert_type(job_spec, Hash)
|
|
131
|
-
assert_type(options, Hash)
|
|
110
|
+
Aspera.assert_type(job_spec, Hash)
|
|
111
|
+
Aspera.assert_type(options, Hash)
|
|
132
112
|
@job_spec = job_spec
|
|
133
113
|
# check necessary options
|
|
134
114
|
missing_options = SUPPORTED_OPTIONS - options.keys
|
|
135
|
-
assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
|
|
115
|
+
Aspera.assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
|
|
136
116
|
@options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
|
|
137
117
|
Log.log.debug{Log.dump(:parameters_options, @options)}
|
|
138
118
|
Log.log.debug{Log.dump(:dismiss_options, options.keys - SUPPORTED_OPTIONS)}
|
|
139
|
-
assert_type(@options[:ascp_args], Array){'ascp_args'}
|
|
140
|
-
assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
|
|
141
|
-
@builder =
|
|
119
|
+
Aspera.assert_type(@options[:ascp_args], Array){'ascp_args'}
|
|
120
|
+
Aspera.assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
|
|
121
|
+
@builder = CommandLineBuilder.new(@job_spec, Spec::DESCRIPTION)
|
|
142
122
|
end
|
|
143
123
|
|
|
144
124
|
# either place source files on command line, or add file list file
|
|
145
125
|
def process_file_list
|
|
146
|
-
# is the file list provided through
|
|
147
|
-
ascp_file_list_provided = self.class.
|
|
126
|
+
# is the file list provided through ascp parameters?
|
|
127
|
+
ascp_file_list_provided = self.class.ascp_args_file_list?(@options[:ascp_args])
|
|
148
128
|
# set if paths is mandatory in ts
|
|
149
129
|
@builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided # cspell:words keepalive
|
|
150
130
|
# get paths in transfer spec (after setting if it is mandatory)
|
|
151
131
|
ts_paths_array = @builder.read_param('paths')
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
132
|
+
file_list_option = nil
|
|
133
|
+
# transfer spec contains paths ?
|
|
134
|
+
if !ts_paths_array.nil?
|
|
135
|
+
Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
|
|
136
|
+
Aspera.assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
|
|
137
|
+
is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
|
|
138
|
+
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
|
|
139
|
+
if self.class.file_list_folder.nil?
|
|
140
|
+
Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
|
|
141
|
+
# not safe for special characters ? (maybe not, depends on OS)
|
|
142
|
+
Log.log.debug('placing source file list on command line (no file list file)')
|
|
143
|
+
@builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
|
|
144
|
+
else
|
|
145
|
+
# safer option: generate a file list file if there is storage defined for it
|
|
146
|
+
if is_pair_list
|
|
147
|
+
file_list_option = '--file-pair-list'
|
|
148
|
+
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']) }
|
|
168
149
|
else
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
|
|
172
|
-
# safer option: generate a file list file if there is storage defined for it
|
|
173
|
-
# if there is one destination in paths, then use file-pair-list
|
|
174
|
-
if is_pair_list
|
|
175
|
-
option = '--file-pair-list'
|
|
176
|
-
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']) }
|
|
177
|
-
else
|
|
178
|
-
option = '--file-list'
|
|
179
|
-
lines = ts_paths_array.map{|i|i['source']}
|
|
180
|
-
end
|
|
181
|
-
file_list_file = Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
|
182
|
-
Log.log.debug{Log.dump(:file_list, lines)}
|
|
183
|
-
File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
|
|
184
|
-
Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
|
|
150
|
+
file_list_option = '--file-list'
|
|
151
|
+
lines = ts_paths_array.map{|i|i['source']}
|
|
185
152
|
end
|
|
153
|
+
file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
|
154
|
+
Log.log.debug{Log.dump(:file_list, lines)}
|
|
155
|
+
File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
|
|
156
|
+
Log.log.debug{"#{file_list_option}=\n#{File.read(file_list_file)}".red}
|
|
186
157
|
end
|
|
187
|
-
else
|
|
188
|
-
option = '--file-list'
|
|
189
158
|
end
|
|
190
|
-
@builder.add_command_line_options(["#{
|
|
159
|
+
@builder.add_command_line_options(["#{file_list_option}=#{file_list_file}"]) unless file_list_option.nil?
|
|
191
160
|
end
|
|
192
161
|
|
|
193
162
|
def remote_certificates
|
|
@@ -200,31 +169,32 @@ module Aspera
|
|
|
200
169
|
# This will need to be cleaned up in aspera core
|
|
201
170
|
@job_spec['ssh_port'] = @builder.read_param('wss_port')
|
|
202
171
|
@job_spec.delete('fasp_port')
|
|
203
|
-
@job_spec.delete('EX_ssh_key_paths')
|
|
204
172
|
@job_spec.delete('sshfp')
|
|
173
|
+
# set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
|
174
|
+
certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs].is_a?(Array)
|
|
205
175
|
# ignore cert for wss ?
|
|
206
|
-
if @options[:
|
|
176
|
+
if @options[:check_ignore_cb]&.call(@job_spec['remote_host'], @job_spec['wss_port'])
|
|
207
177
|
wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
|
|
208
178
|
wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}"
|
|
209
|
-
File.write(wss_cert_file, Rest.
|
|
210
|
-
|
|
179
|
+
File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
|
|
180
|
+
# place in front, as more priority
|
|
181
|
+
certificates_to_use.unshift(wss_cert_file)
|
|
211
182
|
end
|
|
212
|
-
#
|
|
213
|
-
certificates_to_use.
|
|
183
|
+
# when wss is used, only first `-i` is used... Hum...
|
|
184
|
+
certificates_to_use = [certificates_to_use.first] unless certificates_to_use.empty?
|
|
214
185
|
else
|
|
215
186
|
# remove unused parameter (avoid warning)
|
|
216
187
|
@job_spec.delete('wss_port')
|
|
217
188
|
# add SSH bypass keys when authentication is token and no auth is provided
|
|
218
189
|
if @job_spec.key?('token') && !@job_spec.key?('remote_password')
|
|
219
|
-
# @job_spec['remote_password'] = Installation.instance.ssh_cert_uuid # not used: no passphrase
|
|
220
|
-
certificates_to_use.concat(Installation.instance.aspera_token_ssh_key_paths)
|
|
190
|
+
# @job_spec['remote_password'] = Ascp::Installation.instance.ssh_cert_uuid # not used: no passphrase
|
|
191
|
+
certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths)
|
|
221
192
|
end
|
|
222
193
|
end
|
|
223
194
|
return certificates_to_use
|
|
224
195
|
end
|
|
225
196
|
|
|
226
197
|
# translate transfer spec to env vars and command line arguments for ascp
|
|
227
|
-
# NOTE: parameters starting with "EX_" (extended) are not standard
|
|
228
198
|
def ascp_args
|
|
229
199
|
env_args = {
|
|
230
200
|
args: [],
|
|
@@ -236,11 +206,11 @@ module Aspera
|
|
|
236
206
|
@job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
|
|
237
207
|
|
|
238
208
|
# notify multi-session was already used, anyway it was deleted by agent direct
|
|
239
|
-
assert(!@builder.read_param('multi_session'))
|
|
209
|
+
Aspera.assert(!@builder.read_param('multi_session'))
|
|
240
210
|
|
|
241
211
|
# add ssh or wss certificates
|
|
242
|
-
|
|
243
|
-
|
|
212
|
+
# (reverse, to keep order, as we unshift)
|
|
213
|
+
remote_certificates.reverse_each do |cert|
|
|
244
214
|
env_args[:args].unshift('-i', cert)
|
|
245
215
|
end
|
|
246
216
|
|
|
@@ -248,7 +218,7 @@ module Aspera
|
|
|
248
218
|
@builder.process_params
|
|
249
219
|
|
|
250
220
|
base64_destination = false
|
|
251
|
-
# symbol must be index of Installation.paths
|
|
221
|
+
# symbol must be index of Ascp::Installation.paths
|
|
252
222
|
if @builder.read_param('use_ascp4')
|
|
253
223
|
env_args[:ascp_version] = :ascp4
|
|
254
224
|
else
|
|
@@ -258,7 +228,6 @@ module Aspera
|
|
|
258
228
|
# destination will be base64 encoded, put this before source path arguments
|
|
259
229
|
@builder.add_command_line_options(['--dest64']) if base64_destination
|
|
260
230
|
# optional arguments, at the end to override previous ones (to allow override)
|
|
261
|
-
@builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
|
|
262
231
|
@builder.add_command_line_options(@options[:ascp_args])
|
|
263
232
|
# get list of source files to transfer and build arg for ascp
|
|
264
233
|
process_file_list
|
|
@@ -272,8 +241,8 @@ module Aspera
|
|
|
272
241
|
env_args[:args].unshift('-q') if @options[:quiet]
|
|
273
242
|
# add fallback cert and key as arguments if needed
|
|
274
243
|
if ['1', 1, true, 'force'].include?(@job_spec['http_fallback'])
|
|
275
|
-
env_args[:args].unshift('-Y', Installation.instance.path(:fallback_private_key))
|
|
276
|
-
env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
|
|
244
|
+
env_args[:args].unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
|
|
245
|
+
env_args[:args].unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
|
|
277
246
|
end
|
|
278
247
|
Log.log.debug{"ascp args: #{env_args}"}
|
|
279
248
|
return env_args
|
|
@@ -1,12 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/
|
|
3
|
+
require 'aspera/command_line_builder'
|
|
4
4
|
require 'aspera/assert'
|
|
5
|
+
require 'yaml'
|
|
5
6
|
|
|
6
7
|
module Aspera
|
|
7
|
-
module
|
|
8
|
+
module Transfer
|
|
8
9
|
# parameters for Transfer Spec
|
|
9
|
-
class
|
|
10
|
+
class Spec
|
|
10
11
|
# default transfer username for access key based transfers
|
|
11
12
|
ACCESS_KEY_TRANSFER_USER = 'xfer'
|
|
12
13
|
SSH_PORT = 33_001
|
|
@@ -18,35 +19,36 @@ module Aspera
|
|
|
18
19
|
}.freeze
|
|
19
20
|
# reserved tag for Aspera
|
|
20
21
|
TAG_RESERVED = 'aspera'
|
|
21
|
-
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
|
22
|
-
Aspera::Fasp::Parameters.description.each do |name, description|
|
|
23
|
-
next unless description[:enum].is_a?(Array)
|
|
24
|
-
TransferSpec.const_set("#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
|
|
25
|
-
description[:enum].each do |enum|
|
|
26
|
-
TransferSpec.const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
22
|
class << self
|
|
30
23
|
def action_to_direction(tspec, command)
|
|
31
|
-
assert_type(tspec, Hash){'transfer spec'}
|
|
24
|
+
Aspera.assert_type(tspec, Hash){'transfer spec'}
|
|
32
25
|
tspec['direction'] = case command.to_sym
|
|
33
26
|
when :upload then DIRECTION_SEND
|
|
34
27
|
when :download then DIRECTION_RECEIVE
|
|
35
|
-
else error_unexpected_value(command.to_sym)
|
|
28
|
+
else Aspera.error_unexpected_value(command.to_sym)
|
|
36
29
|
end
|
|
37
30
|
return tspec
|
|
38
31
|
end
|
|
39
32
|
|
|
40
33
|
def action(tspec)
|
|
41
|
-
assert_type(tspec, Hash){'transfer spec'}
|
|
42
|
-
assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
|
|
34
|
+
Aspera.assert_type(tspec, Hash){'transfer spec'}
|
|
35
|
+
Aspera.assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
|
|
43
36
|
case tspec['direction']
|
|
44
37
|
when DIRECTION_SEND then :upload
|
|
45
38
|
when DIRECTION_RECEIVE then :download
|
|
46
|
-
else error_unexpected_value(tspec['direction'])
|
|
39
|
+
else Aspera.error_unexpected_value(tspec['direction'])
|
|
47
40
|
end
|
|
48
41
|
end
|
|
49
42
|
end
|
|
43
|
+
DESCRIPTION = CommandLineBuilder.normalize_description(YAML.load_file("#{__FILE__[0..-3]}yaml"))
|
|
44
|
+
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
|
45
|
+
DESCRIPTION.each do |name, description|
|
|
46
|
+
next unless description[:enum].is_a?(Array)
|
|
47
|
+
const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
|
|
48
|
+
description[:enum].each do |enum|
|
|
49
|
+
const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
|
50
|
+
end
|
|
51
|
+
end
|
|
50
52
|
end
|
|
51
53
|
end
|
|
52
54
|
end
|
|
@@ -27,7 +27,7 @@ cipher:
|
|
|
27
27
|
:cli:
|
|
28
28
|
:type: :opt_with_arg
|
|
29
29
|
:switch: "-c"
|
|
30
|
-
:convert: Aspera::
|
|
30
|
+
:convert: Aspera::Transfer::Parameters.convert_remove_hyphen
|
|
31
31
|
content_protection:
|
|
32
32
|
:desc: Enable client-side encryption at rest. (CSEAR, content protection)
|
|
33
33
|
:enum:
|
|
@@ -424,7 +424,7 @@ tags:
|
|
|
424
424
|
:cli:
|
|
425
425
|
:type: :opt_with_arg
|
|
426
426
|
:switch: "--tags64"
|
|
427
|
-
:convert: Aspera::
|
|
427
|
+
:convert: Aspera::Transfer::Parameters.convert_json64
|
|
428
428
|
target_rate_cap_kbps:
|
|
429
429
|
:desc: Returned by upload/download_setup node API.
|
|
430
430
|
:accepted_types: :int
|
|
@@ -479,7 +479,7 @@ source_root:
|
|
|
479
479
|
:cli:
|
|
480
480
|
:type: :opt_with_arg
|
|
481
481
|
:switch: "--source-prefix64"
|
|
482
|
-
:convert: Aspera::
|
|
482
|
+
:convert: Aspera::Transfer::Parameters.convert_base64
|
|
483
483
|
min_rate_cap_kbps:
|
|
484
484
|
:desc: |-
|
|
485
485
|
The highest minimum rate that an incoming transfer can request, in kilobits per second.
|
|
@@ -540,7 +540,7 @@ src_base:
|
|
|
540
540
|
:cli:
|
|
541
541
|
:type: :opt_with_arg
|
|
542
542
|
:switch: "--src-base64"
|
|
543
|
-
:convert: Aspera::
|
|
543
|
+
:convert: Aspera::Transfer::Parameters.convert_base64
|
|
544
544
|
preserve_acls:
|
|
545
545
|
:desc: "Preserve access control lists."
|
|
546
546
|
:enum:
|
|
@@ -610,34 +610,6 @@ remove_empty_source_directory:
|
|
|
610
610
|
- :sdk
|
|
611
611
|
:cli:
|
|
612
612
|
:type: :opt_without_arg
|
|
613
|
-
EX_at_rest_password:
|
|
614
|
-
:desc: Content protection password
|
|
615
|
-
:deprecation: "(4.13) Use standard spec parameter: content_protection_password"
|
|
616
|
-
:agents:
|
|
617
|
-
- :direct
|
|
618
|
-
:cli:
|
|
619
|
-
:type: :envvar
|
|
620
|
-
:variable: ASPERA_SCP_FILEPASS
|
|
621
|
-
EX_proxy_password:
|
|
622
|
-
:desc: |-
|
|
623
|
-
Password used for Aspera proxy server authentication.
|
|
624
|
-
May be overridden by password in URL provided in parameter: proxy.
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
:deprecation: (4.14) Use env var ASPERA_PROXY_PASS
|
|
628
|
-
:agents:
|
|
629
|
-
- :direct
|
|
630
|
-
:cli:
|
|
631
|
-
:type: :envvar
|
|
632
|
-
:variable: ASPERA_PROXY_PASS
|
|
633
|
-
EX_license_text:
|
|
634
|
-
:desc: "License file text override.\nBy default ascp looks for license file near executable."
|
|
635
|
-
:deprecation: (4.14) Use env var ASPERA_SCP_LICENSE
|
|
636
|
-
:agents:
|
|
637
|
-
- :direct
|
|
638
|
-
:cli:
|
|
639
|
-
:type: :envvar
|
|
640
|
-
:variable: ASPERA_SCP_LICENSE
|
|
641
613
|
keepalive:
|
|
642
614
|
:desc: The session is running in persistent session mode.
|
|
643
615
|
:agents:
|
|
@@ -663,49 +635,6 @@ sshfp:
|
|
|
663
635
|
:cli:
|
|
664
636
|
:type: :opt_with_arg
|
|
665
637
|
:switch: "--check-sshfp"
|
|
666
|
-
EX_http_proxy_url:
|
|
667
|
-
:desc: Specify the proxy server address used by HTTP Fallback
|
|
668
|
-
:deprecation: (4.14) TODO, use proxy option ?
|
|
669
|
-
:agents:
|
|
670
|
-
- :direct
|
|
671
|
-
:cli:
|
|
672
|
-
:type: :opt_with_arg
|
|
673
|
-
:switch: "-x"
|
|
674
|
-
EX_ssh_key_paths:
|
|
675
|
-
:desc: Use public key authentication for SSH and specify the private key file paths
|
|
676
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
|
677
|
-
:accepted_types: :array
|
|
678
|
-
:agents:
|
|
679
|
-
- :direct
|
|
680
|
-
:cli:
|
|
681
|
-
:type: :opt_with_arg
|
|
682
|
-
:switch: "-i"
|
|
683
|
-
EX_http_transfer_jpeg:
|
|
684
|
-
:desc: HTTP transfers as JPEG file
|
|
685
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
|
686
|
-
:accepted_types: :int
|
|
687
|
-
:default: '0'
|
|
688
|
-
:agents:
|
|
689
|
-
- :direct
|
|
690
|
-
:cli:
|
|
691
|
-
:type: :opt_with_arg
|
|
692
|
-
:switch: "-j"
|
|
693
|
-
EX_no_read:
|
|
694
|
-
:desc: no read source
|
|
695
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
|
696
|
-
:agents:
|
|
697
|
-
- :direct
|
|
698
|
-
:cli:
|
|
699
|
-
:type: :opt_without_arg
|
|
700
|
-
:switch: "--no-read"
|
|
701
|
-
EX_no_write:
|
|
702
|
-
:desc: no write on destination
|
|
703
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
|
704
|
-
:agents:
|
|
705
|
-
- :direct
|
|
706
|
-
:cli:
|
|
707
|
-
:type: :opt_without_arg
|
|
708
|
-
:switch: "--no-write"
|
|
709
638
|
target_rate_percentage:
|
|
710
639
|
:desc: "TODO: remove ?"
|
|
711
640
|
:cli:
|
|
@@ -754,30 +683,6 @@ obfuscate_file_names:
|
|
|
754
683
|
- :httpgw
|
|
755
684
|
:cli:
|
|
756
685
|
:type: :ignore
|
|
757
|
-
EX_file_list:
|
|
758
|
-
:desc: source file list
|
|
759
|
-
:deprecation: (4.14) Use command line file list, or option transfer_info.ascp_args
|
|
760
|
-
:agents:
|
|
761
|
-
- :direct
|
|
762
|
-
:cli:
|
|
763
|
-
:type: :special
|
|
764
|
-
:switch: "--file-list"
|
|
765
|
-
EX_file_pair_list:
|
|
766
|
-
:desc: source file pair list
|
|
767
|
-
:deprecation: (4.14) Use command line file pair list, or option transfer_info.ascp_args
|
|
768
|
-
:agents:
|
|
769
|
-
- :direct
|
|
770
|
-
:cli:
|
|
771
|
-
:type: :special
|
|
772
|
-
:switch: "--file-pair-list"
|
|
773
|
-
EX_ascp_args:
|
|
774
|
-
:desc: Add native command line arguments to ascp
|
|
775
|
-
:deprecation: (4.13) Use option transfer_info.ascp_args
|
|
776
|
-
:accepted_types: :array
|
|
777
|
-
:agents:
|
|
778
|
-
- :direct
|
|
779
|
-
:cli:
|
|
780
|
-
:type: :special
|
|
781
686
|
wss_enabled:
|
|
782
687
|
:desc: Server has Web Socket service enabled
|
|
783
688
|
:accepted_types: :bool
|