aspera-cli 4.16.0 → 4.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/CHANGELOG.md +50 -19
- data/CONTRIBUTING.md +3 -1
- data/README.md +965 -793
- data/bin/asession +29 -21
- data/lib/aspera/{fasp/agent_alpha.rb → agent/alpha.rb} +26 -25
- data/lib/aspera/{fasp/agent_base.rb → agent/base.rb} +15 -12
- data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
- data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +49 -53
- data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +20 -19
- data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +20 -33
- data/lib/aspera/{fasp/agent_trsdk.rb → agent/trsdk.rb} +11 -11
- data/lib/aspera/api/aoc.rb +586 -0
- data/lib/aspera/api/ats.rb +46 -0
- data/lib/aspera/api/cos_node.rb +95 -0
- data/lib/aspera/api/node.rb +344 -0
- data/lib/aspera/ascmd.rb +46 -10
- data/lib/aspera/{fasp → ascp}/installation.rb +5 -5
- data/lib/aspera/{fasp → ascp}/management.rb +3 -8
- data/lib/aspera/{fasp → ascp}/products.rb +1 -1
- data/lib/aspera/assert.rb +30 -30
- data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
- data/lib/aspera/cli/extended_value.rb +1 -1
- data/lib/aspera/cli/formatter.rb +13 -13
- data/lib/aspera/cli/hints.rb +5 -5
- data/lib/aspera/cli/main.rb +35 -28
- data/lib/aspera/cli/manager.rb +25 -24
- data/lib/aspera/cli/plugin.rb +22 -15
- data/lib/aspera/cli/plugin_factory.rb +61 -0
- data/lib/aspera/cli/plugins/alee.rb +7 -7
- data/lib/aspera/cli/plugins/aoc.rb +83 -77
- data/lib/aspera/cli/plugins/ats.rb +32 -33
- data/lib/aspera/cli/plugins/bss.rb +3 -4
- data/lib/aspera/cli/plugins/config.rb +169 -186
- data/lib/aspera/cli/plugins/console.rb +8 -6
- data/lib/aspera/cli/plugins/cos.rb +19 -18
- data/lib/aspera/cli/plugins/faspex.rb +61 -54
- data/lib/aspera/cli/plugins/faspex5.rb +150 -103
- data/lib/aspera/cli/plugins/node.rb +68 -73
- data/lib/aspera/cli/plugins/orchestrator.rb +34 -44
- data/lib/aspera/cli/plugins/preview.rb +31 -31
- data/lib/aspera/cli/plugins/server.rb +31 -33
- data/lib/aspera/cli/plugins/shares.rb +13 -11
- data/lib/aspera/cli/sync_actions.rb +8 -8
- data/lib/aspera/cli/transfer_agent.rb +32 -19
- data/lib/aspera/cli/transfer_progress.rb +1 -1
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -0
- data/lib/aspera/command_line_builder.rb +14 -14
- data/lib/aspera/coverage.rb +1 -2
- data/lib/aspera/data_repository.rb +1 -1
- data/lib/aspera/environment.rb +2 -3
- data/lib/aspera/faspex_gw.rb +5 -6
- data/lib/aspera/faspex_postproc.rb +1 -1
- data/lib/aspera/id_generator.rb +2 -2
- data/lib/aspera/json_rpc.rb +5 -5
- data/lib/aspera/keychain/encrypted_hash.rb +6 -6
- data/lib/aspera/keychain/macos_security.rb +27 -22
- data/lib/aspera/log.rb +2 -2
- data/lib/aspera/nagios.rb +3 -3
- data/lib/aspera/node_simulator.rb +5 -6
- data/lib/aspera/oauth/base.rb +143 -0
- data/lib/aspera/oauth/factory.rb +124 -0
- data/lib/aspera/oauth/generic.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +51 -0
- data/lib/aspera/oauth/url_json.rb +31 -0
- data/lib/aspera/oauth/web.rb +50 -0
- data/lib/aspera/oauth.rb +5 -331
- data/lib/aspera/open_application.rb +7 -7
- data/lib/aspera/persistency_action_once.rb +4 -4
- data/lib/aspera/persistency_folder.rb +2 -2
- data/lib/aspera/preview/generator.rb +5 -5
- data/lib/aspera/preview/terminal.rb +3 -2
- data/lib/aspera/preview/utils.rb +3 -3
- data/lib/aspera/proxy_auto_config.rb +4 -4
- data/lib/aspera/rest.rb +175 -144
- data/lib/aspera/rest_errors_aspera.rb +3 -3
- data/lib/aspera/resumer.rb +77 -0
- data/lib/aspera/ssh.rb +6 -1
- data/lib/aspera/{fasp → transfer}/error.rb +3 -3
- data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
- data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
- data/lib/aspera/{fasp → transfer}/parameters.rb +58 -89
- data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +18 -16
- data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
- data/lib/aspera/{fasp → transfer}/sync.rb +32 -32
- data/lib/aspera/{fasp → transfer}/uri.rb +9 -8
- data/lib/aspera/web_server_simple.rb +11 -3
- data.tar.gz.sig +0 -0
- metadata +36 -63
- metadata.gz.sig +0 -0
- data/lib/aspera/aoc.rb +0 -601
- data/lib/aspera/ats_api.rb +0 -47
- data/lib/aspera/cos_node.rb +0 -94
- data/lib/aspera/fasp/resume_policy.rb +0 -79
- data/lib/aspera/node.rb +0 -339
@@ -0,0 +1,77 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'singleton'
|
4
|
+
require 'aspera/log'
|
5
|
+
require 'aspera/assert'
|
6
|
+
|
7
|
+
module Aspera
|
8
|
+
# implements a simple resume policy
|
9
|
+
class Resumer
|
10
|
+
# list of supported parameters and default values
|
11
|
+
DEFAULTS = {
|
12
|
+
iter_max: 7,
|
13
|
+
sleep_initial: 2,
|
14
|
+
sleep_factor: 2,
|
15
|
+
sleep_max: 60
|
16
|
+
}.freeze
|
17
|
+
|
18
|
+
# @param params see DEFAULTS
|
19
|
+
def initialize(params=nil)
|
20
|
+
@parameters = DEFAULTS.dup
|
21
|
+
if !params.nil?
|
22
|
+
Aspera.assert_type(params, Hash)
|
23
|
+
params.each do |k, v|
|
24
|
+
Aspera.assert_values(k, DEFAULTS.keys){'resume parameter'}
|
25
|
+
Aspera.assert_type(v, Integer){k}
|
26
|
+
@parameters[k] = v
|
27
|
+
end
|
28
|
+
end
|
29
|
+
Log.log.debug{"resume params=#{@parameters}"}
|
30
|
+
end
|
31
|
+
|
32
|
+
# calls block a number of times (resumes) until success or limit reached
|
33
|
+
# this is re-entrant, one resumer can handle multiple transfers in //
|
34
|
+
def execute_with_resume
|
35
|
+
Aspera.assert(block_given?)
|
36
|
+
# maximum of retry
|
37
|
+
remaining_resumes = @parameters[:iter_max]
|
38
|
+
sleep_seconds = @parameters[:sleep_initial]
|
39
|
+
Log.log.debug{"retries=#{remaining_resumes}"}
|
40
|
+
# try to send the file until ascp is successful
|
41
|
+
loop do
|
42
|
+
Log.log.debug('transfer starting')
|
43
|
+
begin
|
44
|
+
# call provided block
|
45
|
+
yield
|
46
|
+
# exit retry loop if success
|
47
|
+
break
|
48
|
+
rescue Transfer::Error => e
|
49
|
+
Log.log.warn{"An error occurred during transfer: #{e.message}"}
|
50
|
+
# failure in ascp
|
51
|
+
if e.retryable?
|
52
|
+
# exit if we exceed the max number of retry
|
53
|
+
raise Transfer::Error, "Maximum number of retry reached (#{@parameters[:iter_max]})" if remaining_resumes <= 0
|
54
|
+
else
|
55
|
+
# give one chance only to non retryable errors
|
56
|
+
unless remaining_resumes.eql?(@parameters[:iter_max])
|
57
|
+
Log.log.error('non-retryable error'.red.blink)
|
58
|
+
raise e
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
# take this retry in account
|
64
|
+
remaining_resumes -= 1
|
65
|
+
Log.log.warn{"Resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})"}
|
66
|
+
|
67
|
+
# wait a bit before retrying, maybe network condition will be better
|
68
|
+
sleep(sleep_seconds)
|
69
|
+
|
70
|
+
# increase retry period
|
71
|
+
sleep_seconds *= @parameters[:sleep_factor]
|
72
|
+
# cap value
|
73
|
+
sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
|
74
|
+
end # loop
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
data/lib/aspera/ssh.rb
CHANGED
@@ -3,7 +3,7 @@
|
|
3
3
|
require 'net/ssh'
|
4
4
|
|
5
5
|
if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
|
6
|
-
# HACK: deactivate ed25519 and ecdsa private keys from
|
6
|
+
# HACK: deactivate ed25519 and ecdsa private keys from SSH identities, as it usually causes problems
|
7
7
|
old_verbose = $VERBOSE
|
8
8
|
$VERBOSE = nil
|
9
9
|
begin
|
@@ -14,6 +14,11 @@ if ENV.fetch('ASCLI_ENABLE_ED25519', 'false').eql?('false')
|
|
14
14
|
$VERBOSE = old_verbose
|
15
15
|
end
|
16
16
|
|
17
|
+
if RUBY_ENGINE == 'jruby' && ENV.fetch('ASCLI_ENABLE_ECDSHA2', 'false').eql?('false')
|
18
|
+
Net::SSH::Transport::Algorithms::ALGORITHMS.each_value { |a| a.reject! { |a| a =~ /^ecd(sa|h)-sha2/ } }
|
19
|
+
Net::SSH::KnownHosts::SUPPORTED_TYPE.reject! { |t| t =~ /^ecd(sa|h)-sha2/ }
|
20
|
+
end
|
21
|
+
|
17
22
|
module Aspera
|
18
23
|
# A simple wrapper around Net::SSH
|
19
24
|
# executes one command and get its result from stdout
|
@@ -1,9 +1,9 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/
|
3
|
+
require 'aspera/transfer/error_info'
|
4
4
|
|
5
5
|
module Aspera
|
6
|
-
module
|
6
|
+
module Transfer
|
7
7
|
# error raised if transfer fails
|
8
8
|
class Error < StandardError
|
9
9
|
attr_reader :err_code
|
@@ -14,7 +14,7 @@ module Aspera
|
|
14
14
|
end
|
15
15
|
|
16
16
|
def info
|
17
|
-
r =
|
17
|
+
r = ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
|
18
18
|
return r.merge({i: @err_code})
|
19
19
|
end
|
20
20
|
|
@@ -3,7 +3,7 @@
|
|
3
3
|
# cspell:words mnemo PROTO RCVR NOLIC PMTU BRTT VLINK BWMEAS SSEAR FIPS
|
4
4
|
|
5
5
|
module Aspera
|
6
|
-
module
|
6
|
+
module Transfer
|
7
7
|
# from https://www.google.com/search?q=FASP+error+codes
|
8
8
|
# Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
|
9
9
|
# rubocop:disable Layout/MultilineHashKeyLineBreaks
|
@@ -4,19 +4,19 @@ require 'aspera/log'
|
|
4
4
|
require 'aspera/assert'
|
5
5
|
require 'aspera/command_line_builder'
|
6
6
|
require 'aspera/temp_file_manager'
|
7
|
-
require 'aspera/
|
8
|
-
require 'aspera/
|
7
|
+
require 'aspera/transfer/error'
|
8
|
+
require 'aspera/transfer/spec'
|
9
|
+
require 'aspera/ascp/installation'
|
9
10
|
require 'aspera/cli/formatter'
|
10
11
|
require 'aspera/rest'
|
11
12
|
require 'securerandom'
|
12
13
|
require 'base64'
|
13
14
|
require 'json'
|
14
|
-
require 'yaml'
|
15
15
|
require 'fileutils'
|
16
16
|
require 'openssl'
|
17
17
|
|
18
18
|
module Aspera
|
19
|
-
module
|
19
|
+
module Transfer
|
20
20
|
# translate transfer specification to ascp parameter list
|
21
21
|
class Parameters
|
22
22
|
# Agents shown in manual for parameters (sub list)
|
@@ -25,7 +25,7 @@ module Aspera
|
|
25
25
|
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
26
26
|
FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
|
27
27
|
# options that can be provided to the constructor, and then in @options
|
28
|
-
SUPPORTED_OPTIONS = %i[ascp_args wss
|
28
|
+
SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore_cb quiet trusted_certs].freeze
|
29
29
|
|
30
30
|
private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS, :SUPPORTED_OPTIONS
|
31
31
|
|
@@ -34,22 +34,12 @@ module Aspera
|
|
34
34
|
# because of garbage collection takes any file there
|
35
35
|
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
36
36
|
@file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists') # cspell:disable-line
|
37
|
-
@param_description_cache = nil
|
38
|
-
# @return normalized description of transfer spec parameters, direct from yaml
|
39
|
-
def description
|
40
|
-
if @param_description_cache.nil?
|
41
|
-
# config file in same folder with same name as this source
|
42
|
-
description_from_yaml = YAML.load_file("#{__FILE__[0..-3]}yaml")
|
43
|
-
@param_description_cache = Aspera::CommandLineBuilder.normalize_description(description_from_yaml)
|
44
|
-
end
|
45
|
-
return @param_description_cache
|
46
|
-
end
|
47
37
|
|
48
38
|
# @param to_text [bool] replace HTML entities with text equivalent
|
49
39
|
# @return a table suitable to display in manual
|
50
40
|
def man_table
|
51
41
|
result = []
|
52
|
-
|
42
|
+
Spec::DESCRIPTION.each do |name, options|
|
53
43
|
param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
|
54
44
|
# add flags for supported agents in doc
|
55
45
|
SUPPORTED_AGENTS.each do |a|
|
@@ -85,13 +75,7 @@ module Aspera
|
|
85
75
|
param[:description] = param[:description].gsub('/', '\\')
|
86
76
|
result.push(param)
|
87
77
|
end
|
88
|
-
return result.
|
89
|
-
if a[:name].start_with?('EX_').eql?(b[:name].start_with?('EX_'))
|
90
|
-
a[:name] <=> b[:name]
|
91
|
-
else
|
92
|
-
b[:name] <=> a[:name]
|
93
|
-
end
|
94
|
-
end
|
78
|
+
return result.sort_by { |a| a[:name] }
|
95
79
|
end
|
96
80
|
|
97
81
|
# special encoding methods used in YAML (key: :convert)
|
@@ -104,13 +88,9 @@ module Aspera
|
|
104
88
|
def convert_base64(v); Base64.strict_encode64(v); end
|
105
89
|
|
106
90
|
# file list is provided directly with ascp arguments
|
107
|
-
|
108
|
-
|
109
|
-
ascp_args
|
110
|
-
(ts['EX_ascp_args'].is_a?(Array) && ts['EX_ascp_args'].any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
|
111
|
-
(ascp_args.is_a?(Array) && ascp_args.any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
|
112
|
-
ts.key?('EX_file_list') ||
|
113
|
-
ts.key?('EX_file_pair_list')
|
91
|
+
# @param ascp_args [Array,NilClass] ascp arguments
|
92
|
+
def ascp_args_file_list?(ascp_args)
|
93
|
+
ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
|
114
94
|
end
|
115
95
|
|
116
96
|
# temp file list files are created here
|
@@ -127,67 +107,56 @@ module Aspera
|
|
127
107
|
|
128
108
|
# @param options [Hash] key: :wss: bool, :ascp_args: array of strings
|
129
109
|
def initialize(job_spec, options)
|
130
|
-
assert_type(job_spec, Hash)
|
131
|
-
assert_type(options, Hash)
|
110
|
+
Aspera.assert_type(job_spec, Hash)
|
111
|
+
Aspera.assert_type(options, Hash)
|
132
112
|
@job_spec = job_spec
|
133
113
|
# check necessary options
|
134
114
|
missing_options = SUPPORTED_OPTIONS - options.keys
|
135
|
-
assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
|
115
|
+
Aspera.assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
|
136
116
|
@options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
|
137
117
|
Log.log.debug{Log.dump(:parameters_options, @options)}
|
138
118
|
Log.log.debug{Log.dump(:dismiss_options, options.keys - SUPPORTED_OPTIONS)}
|
139
|
-
assert_type(@options[:ascp_args], Array){'ascp_args'}
|
140
|
-
assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
|
141
|
-
@builder =
|
119
|
+
Aspera.assert_type(@options[:ascp_args], Array){'ascp_args'}
|
120
|
+
Aspera.assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
|
121
|
+
@builder = CommandLineBuilder.new(@job_spec, Spec::DESCRIPTION)
|
142
122
|
end
|
143
123
|
|
144
124
|
# either place source files on command line, or add file list file
|
145
125
|
def process_file_list
|
146
|
-
# is the file list provided through
|
147
|
-
ascp_file_list_provided = self.class.
|
126
|
+
# is the file list provided through ascp parameters?
|
127
|
+
ascp_file_list_provided = self.class.ascp_args_file_list?(@options[:ascp_args])
|
148
128
|
# set if paths is mandatory in ts
|
149
129
|
@builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided # cspell:words keepalive
|
150
130
|
# get paths in transfer spec (after setting if it is mandatory)
|
151
131
|
ts_paths_array = @builder.read_param('paths')
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
165
|
-
|
166
|
-
|
167
|
-
|
132
|
+
file_list_option = nil
|
133
|
+
# transfer spec contains paths ?
|
134
|
+
if !ts_paths_array.nil?
|
135
|
+
Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
|
136
|
+
Aspera.assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
|
137
|
+
is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
|
138
|
+
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
|
139
|
+
if self.class.file_list_folder.nil?
|
140
|
+
Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
|
141
|
+
# not safe for special characters ? (maybe not, depends on OS)
|
142
|
+
Log.log.debug('placing source file list on command line (no file list file)')
|
143
|
+
@builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
|
144
|
+
else
|
145
|
+
# safer option: generate a file list file if there is storage defined for it
|
146
|
+
if is_pair_list
|
147
|
+
file_list_option = '--file-pair-list'
|
148
|
+
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']) }
|
168
149
|
else
|
169
|
-
|
170
|
-
|
171
|
-
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
|
172
|
-
# safer option: generate a file list file if there is storage defined for it
|
173
|
-
# if there is one destination in paths, then use file-pair-list
|
174
|
-
if is_pair_list
|
175
|
-
option = '--file-pair-list'
|
176
|
-
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']) }
|
177
|
-
else
|
178
|
-
option = '--file-list'
|
179
|
-
lines = ts_paths_array.map{|i|i['source']}
|
180
|
-
end
|
181
|
-
file_list_file = Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
182
|
-
Log.log.debug{Log.dump(:file_list, lines)}
|
183
|
-
File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
|
184
|
-
Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
|
150
|
+
file_list_option = '--file-list'
|
151
|
+
lines = ts_paths_array.map{|i|i['source']}
|
185
152
|
end
|
153
|
+
file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
154
|
+
Log.log.debug{Log.dump(:file_list, lines)}
|
155
|
+
File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
|
156
|
+
Log.log.debug{"#{file_list_option}=\n#{File.read(file_list_file)}".red}
|
186
157
|
end
|
187
|
-
else
|
188
|
-
option = '--file-list'
|
189
158
|
end
|
190
|
-
@builder.add_command_line_options(["#{
|
159
|
+
@builder.add_command_line_options(["#{file_list_option}=#{file_list_file}"]) unless file_list_option.nil?
|
191
160
|
end
|
192
161
|
|
193
162
|
def remote_certificates
|
@@ -200,31 +169,32 @@ module Aspera
|
|
200
169
|
# This will need to be cleaned up in aspera core
|
201
170
|
@job_spec['ssh_port'] = @builder.read_param('wss_port')
|
202
171
|
@job_spec.delete('fasp_port')
|
203
|
-
@job_spec.delete('EX_ssh_key_paths')
|
204
172
|
@job_spec.delete('sshfp')
|
173
|
+
# set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
174
|
+
certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs].is_a?(Array)
|
205
175
|
# ignore cert for wss ?
|
206
|
-
if @options[:
|
176
|
+
if @options[:check_ignore_cb]&.call(@job_spec['remote_host'], @job_spec['wss_port'])
|
207
177
|
wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
|
208
178
|
wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}"
|
209
|
-
File.write(wss_cert_file, Rest.
|
210
|
-
|
179
|
+
File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
|
180
|
+
# place in front, as more priority
|
181
|
+
certificates_to_use.unshift(wss_cert_file)
|
211
182
|
end
|
212
|
-
#
|
213
|
-
certificates_to_use.
|
183
|
+
# when wss is used, only first `-i` is used... Hum...
|
184
|
+
certificates_to_use = [certificates_to_use.first] unless certificates_to_use.empty?
|
214
185
|
else
|
215
186
|
# remove unused parameter (avoid warning)
|
216
187
|
@job_spec.delete('wss_port')
|
217
188
|
# add SSH bypass keys when authentication is token and no auth is provided
|
218
189
|
if @job_spec.key?('token') && !@job_spec.key?('remote_password')
|
219
|
-
# @job_spec['remote_password'] = Installation.instance.ssh_cert_uuid # not used: no passphrase
|
220
|
-
certificates_to_use.concat(Installation.instance.aspera_token_ssh_key_paths)
|
190
|
+
# @job_spec['remote_password'] = Ascp::Installation.instance.ssh_cert_uuid # not used: no passphrase
|
191
|
+
certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths)
|
221
192
|
end
|
222
193
|
end
|
223
194
|
return certificates_to_use
|
224
195
|
end
|
225
196
|
|
226
197
|
# translate transfer spec to env vars and command line arguments for ascp
|
227
|
-
# NOTE: parameters starting with "EX_" (extended) are not standard
|
228
198
|
def ascp_args
|
229
199
|
env_args = {
|
230
200
|
args: [],
|
@@ -236,11 +206,11 @@ module Aspera
|
|
236
206
|
@job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
|
237
207
|
|
238
208
|
# notify multi-session was already used, anyway it was deleted by agent direct
|
239
|
-
assert(!@builder.read_param('multi_session'))
|
209
|
+
Aspera.assert(!@builder.read_param('multi_session'))
|
240
210
|
|
241
211
|
# add ssh or wss certificates
|
242
|
-
|
243
|
-
|
212
|
+
# (reverse, to keep order, as we unshift)
|
213
|
+
remote_certificates.reverse_each do |cert|
|
244
214
|
env_args[:args].unshift('-i', cert)
|
245
215
|
end
|
246
216
|
|
@@ -248,7 +218,7 @@ module Aspera
|
|
248
218
|
@builder.process_params
|
249
219
|
|
250
220
|
base64_destination = false
|
251
|
-
# symbol must be index of Installation.paths
|
221
|
+
# symbol must be index of Ascp::Installation.paths
|
252
222
|
if @builder.read_param('use_ascp4')
|
253
223
|
env_args[:ascp_version] = :ascp4
|
254
224
|
else
|
@@ -258,7 +228,6 @@ module Aspera
|
|
258
228
|
# destination will be base64 encoded, put this before source path arguments
|
259
229
|
@builder.add_command_line_options(['--dest64']) if base64_destination
|
260
230
|
# optional arguments, at the end to override previous ones (to allow override)
|
261
|
-
@builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
|
262
231
|
@builder.add_command_line_options(@options[:ascp_args])
|
263
232
|
# get list of source files to transfer and build arg for ascp
|
264
233
|
process_file_list
|
@@ -272,8 +241,8 @@ module Aspera
|
|
272
241
|
env_args[:args].unshift('-q') if @options[:quiet]
|
273
242
|
# add fallback cert and key as arguments if needed
|
274
243
|
if ['1', 1, true, 'force'].include?(@job_spec['http_fallback'])
|
275
|
-
env_args[:args].unshift('-Y', Installation.instance.path(:fallback_private_key))
|
276
|
-
env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
|
244
|
+
env_args[:args].unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
|
245
|
+
env_args[:args].unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
|
277
246
|
end
|
278
247
|
Log.log.debug{"ascp args: #{env_args}"}
|
279
248
|
return env_args
|
@@ -1,12 +1,13 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require 'aspera/
|
3
|
+
require 'aspera/command_line_builder'
|
4
4
|
require 'aspera/assert'
|
5
|
+
require 'yaml'
|
5
6
|
|
6
7
|
module Aspera
|
7
|
-
module
|
8
|
+
module Transfer
|
8
9
|
# parameters for Transfer Spec
|
9
|
-
class
|
10
|
+
class Spec
|
10
11
|
# default transfer username for access key based transfers
|
11
12
|
ACCESS_KEY_TRANSFER_USER = 'xfer'
|
12
13
|
SSH_PORT = 33_001
|
@@ -18,35 +19,36 @@ module Aspera
|
|
18
19
|
}.freeze
|
19
20
|
# reserved tag for Aspera
|
20
21
|
TAG_RESERVED = 'aspera'
|
21
|
-
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
22
|
-
Aspera::Fasp::Parameters.description.each do |name, description|
|
23
|
-
next unless description[:enum].is_a?(Array)
|
24
|
-
TransferSpec.const_set("#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
|
25
|
-
description[:enum].each do |enum|
|
26
|
-
TransferSpec.const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
27
|
-
end
|
28
|
-
end
|
29
22
|
class << self
|
30
23
|
def action_to_direction(tspec, command)
|
31
|
-
assert_type(tspec, Hash){'transfer spec'}
|
24
|
+
Aspera.assert_type(tspec, Hash){'transfer spec'}
|
32
25
|
tspec['direction'] = case command.to_sym
|
33
26
|
when :upload then DIRECTION_SEND
|
34
27
|
when :download then DIRECTION_RECEIVE
|
35
|
-
else error_unexpected_value(command.to_sym)
|
28
|
+
else Aspera.error_unexpected_value(command.to_sym)
|
36
29
|
end
|
37
30
|
return tspec
|
38
31
|
end
|
39
32
|
|
40
33
|
def action(tspec)
|
41
|
-
assert_type(tspec, Hash){'transfer spec'}
|
42
|
-
assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
|
34
|
+
Aspera.assert_type(tspec, Hash){'transfer spec'}
|
35
|
+
Aspera.assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
|
43
36
|
case tspec['direction']
|
44
37
|
when DIRECTION_SEND then :upload
|
45
38
|
when DIRECTION_RECEIVE then :download
|
46
|
-
else error_unexpected_value(tspec['direction'])
|
39
|
+
else Aspera.error_unexpected_value(tspec['direction'])
|
47
40
|
end
|
48
41
|
end
|
49
42
|
end
|
43
|
+
DESCRIPTION = CommandLineBuilder.normalize_description(YAML.load_file("#{__FILE__[0..-3]}yaml"))
|
44
|
+
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
45
|
+
DESCRIPTION.each do |name, description|
|
46
|
+
next unless description[:enum].is_a?(Array)
|
47
|
+
const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
|
48
|
+
description[:enum].each do |enum|
|
49
|
+
const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
50
|
+
end
|
51
|
+
end
|
50
52
|
end
|
51
53
|
end
|
52
54
|
end
|
@@ -27,7 +27,7 @@ cipher:
|
|
27
27
|
:cli:
|
28
28
|
:type: :opt_with_arg
|
29
29
|
:switch: "-c"
|
30
|
-
:convert: Aspera::
|
30
|
+
:convert: Aspera::Transfer::Parameters.convert_remove_hyphen
|
31
31
|
content_protection:
|
32
32
|
:desc: Enable client-side encryption at rest. (CSEAR, content protection)
|
33
33
|
:enum:
|
@@ -424,7 +424,7 @@ tags:
|
|
424
424
|
:cli:
|
425
425
|
:type: :opt_with_arg
|
426
426
|
:switch: "--tags64"
|
427
|
-
:convert: Aspera::
|
427
|
+
:convert: Aspera::Transfer::Parameters.convert_json64
|
428
428
|
target_rate_cap_kbps:
|
429
429
|
:desc: Returned by upload/download_setup node API.
|
430
430
|
:accepted_types: :int
|
@@ -479,7 +479,7 @@ source_root:
|
|
479
479
|
:cli:
|
480
480
|
:type: :opt_with_arg
|
481
481
|
:switch: "--source-prefix64"
|
482
|
-
:convert: Aspera::
|
482
|
+
:convert: Aspera::Transfer::Parameters.convert_base64
|
483
483
|
min_rate_cap_kbps:
|
484
484
|
:desc: |-
|
485
485
|
The highest minimum rate that an incoming transfer can request, in kilobits per second.
|
@@ -540,7 +540,7 @@ src_base:
|
|
540
540
|
:cli:
|
541
541
|
:type: :opt_with_arg
|
542
542
|
:switch: "--src-base64"
|
543
|
-
:convert: Aspera::
|
543
|
+
:convert: Aspera::Transfer::Parameters.convert_base64
|
544
544
|
preserve_acls:
|
545
545
|
:desc: "Preserve access control lists."
|
546
546
|
:enum:
|
@@ -610,34 +610,6 @@ remove_empty_source_directory:
|
|
610
610
|
- :sdk
|
611
611
|
:cli:
|
612
612
|
:type: :opt_without_arg
|
613
|
-
EX_at_rest_password:
|
614
|
-
:desc: Content protection password
|
615
|
-
:deprecation: "(4.13) Use standard spec parameter: content_protection_password"
|
616
|
-
:agents:
|
617
|
-
- :direct
|
618
|
-
:cli:
|
619
|
-
:type: :envvar
|
620
|
-
:variable: ASPERA_SCP_FILEPASS
|
621
|
-
EX_proxy_password:
|
622
|
-
:desc: |-
|
623
|
-
Password used for Aspera proxy server authentication.
|
624
|
-
May be overridden by password in URL provided in parameter: proxy.
|
625
|
-
|
626
|
-
|
627
|
-
:deprecation: (4.14) Use env var ASPERA_PROXY_PASS
|
628
|
-
:agents:
|
629
|
-
- :direct
|
630
|
-
:cli:
|
631
|
-
:type: :envvar
|
632
|
-
:variable: ASPERA_PROXY_PASS
|
633
|
-
EX_license_text:
|
634
|
-
:desc: "License file text override.\nBy default ascp looks for license file near executable."
|
635
|
-
:deprecation: (4.14) Use env var ASPERA_SCP_LICENSE
|
636
|
-
:agents:
|
637
|
-
- :direct
|
638
|
-
:cli:
|
639
|
-
:type: :envvar
|
640
|
-
:variable: ASPERA_SCP_LICENSE
|
641
613
|
keepalive:
|
642
614
|
:desc: The session is running in persistent session mode.
|
643
615
|
:agents:
|
@@ -663,49 +635,6 @@ sshfp:
|
|
663
635
|
:cli:
|
664
636
|
:type: :opt_with_arg
|
665
637
|
:switch: "--check-sshfp"
|
666
|
-
EX_http_proxy_url:
|
667
|
-
:desc: Specify the proxy server address used by HTTP Fallback
|
668
|
-
:deprecation: (4.14) TODO, use proxy option ?
|
669
|
-
:agents:
|
670
|
-
- :direct
|
671
|
-
:cli:
|
672
|
-
:type: :opt_with_arg
|
673
|
-
:switch: "-x"
|
674
|
-
EX_ssh_key_paths:
|
675
|
-
:desc: Use public key authentication for SSH and specify the private key file paths
|
676
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
677
|
-
:accepted_types: :array
|
678
|
-
:agents:
|
679
|
-
- :direct
|
680
|
-
:cli:
|
681
|
-
:type: :opt_with_arg
|
682
|
-
:switch: "-i"
|
683
|
-
EX_http_transfer_jpeg:
|
684
|
-
:desc: HTTP transfers as JPEG file
|
685
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
686
|
-
:accepted_types: :int
|
687
|
-
:default: '0'
|
688
|
-
:agents:
|
689
|
-
- :direct
|
690
|
-
:cli:
|
691
|
-
:type: :opt_with_arg
|
692
|
-
:switch: "-j"
|
693
|
-
EX_no_read:
|
694
|
-
:desc: no read source
|
695
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
696
|
-
:agents:
|
697
|
-
- :direct
|
698
|
-
:cli:
|
699
|
-
:type: :opt_without_arg
|
700
|
-
:switch: "--no-read"
|
701
|
-
EX_no_write:
|
702
|
-
:desc: no write on destination
|
703
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
704
|
-
:agents:
|
705
|
-
- :direct
|
706
|
-
:cli:
|
707
|
-
:type: :opt_without_arg
|
708
|
-
:switch: "--no-write"
|
709
638
|
target_rate_percentage:
|
710
639
|
:desc: "TODO: remove ?"
|
711
640
|
:cli:
|
@@ -754,30 +683,6 @@ obfuscate_file_names:
|
|
754
683
|
- :httpgw
|
755
684
|
:cli:
|
756
685
|
:type: :ignore
|
757
|
-
EX_file_list:
|
758
|
-
:desc: source file list
|
759
|
-
:deprecation: (4.14) Use command line file list, or option transfer_info.ascp_args
|
760
|
-
:agents:
|
761
|
-
- :direct
|
762
|
-
:cli:
|
763
|
-
:type: :special
|
764
|
-
:switch: "--file-list"
|
765
|
-
EX_file_pair_list:
|
766
|
-
:desc: source file pair list
|
767
|
-
:deprecation: (4.14) Use command line file pair list, or option transfer_info.ascp_args
|
768
|
-
:agents:
|
769
|
-
- :direct
|
770
|
-
:cli:
|
771
|
-
:type: :special
|
772
|
-
:switch: "--file-pair-list"
|
773
|
-
EX_ascp_args:
|
774
|
-
:desc: Add native command line arguments to ascp
|
775
|
-
:deprecation: (4.13) Use option transfer_info.ascp_args
|
776
|
-
:accepted_types: :array
|
777
|
-
:agents:
|
778
|
-
- :direct
|
779
|
-
:cli:
|
780
|
-
:type: :special
|
781
686
|
wss_enabled:
|
782
687
|
:desc: Server has Web Socket service enabled
|
783
688
|
:accepted_types: :bool
|