aspera-cli 4.15.0 → 4.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/BUGS.md +29 -3
- data/CHANGELOG.md +375 -280
- data/CONTRIBUTING.md +71 -18
- data/README.md +1978 -1656
- data/bin/ascli +13 -31
- data/bin/asession +32 -22
- data/examples/dascli +2 -2
- data/lib/aspera/agent/alpha.rb +117 -0
- data/lib/aspera/agent/base.rb +61 -0
- data/lib/aspera/{fasp/agent_connect.rb → agent/connect.rb} +13 -11
- data/lib/aspera/{fasp/agent_direct.rb → agent/direct.rb} +116 -116
- data/lib/aspera/{fasp/agent_httpgw.rb → agent/httpgw.rb} +21 -19
- data/lib/aspera/{fasp/agent_node.rb → agent/node.rb} +21 -33
- data/lib/aspera/agent/trsdk.rb +188 -0
- data/lib/aspera/api/aoc.rb +586 -0
- data/lib/aspera/api/ats.rb +46 -0
- data/lib/aspera/api/cos_node.rb +95 -0
- data/lib/aspera/api/node.rb +344 -0
- data/lib/aspera/ascmd.rb +47 -14
- data/lib/aspera/{fasp → ascp}/installation.rb +54 -15
- data/lib/aspera/{fasp → ascp}/management.rb +14 -14
- data/lib/aspera/{fasp → ascp}/products.rb +1 -1
- data/lib/aspera/assert.rb +45 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +11 -10
- data/lib/aspera/cli/extended_value.rb +5 -5
- data/lib/aspera/cli/formatter.rb +27 -14
- data/lib/aspera/cli/hints.rb +7 -6
- data/lib/aspera/cli/main.rb +49 -29
- data/lib/aspera/cli/manager.rb +46 -36
- data/lib/aspera/cli/plugin.rb +34 -20
- data/lib/aspera/cli/plugin_factory.rb +61 -0
- data/lib/aspera/cli/plugins/alee.rb +7 -7
- data/lib/aspera/cli/plugins/aoc.rb +168 -132
- data/lib/aspera/cli/plugins/ats.rb +33 -33
- data/lib/aspera/cli/plugins/bss.rb +3 -4
- data/lib/aspera/cli/plugins/config.rb +250 -272
- data/lib/aspera/cli/plugins/console.rb +8 -6
- data/lib/aspera/cli/plugins/cos.rb +20 -19
- data/lib/aspera/cli/plugins/faspex.rb +71 -60
- data/lib/aspera/cli/plugins/faspex5.rb +212 -133
- data/lib/aspera/cli/plugins/node.rb +83 -75
- data/lib/aspera/cli/plugins/orchestrator.rb +36 -44
- data/lib/aspera/cli/plugins/preview.rb +33 -31
- data/lib/aspera/cli/plugins/server.rb +33 -32
- data/lib/aspera/cli/plugins/shares.rb +39 -33
- data/lib/aspera/cli/sync_actions.rb +9 -9
- data/lib/aspera/cli/transfer_agent.rb +45 -25
- data/lib/aspera/cli/transfer_progress.rb +2 -3
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/colors.rb +5 -0
- data/lib/aspera/command_line_builder.rb +16 -14
- data/lib/aspera/coverage.rb +21 -0
- data/lib/aspera/data_repository.rb +33 -2
- data/lib/aspera/environment.rb +5 -4
- data/lib/aspera/faspex_gw.rb +13 -11
- data/lib/aspera/faspex_postproc.rb +6 -5
- data/lib/aspera/id_generator.rb +4 -2
- data/lib/aspera/json_rpc.rb +10 -8
- data/lib/aspera/keychain/encrypted_hash.rb +46 -11
- data/lib/aspera/keychain/macos_security.rb +29 -22
- data/lib/aspera/log.rb +5 -4
- data/lib/aspera/nagios.rb +7 -2
- data/lib/aspera/node_simulator.rb +213 -0
- data/lib/aspera/oauth/base.rb +143 -0
- data/lib/aspera/oauth/factory.rb +124 -0
- data/lib/aspera/oauth/generic.rb +34 -0
- data/lib/aspera/oauth/jwt.rb +51 -0
- data/lib/aspera/oauth/url_json.rb +31 -0
- data/lib/aspera/oauth/web.rb +50 -0
- data/lib/aspera/oauth.rb +5 -328
- data/lib/aspera/open_application.rb +7 -7
- data/lib/aspera/persistency_action_once.rb +13 -14
- data/lib/aspera/persistency_folder.rb +3 -2
- data/lib/aspera/preview/file_types.rb +53 -267
- data/lib/aspera/preview/generator.rb +7 -5
- data/lib/aspera/preview/terminal.rb +17 -7
- data/lib/aspera/preview/utils.rb +8 -7
- data/lib/aspera/proxy_auto_config.rb +6 -3
- data/lib/aspera/rest.rb +187 -140
- data/lib/aspera/rest_error_analyzer.rb +1 -0
- data/lib/aspera/rest_errors_aspera.rb +5 -3
- data/lib/aspera/resumer.rb +77 -0
- data/lib/aspera/secret_hider.rb +5 -2
- data/lib/aspera/ssh.rb +15 -8
- data/lib/aspera/temp_file_manager.rb +1 -1
- data/lib/aspera/{fasp → transfer}/error.rb +3 -3
- data/lib/aspera/{fasp → transfer}/error_info.rb +1 -1
- data/lib/aspera/{fasp → transfer}/faux_file.rb +1 -1
- data/lib/aspera/{fasp → transfer}/parameters.rb +95 -120
- data/lib/aspera/{fasp/transfer_spec.rb → transfer/spec.rb} +23 -19
- data/lib/aspera/{fasp/parameters.yaml → transfer/spec.yaml} +4 -99
- data/lib/aspera/transfer/sync.rb +273 -0
- data/lib/aspera/{fasp → transfer}/uri.rb +10 -9
- data/lib/aspera/web_server_simple.rb +12 -3
- data.tar.gz.sig +0 -0
- metadata +92 -68
- metadata.gz.sig +0 -0
- data/lib/aspera/aoc.rb +0 -606
- data/lib/aspera/ats_api.rb +0 -47
- data/lib/aspera/cos_node.rb +0 -93
- data/lib/aspera/fasp/agent_aspera.rb +0 -126
- data/lib/aspera/fasp/agent_base.rb +0 -48
- data/lib/aspera/fasp/agent_trsdk.rb +0 -146
- data/lib/aspera/fasp/resume_policy.rb +0 -77
- data/lib/aspera/node.rb +0 -338
- data/lib/aspera/sync.rb +0 -219
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/
|
|
3
|
+
require 'aspera/transfer/error_info'
|
|
4
4
|
|
|
5
5
|
module Aspera
|
|
6
|
-
module
|
|
6
|
+
module Transfer
|
|
7
7
|
# error raised if transfer fails
|
|
8
8
|
class Error < StandardError
|
|
9
9
|
attr_reader :err_code
|
|
@@ -14,7 +14,7 @@ module Aspera
|
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
def info
|
|
17
|
-
r =
|
|
17
|
+
r = ERROR_INFO[@err_code] || {r: false, c: 'UNKNOWN', m: 'unknown', a: 'unknown'}
|
|
18
18
|
return r.merge({i: @err_code})
|
|
19
19
|
end
|
|
20
20
|
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
# cspell:words mnemo PROTO RCVR NOLIC PMTU BRTT VLINK BWMEAS SSEAR FIPS
|
|
4
4
|
|
|
5
5
|
module Aspera
|
|
6
|
-
module
|
|
6
|
+
module Transfer
|
|
7
7
|
# from https://www.google.com/search?q=FASP+error+codes
|
|
8
8
|
# Note that the fact that an error is retry-able is not internally defined by protocol, it's client-side responsibility
|
|
9
9
|
# rubocop:disable Layout/MultilineHashKeyLineBreaks
|
|
@@ -1,21 +1,22 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require 'aspera/log'
|
|
4
|
+
require 'aspera/assert'
|
|
4
5
|
require 'aspera/command_line_builder'
|
|
5
6
|
require 'aspera/temp_file_manager'
|
|
6
|
-
require 'aspera/
|
|
7
|
-
require 'aspera/
|
|
7
|
+
require 'aspera/transfer/error'
|
|
8
|
+
require 'aspera/transfer/spec'
|
|
9
|
+
require 'aspera/ascp/installation'
|
|
8
10
|
require 'aspera/cli/formatter'
|
|
9
11
|
require 'aspera/rest'
|
|
10
12
|
require 'securerandom'
|
|
11
13
|
require 'base64'
|
|
12
14
|
require 'json'
|
|
13
|
-
require 'yaml'
|
|
14
15
|
require 'fileutils'
|
|
15
16
|
require 'openssl'
|
|
16
17
|
|
|
17
18
|
module Aspera
|
|
18
|
-
module
|
|
19
|
+
module Transfer
|
|
19
20
|
# translate transfer specification to ascp parameter list
|
|
20
21
|
class Parameters
|
|
21
22
|
# Agents shown in manual for parameters (sub list)
|
|
@@ -23,31 +24,22 @@ module Aspera
|
|
|
23
24
|
# Short names of columns in manual
|
|
24
25
|
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
|
25
26
|
FILE_LIST_OPTIONS = ['--file-list', '--file-pair-list'].freeze
|
|
26
|
-
|
|
27
|
+
# options that can be provided to the constructor, and then in @options
|
|
28
|
+
SUPPORTED_OPTIONS = %i[ascp_args wss check_ignore_cb quiet trusted_certs].freeze
|
|
27
29
|
|
|
28
|
-
private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS
|
|
30
|
+
private_constant :SUPPORTED_AGENTS, :FILE_LIST_OPTIONS, :SUPPORTED_OPTIONS
|
|
29
31
|
|
|
30
32
|
class << self
|
|
31
33
|
# Temp folder for file lists, must contain only file lists
|
|
32
34
|
# because of garbage collection takes any file there
|
|
33
35
|
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
|
34
36
|
@file_list_folder = TempFileManager.instance.new_file_path_global('asession_filelists') # cspell:disable-line
|
|
35
|
-
@param_description_cache = nil
|
|
36
|
-
# @return normalized description of transfer spec parameters, direct from yaml
|
|
37
|
-
def description
|
|
38
|
-
if @param_description_cache.nil?
|
|
39
|
-
# config file in same folder with same name as this source
|
|
40
|
-
description_from_yaml = YAML.load_file("#{__FILE__[0..-3]}yaml")
|
|
41
|
-
@param_description_cache = Aspera::CommandLineBuilder.normalize_description(description_from_yaml)
|
|
42
|
-
end
|
|
43
|
-
return @param_description_cache
|
|
44
|
-
end
|
|
45
37
|
|
|
46
38
|
# @param to_text [bool] replace HTML entities with text equivalent
|
|
47
39
|
# @return a table suitable to display in manual
|
|
48
40
|
def man_table
|
|
49
41
|
result = []
|
|
50
|
-
|
|
42
|
+
Spec::DESCRIPTION.each do |name, options|
|
|
51
43
|
param = {name: name, type: [options[:accepted_types]].flatten.join(','), description: options[:desc]}
|
|
52
44
|
# add flags for supported agents in doc
|
|
53
45
|
SUPPORTED_AGENTS.each do |a|
|
|
@@ -83,13 +75,7 @@ module Aspera
|
|
|
83
75
|
param[:description] = param[:description].gsub('/', '\\')
|
|
84
76
|
result.push(param)
|
|
85
77
|
end
|
|
86
|
-
return result.
|
|
87
|
-
if a[:name].start_with?('EX_').eql?(b[:name].start_with?('EX_'))
|
|
88
|
-
a[:name] <=> b[:name]
|
|
89
|
-
else
|
|
90
|
-
b[:name] <=> a[:name]
|
|
91
|
-
end
|
|
92
|
-
end
|
|
78
|
+
return result.sort_by { |a| a[:name] }
|
|
93
79
|
end
|
|
94
80
|
|
|
95
81
|
# special encoding methods used in YAML (key: :convert)
|
|
@@ -102,13 +88,9 @@ module Aspera
|
|
|
102
88
|
def convert_base64(v); Base64.strict_encode64(v); end
|
|
103
89
|
|
|
104
90
|
# file list is provided directly with ascp arguments
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
ascp_args
|
|
108
|
-
(ts['EX_ascp_args'].is_a?(Array) && ts['EX_ascp_args'].any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
|
|
109
|
-
(ascp_args.is_a?(Array) && ascp_args.any?{|i|FILE_LIST_OPTIONS.include?(i)}) ||
|
|
110
|
-
ts.key?('EX_file_list') ||
|
|
111
|
-
ts.key?('EX_file_pair_list')
|
|
91
|
+
# @param ascp_args [Array,NilClass] ascp arguments
|
|
92
|
+
def ascp_args_file_list?(ascp_args)
|
|
93
|
+
ascp_args&.any?{|i|FILE_LIST_OPTIONS.include?(i)}
|
|
112
94
|
end
|
|
113
95
|
|
|
114
96
|
# temp file list files are created here
|
|
@@ -125,151 +107,144 @@ module Aspera
|
|
|
125
107
|
|
|
126
108
|
# @param options [Hash] key: :wss: bool, :ascp_args: array of strings
|
|
127
109
|
def initialize(job_spec, options)
|
|
110
|
+
Aspera.assert_type(job_spec, Hash)
|
|
111
|
+
Aspera.assert_type(options, Hash)
|
|
128
112
|
@job_spec = job_spec
|
|
129
113
|
# check necessary options
|
|
130
114
|
missing_options = SUPPORTED_OPTIONS - options.keys
|
|
131
|
-
|
|
115
|
+
Aspera.assert(missing_options.empty?){"missing options: #{missing_options.join(', ')}"}
|
|
132
116
|
@options = SUPPORTED_OPTIONS.each_with_object({}){|o, h| h[o] = options[o]}
|
|
133
117
|
Log.log.debug{Log.dump(:parameters_options, @options)}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
118
|
+
Log.log.debug{Log.dump(:dismiss_options, options.keys - SUPPORTED_OPTIONS)}
|
|
119
|
+
Aspera.assert_type(@options[:ascp_args], Array){'ascp_args'}
|
|
120
|
+
Aspera.assert(@options[:ascp_args].all?(String)){'ascp arguments must Strings'}
|
|
121
|
+
@builder = CommandLineBuilder.new(@job_spec, Spec::DESCRIPTION)
|
|
137
122
|
end
|
|
138
123
|
|
|
124
|
+
# either place source files on command line, or add file list file
|
|
139
125
|
def process_file_list
|
|
140
|
-
# is the file list provided through
|
|
141
|
-
ascp_file_list_provided = self.class.
|
|
126
|
+
# is the file list provided through ascp parameters?
|
|
127
|
+
ascp_file_list_provided = self.class.ascp_args_file_list?(@options[:ascp_args])
|
|
142
128
|
# set if paths is mandatory in ts
|
|
143
129
|
@builder.params_definition['paths'][:mandatory] = !@job_spec.key?('keepalive') && !ascp_file_list_provided # cspell:words keepalive
|
|
144
130
|
# get paths in transfer spec (after setting if it is mandatory)
|
|
145
131
|
ts_paths_array = @builder.read_param('paths')
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
132
|
+
file_list_option = nil
|
|
133
|
+
# transfer spec contains paths ?
|
|
134
|
+
if !ts_paths_array.nil?
|
|
135
|
+
Aspera.assert(!ascp_file_list_provided){'file list provided both in transfer spec and ascp file list. Remove one of them.'}
|
|
136
|
+
Aspera.assert(ts_paths_array.all?{|i|i.key?('source')}){"All elements of paths must have a 'source' key"}
|
|
137
|
+
is_pair_list = ts_paths_array.any?{|i|i.key?('destination')}
|
|
138
|
+
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
|
|
139
|
+
if self.class.file_list_folder.nil?
|
|
140
|
+
Aspera.assert(!is_pair_list){'file pair list is not supported when file list folder is not set'}
|
|
141
|
+
# not safe for special characters ? (maybe not, depends on OS)
|
|
142
|
+
Log.log.debug('placing source file list on command line (no file list file)')
|
|
143
|
+
@builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
|
|
144
|
+
else
|
|
145
|
+
# safer option: generate a file list file if there is storage defined for it
|
|
146
|
+
if is_pair_list
|
|
147
|
+
file_list_option = '--file-pair-list'
|
|
148
|
+
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination']) }
|
|
162
149
|
else
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
raise "All elements of paths must be consistent with 'destination' key" if is_pair_list && !ts_paths_array.all?{|i|i.key?('destination')}
|
|
166
|
-
# safer option: generate a file list file if there is storage defined for it
|
|
167
|
-
# if there is one destination in paths, then use file-pair-list
|
|
168
|
-
if is_pair_list
|
|
169
|
-
option = '--file-pair-list'
|
|
170
|
-
lines = ts_paths_array.each_with_object([]){|e, m|m.push(e['source'], e['destination'] || e['source']) }
|
|
171
|
-
else
|
|
172
|
-
option = '--file-list'
|
|
173
|
-
lines = ts_paths_array.map{|i|i['source']}
|
|
174
|
-
end
|
|
175
|
-
file_list_file = Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
|
176
|
-
Log.log.debug{Log.dump(:file_list, lines)}
|
|
177
|
-
File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
|
|
178
|
-
Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
|
|
150
|
+
file_list_option = '--file-list'
|
|
151
|
+
lines = ts_paths_array.map{|i|i['source']}
|
|
179
152
|
end
|
|
153
|
+
file_list_file = TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
|
154
|
+
Log.log.debug{Log.dump(:file_list, lines)}
|
|
155
|
+
File.write(file_list_file, lines.join("\n"), encoding: 'UTF-8')
|
|
156
|
+
Log.log.debug{"#{file_list_option}=\n#{File.read(file_list_file)}".red}
|
|
180
157
|
end
|
|
181
|
-
else
|
|
182
|
-
option = '--file-list'
|
|
183
158
|
end
|
|
184
|
-
@builder.add_command_line_options(["#{
|
|
159
|
+
@builder.add_command_line_options(["#{file_list_option}=#{file_list_file}"]) unless file_list_option.nil?
|
|
185
160
|
end
|
|
186
161
|
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
def ascp_args
|
|
190
|
-
env_args = {
|
|
191
|
-
args: [],
|
|
192
|
-
env: {},
|
|
193
|
-
ascp_version: :ascp
|
|
194
|
-
}
|
|
195
|
-
|
|
196
|
-
# special cases
|
|
197
|
-
@job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
|
|
198
|
-
|
|
199
|
-
# notify multi-session was already used, anyway it was deleted by agent direct
|
|
200
|
-
raise 'internal error' if @builder.read_param('multi_session')
|
|
201
|
-
|
|
162
|
+
def remote_certificates
|
|
163
|
+
certificates_to_use = []
|
|
202
164
|
# use web socket secure for session ?
|
|
203
165
|
if @builder.read_param('wss_enabled') && (@options[:wss] || !@job_spec.key?('fasp_port'))
|
|
204
166
|
# by default use web socket session if available, unless removed by user
|
|
205
167
|
@builder.add_command_line_options(['--ws-connect'])
|
|
206
|
-
# TODO: option to give order ssh,ws (legacy http is implied
|
|
168
|
+
# TODO: option to give order ssh,ws (legacy http is implied by ssh)
|
|
207
169
|
# This will need to be cleaned up in aspera core
|
|
208
170
|
@job_spec['ssh_port'] = @builder.read_param('wss_port')
|
|
209
171
|
@job_spec.delete('fasp_port')
|
|
210
|
-
@job_spec.delete('EX_ssh_key_paths')
|
|
211
172
|
@job_spec.delete('sshfp')
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
173
|
+
# set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
|
174
|
+
certificates_to_use.concat(@options[:trusted_certs]) if @options[:trusted_certs].is_a?(Array)
|
|
175
|
+
# ignore cert for wss ?
|
|
176
|
+
if @options[:check_ignore_cb]&.call(@job_spec['remote_host'], @job_spec['wss_port'])
|
|
216
177
|
wss_cert_file = TempFileManager.instance.new_file_path_global('wss_cert')
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
else
|
|
222
|
-
# set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
|
223
|
-
@options[:trusted_certs].each do |file|
|
|
224
|
-
env_args[:args].unshift('-i', file)
|
|
225
|
-
Log.log.debug{"trusted certs for wss: #{file}"}
|
|
226
|
-
end
|
|
178
|
+
wss_url = "https://#{@job_spec['remote_host']}:#{@job_spec['wss_port']}"
|
|
179
|
+
File.write(wss_cert_file, Rest.remote_certificate_chain(wss_url))
|
|
180
|
+
# place in front, as more priority
|
|
181
|
+
certificates_to_use.unshift(wss_cert_file)
|
|
227
182
|
end
|
|
183
|
+
# when wss is used, only first `-i` is used... Hum...
|
|
184
|
+
certificates_to_use = [certificates_to_use.first] unless certificates_to_use.empty?
|
|
228
185
|
else
|
|
229
186
|
# remove unused parameter (avoid warning)
|
|
230
187
|
@job_spec.delete('wss_port')
|
|
231
188
|
# add SSH bypass keys when authentication is token and no auth is provided
|
|
232
189
|
if @job_spec.key?('token') && !@job_spec.key?('remote_password')
|
|
233
|
-
# @job_spec['remote_password'] = Installation.instance.ssh_cert_uuid # not used: no passphrase
|
|
234
|
-
Installation.instance.aspera_token_ssh_key_paths
|
|
190
|
+
# @job_spec['remote_password'] = Ascp::Installation.instance.ssh_cert_uuid # not used: no passphrase
|
|
191
|
+
certificates_to_use.concat(Ascp::Installation.instance.aspera_token_ssh_key_paths)
|
|
235
192
|
end
|
|
236
193
|
end
|
|
194
|
+
return certificates_to_use
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# translate transfer spec to env vars and command line arguments for ascp
|
|
198
|
+
def ascp_args
|
|
199
|
+
env_args = {
|
|
200
|
+
args: [],
|
|
201
|
+
env: {},
|
|
202
|
+
ascp_version: :ascp
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
# special cases
|
|
206
|
+
@job_spec.delete('source_root') if @job_spec.key?('source_root') && @job_spec['source_root'].empty?
|
|
207
|
+
|
|
208
|
+
# notify multi-session was already used, anyway it was deleted by agent direct
|
|
209
|
+
Aspera.assert(!@builder.read_param('multi_session'))
|
|
210
|
+
|
|
211
|
+
# add ssh or wss certificates
|
|
212
|
+
# (reverse, to keep order, as we unshift)
|
|
213
|
+
remote_certificates.reverse_each do |cert|
|
|
214
|
+
env_args[:args].unshift('-i', cert)
|
|
215
|
+
end
|
|
237
216
|
|
|
238
217
|
# process parameters as specified in table
|
|
239
218
|
@builder.process_params
|
|
240
219
|
|
|
241
|
-
|
|
220
|
+
base64_destination = false
|
|
221
|
+
# symbol must be index of Ascp::Installation.paths
|
|
242
222
|
if @builder.read_param('use_ascp4')
|
|
243
223
|
env_args[:ascp_version] = :ascp4
|
|
244
224
|
else
|
|
245
225
|
env_args[:ascp_version] = :ascp
|
|
246
|
-
|
|
247
|
-
@builder.add_command_line_options(['--dest64'])
|
|
226
|
+
base64_destination = true
|
|
248
227
|
end
|
|
249
|
-
#
|
|
250
|
-
|
|
228
|
+
# destination will be base64 encoded, put this before source path arguments
|
|
229
|
+
@builder.add_command_line_options(['--dest64']) if base64_destination
|
|
251
230
|
# optional arguments, at the end to override previous ones (to allow override)
|
|
252
|
-
@builder.add_command_line_options(@builder.read_param('EX_ascp_args'))
|
|
253
231
|
@builder.add_command_line_options(@options[:ascp_args])
|
|
232
|
+
# get list of source files to transfer and build arg for ascp
|
|
233
|
+
process_file_list
|
|
254
234
|
# process destination folder
|
|
255
235
|
destination_folder = @builder.read_param('destination_root') || '/'
|
|
256
236
|
# ascp4 does not support base64 encoding of destination
|
|
257
|
-
destination_folder = Base64.strict_encode64(destination_folder)
|
|
237
|
+
destination_folder = Base64.strict_encode64(destination_folder) if base64_destination
|
|
258
238
|
# destination MUST be last command line argument to ascp
|
|
259
239
|
@builder.add_command_line_options([destination_folder])
|
|
260
|
-
|
|
261
|
-
Log.log.debug{"ascp args: #{env_args}"}
|
|
262
|
-
|
|
263
240
|
@builder.add_env_args(env_args)
|
|
264
|
-
|
|
265
241
|
env_args[:args].unshift('-q') if @options[:quiet]
|
|
266
|
-
|
|
267
242
|
# add fallback cert and key as arguments if needed
|
|
268
243
|
if ['1', 1, true, 'force'].include?(@job_spec['http_fallback'])
|
|
269
|
-
env_args[:args].unshift('-Y', Installation.instance.path(:fallback_private_key))
|
|
270
|
-
env_args[:args].unshift('-I', Installation.instance.path(:fallback_certificate))
|
|
244
|
+
env_args[:args].unshift('-Y', Ascp::Installation.instance.path(:fallback_private_key))
|
|
245
|
+
env_args[:args].unshift('-I', Ascp::Installation.instance.path(:fallback_certificate))
|
|
271
246
|
end
|
|
272
|
-
|
|
247
|
+
Log.log.debug{"ascp args: #{env_args}"}
|
|
273
248
|
return env_args
|
|
274
249
|
end
|
|
275
250
|
end # Parameters
|
|
@@ -1,11 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require 'aspera/
|
|
3
|
+
require 'aspera/command_line_builder'
|
|
4
|
+
require 'aspera/assert'
|
|
5
|
+
require 'yaml'
|
|
4
6
|
|
|
5
7
|
module Aspera
|
|
6
|
-
module
|
|
8
|
+
module Transfer
|
|
7
9
|
# parameters for Transfer Spec
|
|
8
|
-
class
|
|
10
|
+
class Spec
|
|
9
11
|
# default transfer username for access key based transfers
|
|
10
12
|
ACCESS_KEY_TRANSFER_USER = 'xfer'
|
|
11
13
|
SSH_PORT = 33_001
|
|
@@ -17,32 +19,34 @@ module Aspera
|
|
|
17
19
|
}.freeze
|
|
18
20
|
# reserved tag for Aspera
|
|
19
21
|
TAG_RESERVED = 'aspera'
|
|
20
|
-
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
|
21
|
-
Aspera::Fasp::Parameters.description.each do |name, description|
|
|
22
|
-
next unless description[:enum].is_a?(Array)
|
|
23
|
-
TransferSpec.const_set("#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
|
|
24
|
-
description[:enum].each do |enum|
|
|
25
|
-
TransferSpec.const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
|
26
|
-
end
|
|
27
|
-
end
|
|
28
22
|
class << self
|
|
29
23
|
def action_to_direction(tspec, command)
|
|
30
|
-
|
|
24
|
+
Aspera.assert_type(tspec, Hash){'transfer spec'}
|
|
31
25
|
tspec['direction'] = case command.to_sym
|
|
32
26
|
when :upload then DIRECTION_SEND
|
|
33
27
|
when :download then DIRECTION_RECEIVE
|
|
34
|
-
else
|
|
28
|
+
else Aspera.error_unexpected_value(command.to_sym)
|
|
35
29
|
end
|
|
36
30
|
return tspec
|
|
37
31
|
end
|
|
38
32
|
|
|
39
33
|
def action(tspec)
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
34
|
+
Aspera.assert_type(tspec, Hash){'transfer spec'}
|
|
35
|
+
Aspera.assert_values(tspec['direction'], [DIRECTION_SEND, DIRECTION_RECEIVE]){'direction'}
|
|
36
|
+
case tspec['direction']
|
|
37
|
+
when DIRECTION_SEND then :upload
|
|
38
|
+
when DIRECTION_RECEIVE then :download
|
|
39
|
+
else Aspera.error_unexpected_value(tspec['direction'])
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
DESCRIPTION = CommandLineBuilder.normalize_description(YAML.load_file("#{__FILE__[0..-3]}yaml"))
|
|
44
|
+
# define constants for enums of parameters: <parameter>_<enum>, e.g. CIPHER_AES_128, DIRECTION_SEND, ...
|
|
45
|
+
DESCRIPTION.each do |name, description|
|
|
46
|
+
next unless description[:enum].is_a?(Array)
|
|
47
|
+
const_set(:"#{name.to_s.upcase}_ENUM_VALUES", description[:enum])
|
|
48
|
+
description[:enum].each do |enum|
|
|
49
|
+
const_set("#{name.to_s.upcase}_#{enum.upcase.gsub(/[^A-Z0-9]/, '_')}", enum.freeze)
|
|
46
50
|
end
|
|
47
51
|
end
|
|
48
52
|
end
|
|
@@ -27,7 +27,7 @@ cipher:
|
|
|
27
27
|
:cli:
|
|
28
28
|
:type: :opt_with_arg
|
|
29
29
|
:switch: "-c"
|
|
30
|
-
:convert: Aspera::
|
|
30
|
+
:convert: Aspera::Transfer::Parameters.convert_remove_hyphen
|
|
31
31
|
content_protection:
|
|
32
32
|
:desc: Enable client-side encryption at rest. (CSEAR, content protection)
|
|
33
33
|
:enum:
|
|
@@ -424,7 +424,7 @@ tags:
|
|
|
424
424
|
:cli:
|
|
425
425
|
:type: :opt_with_arg
|
|
426
426
|
:switch: "--tags64"
|
|
427
|
-
:convert: Aspera::
|
|
427
|
+
:convert: Aspera::Transfer::Parameters.convert_json64
|
|
428
428
|
target_rate_cap_kbps:
|
|
429
429
|
:desc: Returned by upload/download_setup node API.
|
|
430
430
|
:accepted_types: :int
|
|
@@ -479,7 +479,7 @@ source_root:
|
|
|
479
479
|
:cli:
|
|
480
480
|
:type: :opt_with_arg
|
|
481
481
|
:switch: "--source-prefix64"
|
|
482
|
-
:convert: Aspera::
|
|
482
|
+
:convert: Aspera::Transfer::Parameters.convert_base64
|
|
483
483
|
min_rate_cap_kbps:
|
|
484
484
|
:desc: |-
|
|
485
485
|
The highest minimum rate that an incoming transfer can request, in kilobits per second.
|
|
@@ -540,7 +540,7 @@ src_base:
|
|
|
540
540
|
:cli:
|
|
541
541
|
:type: :opt_with_arg
|
|
542
542
|
:switch: "--src-base64"
|
|
543
|
-
:convert: Aspera::
|
|
543
|
+
:convert: Aspera::Transfer::Parameters.convert_base64
|
|
544
544
|
preserve_acls:
|
|
545
545
|
:desc: "Preserve access control lists."
|
|
546
546
|
:enum:
|
|
@@ -610,34 +610,6 @@ remove_empty_source_directory:
|
|
|
610
610
|
- :sdk
|
|
611
611
|
:cli:
|
|
612
612
|
:type: :opt_without_arg
|
|
613
|
-
EX_at_rest_password:
|
|
614
|
-
:desc: Content protection password
|
|
615
|
-
:deprecation: "(4.13) Use standard spec parameter: content_protection_password"
|
|
616
|
-
:agents:
|
|
617
|
-
- :direct
|
|
618
|
-
:cli:
|
|
619
|
-
:type: :envvar
|
|
620
|
-
:variable: ASPERA_SCP_FILEPASS
|
|
621
|
-
EX_proxy_password:
|
|
622
|
-
:desc: |-
|
|
623
|
-
Password used for Aspera proxy server authentication.
|
|
624
|
-
May be overridden by password in URL provided in parameter: proxy.
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
:deprecation: (4.14) Use env var ASPERA_PROXY_PASS
|
|
628
|
-
:agents:
|
|
629
|
-
- :direct
|
|
630
|
-
:cli:
|
|
631
|
-
:type: :envvar
|
|
632
|
-
:variable: ASPERA_PROXY_PASS
|
|
633
|
-
EX_license_text:
|
|
634
|
-
:desc: "License file text override.\nBy default ascp looks for license file near executable."
|
|
635
|
-
:deprecation: (4.14) Use env var ASPERA_SCP_LICENSE
|
|
636
|
-
:agents:
|
|
637
|
-
- :direct
|
|
638
|
-
:cli:
|
|
639
|
-
:type: :envvar
|
|
640
|
-
:variable: ASPERA_SCP_LICENSE
|
|
641
613
|
keepalive:
|
|
642
614
|
:desc: The session is running in persistent session mode.
|
|
643
615
|
:agents:
|
|
@@ -663,49 +635,6 @@ sshfp:
|
|
|
663
635
|
:cli:
|
|
664
636
|
:type: :opt_with_arg
|
|
665
637
|
:switch: "--check-sshfp"
|
|
666
|
-
EX_http_proxy_url:
|
|
667
|
-
:desc: Specify the proxy server address used by HTTP Fallback
|
|
668
|
-
:deprecation: (4.14) TODO, use proxy option ?
|
|
669
|
-
:agents:
|
|
670
|
-
- :direct
|
|
671
|
-
:cli:
|
|
672
|
-
:type: :opt_with_arg
|
|
673
|
-
:switch: "-x"
|
|
674
|
-
EX_ssh_key_paths:
|
|
675
|
-
:desc: Use public key authentication for SSH and specify the private key file paths
|
|
676
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
|
677
|
-
:accepted_types: :array
|
|
678
|
-
:agents:
|
|
679
|
-
- :direct
|
|
680
|
-
:cli:
|
|
681
|
-
:type: :opt_with_arg
|
|
682
|
-
:switch: "-i"
|
|
683
|
-
EX_http_transfer_jpeg:
|
|
684
|
-
:desc: HTTP transfers as JPEG file
|
|
685
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
|
686
|
-
:accepted_types: :int
|
|
687
|
-
:default: '0'
|
|
688
|
-
:agents:
|
|
689
|
-
- :direct
|
|
690
|
-
:cli:
|
|
691
|
-
:type: :opt_with_arg
|
|
692
|
-
:switch: "-j"
|
|
693
|
-
EX_no_read:
|
|
694
|
-
:desc: no read source
|
|
695
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
|
696
|
-
:agents:
|
|
697
|
-
- :direct
|
|
698
|
-
:cli:
|
|
699
|
-
:type: :opt_without_arg
|
|
700
|
-
:switch: "--no-read"
|
|
701
|
-
EX_no_write:
|
|
702
|
-
:desc: no write on destination
|
|
703
|
-
:deprecation: (4.14) Use option transfer_info.ascp_args
|
|
704
|
-
:agents:
|
|
705
|
-
- :direct
|
|
706
|
-
:cli:
|
|
707
|
-
:type: :opt_without_arg
|
|
708
|
-
:switch: "--no-write"
|
|
709
638
|
target_rate_percentage:
|
|
710
639
|
:desc: "TODO: remove ?"
|
|
711
640
|
:cli:
|
|
@@ -754,30 +683,6 @@ obfuscate_file_names:
|
|
|
754
683
|
- :httpgw
|
|
755
684
|
:cli:
|
|
756
685
|
:type: :ignore
|
|
757
|
-
EX_file_list:
|
|
758
|
-
:desc: source file list
|
|
759
|
-
:deprecation: (4.14) Use command line file list, or option transfer_info.ascp_args
|
|
760
|
-
:agents:
|
|
761
|
-
- :direct
|
|
762
|
-
:cli:
|
|
763
|
-
:type: :special
|
|
764
|
-
:switch: "--file-list"
|
|
765
|
-
EX_file_pair_list:
|
|
766
|
-
:desc: source file pair list
|
|
767
|
-
:deprecation: (4.14) Use command line file pair list, or option transfer_info.ascp_args
|
|
768
|
-
:agents:
|
|
769
|
-
- :direct
|
|
770
|
-
:cli:
|
|
771
|
-
:type: :special
|
|
772
|
-
:switch: "--file-pair-list"
|
|
773
|
-
EX_ascp_args:
|
|
774
|
-
:desc: Add native command line arguments to ascp
|
|
775
|
-
:deprecation: (4.13) Use option transfer_info.ascp_args
|
|
776
|
-
:accepted_types: :array
|
|
777
|
-
:agents:
|
|
778
|
-
- :direct
|
|
779
|
-
:cli:
|
|
780
|
-
:type: :special
|
|
781
686
|
wss_enabled:
|
|
782
687
|
:desc: Server has Web Socket service enabled
|
|
783
688
|
:accepted_types: :bool
|