aspera-cli 4.6.0 → 4.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
require 'English'
|
1
3
|
require 'singleton'
|
2
4
|
require 'aspera/log'
|
3
5
|
require 'aspera/environment'
|
@@ -6,6 +8,7 @@ require 'xmlsimple'
|
|
6
8
|
require 'zlib'
|
7
9
|
require 'base64'
|
8
10
|
require 'fileutils'
|
11
|
+
require 'openssl'
|
9
12
|
|
10
13
|
module Aspera
|
11
14
|
module Fasp
|
@@ -18,6 +21,7 @@ module Aspera
|
|
18
21
|
# Installation.instance.ascp_path=""
|
19
22
|
class Installation
|
20
23
|
include Singleton
|
24
|
+
# known product names
|
21
25
|
PRODUCT_CONNECT='Aspera Connect'
|
22
26
|
PRODUCT_CLI_V1='Aspera CLI'
|
23
27
|
PRODUCT_DRIVE='Aspera Drive'
|
@@ -25,23 +29,48 @@ module Aspera
|
|
25
29
|
# protobuf generated files from sdk
|
26
30
|
EXT_RUBY_PROTOBUF='_pb.rb'
|
27
31
|
RB_SDK_FOLDER='lib'
|
28
|
-
|
29
|
-
|
32
|
+
ONE_YEAR_SECONDS=365 * 24 * 60 * 60
|
33
|
+
DEFAULT_ASPERA_CONF=<<~END_OF_CONFIG_FILE
|
34
|
+
<?xml version='1.0' encoding='UTF-8'?>
|
35
|
+
<CONF version="2">
|
36
|
+
<default>
|
37
|
+
<file_system>
|
38
|
+
<resume_suffix>.aspera-ckpt</resume_suffix>
|
39
|
+
<partial_file_suffix>.partial</partial_file_suffix>
|
40
|
+
</file_system>
|
41
|
+
</default>
|
42
|
+
</CONF>
|
43
|
+
END_OF_CONFIG_FILE
|
44
|
+
private_constant :PRODUCT_CONNECT,:PRODUCT_CLI_V1,:PRODUCT_DRIVE,:PRODUCT_ENTSRV,:EXT_RUBY_PROTOBUF,:RB_SDK_FOLDER,:ONE_YEAR_SECONDS,:DEFAULT_ASPERA_CONF
|
30
45
|
# set ascp executable path
|
31
46
|
def ascp_path=(v)
|
32
47
|
@path_to_ascp=v
|
33
48
|
end
|
34
49
|
|
50
|
+
def ascp_path
|
51
|
+
path(:ascp)
|
52
|
+
end
|
53
|
+
|
35
54
|
def sdk_ruby_folder
|
36
|
-
ruby_pb_folder=File.join(
|
55
|
+
ruby_pb_folder=File.join(sdk_folder,RB_SDK_FOLDER)
|
37
56
|
FileUtils.mkdir_p(ruby_pb_folder) unless Dir.exist?(ruby_pb_folder)
|
38
57
|
return ruby_pb_folder
|
39
58
|
end
|
40
59
|
|
41
60
|
# location of SDK files
|
42
|
-
def
|
43
|
-
@
|
44
|
-
|
61
|
+
def sdk_folder=(v)
|
62
|
+
@sdk_dir=v
|
63
|
+
sdk_folder
|
64
|
+
end
|
65
|
+
|
66
|
+
# backward compatibility in sample program
|
67
|
+
alias folder= sdk_folder=
|
68
|
+
|
69
|
+
# @return the path to folder where SDK is installed
|
70
|
+
def sdk_folder
|
71
|
+
raise 'SDK path was ot initialized' if @sdk_dir.nil?
|
72
|
+
FileUtils.mkdir_p(@sdk_dir) unless Dir.exist?(@sdk_dir)
|
73
|
+
@sdk_dir
|
45
74
|
end
|
46
75
|
|
47
76
|
# find ascp in named product (use value : FIRST_FOUND='FIRST' to just use first one)
|
@@ -64,9 +93,9 @@ module Aspera
|
|
64
93
|
scan_locations=product_locations.clone
|
65
94
|
# add SDK as first search path
|
66
95
|
scan_locations.unshift({
|
67
|
-
:
|
68
|
-
:
|
69
|
-
:
|
96
|
+
expected: 'SDK',
|
97
|
+
app_root: sdk_folder,
|
98
|
+
sub_bin: ''
|
70
99
|
})
|
71
100
|
# search installed products: with ascp
|
72
101
|
@found_products=scan_locations.select! do |item|
|
@@ -107,45 +136,36 @@ module Aspera
|
|
107
136
|
when :transferd
|
108
137
|
file=transferd_filepath
|
109
138
|
when :ssh_bypass_key_dsa
|
110
|
-
file=File.join(
|
139
|
+
file=File.join(sdk_folder,'aspera_bypass_dsa.pem')
|
111
140
|
File.write(file,get_key('dsa',1)) unless File.exist?(file)
|
112
141
|
File.chmod(0400,file)
|
113
142
|
when :ssh_bypass_key_rsa
|
114
|
-
file=File.join(
|
143
|
+
file=File.join(sdk_folder,'aspera_bypass_rsa.pem')
|
115
144
|
File.write(file,get_key('rsa',2)) unless File.exist?(file)
|
116
145
|
File.chmod(0400,file)
|
117
146
|
when :aspera_license
|
118
|
-
file=File.join(
|
119
|
-
File.write(file,Base64.strict_encode64("#{Zlib::Inflate.inflate(DataRepository.instance.
|
147
|
+
file=File.join(sdk_folder,'aspera-license')
|
148
|
+
File.write(file,Base64.strict_encode64("#{Zlib::Inflate.inflate(DataRepository.instance.data(6))}==SIGNATURE==\n#{Base64.strict_encode64(DataRepository.instance.data(7))}")) unless File.exist?(file)
|
120
149
|
File.chmod(0400,file)
|
121
150
|
when :aspera_conf
|
122
|
-
file=File.join(
|
123
|
-
File.write(file
|
124
|
-
<CONF version="2">
|
125
|
-
<default>
|
126
|
-
<file_system>
|
127
|
-
<resume_suffix>.aspera-ckpt</resume_suffix>
|
128
|
-
<partial_file_suffix>.partial</partial_file_suffix>
|
129
|
-
</file_system>
|
130
|
-
</default>
|
131
|
-
</CONF>
|
132
|
-
}) unless File.exist?(file)
|
151
|
+
file=File.join(sdk_folder,'aspera.conf')
|
152
|
+
File.write(file,DEFAULT_ASPERA_CONF) unless File.exist?(file)
|
133
153
|
File.chmod(0400,file)
|
134
154
|
when :fallback_cert,:fallback_key
|
135
|
-
file_key=File.join(
|
136
|
-
file_cert=File.join(
|
137
|
-
if !File.exist?(file_key)
|
155
|
+
file_key=File.join(sdk_folder,'aspera_fallback_key.pem')
|
156
|
+
file_cert=File.join(sdk_folder,'aspera_fallback_cert.pem')
|
157
|
+
if !File.exist?(file_key) || !File.exist?(file_cert)
|
138
158
|
require 'openssl'
|
139
159
|
# create new self signed certificate for http fallback
|
140
160
|
private_key = OpenSSL::PKey::RSA.new(1024)
|
141
161
|
cert = OpenSSL::X509::Certificate.new
|
142
|
-
cert.subject = cert.issuer = OpenSSL::X509::Name.parse(
|
162
|
+
cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=US/ST=California/L=Emeryville/O=Aspera Inc./OU=Corporate/CN=Aspera Inc./emailAddress=info@asperasoft.com')
|
143
163
|
cert.not_before = Time.now
|
144
|
-
cert.not_after = Time.now +
|
164
|
+
cert.not_after = Time.now + ONE_YEAR_SECONDS
|
145
165
|
cert.public_key = private_key.public_key
|
146
166
|
cert.serial = 0x0
|
147
167
|
cert.version = 2
|
148
|
-
cert.sign(private_key, OpenSSL::Digest
|
168
|
+
cert.sign(private_key, OpenSSL::Digest.new('SHA1'))
|
149
169
|
File.write(file_key,private_key.to_pem)
|
150
170
|
File.write(file_cert,cert.to_pem)
|
151
171
|
File.chmod(0400,file_key)
|
@@ -167,7 +187,7 @@ module Aspera
|
|
167
187
|
uri_file=File.join(folder,"http#{ext}.uri")
|
168
188
|
Log.log.debug("checking connect port file: #{uri_file}")
|
169
189
|
if File.exist?(uri_file)
|
170
|
-
return File.open(uri_file)
|
190
|
+
return File.open(uri_file, &:gets).strip
|
171
191
|
end
|
172
192
|
end
|
173
193
|
raise "no connect uri file found in #{folder}"
|
@@ -181,7 +201,7 @@ module Aspera
|
|
181
201
|
|
182
202
|
# default bypass key phrase
|
183
203
|
def bypass_pass
|
184
|
-
return
|
204
|
+
return format('%08x-%04x-%04x-%04x-%04x%08x',*DataRepository.instance.data(3).unpack('NnnnnN'))
|
185
205
|
end
|
186
206
|
|
187
207
|
def bypass_keys
|
@@ -195,13 +215,13 @@ module Aspera
|
|
195
215
|
|
196
216
|
# Check that specified path is ascp and get version
|
197
217
|
def get_exe_version(exe_path,vers_arg)
|
198
|
-
raise
|
218
|
+
raise 'ERROR: nil arg' if exe_path.nil?
|
199
219
|
return nil unless File.exist?(exe_path)
|
200
220
|
exe_version=nil
|
201
|
-
cmd_out=%x
|
202
|
-
raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless
|
221
|
+
cmd_out=%x("#{exe_path}" #{vers_arg})
|
222
|
+
raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $CHILD_STATUS == 0
|
203
223
|
# get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
|
204
|
-
m=cmd_out.match(/ version ([0-9
|
224
|
+
m=cmd_out.match(/ version ([0-9.]+)/)
|
205
225
|
exe_version=m[1] unless m.nil?
|
206
226
|
return exe_version
|
207
227
|
end
|
@@ -220,9 +240,9 @@ module Aspera
|
|
220
240
|
Aspera::Rest.new(base_url: sdk_url, redirect_max: 3).call(operation: 'GET',save_to_file: sdk_zip_path)
|
221
241
|
end
|
222
242
|
# rename old install
|
223
|
-
if !
|
224
|
-
Log.log.warn(
|
225
|
-
File.rename(
|
243
|
+
if !Dir.empty?(sdk_folder)
|
244
|
+
Log.log.warn('Previous install exists, renaming folder.')
|
245
|
+
File.rename(sdk_folder,"#{sdk_folder}.#{Time.now.strftime('%Y%m%d%H%M%S')}")
|
226
246
|
# TODO: delete old archives ?
|
227
247
|
end
|
228
248
|
# SDK is organized by architecture
|
@@ -234,31 +254,30 @@ module Aspera
|
|
234
254
|
next if entry.name.end_with?('/')
|
235
255
|
dest_folder=nil
|
236
256
|
# binaries
|
237
|
-
dest_folder=
|
257
|
+
dest_folder=sdk_folder if entry.name.include?(arch_filter)
|
238
258
|
# ruby adapters
|
239
259
|
dest_folder=sdk_ruby_folder if entry.name.end_with?(EXT_RUBY_PROTOBUF)
|
240
|
-
if
|
241
|
-
|
242
|
-
|
243
|
-
end
|
260
|
+
next if dest_folder.nil?
|
261
|
+
File.open(File.join(dest_folder,File.basename(entry.name)), 'wb') do |output_stream|
|
262
|
+
IO.copy_stream(entry.get_input_stream, output_stream)
|
244
263
|
end
|
245
264
|
end
|
246
265
|
end
|
247
266
|
File.unlink(sdk_zip_path) rescue nil # Windows may give error
|
248
267
|
# ensure license file are generated so that ascp invokation for version works
|
249
|
-
|
250
|
-
|
251
|
-
ascp_path=File.join(
|
268
|
+
path(:aspera_license)
|
269
|
+
path(:aspera_conf)
|
270
|
+
ascp_path=File.join(sdk_folder,ascp_filename)
|
252
271
|
raise "No #{ascp_filename} found in SDK archive" unless File.exist?(ascp_path)
|
253
272
|
FileUtils.chmod(0755,ascp_path)
|
254
273
|
FileUtils.chmod(0755,ascp_path.gsub('ascp','ascp4'))
|
255
|
-
ascp_version=get_ascp_version(File.join(
|
274
|
+
ascp_version=get_ascp_version(File.join(sdk_folder,ascp_filename))
|
256
275
|
trd_path=transferd_filepath
|
257
276
|
Log.log.warn("No #{trd_path} in SDK archive") unless File.exist?(trd_path)
|
258
277
|
FileUtils.chmod(0755,trd_path) if File.exist?(trd_path)
|
259
278
|
transferd_version=get_exe_version(trd_path,'version')
|
260
279
|
sdk_version = transferd_version||ascp_version
|
261
|
-
File.write(File.join(
|
280
|
+
File.write(File.join(sdk_folder,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")
|
262
281
|
return sdk_version
|
263
282
|
end
|
264
283
|
|
@@ -276,7 +295,7 @@ module Aspera
|
|
276
295
|
|
277
296
|
def initialize
|
278
297
|
@path_to_ascp=nil
|
279
|
-
@
|
298
|
+
@sdk_dir=nil
|
280
299
|
@found_products=nil
|
281
300
|
end
|
282
301
|
|
@@ -288,20 +307,13 @@ module Aspera
|
|
288
307
|
return found.first
|
289
308
|
end
|
290
309
|
|
291
|
-
# @return the path to folder where SDK is installed
|
292
|
-
def folder_path
|
293
|
-
raise "Undefined path to SDK" if @sdk_folder.nil?
|
294
|
-
FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
|
295
|
-
@sdk_folder
|
296
|
-
end
|
297
|
-
|
298
310
|
# filename for ascp with optional extension (Windows)
|
299
311
|
def ascp_filename
|
300
312
|
return 'ascp'+Environment.exe_extension
|
301
313
|
end
|
302
314
|
|
303
315
|
def transferd_filepath
|
304
|
-
return File.join(
|
316
|
+
return File.join(sdk_folder,'asperatransferd'+Environment.exe_extension)
|
305
317
|
end
|
306
318
|
|
307
319
|
# @return product folders depending on OS fields
|
@@ -313,49 +325,49 @@ module Aspera
|
|
313
325
|
def product_locations
|
314
326
|
case Aspera::Environment.os
|
315
327
|
when Aspera::Environment::OS_WINDOWS; return [{
|
316
|
-
:
|
317
|
-
:
|
318
|
-
:
|
319
|
-
:
|
328
|
+
expected: PRODUCT_CONNECT,
|
329
|
+
app_root: File.join(ENV['LOCALAPPDATA'],'Programs','Aspera','Aspera Connect'),
|
330
|
+
log_root: File.join(ENV['LOCALAPPDATA'],'Aspera','Aspera Connect','var','log'),
|
331
|
+
run_root: File.join(ENV['LOCALAPPDATA'],'Aspera','Aspera Connect')
|
320
332
|
},{
|
321
|
-
:
|
322
|
-
:
|
323
|
-
:
|
333
|
+
expected: PRODUCT_CLI_V1,
|
334
|
+
app_root: File.join('C:','Program Files','Aspera','cli'),
|
335
|
+
log_root: File.join('C:','Program Files','Aspera','cli','var','log')
|
324
336
|
},{
|
325
|
-
:
|
326
|
-
:
|
327
|
-
:
|
337
|
+
expected: PRODUCT_ENTSRV,
|
338
|
+
app_root: File.join('C:','Program Files','Aspera','Enterprise Server'),
|
339
|
+
log_root: File.join('C:','Program Files','Aspera','Enterprise Server','var','log')
|
328
340
|
}]
|
329
341
|
when Aspera::Environment::OS_X; return [{
|
330
|
-
:
|
331
|
-
:
|
332
|
-
:
|
333
|
-
:
|
334
|
-
:
|
342
|
+
expected: PRODUCT_CONNECT,
|
343
|
+
app_root: File.join(Dir.home,'Applications','Aspera Connect.app'),
|
344
|
+
log_root: File.join(Dir.home,'Library','Logs','Aspera_Connect'),
|
345
|
+
run_root: File.join(Dir.home,'Library','Application Support','Aspera','Aspera Connect'),
|
346
|
+
sub_bin: File.join('Contents','Resources')
|
335
347
|
},{
|
336
|
-
:
|
337
|
-
:
|
338
|
-
:
|
348
|
+
expected: PRODUCT_CLI_V1,
|
349
|
+
app_root: File.join(Dir.home,'Applications','Aspera CLI'),
|
350
|
+
log_root: File.join(Dir.home,'Library','Logs','Aspera')
|
339
351
|
},{
|
340
|
-
:
|
341
|
-
:
|
342
|
-
:
|
352
|
+
expected: PRODUCT_ENTSRV,
|
353
|
+
app_root: File.join('','Library','Aspera'),
|
354
|
+
log_root: File.join(Dir.home,'Library','Logs','Aspera')
|
343
355
|
},{
|
344
|
-
:
|
345
|
-
:
|
346
|
-
:
|
347
|
-
:
|
356
|
+
expected: PRODUCT_DRIVE,
|
357
|
+
app_root: File.join('','Applications','Aspera Drive.app'),
|
358
|
+
log_root: File.join(Dir.home,'Library','Logs','Aspera_Drive'),
|
359
|
+
sub_bin: File.join('Contents','Resources')
|
348
360
|
}]
|
349
|
-
else; return [{
|
350
|
-
:
|
351
|
-
:
|
352
|
-
:
|
361
|
+
else; return [{ # other: Linux and Unix family
|
362
|
+
expected: PRODUCT_CONNECT,
|
363
|
+
app_root: File.join(Dir.home,'.aspera','connect'),
|
364
|
+
run_root: File.join(Dir.home,'.aspera','connect')
|
353
365
|
},{
|
354
|
-
:
|
355
|
-
:
|
366
|
+
expected: PRODUCT_CLI_V1,
|
367
|
+
app_root: File.join(Dir.home,'.aspera','cli')
|
356
368
|
},{
|
357
|
-
:
|
358
|
-
:
|
369
|
+
expected: PRODUCT_ENTSRV,
|
370
|
+
app_root: File.join('','opt','aspera')
|
359
371
|
}]
|
360
372
|
end
|
361
373
|
end
|
@@ -365,10 +377,10 @@ module Aspera
|
|
365
377
|
# @param id in repository 1 for dsa, 2 for rsa
|
366
378
|
def get_key(type,id)
|
367
379
|
hf=['begin','end'].map{|t|"-----#{t} #{type} private key-----".upcase}
|
368
|
-
bin=Base64.strict_encode64(DataRepository.instance.
|
369
|
-
|
380
|
+
bin=Base64.strict_encode64(DataRepository.instance.data(id))
|
381
|
+
# validate valie and generate key in connonical format
|
382
|
+
OpenSSL::PKey.const_get(type.upcase).send(:new,hf.insert(1,bin).join("\n")).to_pem
|
370
383
|
end
|
371
|
-
|
372
384
|
end # Installation
|
373
385
|
end
|
374
386
|
end
|
data/lib/aspera/fasp/listener.rb
CHANGED
@@ -1,3 +1,4 @@
|
|
1
|
+
# frozen_string_literal: true
|
1
2
|
require 'aspera/log'
|
2
3
|
require 'aspera/command_line_builder'
|
3
4
|
require 'aspera/temp_file_manager'
|
@@ -5,7 +6,6 @@ require 'securerandom'
|
|
5
6
|
require 'base64'
|
6
7
|
require 'json'
|
7
8
|
require 'yaml'
|
8
|
-
require 'securerandom'
|
9
9
|
require 'fileutils'
|
10
10
|
require 'openssl'
|
11
11
|
|
@@ -13,68 +13,79 @@ module Aspera
|
|
13
13
|
module Fasp
|
14
14
|
# translate transfer specification to ascp parameter list
|
15
15
|
class Parameters
|
16
|
-
private
|
17
|
-
# Temp folder for file lists, must contain only file lists
|
18
|
-
# because of garbage collection takes any file there
|
19
|
-
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
20
|
-
@@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
|
21
|
-
@@param_description_cache=nil
|
22
|
-
# @return normaiwed description of transfer spec parameters
|
23
|
-
def self.description
|
24
|
-
return @@param_description_cache unless @@param_description_cache.nil?
|
25
|
-
# config file in same folder with same name as this source
|
26
|
-
@@param_description_cache=YAML.load_file("#{__FILE__[0..-3]}yaml")
|
27
|
-
Aspera::CommandLineBuilder.normalize_description(@@param_description_cache)
|
28
|
-
end
|
29
|
-
|
30
16
|
# Agents shown in manual for parameters (sub list)
|
31
17
|
SUPPORTED_AGENTS=[:direct,:node,:connect]
|
32
18
|
# Short names of columns in manual
|
33
19
|
SUPPORTED_AGENTS_SHORT=SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
34
20
|
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
#
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
21
|
+
class << self
|
22
|
+
# Temp folder for file lists, must contain only file lists
|
23
|
+
# because of garbage collection takes any file there
|
24
|
+
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
25
|
+
@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
|
26
|
+
@param_description_cache=nil
|
27
|
+
# @return normalized description of transfer spec parameters, direct from yaml
|
28
|
+
def description
|
29
|
+
return @param_description_cache unless @param_description_cache.nil?
|
30
|
+
# config file in same folder with same name as this source
|
31
|
+
@param_description_cache=YAML.load_file("#{__FILE__[0..-3]}yaml")
|
32
|
+
Aspera::CommandLineBuilder.normalize_description(@param_description_cache)
|
33
|
+
end
|
34
|
+
|
35
|
+
# @return a table suitable to display in manual
|
36
|
+
def man_table
|
37
|
+
result=[]
|
38
|
+
description.each do |k,i|
|
39
|
+
param={name: k, type: [i[:accepted_types]].flatten.join(','),description: i[:desc]}
|
40
|
+
SUPPORTED_AGENTS.each do |a|
|
41
|
+
param[a.to_s[0].to_sym]=i[:tragents].nil? || i[:tragents].include?(a) ? 'Y' : ''
|
42
|
+
end
|
43
|
+
# only keep lines that are usable in supported agents
|
44
|
+
next if SUPPORTED_AGENTS_SHORT.inject(true){|m,j|m && param[j].empty?}
|
45
|
+
param[:cli]=
|
46
|
+
case i[:cltype]
|
47
|
+
when :envvar then 'env:'+i[:clvarname]
|
48
|
+
when :opt_without_arg,:opt_with_arg then i[:clswitch]
|
49
|
+
else ''
|
50
|
+
end
|
51
|
+
if i.has_key?(:enum)
|
52
|
+
param[:description] += "\nAllowed values: #{i[:enum].join(', ')}"
|
53
|
+
end
|
54
|
+
result.push(param)
|
53
55
|
end
|
54
|
-
result
|
56
|
+
return result
|
55
57
|
end
|
56
|
-
return result
|
57
|
-
end
|
58
58
|
|
59
|
-
|
60
|
-
|
61
|
-
v.tr('-','')
|
62
|
-
end
|
59
|
+
# special encoding methods used in YAML (key: :clconvert)
|
60
|
+
def clconv_remove_hyphen(v); v.tr('-',''); end
|
63
61
|
|
64
|
-
|
65
|
-
|
66
|
-
Base64.strict_encode64(v)
|
67
|
-
end
|
62
|
+
# special encoding methods used in YAML (key: :clconvert)
|
63
|
+
def clconv_json64(v); Base64.strict_encode64(JSON.generate(v)); end
|
68
64
|
|
69
|
-
|
70
|
-
|
71
|
-
Base64.strict_encode64(JSON.generate(v))
|
72
|
-
end
|
65
|
+
# special encoding methods used in YAML (key: :clconvert)
|
66
|
+
def clconv_base64(v); Base64.strict_encode64(v); end
|
73
67
|
|
74
|
-
|
75
|
-
|
76
|
-
|
68
|
+
def ts_has_file_list(ts)
|
69
|
+
ts.has_key?('EX_ascp_args') and ts['EX_ascp_args'].is_a?(Array) and ['--file-list','--file-pair-list'].any?{|i|ts['EX_ascp_args'].include?(i)}
|
70
|
+
end
|
77
71
|
|
72
|
+
def ts_to_env_args(transfer_spec,options)
|
73
|
+
return Parameters.new(transfer_spec,options).ascp_args()
|
74
|
+
end
|
75
|
+
|
76
|
+
# temp file list files are created here
|
77
|
+
def file_list_folder=(v)
|
78
|
+
@file_list_folder=v
|
79
|
+
return if @file_list_folder.nil?
|
80
|
+
FileUtils.mkdir_p(@file_list_folder)
|
81
|
+
TempFileManager.instance.cleanup_expired(@file_list_folder)
|
82
|
+
end
|
83
|
+
|
84
|
+
# static methods
|
85
|
+
attr_reader :file_list_folder
|
86
|
+
end # self
|
87
|
+
|
88
|
+
# @param options [Hash] key: :wss: bool
|
78
89
|
def initialize(job_spec,options)
|
79
90
|
@job_spec=job_spec
|
80
91
|
@options=options
|
@@ -82,28 +93,26 @@ module Aspera
|
|
82
93
|
Log.log.debug("agent options: #{@options}")
|
83
94
|
end
|
84
95
|
|
85
|
-
public
|
86
|
-
|
87
96
|
# translate transfer spec to env vars and command line arguments for ascp
|
88
97
|
# NOTE: parameters starting with "EX_" (extended) are not standard
|
89
|
-
def ascp_args
|
98
|
+
def ascp_args
|
90
99
|
env_args={
|
91
|
-
:
|
92
|
-
:
|
93
|
-
:
|
100
|
+
args: [],
|
101
|
+
env: {},
|
102
|
+
ascp_version: :ascp
|
94
103
|
}
|
95
104
|
# some ssh credentials are required to avoid interactive password input
|
96
|
-
if !@job_spec.has_key?('remote_password')
|
97
|
-
!@job_spec.has_key?('ssh_private_key')
|
98
|
-
!@job_spec.has_key?('EX_ssh_key_paths')
|
99
|
-
raise Fasp::Error
|
105
|
+
if !@job_spec.has_key?('remote_password') &&
|
106
|
+
!@job_spec.has_key?('ssh_private_key') &&
|
107
|
+
!@job_spec.has_key?('EX_ssh_key_paths')
|
108
|
+
raise Fasp::Error, 'required: password or ssh key (value or path)'
|
100
109
|
end
|
101
110
|
|
102
111
|
# special cases
|
103
|
-
@job_spec.delete('source_root') if @job_spec.has_key?('source_root')
|
112
|
+
@job_spec.delete('source_root') if @job_spec.has_key?('source_root') && @job_spec['source_root'].empty?
|
104
113
|
|
105
114
|
# use web socket session initiation ?
|
106
|
-
if @builder.process_param('wss_enabled',:get_value)
|
115
|
+
if @builder.process_param('wss_enabled',:get_value) && (@options[:wss] || !@job_spec.has_key?('fasp_port'))
|
107
116
|
# by default use web socket session if available, unless removed by user
|
108
117
|
@builder.add_command_line_options(['--ws-connect'])
|
109
118
|
# TODO: option to give order ssh,ws (legacy http is implied bu ssh)
|
@@ -112,9 +121,9 @@ module Aspera
|
|
112
121
|
@job_spec.delete('fasp_port')
|
113
122
|
@job_spec.delete('EX_ssh_key_paths')
|
114
123
|
@job_spec.delete('sshfp')
|
115
|
-
# set
|
124
|
+
# set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
116
125
|
@job_spec['EX_ssh_key_paths']=[OpenSSL::X509::DEFAULT_CERT_FILE]
|
117
|
-
Log.log.debug(
|
126
|
+
Log.log.debug('CA certs: EX_ssh_key_paths <- DEFAULT_CERT_FILE from openssl')
|
118
127
|
else
|
119
128
|
# remove unused parameter (avoid warning)
|
120
129
|
@job_spec.delete('wss_port')
|
@@ -135,17 +144,17 @@ module Aspera
|
|
135
144
|
file_list_provided=self.class.ts_has_file_list(@job_spec)
|
136
145
|
@builder.params_definition['paths'][:mandatory]=!@job_spec.has_key?('keepalive') and !file_list_provided
|
137
146
|
paths_array=@builder.process_param('paths',:get_value)
|
138
|
-
if file_list_provided
|
139
|
-
Log.log.warn(
|
147
|
+
if file_list_provided && !paths_array.nil?
|
148
|
+
Log.log.warn('file list provided both in transfer spec and ascp file list. Keeping file list only.')
|
140
149
|
paths_array=nil
|
141
150
|
end
|
142
|
-
if !
|
151
|
+
if !paths_array.nil?
|
143
152
|
# it's an array
|
144
|
-
raise
|
153
|
+
raise 'paths is empty in transfer spec' if paths_array.empty?
|
145
154
|
# use file list if there is storage defined for it.
|
146
|
-
if
|
155
|
+
if self.class.file_list_folder.nil?
|
147
156
|
# not safe for special characters ? (maybe not, depends on OS)
|
148
|
-
Log.log.debug(
|
157
|
+
Log.log.debug('placing source file list on command line (no file list file)')
|
149
158
|
@builder.add_command_line_options(paths_array.map{|i|i['source']})
|
150
159
|
else
|
151
160
|
file_list_file=@builder.process_param('EX_file_list',:get_value)
|
@@ -161,13 +170,13 @@ module Aspera
|
|
161
170
|
# TODO: well, we test only the first one, but anyway it shall be consistent
|
162
171
|
if paths_array.first.has_key?('destination')
|
163
172
|
option='--file-pair-list'
|
164
|
-
lines=paths_array.
|
173
|
+
lines=paths_array.each_with_object([]){|e,m|m.push(e['source'],e['destination']);}
|
165
174
|
else
|
166
175
|
option='--file-list'
|
167
176
|
lines=paths_array.map{|i|i['source']}
|
168
177
|
end
|
169
|
-
file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(
|
170
|
-
File.
|
178
|
+
file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
179
|
+
File.write(file_list_file, lines.join("\n"))
|
171
180
|
Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
|
172
181
|
end
|
173
182
|
end
|
@@ -187,24 +196,6 @@ module Aspera
|
|
187
196
|
|
188
197
|
return env_args
|
189
198
|
end
|
190
|
-
|
191
|
-
# temp file list files are created here
|
192
|
-
def self.file_list_folder=(v)
|
193
|
-
@@file_list_folder=v
|
194
|
-
if !@@file_list_folder.nil?
|
195
|
-
FileUtils.mkdir_p(@@file_list_folder)
|
196
|
-
TempFileManager.instance.cleanup_expired(@@file_list_folder)
|
197
|
-
end
|
198
|
-
end
|
199
|
-
|
200
|
-
# static methods
|
201
|
-
class << self
|
202
|
-
def file_list_folder; @@file_list_folder;end
|
203
|
-
|
204
|
-
def ts_to_env_args(transfer_spec,options)
|
205
|
-
return Parameters.new(transfer_spec,options).ascp_args()
|
206
|
-
end
|
207
|
-
end
|
208
199
|
end # Parameters
|
209
200
|
end
|
210
201
|
end
|