aspera-cli 4.6.0 → 4.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +427 -300
- data/bin/ascli +2 -1
- data/bin/asession +1 -0
- data/docs/test_env.conf +2 -0
- data/examples/aoc.rb +4 -3
- data/examples/faspex4.rb +21 -19
- data/examples/proxy.pac +1 -1
- data/examples/transfer.rb +15 -15
- data/lib/aspera/aoc.rb +135 -124
- data/lib/aspera/ascmd.rb +85 -75
- data/lib/aspera/ats_api.rb +11 -10
- data/lib/aspera/cli/basic_auth_plugin.rb +13 -14
- data/lib/aspera/cli/extended_value.rb +42 -33
- data/lib/aspera/cli/formater.rb +138 -111
- data/lib/aspera/cli/info.rb +17 -0
- data/lib/aspera/cli/listener/line_dump.rb +3 -2
- data/lib/aspera/cli/listener/logger.rb +2 -1
- data/lib/aspera/cli/listener/progress.rb +16 -18
- data/lib/aspera/cli/listener/progress_multi.rb +13 -16
- data/lib/aspera/cli/main.rb +122 -130
- data/lib/aspera/cli/manager.rb +146 -154
- data/lib/aspera/cli/plugin.rb +38 -34
- data/lib/aspera/cli/plugins/alee.rb +6 -6
- data/lib/aspera/cli/plugins/aoc.rb +273 -276
- data/lib/aspera/cli/plugins/ats.rb +82 -76
- data/lib/aspera/cli/plugins/bss.rb +14 -16
- data/lib/aspera/cli/plugins/config.rb +350 -306
- data/lib/aspera/cli/plugins/console.rb +23 -19
- data/lib/aspera/cli/plugins/cos.rb +18 -18
- data/lib/aspera/cli/plugins/faspex.rb +180 -159
- data/lib/aspera/cli/plugins/faspex5.rb +64 -54
- data/lib/aspera/cli/plugins/node.rb +147 -140
- data/lib/aspera/cli/plugins/orchestrator.rb +68 -66
- data/lib/aspera/cli/plugins/preview.rb +92 -96
- data/lib/aspera/cli/plugins/server.rb +79 -75
- data/lib/aspera/cli/plugins/shares.rb +23 -24
- data/lib/aspera/cli/plugins/sync.rb +20 -22
- data/lib/aspera/cli/transfer_agent.rb +40 -39
- data/lib/aspera/cli/version.rb +2 -1
- data/lib/aspera/colors.rb +35 -27
- data/lib/aspera/command_line_builder.rb +48 -34
- data/lib/aspera/cos_node.rb +29 -21
- data/lib/aspera/data_repository.rb +3 -2
- data/lib/aspera/environment.rb +50 -45
- data/lib/aspera/fasp/agent_base.rb +22 -20
- data/lib/aspera/fasp/agent_connect.rb +13 -11
- data/lib/aspera/fasp/agent_direct.rb +48 -59
- data/lib/aspera/fasp/agent_httpgw.rb +33 -39
- data/lib/aspera/fasp/agent_node.rb +15 -13
- data/lib/aspera/fasp/agent_trsdk.rb +12 -14
- data/lib/aspera/fasp/error.rb +2 -1
- data/lib/aspera/fasp/error_info.rb +68 -52
- data/lib/aspera/fasp/installation.rb +106 -94
- data/lib/aspera/fasp/listener.rb +1 -0
- data/lib/aspera/fasp/parameters.rb +83 -92
- data/lib/aspera/fasp/parameters.yaml +305 -249
- data/lib/aspera/fasp/resume_policy.rb +11 -14
- data/lib/aspera/fasp/transfer_spec.rb +26 -0
- data/lib/aspera/fasp/uri.rb +22 -21
- data/lib/aspera/faspex_gw.rb +55 -90
- data/lib/aspera/hash_ext.rb +4 -3
- data/lib/aspera/id_generator.rb +8 -7
- data/lib/aspera/keychain/encrypted_hash.rb +17 -16
- data/lib/aspera/keychain/macos_security.rb +6 -10
- data/lib/aspera/log.rb +25 -20
- data/lib/aspera/nagios.rb +13 -12
- data/lib/aspera/node.rb +30 -22
- data/lib/aspera/oauth.rb +175 -226
- data/lib/aspera/open_application.rb +4 -3
- data/lib/aspera/persistency_action_once.rb +6 -6
- data/lib/aspera/persistency_folder.rb +5 -9
- data/lib/aspera/preview/file_types.rb +6 -5
- data/lib/aspera/preview/generator.rb +25 -24
- data/lib/aspera/preview/options.rb +16 -14
- data/lib/aspera/preview/utils.rb +98 -98
- data/lib/aspera/{proxy_auto_config.erb.js → proxy_auto_config.js} +23 -31
- data/lib/aspera/proxy_auto_config.rb +111 -20
- data/lib/aspera/rest.rb +115 -113
- data/lib/aspera/rest_call_error.rb +2 -2
- data/lib/aspera/rest_error_analyzer.rb +23 -25
- data/lib/aspera/rest_errors_aspera.rb +15 -14
- data/lib/aspera/ssh.rb +12 -10
- data/lib/aspera/sync.rb +42 -41
- data/lib/aspera/temp_file_manager.rb +18 -14
- data/lib/aspera/timer_limiter.rb +2 -1
- data/lib/aspera/uri_reader.rb +7 -5
- data/lib/aspera/web_auth.rb +79 -76
- metadata +64 -21
- data/docs/Makefile +0 -65
- data/docs/README.erb.md +0 -4424
- data/docs/README.md +0 -13
- data/docs/diagrams.txt +0 -49
- data/docs/doc_tools.rb +0 -58
- data/lib/aspera/cli/plugins/shares2.rb +0 -114
- data/lib/aspera/fasp/default.rb +0 -17
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require 'English'
|
|
1
3
|
require 'singleton'
|
|
2
4
|
require 'aspera/log'
|
|
3
5
|
require 'aspera/environment'
|
|
@@ -6,6 +8,7 @@ require 'xmlsimple'
|
|
|
6
8
|
require 'zlib'
|
|
7
9
|
require 'base64'
|
|
8
10
|
require 'fileutils'
|
|
11
|
+
require 'openssl'
|
|
9
12
|
|
|
10
13
|
module Aspera
|
|
11
14
|
module Fasp
|
|
@@ -18,6 +21,7 @@ module Aspera
|
|
|
18
21
|
# Installation.instance.ascp_path=""
|
|
19
22
|
class Installation
|
|
20
23
|
include Singleton
|
|
24
|
+
# known product names
|
|
21
25
|
PRODUCT_CONNECT='Aspera Connect'
|
|
22
26
|
PRODUCT_CLI_V1='Aspera CLI'
|
|
23
27
|
PRODUCT_DRIVE='Aspera Drive'
|
|
@@ -25,23 +29,48 @@ module Aspera
|
|
|
25
29
|
# protobuf generated files from sdk
|
|
26
30
|
EXT_RUBY_PROTOBUF='_pb.rb'
|
|
27
31
|
RB_SDK_FOLDER='lib'
|
|
28
|
-
|
|
29
|
-
|
|
32
|
+
ONE_YEAR_SECONDS=365 * 24 * 60 * 60
|
|
33
|
+
DEFAULT_ASPERA_CONF=<<~END_OF_CONFIG_FILE
|
|
34
|
+
<?xml version='1.0' encoding='UTF-8'?>
|
|
35
|
+
<CONF version="2">
|
|
36
|
+
<default>
|
|
37
|
+
<file_system>
|
|
38
|
+
<resume_suffix>.aspera-ckpt</resume_suffix>
|
|
39
|
+
<partial_file_suffix>.partial</partial_file_suffix>
|
|
40
|
+
</file_system>
|
|
41
|
+
</default>
|
|
42
|
+
</CONF>
|
|
43
|
+
END_OF_CONFIG_FILE
|
|
44
|
+
private_constant :PRODUCT_CONNECT,:PRODUCT_CLI_V1,:PRODUCT_DRIVE,:PRODUCT_ENTSRV,:EXT_RUBY_PROTOBUF,:RB_SDK_FOLDER,:ONE_YEAR_SECONDS,:DEFAULT_ASPERA_CONF
|
|
30
45
|
# set ascp executable path
|
|
31
46
|
def ascp_path=(v)
|
|
32
47
|
@path_to_ascp=v
|
|
33
48
|
end
|
|
34
49
|
|
|
50
|
+
def ascp_path
|
|
51
|
+
path(:ascp)
|
|
52
|
+
end
|
|
53
|
+
|
|
35
54
|
def sdk_ruby_folder
|
|
36
|
-
ruby_pb_folder=File.join(
|
|
55
|
+
ruby_pb_folder=File.join(sdk_folder,RB_SDK_FOLDER)
|
|
37
56
|
FileUtils.mkdir_p(ruby_pb_folder) unless Dir.exist?(ruby_pb_folder)
|
|
38
57
|
return ruby_pb_folder
|
|
39
58
|
end
|
|
40
59
|
|
|
41
60
|
# location of SDK files
|
|
42
|
-
def
|
|
43
|
-
@
|
|
44
|
-
|
|
61
|
+
def sdk_folder=(v)
|
|
62
|
+
@sdk_dir=v
|
|
63
|
+
sdk_folder
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# backward compatibility in sample program
|
|
67
|
+
alias folder= sdk_folder=
|
|
68
|
+
|
|
69
|
+
# @return the path to folder where SDK is installed
|
|
70
|
+
def sdk_folder
|
|
71
|
+
raise 'SDK path was ot initialized' if @sdk_dir.nil?
|
|
72
|
+
FileUtils.mkdir_p(@sdk_dir) unless Dir.exist?(@sdk_dir)
|
|
73
|
+
@sdk_dir
|
|
45
74
|
end
|
|
46
75
|
|
|
47
76
|
# find ascp in named product (use value : FIRST_FOUND='FIRST' to just use first one)
|
|
@@ -64,9 +93,9 @@ module Aspera
|
|
|
64
93
|
scan_locations=product_locations.clone
|
|
65
94
|
# add SDK as first search path
|
|
66
95
|
scan_locations.unshift({
|
|
67
|
-
:
|
|
68
|
-
:
|
|
69
|
-
:
|
|
96
|
+
expected: 'SDK',
|
|
97
|
+
app_root: sdk_folder,
|
|
98
|
+
sub_bin: ''
|
|
70
99
|
})
|
|
71
100
|
# search installed products: with ascp
|
|
72
101
|
@found_products=scan_locations.select! do |item|
|
|
@@ -107,45 +136,36 @@ module Aspera
|
|
|
107
136
|
when :transferd
|
|
108
137
|
file=transferd_filepath
|
|
109
138
|
when :ssh_bypass_key_dsa
|
|
110
|
-
file=File.join(
|
|
139
|
+
file=File.join(sdk_folder,'aspera_bypass_dsa.pem')
|
|
111
140
|
File.write(file,get_key('dsa',1)) unless File.exist?(file)
|
|
112
141
|
File.chmod(0400,file)
|
|
113
142
|
when :ssh_bypass_key_rsa
|
|
114
|
-
file=File.join(
|
|
143
|
+
file=File.join(sdk_folder,'aspera_bypass_rsa.pem')
|
|
115
144
|
File.write(file,get_key('rsa',2)) unless File.exist?(file)
|
|
116
145
|
File.chmod(0400,file)
|
|
117
146
|
when :aspera_license
|
|
118
|
-
file=File.join(
|
|
119
|
-
File.write(file,Base64.strict_encode64("#{Zlib::Inflate.inflate(DataRepository.instance.
|
|
147
|
+
file=File.join(sdk_folder,'aspera-license')
|
|
148
|
+
File.write(file,Base64.strict_encode64("#{Zlib::Inflate.inflate(DataRepository.instance.data(6))}==SIGNATURE==\n#{Base64.strict_encode64(DataRepository.instance.data(7))}")) unless File.exist?(file)
|
|
120
149
|
File.chmod(0400,file)
|
|
121
150
|
when :aspera_conf
|
|
122
|
-
file=File.join(
|
|
123
|
-
File.write(file
|
|
124
|
-
<CONF version="2">
|
|
125
|
-
<default>
|
|
126
|
-
<file_system>
|
|
127
|
-
<resume_suffix>.aspera-ckpt</resume_suffix>
|
|
128
|
-
<partial_file_suffix>.partial</partial_file_suffix>
|
|
129
|
-
</file_system>
|
|
130
|
-
</default>
|
|
131
|
-
</CONF>
|
|
132
|
-
}) unless File.exist?(file)
|
|
151
|
+
file=File.join(sdk_folder,'aspera.conf')
|
|
152
|
+
File.write(file,DEFAULT_ASPERA_CONF) unless File.exist?(file)
|
|
133
153
|
File.chmod(0400,file)
|
|
134
154
|
when :fallback_cert,:fallback_key
|
|
135
|
-
file_key=File.join(
|
|
136
|
-
file_cert=File.join(
|
|
137
|
-
if !File.exist?(file_key)
|
|
155
|
+
file_key=File.join(sdk_folder,'aspera_fallback_key.pem')
|
|
156
|
+
file_cert=File.join(sdk_folder,'aspera_fallback_cert.pem')
|
|
157
|
+
if !File.exist?(file_key) || !File.exist?(file_cert)
|
|
138
158
|
require 'openssl'
|
|
139
159
|
# create new self signed certificate for http fallback
|
|
140
160
|
private_key = OpenSSL::PKey::RSA.new(1024)
|
|
141
161
|
cert = OpenSSL::X509::Certificate.new
|
|
142
|
-
cert.subject = cert.issuer = OpenSSL::X509::Name.parse(
|
|
162
|
+
cert.subject = cert.issuer = OpenSSL::X509::Name.parse('/C=US/ST=California/L=Emeryville/O=Aspera Inc./OU=Corporate/CN=Aspera Inc./emailAddress=info@asperasoft.com')
|
|
143
163
|
cert.not_before = Time.now
|
|
144
|
-
cert.not_after = Time.now +
|
|
164
|
+
cert.not_after = Time.now + ONE_YEAR_SECONDS
|
|
145
165
|
cert.public_key = private_key.public_key
|
|
146
166
|
cert.serial = 0x0
|
|
147
167
|
cert.version = 2
|
|
148
|
-
cert.sign(private_key, OpenSSL::Digest
|
|
168
|
+
cert.sign(private_key, OpenSSL::Digest.new('SHA1'))
|
|
149
169
|
File.write(file_key,private_key.to_pem)
|
|
150
170
|
File.write(file_cert,cert.to_pem)
|
|
151
171
|
File.chmod(0400,file_key)
|
|
@@ -167,7 +187,7 @@ module Aspera
|
|
|
167
187
|
uri_file=File.join(folder,"http#{ext}.uri")
|
|
168
188
|
Log.log.debug("checking connect port file: #{uri_file}")
|
|
169
189
|
if File.exist?(uri_file)
|
|
170
|
-
return File.open(uri_file)
|
|
190
|
+
return File.open(uri_file, &:gets).strip
|
|
171
191
|
end
|
|
172
192
|
end
|
|
173
193
|
raise "no connect uri file found in #{folder}"
|
|
@@ -181,7 +201,7 @@ module Aspera
|
|
|
181
201
|
|
|
182
202
|
# default bypass key phrase
|
|
183
203
|
def bypass_pass
|
|
184
|
-
return
|
|
204
|
+
return format('%08x-%04x-%04x-%04x-%04x%08x',*DataRepository.instance.data(3).unpack('NnnnnN'))
|
|
185
205
|
end
|
|
186
206
|
|
|
187
207
|
def bypass_keys
|
|
@@ -195,13 +215,13 @@ module Aspera
|
|
|
195
215
|
|
|
196
216
|
# Check that specified path is ascp and get version
|
|
197
217
|
def get_exe_version(exe_path,vers_arg)
|
|
198
|
-
raise
|
|
218
|
+
raise 'ERROR: nil arg' if exe_path.nil?
|
|
199
219
|
return nil unless File.exist?(exe_path)
|
|
200
220
|
exe_version=nil
|
|
201
|
-
cmd_out=%x
|
|
202
|
-
raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless
|
|
221
|
+
cmd_out=%x("#{exe_path}" #{vers_arg})
|
|
222
|
+
raise "An error occured when testing #{ascp_filename}: #{cmd_out}" unless $CHILD_STATUS == 0
|
|
203
223
|
# get version from ascp, only after full extract, as windows requires DLLs (SSL/TLS/etc...)
|
|
204
|
-
m=cmd_out.match(/ version ([0-9
|
|
224
|
+
m=cmd_out.match(/ version ([0-9.]+)/)
|
|
205
225
|
exe_version=m[1] unless m.nil?
|
|
206
226
|
return exe_version
|
|
207
227
|
end
|
|
@@ -220,9 +240,9 @@ module Aspera
|
|
|
220
240
|
Aspera::Rest.new(base_url: sdk_url, redirect_max: 3).call(operation: 'GET',save_to_file: sdk_zip_path)
|
|
221
241
|
end
|
|
222
242
|
# rename old install
|
|
223
|
-
if !
|
|
224
|
-
Log.log.warn(
|
|
225
|
-
File.rename(
|
|
243
|
+
if !Dir.empty?(sdk_folder)
|
|
244
|
+
Log.log.warn('Previous install exists, renaming folder.')
|
|
245
|
+
File.rename(sdk_folder,"#{sdk_folder}.#{Time.now.strftime('%Y%m%d%H%M%S')}")
|
|
226
246
|
# TODO: delete old archives ?
|
|
227
247
|
end
|
|
228
248
|
# SDK is organized by architecture
|
|
@@ -234,31 +254,30 @@ module Aspera
|
|
|
234
254
|
next if entry.name.end_with?('/')
|
|
235
255
|
dest_folder=nil
|
|
236
256
|
# binaries
|
|
237
|
-
dest_folder=
|
|
257
|
+
dest_folder=sdk_folder if entry.name.include?(arch_filter)
|
|
238
258
|
# ruby adapters
|
|
239
259
|
dest_folder=sdk_ruby_folder if entry.name.end_with?(EXT_RUBY_PROTOBUF)
|
|
240
|
-
if
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
end
|
|
260
|
+
next if dest_folder.nil?
|
|
261
|
+
File.open(File.join(dest_folder,File.basename(entry.name)), 'wb') do |output_stream|
|
|
262
|
+
IO.copy_stream(entry.get_input_stream, output_stream)
|
|
244
263
|
end
|
|
245
264
|
end
|
|
246
265
|
end
|
|
247
266
|
File.unlink(sdk_zip_path) rescue nil # Windows may give error
|
|
248
267
|
# ensure license file are generated so that ascp invokation for version works
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
ascp_path=File.join(
|
|
268
|
+
path(:aspera_license)
|
|
269
|
+
path(:aspera_conf)
|
|
270
|
+
ascp_path=File.join(sdk_folder,ascp_filename)
|
|
252
271
|
raise "No #{ascp_filename} found in SDK archive" unless File.exist?(ascp_path)
|
|
253
272
|
FileUtils.chmod(0755,ascp_path)
|
|
254
273
|
FileUtils.chmod(0755,ascp_path.gsub('ascp','ascp4'))
|
|
255
|
-
ascp_version=get_ascp_version(File.join(
|
|
274
|
+
ascp_version=get_ascp_version(File.join(sdk_folder,ascp_filename))
|
|
256
275
|
trd_path=transferd_filepath
|
|
257
276
|
Log.log.warn("No #{trd_path} in SDK archive") unless File.exist?(trd_path)
|
|
258
277
|
FileUtils.chmod(0755,trd_path) if File.exist?(trd_path)
|
|
259
278
|
transferd_version=get_exe_version(trd_path,'version')
|
|
260
279
|
sdk_version = transferd_version||ascp_version
|
|
261
|
-
File.write(File.join(
|
|
280
|
+
File.write(File.join(sdk_folder,PRODUCT_INFO),"<product><name>IBM Aspera SDK</name><version>#{sdk_version}</version></product>")
|
|
262
281
|
return sdk_version
|
|
263
282
|
end
|
|
264
283
|
|
|
@@ -276,7 +295,7 @@ module Aspera
|
|
|
276
295
|
|
|
277
296
|
def initialize
|
|
278
297
|
@path_to_ascp=nil
|
|
279
|
-
@
|
|
298
|
+
@sdk_dir=nil
|
|
280
299
|
@found_products=nil
|
|
281
300
|
end
|
|
282
301
|
|
|
@@ -288,20 +307,13 @@ module Aspera
|
|
|
288
307
|
return found.first
|
|
289
308
|
end
|
|
290
309
|
|
|
291
|
-
# @return the path to folder where SDK is installed
|
|
292
|
-
def folder_path
|
|
293
|
-
raise "Undefined path to SDK" if @sdk_folder.nil?
|
|
294
|
-
FileUtils.mkdir_p(@sdk_folder) unless Dir.exist?(@sdk_folder)
|
|
295
|
-
@sdk_folder
|
|
296
|
-
end
|
|
297
|
-
|
|
298
310
|
# filename for ascp with optional extension (Windows)
|
|
299
311
|
def ascp_filename
|
|
300
312
|
return 'ascp'+Environment.exe_extension
|
|
301
313
|
end
|
|
302
314
|
|
|
303
315
|
def transferd_filepath
|
|
304
|
-
return File.join(
|
|
316
|
+
return File.join(sdk_folder,'asperatransferd'+Environment.exe_extension)
|
|
305
317
|
end
|
|
306
318
|
|
|
307
319
|
# @return product folders depending on OS fields
|
|
@@ -313,49 +325,49 @@ module Aspera
|
|
|
313
325
|
def product_locations
|
|
314
326
|
case Aspera::Environment.os
|
|
315
327
|
when Aspera::Environment::OS_WINDOWS; return [{
|
|
316
|
-
:
|
|
317
|
-
:
|
|
318
|
-
:
|
|
319
|
-
:
|
|
328
|
+
expected: PRODUCT_CONNECT,
|
|
329
|
+
app_root: File.join(ENV['LOCALAPPDATA'],'Programs','Aspera','Aspera Connect'),
|
|
330
|
+
log_root: File.join(ENV['LOCALAPPDATA'],'Aspera','Aspera Connect','var','log'),
|
|
331
|
+
run_root: File.join(ENV['LOCALAPPDATA'],'Aspera','Aspera Connect')
|
|
320
332
|
},{
|
|
321
|
-
:
|
|
322
|
-
:
|
|
323
|
-
:
|
|
333
|
+
expected: PRODUCT_CLI_V1,
|
|
334
|
+
app_root: File.join('C:','Program Files','Aspera','cli'),
|
|
335
|
+
log_root: File.join('C:','Program Files','Aspera','cli','var','log')
|
|
324
336
|
},{
|
|
325
|
-
:
|
|
326
|
-
:
|
|
327
|
-
:
|
|
337
|
+
expected: PRODUCT_ENTSRV,
|
|
338
|
+
app_root: File.join('C:','Program Files','Aspera','Enterprise Server'),
|
|
339
|
+
log_root: File.join('C:','Program Files','Aspera','Enterprise Server','var','log')
|
|
328
340
|
}]
|
|
329
341
|
when Aspera::Environment::OS_X; return [{
|
|
330
|
-
:
|
|
331
|
-
:
|
|
332
|
-
:
|
|
333
|
-
:
|
|
334
|
-
:
|
|
342
|
+
expected: PRODUCT_CONNECT,
|
|
343
|
+
app_root: File.join(Dir.home,'Applications','Aspera Connect.app'),
|
|
344
|
+
log_root: File.join(Dir.home,'Library','Logs','Aspera_Connect'),
|
|
345
|
+
run_root: File.join(Dir.home,'Library','Application Support','Aspera','Aspera Connect'),
|
|
346
|
+
sub_bin: File.join('Contents','Resources')
|
|
335
347
|
},{
|
|
336
|
-
:
|
|
337
|
-
:
|
|
338
|
-
:
|
|
348
|
+
expected: PRODUCT_CLI_V1,
|
|
349
|
+
app_root: File.join(Dir.home,'Applications','Aspera CLI'),
|
|
350
|
+
log_root: File.join(Dir.home,'Library','Logs','Aspera')
|
|
339
351
|
},{
|
|
340
|
-
:
|
|
341
|
-
:
|
|
342
|
-
:
|
|
352
|
+
expected: PRODUCT_ENTSRV,
|
|
353
|
+
app_root: File.join('','Library','Aspera'),
|
|
354
|
+
log_root: File.join(Dir.home,'Library','Logs','Aspera')
|
|
343
355
|
},{
|
|
344
|
-
:
|
|
345
|
-
:
|
|
346
|
-
:
|
|
347
|
-
:
|
|
356
|
+
expected: PRODUCT_DRIVE,
|
|
357
|
+
app_root: File.join('','Applications','Aspera Drive.app'),
|
|
358
|
+
log_root: File.join(Dir.home,'Library','Logs','Aspera_Drive'),
|
|
359
|
+
sub_bin: File.join('Contents','Resources')
|
|
348
360
|
}]
|
|
349
|
-
else; return [{
|
|
350
|
-
:
|
|
351
|
-
:
|
|
352
|
-
:
|
|
361
|
+
else; return [{ # other: Linux and Unix family
|
|
362
|
+
expected: PRODUCT_CONNECT,
|
|
363
|
+
app_root: File.join(Dir.home,'.aspera','connect'),
|
|
364
|
+
run_root: File.join(Dir.home,'.aspera','connect')
|
|
353
365
|
},{
|
|
354
|
-
:
|
|
355
|
-
:
|
|
366
|
+
expected: PRODUCT_CLI_V1,
|
|
367
|
+
app_root: File.join(Dir.home,'.aspera','cli')
|
|
356
368
|
},{
|
|
357
|
-
:
|
|
358
|
-
:
|
|
369
|
+
expected: PRODUCT_ENTSRV,
|
|
370
|
+
app_root: File.join('','opt','aspera')
|
|
359
371
|
}]
|
|
360
372
|
end
|
|
361
373
|
end
|
|
@@ -365,10 +377,10 @@ module Aspera
|
|
|
365
377
|
# @param id in repository 1 for dsa, 2 for rsa
|
|
366
378
|
def get_key(type,id)
|
|
367
379
|
hf=['begin','end'].map{|t|"-----#{t} #{type} private key-----".upcase}
|
|
368
|
-
bin=Base64.strict_encode64(DataRepository.instance.
|
|
369
|
-
|
|
380
|
+
bin=Base64.strict_encode64(DataRepository.instance.data(id))
|
|
381
|
+
# validate valie and generate key in connonical format
|
|
382
|
+
OpenSSL::PKey.const_get(type.upcase).send(:new,hf.insert(1,bin).join("\n")).to_pem
|
|
370
383
|
end
|
|
371
|
-
|
|
372
384
|
end # Installation
|
|
373
385
|
end
|
|
374
386
|
end
|
data/lib/aspera/fasp/listener.rb
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
1
2
|
require 'aspera/log'
|
|
2
3
|
require 'aspera/command_line_builder'
|
|
3
4
|
require 'aspera/temp_file_manager'
|
|
@@ -5,7 +6,6 @@ require 'securerandom'
|
|
|
5
6
|
require 'base64'
|
|
6
7
|
require 'json'
|
|
7
8
|
require 'yaml'
|
|
8
|
-
require 'securerandom'
|
|
9
9
|
require 'fileutils'
|
|
10
10
|
require 'openssl'
|
|
11
11
|
|
|
@@ -13,68 +13,79 @@ module Aspera
|
|
|
13
13
|
module Fasp
|
|
14
14
|
# translate transfer specification to ascp parameter list
|
|
15
15
|
class Parameters
|
|
16
|
-
private
|
|
17
|
-
# Temp folder for file lists, must contain only file lists
|
|
18
|
-
# because of garbage collection takes any file there
|
|
19
|
-
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
|
20
|
-
@@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
|
|
21
|
-
@@param_description_cache=nil
|
|
22
|
-
# @return normaiwed description of transfer spec parameters
|
|
23
|
-
def self.description
|
|
24
|
-
return @@param_description_cache unless @@param_description_cache.nil?
|
|
25
|
-
# config file in same folder with same name as this source
|
|
26
|
-
@@param_description_cache=YAML.load_file("#{__FILE__[0..-3]}yaml")
|
|
27
|
-
Aspera::CommandLineBuilder.normalize_description(@@param_description_cache)
|
|
28
|
-
end
|
|
29
|
-
|
|
30
16
|
# Agents shown in manual for parameters (sub list)
|
|
31
17
|
SUPPORTED_AGENTS=[:direct,:node,:connect]
|
|
32
18
|
# Short names of columns in manual
|
|
33
19
|
SUPPORTED_AGENTS_SHORT=SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
|
34
20
|
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
#
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
21
|
+
class << self
|
|
22
|
+
# Temp folder for file lists, must contain only file lists
|
|
23
|
+
# because of garbage collection takes any file there
|
|
24
|
+
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
|
25
|
+
@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
|
|
26
|
+
@param_description_cache=nil
|
|
27
|
+
# @return normalized description of transfer spec parameters, direct from yaml
|
|
28
|
+
def description
|
|
29
|
+
return @param_description_cache unless @param_description_cache.nil?
|
|
30
|
+
# config file in same folder with same name as this source
|
|
31
|
+
@param_description_cache=YAML.load_file("#{__FILE__[0..-3]}yaml")
|
|
32
|
+
Aspera::CommandLineBuilder.normalize_description(@param_description_cache)
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# @return a table suitable to display in manual
|
|
36
|
+
def man_table
|
|
37
|
+
result=[]
|
|
38
|
+
description.each do |k,i|
|
|
39
|
+
param={name: k, type: [i[:accepted_types]].flatten.join(','),description: i[:desc]}
|
|
40
|
+
SUPPORTED_AGENTS.each do |a|
|
|
41
|
+
param[a.to_s[0].to_sym]=i[:tragents].nil? || i[:tragents].include?(a) ? 'Y' : ''
|
|
42
|
+
end
|
|
43
|
+
# only keep lines that are usable in supported agents
|
|
44
|
+
next if SUPPORTED_AGENTS_SHORT.inject(true){|m,j|m && param[j].empty?}
|
|
45
|
+
param[:cli]=
|
|
46
|
+
case i[:cltype]
|
|
47
|
+
when :envvar then 'env:'+i[:clvarname]
|
|
48
|
+
when :opt_without_arg,:opt_with_arg then i[:clswitch]
|
|
49
|
+
else ''
|
|
50
|
+
end
|
|
51
|
+
if i.has_key?(:enum)
|
|
52
|
+
param[:description] += "\nAllowed values: #{i[:enum].join(', ')}"
|
|
53
|
+
end
|
|
54
|
+
result.push(param)
|
|
53
55
|
end
|
|
54
|
-
result
|
|
56
|
+
return result
|
|
55
57
|
end
|
|
56
|
-
return result
|
|
57
|
-
end
|
|
58
58
|
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
v.tr('-','')
|
|
62
|
-
end
|
|
59
|
+
# special encoding methods used in YAML (key: :clconvert)
|
|
60
|
+
def clconv_remove_hyphen(v); v.tr('-',''); end
|
|
63
61
|
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
Base64.strict_encode64(v)
|
|
67
|
-
end
|
|
62
|
+
# special encoding methods used in YAML (key: :clconvert)
|
|
63
|
+
def clconv_json64(v); Base64.strict_encode64(JSON.generate(v)); end
|
|
68
64
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
Base64.strict_encode64(JSON.generate(v))
|
|
72
|
-
end
|
|
65
|
+
# special encoding methods used in YAML (key: :clconvert)
|
|
66
|
+
def clconv_base64(v); Base64.strict_encode64(v); end
|
|
73
67
|
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
68
|
+
def ts_has_file_list(ts)
|
|
69
|
+
ts.has_key?('EX_ascp_args') and ts['EX_ascp_args'].is_a?(Array) and ['--file-list','--file-pair-list'].any?{|i|ts['EX_ascp_args'].include?(i)}
|
|
70
|
+
end
|
|
77
71
|
|
|
72
|
+
def ts_to_env_args(transfer_spec,options)
|
|
73
|
+
return Parameters.new(transfer_spec,options).ascp_args()
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
# temp file list files are created here
|
|
77
|
+
def file_list_folder=(v)
|
|
78
|
+
@file_list_folder=v
|
|
79
|
+
return if @file_list_folder.nil?
|
|
80
|
+
FileUtils.mkdir_p(@file_list_folder)
|
|
81
|
+
TempFileManager.instance.cleanup_expired(@file_list_folder)
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# static methods
|
|
85
|
+
attr_reader :file_list_folder
|
|
86
|
+
end # self
|
|
87
|
+
|
|
88
|
+
# @param options [Hash] key: :wss: bool
|
|
78
89
|
def initialize(job_spec,options)
|
|
79
90
|
@job_spec=job_spec
|
|
80
91
|
@options=options
|
|
@@ -82,28 +93,26 @@ module Aspera
|
|
|
82
93
|
Log.log.debug("agent options: #{@options}")
|
|
83
94
|
end
|
|
84
95
|
|
|
85
|
-
public
|
|
86
|
-
|
|
87
96
|
# translate transfer spec to env vars and command line arguments for ascp
|
|
88
97
|
# NOTE: parameters starting with "EX_" (extended) are not standard
|
|
89
|
-
def ascp_args
|
|
98
|
+
def ascp_args
|
|
90
99
|
env_args={
|
|
91
|
-
:
|
|
92
|
-
:
|
|
93
|
-
:
|
|
100
|
+
args: [],
|
|
101
|
+
env: {},
|
|
102
|
+
ascp_version: :ascp
|
|
94
103
|
}
|
|
95
104
|
# some ssh credentials are required to avoid interactive password input
|
|
96
|
-
if !@job_spec.has_key?('remote_password')
|
|
97
|
-
!@job_spec.has_key?('ssh_private_key')
|
|
98
|
-
!@job_spec.has_key?('EX_ssh_key_paths')
|
|
99
|
-
raise Fasp::Error
|
|
105
|
+
if !@job_spec.has_key?('remote_password') &&
|
|
106
|
+
!@job_spec.has_key?('ssh_private_key') &&
|
|
107
|
+
!@job_spec.has_key?('EX_ssh_key_paths')
|
|
108
|
+
raise Fasp::Error, 'required: password or ssh key (value or path)'
|
|
100
109
|
end
|
|
101
110
|
|
|
102
111
|
# special cases
|
|
103
|
-
@job_spec.delete('source_root') if @job_spec.has_key?('source_root')
|
|
112
|
+
@job_spec.delete('source_root') if @job_spec.has_key?('source_root') && @job_spec['source_root'].empty?
|
|
104
113
|
|
|
105
114
|
# use web socket session initiation ?
|
|
106
|
-
if @builder.process_param('wss_enabled',:get_value)
|
|
115
|
+
if @builder.process_param('wss_enabled',:get_value) && (@options[:wss] || !@job_spec.has_key?('fasp_port'))
|
|
107
116
|
# by default use web socket session if available, unless removed by user
|
|
108
117
|
@builder.add_command_line_options(['--ws-connect'])
|
|
109
118
|
# TODO: option to give order ssh,ws (legacy http is implied bu ssh)
|
|
@@ -112,9 +121,9 @@ module Aspera
|
|
|
112
121
|
@job_spec.delete('fasp_port')
|
|
113
122
|
@job_spec.delete('EX_ssh_key_paths')
|
|
114
123
|
@job_spec.delete('sshfp')
|
|
115
|
-
# set
|
|
124
|
+
# set location for CA bundle to be the one of Ruby, see env var SSL_CERT_FILE / SSL_CERT_DIR
|
|
116
125
|
@job_spec['EX_ssh_key_paths']=[OpenSSL::X509::DEFAULT_CERT_FILE]
|
|
117
|
-
Log.log.debug(
|
|
126
|
+
Log.log.debug('CA certs: EX_ssh_key_paths <- DEFAULT_CERT_FILE from openssl')
|
|
118
127
|
else
|
|
119
128
|
# remove unused parameter (avoid warning)
|
|
120
129
|
@job_spec.delete('wss_port')
|
|
@@ -135,17 +144,17 @@ module Aspera
|
|
|
135
144
|
file_list_provided=self.class.ts_has_file_list(@job_spec)
|
|
136
145
|
@builder.params_definition['paths'][:mandatory]=!@job_spec.has_key?('keepalive') and !file_list_provided
|
|
137
146
|
paths_array=@builder.process_param('paths',:get_value)
|
|
138
|
-
if file_list_provided
|
|
139
|
-
Log.log.warn(
|
|
147
|
+
if file_list_provided && !paths_array.nil?
|
|
148
|
+
Log.log.warn('file list provided both in transfer spec and ascp file list. Keeping file list only.')
|
|
140
149
|
paths_array=nil
|
|
141
150
|
end
|
|
142
|
-
if !
|
|
151
|
+
if !paths_array.nil?
|
|
143
152
|
# it's an array
|
|
144
|
-
raise
|
|
153
|
+
raise 'paths is empty in transfer spec' if paths_array.empty?
|
|
145
154
|
# use file list if there is storage defined for it.
|
|
146
|
-
if
|
|
155
|
+
if self.class.file_list_folder.nil?
|
|
147
156
|
# not safe for special characters ? (maybe not, depends on OS)
|
|
148
|
-
Log.log.debug(
|
|
157
|
+
Log.log.debug('placing source file list on command line (no file list file)')
|
|
149
158
|
@builder.add_command_line_options(paths_array.map{|i|i['source']})
|
|
150
159
|
else
|
|
151
160
|
file_list_file=@builder.process_param('EX_file_list',:get_value)
|
|
@@ -161,13 +170,13 @@ module Aspera
|
|
|
161
170
|
# TODO: well, we test only the first one, but anyway it shall be consistent
|
|
162
171
|
if paths_array.first.has_key?('destination')
|
|
163
172
|
option='--file-pair-list'
|
|
164
|
-
lines=paths_array.
|
|
173
|
+
lines=paths_array.each_with_object([]){|e,m|m.push(e['source'],e['destination']);}
|
|
165
174
|
else
|
|
166
175
|
option='--file-list'
|
|
167
176
|
lines=paths_array.map{|i|i['source']}
|
|
168
177
|
end
|
|
169
|
-
file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(
|
|
170
|
-
File.
|
|
178
|
+
file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
|
179
|
+
File.write(file_list_file, lines.join("\n"))
|
|
171
180
|
Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
|
|
172
181
|
end
|
|
173
182
|
end
|
|
@@ -187,24 +196,6 @@ module Aspera
|
|
|
187
196
|
|
|
188
197
|
return env_args
|
|
189
198
|
end
|
|
190
|
-
|
|
191
|
-
# temp file list files are created here
|
|
192
|
-
def self.file_list_folder=(v)
|
|
193
|
-
@@file_list_folder=v
|
|
194
|
-
if !@@file_list_folder.nil?
|
|
195
|
-
FileUtils.mkdir_p(@@file_list_folder)
|
|
196
|
-
TempFileManager.instance.cleanup_expired(@@file_list_folder)
|
|
197
|
-
end
|
|
198
|
-
end
|
|
199
|
-
|
|
200
|
-
# static methods
|
|
201
|
-
class << self
|
|
202
|
-
def file_list_folder; @@file_list_folder;end
|
|
203
|
-
|
|
204
|
-
def ts_to_env_args(transfer_spec,options)
|
|
205
|
-
return Parameters.new(transfer_spec,options).ascp_args()
|
|
206
|
-
end
|
|
207
|
-
end
|
|
208
199
|
end # Parameters
|
|
209
200
|
end
|
|
210
201
|
end
|