aspera-cli 4.8.0 → 4.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- checksums.yaml.gz.sig +0 -0
- data/README.md +445 -160
- data/docs/test_env.conf +1 -5
- data/examples/dascli +1 -4
- data/examples/{transfer.rb → node.rb} +17 -46
- data/examples/server.rb +93 -0
- data/lib/aspera/aoc.rb +4 -2
- data/lib/aspera/ats_api.rb +3 -1
- data/lib/aspera/cli/extended_value.rb +1 -0
- data/lib/aspera/cli/formater.rb +3 -1
- data/lib/aspera/cli/info.rb +1 -1
- data/lib/aspera/cli/main.rb +14 -11
- data/lib/aspera/cli/manager.rb +4 -4
- data/lib/aspera/cli/plugin.rb +50 -9
- data/lib/aspera/cli/plugins/aoc.rb +88 -52
- data/lib/aspera/cli/plugins/config.rb +5 -0
- data/lib/aspera/cli/plugins/faspex.rb +5 -4
- data/lib/aspera/cli/plugins/node.rb +3 -2
- data/lib/aspera/cli/plugins/server.rb +7 -108
- data/lib/aspera/cli/plugins/shares.rb +21 -1
- data/lib/aspera/cli/transfer_agent.rb +21 -14
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/environment.rb +15 -2
- data/lib/aspera/fasp/agent_base.rb +9 -7
- data/lib/aspera/fasp/installation.rb +15 -19
- data/lib/aspera/fasp/parameters.rb +38 -30
- data/lib/aspera/fasp/parameters.yaml +69 -17
- data/lib/aspera/hash_ext.rb +14 -2
- data/lib/aspera/id_generator.rb +12 -10
- data/lib/aspera/keychain/encrypted_hash.rb +3 -3
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/nagios.rb +26 -19
- data/lib/aspera/oauth.rb +4 -4
- data/lib/aspera/open_application.rb +21 -19
- data/lib/aspera/persistency_folder.rb +3 -0
- data/lib/aspera/preview/image_error.png +0 -0
- data/lib/aspera/preview/video_error.png +0 -0
- data/lib/aspera/proxy_auto_config.rb +6 -4
- data/lib/aspera/rest_error_analyzer.rb +15 -13
- data/lib/aspera/rest_errors_aspera.rb +42 -40
- data/lib/aspera/secret_hider.rb +11 -5
- data/lib/aspera/ssh.rb +1 -0
- data/lib/aspera/uri_reader.rb +15 -13
- data.tar.gz.sig +0 -0
- metadata +4 -3
- metadata.gz.sig +0 -0
@@ -14,6 +14,7 @@ module Aspera
|
|
14
14
|
FILE_LIST_FROM_ARGS = '@args'
|
15
15
|
# special value for --sources : read file list from transfer spec (--ts)
|
16
16
|
FILE_LIST_FROM_TRANSFER_SPEC = '@ts'
|
17
|
+
FILE_LIST_OPTIONS=[FILE_LIST_FROM_ARGS,FILE_LIST_FROM_TRANSFER_SPEC,'Array'].freeze
|
17
18
|
DEFAULT_TRANSFER_NOTIF_TMPL = <<~END_OF_TEMPLATE
|
18
19
|
From: <%=from_name%> <<%=from_email%>>
|
19
20
|
To: <<%=to%>>
|
@@ -24,9 +25,20 @@ module Aspera
|
|
24
25
|
<%=ts.to_yaml%>
|
25
26
|
END_OF_TEMPLATE
|
26
27
|
#% (formating bug in eclipse)
|
27
|
-
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:
|
28
|
+
private_constant :FILE_LIST_FROM_ARGS,:FILE_LIST_FROM_TRANSFER_SPEC,:FILE_LIST_OPTIONS,
|
29
|
+
:DEFAULT_TRANSFER_NOTIF_TMPL
|
28
30
|
TRANSFER_AGENTS = %i[direct node connect httpgw trsdk].freeze
|
29
31
|
|
32
|
+
class << self
|
33
|
+
# @return :success if all sessions statuses returned by "start" are success
|
34
|
+
# else return the first error exception object
|
35
|
+
def session_status(statuses)
|
36
|
+
error_statuses = statuses.reject{|i|i.eql?(:success)}
|
37
|
+
return :success if error_statuses.empty?
|
38
|
+
return error_statuses.first
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
30
42
|
# @param env external objects: option manager, config file manager
|
31
43
|
def initialize(opt_mgr,config)
|
32
44
|
@opt_mgr = opt_mgr
|
@@ -41,11 +53,11 @@ module Aspera
|
|
41
53
|
@opt_mgr.set_obj_attr(:ts,self,:option_transfer_spec)
|
42
54
|
@opt_mgr.add_opt_simple(:ts,"override transfer spec values (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:ts)}")
|
43
55
|
@opt_mgr.add_opt_simple(:local_resume,"set resume policy (Hash, use @json: prefix), current=#{@opt_mgr.get_option(:local_resume)}")
|
44
|
-
@opt_mgr.add_opt_simple(:to_folder,'destination folder for
|
45
|
-
@opt_mgr.add_opt_simple(:sources,
|
46
|
-
@opt_mgr.add_opt_simple(:transfer_info,'parameters for transfer agent')
|
56
|
+
@opt_mgr.add_opt_simple(:to_folder,'destination folder for transfered files')
|
57
|
+
@opt_mgr.add_opt_simple(:sources,"how list of transfered files is provided (#{FILE_LIST_OPTIONS.join(',')})")
|
47
58
|
@opt_mgr.add_opt_list(:src_type,%i[list pair],'type of file list')
|
48
59
|
@opt_mgr.add_opt_list(:transfer,TRANSFER_AGENTS,'type of transfer agent')
|
60
|
+
@opt_mgr.add_opt_simple(:transfer_info,'parameters for transfer agent')
|
49
61
|
@opt_mgr.add_opt_list(:progress,%i[none native multi],'type of progress bar')
|
50
62
|
@opt_mgr.set_option(:transfer,:direct)
|
51
63
|
@opt_mgr.set_option(:src_type,:list)
|
@@ -102,7 +114,8 @@ module Aspera
|
|
102
114
|
# param: 'send' or 'receive'
|
103
115
|
def destination_folder(direction)
|
104
116
|
dest_folder = @opt_mgr.get_option(:to_folder)
|
105
|
-
|
117
|
+
# do not expand path, if user wants to expand path: user @path:
|
118
|
+
return dest_folder unless dest_folder.nil?
|
106
119
|
dest_folder = @transfer_spec_cmdline['destination_root']
|
107
120
|
return dest_folder unless dest_folder.nil?
|
108
121
|
# default: / on remote, . on local
|
@@ -134,7 +147,9 @@ module Aspera
|
|
134
147
|
"--sources=#{FILE_LIST_FROM_TRANSFER_SPEC} to use transfer spec" if !file_list.is_a?(Array) || file_list.empty?
|
135
148
|
when FILE_LIST_FROM_TRANSFER_SPEC
|
136
149
|
Log.log.debug('assume list provided in transfer spec')
|
137
|
-
|
150
|
+
special_case_direct_with_list =
|
151
|
+
@opt_mgr.get_option(:transfer,is_type: :mandatory).eql?(:direct) &&
|
152
|
+
Fasp::Parameters.ts_has_ascp_file_list(@transfer_spec_cmdline)
|
138
153
|
raise CliBadArgument,'transfer spec on command line must have sources' if @transfer_paths.nil? && !special_case_direct_with_list
|
139
154
|
# here we assume check of sources is made in transfer agent
|
140
155
|
return @transfer_paths
|
@@ -222,14 +237,6 @@ module Aspera
|
|
222
237
|
@config.send_email_template(email_vars,DEFAULT_TRANSFER_NOTIF_TMPL)
|
223
238
|
end
|
224
239
|
|
225
|
-
# @return :success if all sessions statuses returned by "start" are success
|
226
|
-
# else return the first error exception object
|
227
|
-
def self.session_status(statuses)
|
228
|
-
error_statuses = statuses.reject{|i|i.eql?(:success)}
|
229
|
-
return :success if error_statuses.empty?
|
230
|
-
return error_statuses.first
|
231
|
-
end
|
232
|
-
|
233
240
|
# shut down if agent requires it
|
234
241
|
def shutdown
|
235
242
|
@agent.shutdown if @agent.respond_to?(:shutdown)
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/environment.rb
CHANGED
@@ -46,9 +46,11 @@ module Aspera
|
|
46
46
|
return CPU_PPC64
|
47
47
|
when /s390/
|
48
48
|
return CPU_S390
|
49
|
-
|
50
|
-
|
49
|
+
when /arm/
|
50
|
+
# arm on mac has rosetta 2
|
51
|
+
return CPU_X86_64 if os.eql?(OS_X)
|
51
52
|
end
|
53
|
+
raise "Unknown CPU: #{RbConfig::CONFIG['host_cpu']}"
|
52
54
|
end
|
53
55
|
|
54
56
|
def architecture
|
@@ -76,6 +78,17 @@ module Aspera
|
|
76
78
|
def secure_eval(code)
|
77
79
|
Kernel.send('lave'.reverse,code,empty_binding, __FILE__, __LINE__)
|
78
80
|
end
|
81
|
+
|
82
|
+
# value is provided in block
|
83
|
+
def write_file_restricted(path,force: false)
|
84
|
+
raise 'coding error, missing content block' unless block_given?
|
85
|
+
if force || !File.exist?(path)
|
86
|
+
File.unlink(path) rescue nil # Windows may give error
|
87
|
+
File.write(path,yield)
|
88
|
+
File.chmod(0400,path)
|
89
|
+
end
|
90
|
+
return path
|
91
|
+
end
|
79
92
|
end
|
80
93
|
end
|
81
94
|
end
|
@@ -15,6 +15,15 @@ module Aspera
|
|
15
15
|
EXPECTED_METHODS = %i[text struct enhanced].freeze
|
16
16
|
private_constant :INTEGER_FIELDS,:BOOLEAN_FIELDS,:EXPECTED_METHODS
|
17
17
|
|
18
|
+
class << self
|
19
|
+
# This checks the validity of the value returned by wait_for_transfers_completion
|
20
|
+
# it must be a list of :success or exception
|
21
|
+
def validate_status_list(statuses)
|
22
|
+
raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
|
23
|
+
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
18
27
|
private
|
19
28
|
|
20
29
|
# translates legacy event into enhanced (JSON) event
|
@@ -74,13 +83,6 @@ module Aspera
|
|
74
83
|
self
|
75
84
|
end
|
76
85
|
|
77
|
-
# This checks the validity of the value returned by wait_for_transfers_completion
|
78
|
-
# it must be a list of :success or exception
|
79
|
-
def self.validate_status_list(statuses)
|
80
|
-
raise "internal error: bad statuses type: #{statuses.class}" unless statuses.is_a?(Array)
|
81
|
-
raise "internal error: bad statuses content: #{statuses}" unless statuses.select{|i|!i.eql?(:success) && !i.is_a?(StandardError)}.empty?
|
82
|
-
end
|
83
|
-
|
84
86
|
# the following methods must be implemented by subclass:
|
85
87
|
# start_transfer(transfer_spec,options) : start and wait for completion
|
86
88
|
# wait_for_transfers_completion : wait for termination of all transfers, @return list of : :success or error message
|
@@ -139,28 +139,20 @@ module Aspera
|
|
139
139
|
when :transferd
|
140
140
|
file = transferd_filepath
|
141
141
|
when :ssh_bypass_key_dsa
|
142
|
-
file
|
143
|
-
File.write(file,get_key('dsa',1)) unless File.exist?(file)
|
144
|
-
File.chmod(0400,file)
|
142
|
+
file=Environment.write_file_restricted(File.join(sdk_folder,'aspera_bypass_dsa.pem')) {get_key('dsa',1)}
|
145
143
|
when :ssh_bypass_key_rsa
|
146
|
-
file
|
147
|
-
File.write(file,get_key('rsa',2)) unless File.exist?(file)
|
148
|
-
File.chmod(0400,file)
|
144
|
+
file=Environment.write_file_restricted(File.join(sdk_folder,'aspera_bypass_rsa.pem')) {get_key('rsa',2)}
|
149
145
|
when :aspera_license
|
150
|
-
file
|
151
|
-
unless File.exist?(file)
|
146
|
+
file=Environment.write_file_restricted(File.join(sdk_folder,'aspera-license')) do
|
152
147
|
clear=[
|
153
148
|
Zlib::Inflate.inflate(DataRepository.instance.data(6)),
|
154
149
|
"==SIGNATURE==\n",
|
155
150
|
Base64.strict_encode64(DataRepository.instance.data(7))
|
156
151
|
]
|
157
|
-
|
152
|
+
Base64.strict_encode64(clear.join)
|
158
153
|
end
|
159
|
-
File.chmod(0400,file)
|
160
154
|
when :aspera_conf
|
161
|
-
file
|
162
|
-
File.write(file,DEFAULT_ASPERA_CONF) unless File.exist?(file)
|
163
|
-
File.chmod(0400,file)
|
155
|
+
file=Environment.write_file_restricted(File.join(sdk_folder,'aspera.conf')) {DEFAULT_ASPERA_CONF}
|
164
156
|
when :fallback_cert,:fallback_key
|
165
157
|
file_key = File.join(sdk_folder,'aspera_fallback_key.pem')
|
166
158
|
file_cert = File.join(sdk_folder,'aspera_fallback_cert.pem')
|
@@ -176,10 +168,8 @@ module Aspera
|
|
176
168
|
cert.serial = 0x0
|
177
169
|
cert.version = 2
|
178
170
|
cert.sign(private_key, OpenSSL::Digest.new('SHA1'))
|
179
|
-
|
180
|
-
|
181
|
-
File.chmod(0400,file_key)
|
182
|
-
File.chmod(0400,file_cert)
|
171
|
+
Environment.write_file_restricted(file_key, force: true) {private_key.to_pem}
|
172
|
+
Environment.write_file_restricted(file_cert, force: true) {cert.to_pem}
|
183
173
|
end
|
184
174
|
file = k.eql?(:fallback_cert) ? file_cert : file_key
|
185
175
|
else
|
@@ -240,6 +230,8 @@ module Aspera
|
|
240
230
|
# extracts ascp binary for current system architecture
|
241
231
|
# @return ascp version (from execution)
|
242
232
|
def install_sdk(sdk_url)
|
233
|
+
# SDK is organized by architecture, check this first, in case architecture is not supported
|
234
|
+
arch_filter = "#{Environment.architecture}/"
|
243
235
|
require 'zip'
|
244
236
|
sdk_zip_path = File.join(Dir.tmpdir,'sdk.zip')
|
245
237
|
if sdk_url.start_with?('file:')
|
@@ -255,8 +247,6 @@ module Aspera
|
|
255
247
|
File.rename(sdk_folder,"#{sdk_folder}.#{Time.now.strftime('%Y%m%d%H%M%S')}")
|
256
248
|
# TODO: delete old archives ?
|
257
249
|
end
|
258
|
-
# SDK is organized by architecture
|
259
|
-
arch_filter = "#{Environment.architecture}/"
|
260
250
|
# extract files from archive
|
261
251
|
Zip::File.open(sdk_zip_path) do |zip_file|
|
262
252
|
zip_file.each do |entry|
|
@@ -354,6 +344,12 @@ module Aspera
|
|
354
344
|
log_root: File.join(Dir.home,'Library','Logs','Aspera_Connect'),
|
355
345
|
run_root: File.join(Dir.home,'Library','Application Support','Aspera','Aspera Connect'),
|
356
346
|
sub_bin: File.join('Contents','Resources')
|
347
|
+
},{
|
348
|
+
expected: PRODUCT_CONNECT,
|
349
|
+
app_root: File.join('','Applications','Aspera Connect.app'),
|
350
|
+
log_root: File.join(Dir.home,'Library','Logs','Aspera_Connect'),
|
351
|
+
run_root: File.join(Dir.home,'Library','Application Support','Aspera','Aspera Connect'),
|
352
|
+
sub_bin: File.join('Contents','Resources')
|
357
353
|
},{
|
358
354
|
expected: PRODUCT_CLI_V1,
|
359
355
|
app_root: File.join(Dir.home,'Applications','Aspera CLI'),
|
@@ -19,6 +19,8 @@ module Aspera
|
|
19
19
|
# Short names of columns in manual
|
20
20
|
SUPPORTED_AGENTS_SHORT = SUPPORTED_AGENTS.map{|a|a.to_s[0].to_sym}
|
21
21
|
|
22
|
+
private_constant :SUPPORTED_AGENTS
|
23
|
+
|
22
24
|
class << self
|
23
25
|
# Temp folder for file lists, must contain only file lists
|
24
26
|
# because of garbage collection takes any file there
|
@@ -40,6 +42,7 @@ module Aspera
|
|
40
42
|
result = []
|
41
43
|
description.each do |k,i|
|
42
44
|
param = {name: k, type: [i[:accepted_types]].flatten.join(','),description: i[:desc]}
|
45
|
+
# add flags for supported agents in doc
|
43
46
|
SUPPORTED_AGENTS.each do |a|
|
44
47
|
param[a.to_s[0].to_sym] = i[:tragents].nil? || i[:tragents].include?(a) ? 'Y' : ''
|
45
48
|
end
|
@@ -68,8 +71,11 @@ module Aspera
|
|
68
71
|
# special encoding methods used in YAML (key: :clconvert)
|
69
72
|
def clconv_base64(v); Base64.strict_encode64(v); end
|
70
73
|
|
71
|
-
|
72
|
-
|
74
|
+
# file list is provided directly with ascp arguments
|
75
|
+
def ts_has_ascp_file_list(ts)
|
76
|
+
(ts['EX_ascp_args'].is_a?(Array) && ['--file-list','--file-pair-list'].any?{|i|ts['EX_ascp_args'].include?(i)}) ||
|
77
|
+
ts.has_key?('EX_file_list') ||
|
78
|
+
ts.has_key?('EX_file_pair_list')
|
73
79
|
end
|
74
80
|
|
75
81
|
def ts_to_env_args(transfer_spec,options)
|
@@ -143,48 +149,50 @@ module Aspera
|
|
143
149
|
# destination will be base64 encoded, put before path arguments
|
144
150
|
@builder.add_command_line_options(['--dest64'])
|
145
151
|
end
|
146
|
-
#
|
147
|
-
|
148
|
-
|
149
|
-
|
150
|
-
|
151
|
-
|
152
|
-
|
153
|
-
|
154
|
-
|
155
|
-
|
156
|
-
|
157
|
-
#
|
158
|
-
|
159
|
-
|
160
|
-
|
161
|
-
@builder.add_command_line_options(paths_array.map{|i|i['source']})
|
152
|
+
# process file lists
|
153
|
+
begin
|
154
|
+
# is the file list provided through EX_ parameters?
|
155
|
+
ascp_file_list_provided = self.class.ts_has_ascp_file_list(@job_spec)
|
156
|
+
# set if paths is mandatory in ts
|
157
|
+
@builder.params_definition['paths'][:mandatory] = !@job_spec.has_key?('keepalive') && !ascp_file_list_provided
|
158
|
+
# get paths in transfer spec (after setting if it is mandatory)
|
159
|
+
ts_paths_array = @builder.process_param('paths',:get_value)
|
160
|
+
if ascp_file_list_provided && !ts_paths_array.nil?
|
161
|
+
raise 'file list provided both in transfer spec and ascp file list. Remove one of them.'
|
162
|
+
end
|
163
|
+
# option 1: EX_file_list
|
164
|
+
file_list_file = @builder.process_param('EX_file_list',:get_value)
|
165
|
+
if !file_list_file.nil?
|
166
|
+
option = '--file-list'
|
162
167
|
else
|
163
|
-
|
168
|
+
# option 2: EX_file_pair_list
|
169
|
+
file_list_file = @builder.process_param('EX_file_pair_list',:get_value)
|
164
170
|
if !file_list_file.nil?
|
165
|
-
option = '--file-list'
|
166
|
-
|
167
|
-
|
168
|
-
if !
|
169
|
-
option
|
170
|
-
else
|
171
|
-
# safer option: file list
|
171
|
+
option = '--file-pair-list'
|
172
|
+
elsif !ts_paths_array.nil?
|
173
|
+
# option 3: in TS, it is an array
|
174
|
+
if !self.class.file_list_folder.nil?
|
175
|
+
# safer option: generate a file list file if there is storage defined for it
|
172
176
|
# if there is destination in paths, then use filepairlist
|
173
177
|
# TODO: well, we test only the first one, but anyway it shall be consistent
|
174
|
-
if
|
178
|
+
if ts_paths_array.first.has_key?('destination')
|
175
179
|
option = '--file-pair-list'
|
176
|
-
lines =
|
180
|
+
lines = ts_paths_array.each_with_object([]){|e,m|m.push(e['source'],e['destination']);}
|
177
181
|
else
|
178
182
|
option = '--file-list'
|
179
|
-
lines =
|
183
|
+
lines = ts_paths_array.map{|i|i['source']}
|
180
184
|
end
|
181
185
|
file_list_file = Aspera::TempFileManager.instance.new_file_path_in_folder(self.class.file_list_folder)
|
182
186
|
File.write(file_list_file, lines.join("\n"))
|
183
187
|
Log.log.debug{"#{option}=\n#{File.read(file_list_file)}".red}
|
188
|
+
else
|
189
|
+
# not safe for special characters ? (maybe not, depends on OS)
|
190
|
+
Log.log.debug('placing source file list on command line (no file list file)')
|
191
|
+
@builder.add_command_line_options(ts_paths_array.map{|i|i['source']})
|
184
192
|
end
|
185
193
|
end
|
186
|
-
@builder.add_command_line_options(["#{option}=#{file_list_file}"])
|
187
194
|
end
|
195
|
+
@builder.add_command_line_options(["#{option}=#{file_list_file}"]) unless option.nil?
|
188
196
|
end
|
189
197
|
# optional args, at the end to override previous ones (to allow override)
|
190
198
|
@builder.add_command_line_options(@builder.process_param('EX_ascp_args',:get_value))
|
@@ -49,7 +49,13 @@ create_dir:
|
|
49
49
|
:cltype: :opt_without_arg
|
50
50
|
:clswitch: "-d"
|
51
51
|
delete_before_transfer:
|
52
|
-
:desc:
|
52
|
+
:desc: |-
|
53
|
+
Before transfer, delete files that exist at the destination but not at the source.
|
54
|
+
The source and destination arguments must be directories that have matching names.
|
55
|
+
Objects on the destination that have the same name but different type or size as objects
|
56
|
+
on the source are not deleted.
|
57
|
+
|
58
|
+
|
53
59
|
:cltype: :opt_without_arg
|
54
60
|
delete_source: # duplicate of remove_after_transfer ?
|
55
61
|
:desc: Remove SRC files after transfer success
|
@@ -138,16 +144,24 @@ https_fallback_port:
|
|
138
144
|
:cltype: :opt_with_arg
|
139
145
|
:clswitch: "-t"
|
140
146
|
move_after_transfer:
|
147
|
+
:desc: The relative path to which the files will be moved after the transfer at the source side.
|
141
148
|
:cltype: :opt_with_arg
|
142
149
|
multi_session:
|
143
|
-
:desc:
|
144
|
-
|
145
|
-
|
150
|
+
:desc: |
|
151
|
+
Use multi-session transfer. max 128.
|
152
|
+
Each participant on one host needs an independent UDP (-O) port.
|
153
|
+
Large files are split between sessions only when transferring with resume_policy=none.
|
154
|
+
|
155
|
+
|
146
156
|
:accepted_types: :int
|
147
157
|
:cltype: :ignore
|
148
158
|
:clswitch: "-C"
|
149
159
|
multi_session_threshold:
|
150
|
-
:desc:
|
160
|
+
:desc: |-
|
161
|
+
Split files across multiple ascp sessions if their size in bytes is greater than or equal to the specified value.
|
162
|
+
(0=no file is split)
|
163
|
+
|
164
|
+
|
151
165
|
:accepted_types: :int
|
152
166
|
:tragents:
|
153
167
|
- :direct
|
@@ -163,6 +177,18 @@ overwrite:
|
|
163
177
|
- older
|
164
178
|
- diff+older
|
165
179
|
:cltype: :opt_with_arg
|
180
|
+
password:
|
181
|
+
:desc: |-
|
182
|
+
Password for local Windows user when transfer user associated with node api user is not the same as the one running asperanoded.
|
183
|
+
Allows impersonating the transfer user and have access to resources (e.g. network shares).
|
184
|
+
Windows only, node api only.
|
185
|
+
|
186
|
+
|
187
|
+
:required: false
|
188
|
+
:accepted_types: :string
|
189
|
+
:cltype: :ignore
|
190
|
+
:tragents:
|
191
|
+
- :node
|
166
192
|
paths:
|
167
193
|
:desc: Array of path to the source (required) and a path to the destination (optional).
|
168
194
|
:required: true
|
@@ -233,10 +259,13 @@ remove_skipped:
|
|
233
259
|
- :node
|
234
260
|
:cltype: :opt_without_arg
|
235
261
|
proxy:
|
236
|
-
:desc:
|
237
|
-
|
238
|
-
|
239
|
-
|
262
|
+
:desc: |-
|
263
|
+
Specify the address of the Aspera high-speed proxy server.
|
264
|
+
dnat(s)://[user[:password]@]server:port
|
265
|
+
Default ports for DNAT and DNATS protocols are 9091 and 9092.
|
266
|
+
Password, if specified here, overrides the value of environment variable ASPERA_PROXY_PASS.
|
267
|
+
|
268
|
+
|
240
269
|
:tragents:
|
241
270
|
- :direct
|
242
271
|
- :sdk
|
@@ -279,9 +308,12 @@ ssh_port:
|
|
279
308
|
:cltype: :opt_with_arg
|
280
309
|
:clswitch: "-P"
|
281
310
|
ssh_private_key:
|
282
|
-
:desc:
|
283
|
-
|
284
|
-
|
311
|
+
:desc: |-
|
312
|
+
Private key used for SSH authentication.
|
313
|
+
Shall look like: -----BEGIN RSA PRIV4TE KEY-----\nMII...
|
314
|
+
Note the JSON encoding: \n for newlines.
|
315
|
+
|
316
|
+
|
285
317
|
:tragents:
|
286
318
|
- :direct
|
287
319
|
- :sdk
|
@@ -355,8 +387,11 @@ use_system_ssh:
|
|
355
387
|
:cltype: :ignore
|
356
388
|
:clswitch: "-SSH"
|
357
389
|
source_root:
|
358
|
-
:desc:
|
359
|
-
|
390
|
+
:desc: |-
|
391
|
+
Path to be prepended to each source path.
|
392
|
+
This is either a conventional path or it can be a URI but only if there is no root defined.
|
393
|
+
|
394
|
+
|
360
395
|
:cltype: :opt_with_arg
|
361
396
|
:clswitch: "--source-prefix64"
|
362
397
|
:clconvert: Aspera::Fasp::Parameters.clconv_base64
|
@@ -377,6 +412,20 @@ apply_local_docroot:
|
|
377
412
|
- :direct
|
378
413
|
- :sdk
|
379
414
|
:cltype: :opt_without_arg
|
415
|
+
src_base:
|
416
|
+
:desc: |-
|
417
|
+
Specify the prefix to be stripped off from each source object.
|
418
|
+
The remaining portion of the source path is kept intact at the destination.
|
419
|
+
Special care must be taken when used with cloud storage.
|
420
|
+
|
421
|
+
|
422
|
+
:tragents:
|
423
|
+
- :direct
|
424
|
+
- :node
|
425
|
+
- :sdk
|
426
|
+
:cltype: :opt_with_arg
|
427
|
+
:clswitch: "--src-base64"
|
428
|
+
:clconvert: Aspera::Fasp::Parameters.clconv_base64
|
380
429
|
preserve_acls:
|
381
430
|
:desc: "Preserve access control lists."
|
382
431
|
:enum:
|
@@ -440,14 +489,17 @@ remove_empty_source_directory:
|
|
440
489
|
- :sdk
|
441
490
|
:cltype: :opt_without_arg
|
442
491
|
EX_at_rest_password:
|
443
|
-
:desc: "
|
492
|
+
:desc: "DEPRECATED: Prefer to use standard parameter: content_protection_password"
|
444
493
|
:tragents:
|
445
494
|
- :direct
|
446
495
|
:cltype: :envvar
|
447
496
|
:clvarname: ASPERA_SCP_FILEPASS
|
448
497
|
EX_proxy_password:
|
449
|
-
:desc:
|
450
|
-
|
498
|
+
:desc: |-
|
499
|
+
Password used for Aspera proxy server authentication.
|
500
|
+
May be overridden by password in URL EX_fasp_proxy_url.
|
501
|
+
|
502
|
+
|
451
503
|
:tragents:
|
452
504
|
- :direct
|
453
505
|
:cltype: :envvar
|
data/lib/aspera/hash_ext.rb
CHANGED
@@ -10,18 +10,30 @@ class ::Hash
|
|
10
10
|
end
|
11
11
|
end
|
12
12
|
|
13
|
+
# in 2.5
|
14
|
+
unless Hash.method_defined?(:transform_keys)
|
15
|
+
class Hash
|
16
|
+
def transform_keys
|
17
|
+
return each_with_object({}){|(k,v),memo|memo[yield(k)]=v} if block_given?
|
18
|
+
raise 'missing block'
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
# rails
|
13
24
|
unless Hash.method_defined?(:symbolize_keys)
|
14
25
|
class Hash
|
15
26
|
def symbolize_keys
|
16
|
-
return
|
27
|
+
return transform_keys(&:to_sym)
|
17
28
|
end
|
18
29
|
end
|
19
30
|
end
|
20
31
|
|
32
|
+
# rails
|
21
33
|
unless Hash.method_defined?(:stringify_keys)
|
22
34
|
class Hash
|
23
35
|
def stringify_keys
|
24
|
-
return
|
36
|
+
return transform_keys(&:to_s)
|
25
37
|
end
|
26
38
|
end
|
27
39
|
end
|
data/lib/aspera/id_generator.rb
CHANGED
@@ -8,17 +8,19 @@ module Aspera
|
|
8
8
|
WINDOWS_PROTECTED_CHAR = %r{[/:"<>\\*?]}.freeze
|
9
9
|
PROTECTED_CHAR_REPLACE = '_'
|
10
10
|
private_constant :ID_SEPARATOR,:PROTECTED_CHAR_REPLACE,:WINDOWS_PROTECTED_CHAR
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
11
|
+
class << self
|
12
|
+
def from_list(object_id)
|
13
|
+
if object_id.is_a?(Array)
|
14
|
+
object_id = object_id.compact.map do |i|
|
15
|
+
i.is_a?(String) && i.start_with?('https://') ? URI.parse(i).host : i.to_s
|
16
|
+
end.join(ID_SEPARATOR)
|
17
|
+
end
|
18
|
+
raise 'id must be a String' unless object_id.is_a?(String)
|
19
|
+
return object_id.
|
20
|
+
gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
|
21
|
+
gsub('.',PROTECTED_CHAR_REPLACE). # keep dot for extension only (nicer)
|
22
|
+
downcase
|
16
23
|
end
|
17
|
-
raise 'id must be a String' unless object_id.is_a?(String)
|
18
|
-
return object_id.
|
19
|
-
gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
|
20
|
-
gsub('.',PROTECTED_CHAR_REPLACE). # keep dot for extension only (nicer)
|
21
|
-
downcase
|
22
24
|
end
|
23
25
|
end
|
24
26
|
end
|
@@ -117,17 +117,17 @@ module Aspera
|
|
117
117
|
result = options.clone
|
118
118
|
case info
|
119
119
|
when NilClass
|
120
|
-
raise "no such secret: #{
|
120
|
+
raise "no such secret: [#{url}|#{username}] in #{@all_secrets.keys.join(',')}"
|
121
121
|
when String
|
122
122
|
result[:secret] = info
|
123
123
|
result[:description] = ''
|
124
124
|
when Hash
|
125
125
|
info=info.symbolize_keys
|
126
126
|
key = identifier(options)
|
127
|
-
plain = SimpleCipher.new(key).decrypt(info[:secret])
|
127
|
+
plain = SimpleCipher.new(key).decrypt(info[:secret]) rescue info[:secret]
|
128
128
|
result[:secret] = plain
|
129
129
|
result[:description] = info[:description]
|
130
|
-
else raise
|
130
|
+
else raise "#{info.class} is not an expected type"
|
131
131
|
end
|
132
132
|
return result
|
133
133
|
end
|
data/lib/aspera/log.rb
CHANGED
@@ -79,7 +79,7 @@ module Aspera
|
|
79
79
|
@logger = Logger.new($stdout)
|
80
80
|
when :syslog
|
81
81
|
require 'syslog/logger'
|
82
|
-
@logger = Syslog::Logger.new(@program_name)
|
82
|
+
@logger = Syslog::Logger.new(@program_name, Syslog::LOG_LOCAL2)
|
83
83
|
else
|
84
84
|
raise "unknown log type: #{new_logtype.class} #{new_logtype}"
|
85
85
|
end
|
data/lib/aspera/nagios.rb
CHANGED
@@ -18,6 +18,32 @@ module Aspera
|
|
18
18
|
define_method(name){|comp,msg|@data.push({code: code,comp: comp,msg: msg})}
|
19
19
|
end
|
20
20
|
|
21
|
+
class << self
|
22
|
+
# process results of a analysis and display status and exit with code
|
23
|
+
def process(data)
|
24
|
+
raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
|
25
|
+
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
|
26
|
+
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
27
|
+
# keep only errors in case of problem, other ok are assumed so
|
28
|
+
data = res_errors unless res_errors.empty?
|
29
|
+
# first is most critical
|
30
|
+
data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
31
|
+
# build message: if multiple components: concatenate
|
32
|
+
#message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
33
|
+
message = data.
|
34
|
+
map{|i|i['component']}.
|
35
|
+
uniq.
|
36
|
+
map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.
|
37
|
+
join(', ').
|
38
|
+
tr("\n",' ')
|
39
|
+
status = data.first['status'].upcase
|
40
|
+
# display status for nagios
|
41
|
+
puts("#{status} - [#{message}]\n")
|
42
|
+
# provide exit code to nagios
|
43
|
+
Process.exit(LEVELS.index(data.first['status'].to_sym))
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
21
47
|
attr_reader :data
|
22
48
|
def initialize
|
23
49
|
@data = []
|
@@ -50,24 +76,5 @@ module Aspera
|
|
50
76
|
raise 'missing result' if @data.empty?
|
51
77
|
{type: :object_list,data: @data.map{|i|{'status' => LEVELS[i[:code]].to_s,'component' => i[:comp],'message' => i[:msg]}}}
|
52
78
|
end
|
53
|
-
|
54
|
-
# process results of a analysis and display status and exit with code
|
55
|
-
def self.process(data)
|
56
|
-
raise 'INTERNAL ERROR, result must be list and not empty' unless data.is_a?(Array) && !data.empty?
|
57
|
-
%w[status component message].each{|c|raise "INTERNAL ERROR, result must have #{c}" unless data.first.has_key?(c)}
|
58
|
-
res_errors = data.reject{|s|s['status'].eql?('ok')}
|
59
|
-
# keep only errors in case of problem, other ok are assumed so
|
60
|
-
data = res_errors unless res_errors.empty?
|
61
|
-
# first is most critical
|
62
|
-
data.sort!{|a,b|LEVELS.index(a['status'].to_sym) <=> LEVELS.index(b['status'].to_sym)}
|
63
|
-
# build message: if multiple components: concatenate
|
64
|
-
#message = data.map{|i|"#{i['component']}:#{i['message']}"}.join(', ').gsub("\n",' ')
|
65
|
-
message = data.map{|i|i['component']}.uniq.map{|comp|comp + ':' + data.select{|d|d['component'].eql?(comp)}.map{|d|d['message']}.join(',')}.join(', ').tr("\n",' ')
|
66
|
-
status = data.first['status'].upcase
|
67
|
-
# display status for nagios
|
68
|
-
puts("#{status} - [#{message}]\n")
|
69
|
-
# provide exit code to nagios
|
70
|
-
Process.exit(LEVELS.index(data.first['status'].to_sym))
|
71
|
-
end
|
72
79
|
end
|
73
80
|
end
|