aspera-cli 4.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +3592 -0
- data/bin/ascli +7 -0
- data/bin/asession +89 -0
- data/docs/Makefile +59 -0
- data/docs/README.erb.md +3012 -0
- data/docs/README.md +13 -0
- data/docs/diagrams.txt +49 -0
- data/docs/secrets.make +38 -0
- data/docs/test_env.conf +117 -0
- data/docs/transfer_spec.html +99 -0
- data/examples/aoc.rb +17 -0
- data/examples/proxy.pac +60 -0
- data/examples/transfer.rb +115 -0
- data/lib/aspera/api_detector.rb +60 -0
- data/lib/aspera/ascmd.rb +151 -0
- data/lib/aspera/ats_api.rb +43 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +38 -0
- data/lib/aspera/cli/extended_value.rb +88 -0
- data/lib/aspera/cli/formater.rb +238 -0
- data/lib/aspera/cli/listener/line_dump.rb +17 -0
- data/lib/aspera/cli/listener/logger.rb +20 -0
- data/lib/aspera/cli/listener/progress.rb +52 -0
- data/lib/aspera/cli/listener/progress_multi.rb +91 -0
- data/lib/aspera/cli/main.rb +304 -0
- data/lib/aspera/cli/manager.rb +440 -0
- data/lib/aspera/cli/plugin.rb +90 -0
- data/lib/aspera/cli/plugins/alee.rb +24 -0
- data/lib/aspera/cli/plugins/ats.rb +231 -0
- data/lib/aspera/cli/plugins/bss.rb +71 -0
- data/lib/aspera/cli/plugins/config.rb +806 -0
- data/lib/aspera/cli/plugins/console.rb +62 -0
- data/lib/aspera/cli/plugins/cos.rb +106 -0
- data/lib/aspera/cli/plugins/faspex.rb +377 -0
- data/lib/aspera/cli/plugins/faspex5.rb +93 -0
- data/lib/aspera/cli/plugins/node.rb +438 -0
- data/lib/aspera/cli/plugins/oncloud.rb +937 -0
- data/lib/aspera/cli/plugins/orchestrator.rb +169 -0
- data/lib/aspera/cli/plugins/preview.rb +464 -0
- data/lib/aspera/cli/plugins/server.rb +216 -0
- data/lib/aspera/cli/plugins/shares.rb +63 -0
- data/lib/aspera/cli/plugins/shares2.rb +114 -0
- data/lib/aspera/cli/plugins/sync.rb +65 -0
- data/lib/aspera/cli/plugins/xnode.rb +115 -0
- data/lib/aspera/cli/transfer_agent.rb +251 -0
- data/lib/aspera/cli/version.rb +5 -0
- data/lib/aspera/colors.rb +39 -0
- data/lib/aspera/command_line_builder.rb +137 -0
- data/lib/aspera/fasp/aoc.rb +24 -0
- data/lib/aspera/fasp/connect.rb +99 -0
- data/lib/aspera/fasp/error.rb +21 -0
- data/lib/aspera/fasp/error_info.rb +60 -0
- data/lib/aspera/fasp/http_gw.rb +81 -0
- data/lib/aspera/fasp/installation.rb +240 -0
- data/lib/aspera/fasp/listener.rb +11 -0
- data/lib/aspera/fasp/local.rb +377 -0
- data/lib/aspera/fasp/manager.rb +69 -0
- data/lib/aspera/fasp/node.rb +88 -0
- data/lib/aspera/fasp/parameters.rb +235 -0
- data/lib/aspera/fasp/resume_policy.rb +76 -0
- data/lib/aspera/fasp/uri.rb +51 -0
- data/lib/aspera/faspex_gw.rb +196 -0
- data/lib/aspera/hash_ext.rb +28 -0
- data/lib/aspera/log.rb +80 -0
- data/lib/aspera/nagios.rb +71 -0
- data/lib/aspera/node.rb +14 -0
- data/lib/aspera/oauth.rb +319 -0
- data/lib/aspera/on_cloud.rb +421 -0
- data/lib/aspera/open_application.rb +72 -0
- data/lib/aspera/persistency_action_once.rb +42 -0
- data/lib/aspera/persistency_folder.rb +91 -0
- data/lib/aspera/preview/file_types.rb +300 -0
- data/lib/aspera/preview/generator.rb +258 -0
- data/lib/aspera/preview/image_error.png +0 -0
- data/lib/aspera/preview/options.rb +35 -0
- data/lib/aspera/preview/utils.rb +131 -0
- data/lib/aspera/preview/video_error.png +0 -0
- data/lib/aspera/proxy_auto_config.erb.js +287 -0
- data/lib/aspera/proxy_auto_config.rb +34 -0
- data/lib/aspera/rest.rb +296 -0
- data/lib/aspera/rest_call_error.rb +13 -0
- data/lib/aspera/rest_error_analyzer.rb +98 -0
- data/lib/aspera/rest_errors_aspera.rb +58 -0
- data/lib/aspera/ssh.rb +53 -0
- data/lib/aspera/sync.rb +82 -0
- data/lib/aspera/temp_file_manager.rb +37 -0
- data/lib/aspera/uri_reader.rb +25 -0
- metadata +288 -0
@@ -0,0 +1,88 @@
|
|
1
|
+
require 'aspera/fasp/manager'
|
2
|
+
require 'aspera/log'
|
3
|
+
require 'tty-spinner'
|
4
|
+
|
5
|
+
module Aspera
|
6
|
+
module Fasp
|
7
|
+
# this singleton class is used by the CLI to provide a common interface to start a transfer
|
8
|
+
# before using it, the use must set the `node_api` member.
|
9
|
+
class Node < Manager
|
10
|
+
def initialize(node_api)
|
11
|
+
super()
|
12
|
+
@node_api=node_api
|
13
|
+
# TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
|
14
|
+
@transfer_id=nil
|
15
|
+
end
|
16
|
+
|
17
|
+
# used internally to ensure node api is set before using.
|
18
|
+
def node_api_
|
19
|
+
raise StandardError,'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
|
20
|
+
return @node_api
|
21
|
+
end
|
22
|
+
# use this to read the node_api end point.
|
23
|
+
attr_reader :node_api
|
24
|
+
|
25
|
+
# use this to set the node_api end point before using the class.
|
26
|
+
def node_api=(new_value)
|
27
|
+
if !@node_api.nil? and !new_value.nil?
|
28
|
+
Log.log.warn('overriding existing node api value')
|
29
|
+
end
|
30
|
+
@node_api=new_value
|
31
|
+
end
|
32
|
+
|
33
|
+
# generic method
|
34
|
+
def start_transfer(transfer_spec,options=nil)
|
35
|
+
if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
|
36
|
+
transfer_spec['tags']['aspera']['xfer_retry']||=150
|
37
|
+
end
|
38
|
+
# optimisation in case of sending to the same node
|
39
|
+
if transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
|
40
|
+
transfer_spec['remote_host']='localhost'
|
41
|
+
end
|
42
|
+
resp=node_api_.create('ops/transfers',transfer_spec)[:data]
|
43
|
+
@transfer_id=resp['id']
|
44
|
+
Log.log.debug("tr_id=#{@transfer_id}")
|
45
|
+
return @transfer_id
|
46
|
+
end
|
47
|
+
|
48
|
+
# generic method
|
49
|
+
def wait_for_transfers_completion
|
50
|
+
started=false
|
51
|
+
spinner=nil
|
52
|
+
# lets emulate management events to display progress bar
|
53
|
+
loop do
|
54
|
+
# status is empty sometimes with status 200...
|
55
|
+
trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
|
56
|
+
case trdata['status']
|
57
|
+
when 'completed'
|
58
|
+
notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'DONE'})
|
59
|
+
break
|
60
|
+
when 'waiting','partially_completed','unknown','waiting(read error)'
|
61
|
+
if spinner.nil?
|
62
|
+
spinner = TTY::Spinner.new("[:spinner] :title", format: :classic)
|
63
|
+
spinner.start
|
64
|
+
end
|
65
|
+
spinner.update(title: trdata['status'])
|
66
|
+
spinner.spin
|
67
|
+
#puts trdata
|
68
|
+
when 'running'
|
69
|
+
#puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
70
|
+
if !started and trdata['precalc'].is_a?(Hash) and
|
71
|
+
trdata['precalc']['status'].eql?('ready')
|
72
|
+
notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'NOTIFICATION','PreTransferBytes'=>trdata['precalc']['bytes_expected']})
|
73
|
+
started=true
|
74
|
+
else
|
75
|
+
notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'STATS','Bytescont'=>trdata['bytes_transferred']})
|
76
|
+
end
|
77
|
+
else
|
78
|
+
Log.log.warn("trdata -> #{trdata}")
|
79
|
+
raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
|
80
|
+
end
|
81
|
+
sleep 1
|
82
|
+
end
|
83
|
+
#TODO get status of sessions
|
84
|
+
return []
|
85
|
+
end
|
86
|
+
end
|
87
|
+
end
|
88
|
+
end
|
@@ -0,0 +1,235 @@
|
|
1
|
+
require 'aspera/log'
|
2
|
+
require 'aspera/command_line_builder'
|
3
|
+
require 'aspera/temp_file_manager'
|
4
|
+
require 'securerandom'
|
5
|
+
require 'base64'
|
6
|
+
require 'json'
|
7
|
+
require 'securerandom'
|
8
|
+
require 'fileutils'
|
9
|
+
|
10
|
+
module Aspera
|
11
|
+
module Fasp
|
12
|
+
# translate transfer specification to ascp parameter list
|
13
|
+
class Parameters
|
14
|
+
private
|
15
|
+
# temp folder for file lists, must contain only file lists
|
16
|
+
# because of garbage collection takes any file there
|
17
|
+
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
18
|
+
@@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
|
19
|
+
SEC_IN_DAY=86400
|
20
|
+
# assume no transfer last longer than this
|
21
|
+
# (garbage collect file list which were not deleted after transfer)
|
22
|
+
FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
|
23
|
+
PARAM_DEFINITION={
|
24
|
+
# parameters with env vars
|
25
|
+
'remote_password' => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
|
26
|
+
'token' => { :type => :envvar, :variable=>'ASPERA_SCP_TOKEN'},
|
27
|
+
'cookie' => { :type => :envvar, :variable=>'ASPERA_SCP_COOKIE'},
|
28
|
+
'ssh_private_key' => { :type => :envvar, :variable=>'ASPERA_SCP_KEY'},
|
29
|
+
'EX_at_rest_password' => { :type => :envvar, :variable=>'ASPERA_SCP_FILEPASS'},
|
30
|
+
'EX_proxy_password' => { :type => :envvar, :variable=>'ASPERA_PROXY_PASS'},
|
31
|
+
'EX_license_text' => { :type => :envvar, :variable=>'ASPERA_SCP_LICENSE'},
|
32
|
+
# bool params
|
33
|
+
'create_dir' => { :type => :opt_without_arg, :option_switch=>'-d'},
|
34
|
+
'precalculate_job_size' => { :type => :opt_without_arg},
|
35
|
+
'keepalive' => { :type => :opt_without_arg},
|
36
|
+
'delete_before_transfer' => { :type => :opt_without_arg}, #TODO: doc readme
|
37
|
+
'preserve_access_time' => { :type => :opt_without_arg}, #TODO: doc
|
38
|
+
'preserve_creation_time' => { :type => :opt_without_arg}, #TODO: doc
|
39
|
+
'preserve_times' => { :type => :opt_without_arg}, #TODO: doc
|
40
|
+
'preserve_modification_time'=> { :type => :opt_without_arg}, #TODO: doc
|
41
|
+
'remove_empty_directories'=> { :type => :opt_without_arg}, #TODO: doc
|
42
|
+
'remove_after_transfer' => { :type => :opt_without_arg}, #TODO: doc
|
43
|
+
'remove_empty_source_directory'=> { :type => :opt_without_arg}, #TODO: doc
|
44
|
+
# value params
|
45
|
+
'cipher' => { :type => :opt_with_arg, :option_switch=>'-c',:accepted_types=>String,:encode=>lambda{|cipher|cipher.tr('-','')}},
|
46
|
+
'resume_policy' => { :type => :opt_with_arg, :option_switch=>'-k',:accepted_types=>String,:default=>'sparse_csum',:translate_values=>{'none'=>0,'attrs'=>1,'sparse_csum'=>2,'full_csum'=>3}},
|
47
|
+
'direction' => { :type => :opt_with_arg, :option_switch=>'--mode',:accepted_types=>String,:translate_values=>{'receive'=>'recv','send'=>'send'}},
|
48
|
+
'remote_user' => { :type => :opt_with_arg, :option_switch=>'--user',:accepted_types=>String},
|
49
|
+
'remote_host' => { :type => :opt_with_arg, :option_switch=>'--host',:accepted_types=>String},
|
50
|
+
'ssh_port' => { :type => :opt_with_arg, :option_switch=>'-P',:accepted_types=>Integer},
|
51
|
+
'fasp_port' => { :type => :opt_with_arg, :option_switch=>'-O',:accepted_types=>Integer},
|
52
|
+
'dgram_size' => { :type => :opt_with_arg, :option_switch=>'-Z',:accepted_types=>Integer},
|
53
|
+
'target_rate_kbps' => { :type => :opt_with_arg, :option_switch=>'-l',:accepted_types=>Integer},
|
54
|
+
'min_rate_kbps' => { :type => :opt_with_arg, :option_switch=>'-m',:accepted_types=>Integer},
|
55
|
+
'rate_policy' => { :type => :opt_with_arg, :option_switch=>'--policy',:accepted_types=>String},
|
56
|
+
'http_fallback' => { :type => :opt_with_arg, :option_switch=>'-y',:accepted_types=>[String,*Aspera::CommandLineBuilder::BOOLEAN_CLASSES],:translate_values=>{'force'=>'F',true=>1,false=>0}},
|
57
|
+
'http_fallback_port' => { :type => :opt_with_arg, :option_switch=>'-t',:accepted_types=>Integer},
|
58
|
+
'source_root' => { :type => :opt_with_arg, :option_switch=>'--source-prefix64',:accepted_types=>String,:encode=>lambda{|prefix|Base64.strict_encode64(prefix)}},
|
59
|
+
'sshfp' => { :type => :opt_with_arg, :option_switch=>'--check-sshfp',:accepted_types=>String},
|
60
|
+
'symlink_policy' => { :type => :opt_with_arg, :option_switch=>'--symbolic-links',:accepted_types=>String},
|
61
|
+
'overwrite' => { :type => :opt_with_arg, :accepted_types=>String},
|
62
|
+
'exclude_newer_than' => { :type => :opt_with_arg, :accepted_types=>Integer},
|
63
|
+
'exclude_older_than' => { :type => :opt_with_arg, :accepted_types=>Integer},
|
64
|
+
'preserve_acls' => { :type => :opt_with_arg, :accepted_types=>String},
|
65
|
+
'move_after_transfer' => { :type => :opt_with_arg, :accepted_types=>String},
|
66
|
+
'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>String},
|
67
|
+
# non standard parameters
|
68
|
+
'EX_fasp_proxy_url' => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
|
69
|
+
'EX_http_proxy_url' => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
|
70
|
+
'EX_ssh_key_paths' => { :type => :opt_with_arg, :option_switch=>'-i',:accepted_types=>Array},
|
71
|
+
'EX_http_transfer_jpeg' => { :type => :opt_with_arg, :option_switch=>'-j',:accepted_types=>Integer},
|
72
|
+
'EX_multi_session_part' => { :type => :opt_with_arg, :option_switch=>'-C',:accepted_types=>String},
|
73
|
+
'EX_no_read' => { :type => :opt_without_arg, :option_switch=>'--no-read'},
|
74
|
+
'EX_no_write' => { :type => :opt_without_arg, :option_switch=>'--no-write'},
|
75
|
+
'EX_apply_local_docroot' => { :type => :opt_without_arg, :option_switch=>'--apply-local-docroot'},
|
76
|
+
# TODO: manage those parameters, some are for connect only ? node api ?
|
77
|
+
'target_rate_cap_kbps' => { :type => :ignore, :accepted_types=>Integer},
|
78
|
+
'target_rate_percentage' => { :type => :ignore, :accepted_types=>String}, # -wf -l<rate>p
|
79
|
+
'min_rate_cap_kbps' => { :type => :ignore, :accepted_types=>Integer},
|
80
|
+
'rate_policy_allowed' => { :type => :ignore, :accepted_types=>String},
|
81
|
+
'fasp_url' => { :type => :ignore, :accepted_types=>String},
|
82
|
+
'lock_rate_policy' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
|
83
|
+
'lock_min_rate' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
|
84
|
+
'lock_target_rate' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
|
85
|
+
#'authentication' => { :type => :ignore, :accepted_types=>String}, # = token
|
86
|
+
'https_fallback_port' => { :type => :ignore, :accepted_types=>Integer}, # same as http fallback, option -t ?
|
87
|
+
'content_protection' => { :type => :ignore, :accepted_types=>String},
|
88
|
+
'cipher_allowed' => { :type => :ignore, :accepted_types=>String},
|
89
|
+
'multi_session' => { :type => :ignore, :accepted_types=>Integer}, # managed
|
90
|
+
# optional tags ( additional option to generate: {:space=>' ',:object_nl=>' ',:space_before=>'+',:array_nl=>'1'} )
|
91
|
+
'tags' => { :type => :opt_with_arg, :option_switch=>'--tags64',:accepted_types=>Hash,:encode=>lambda{|tags|Base64.strict_encode64(JSON.generate(tags))}},
|
92
|
+
# special processing @builder.process_param( called individually
|
93
|
+
'use_ascp4' => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
|
94
|
+
'paths' => { :type => :defer, :accepted_types=>Array},
|
95
|
+
'EX_file_list' => { :type => :defer, :option_switch=>'--file-list', :accepted_types=>String},
|
96
|
+
'EX_file_pair_list' => { :type => :defer, :option_switch=>'--file-pair-list', :accepted_types=>String},
|
97
|
+
'EX_ascp_args' => { :type => :defer, :accepted_types=>Array},
|
98
|
+
'destination_root' => { :type => :defer, :accepted_types=>String},
|
99
|
+
'wss_enabled' => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
|
100
|
+
'wss_port' => { :type => :defer, :accepted_types=>Integer},
|
101
|
+
}
|
102
|
+
|
103
|
+
private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC,:PARAM_DEFINITION
|
104
|
+
|
105
|
+
def initialize(job_spec,options)
|
106
|
+
@job_spec=job_spec
|
107
|
+
@builder=Aspera::CommandLineBuilder.new(@job_spec,PARAM_DEFINITION)
|
108
|
+
@options=options
|
109
|
+
end
|
110
|
+
|
111
|
+
public
|
112
|
+
|
113
|
+
# translate transfer spec to env vars and command line arguments for ascp
|
114
|
+
# NOTE: parameters starting with "EX_" (extended) are not standard
|
115
|
+
def ascp_args()
|
116
|
+
env_args={
|
117
|
+
:args=>[],
|
118
|
+
:env=>{},
|
119
|
+
:ascp_version=>:ascp
|
120
|
+
}
|
121
|
+
# some ssh credentials are required to avoid interactive password input
|
122
|
+
if !@job_spec.has_key?('remote_password') and
|
123
|
+
!@job_spec.has_key?('ssh_private_key') and
|
124
|
+
!@job_spec.has_key?('EX_ssh_key_paths') then
|
125
|
+
raise Fasp::Error.new('required: password or ssh key (value or path)')
|
126
|
+
end
|
127
|
+
|
128
|
+
# special cases
|
129
|
+
@job_spec.delete('source_root') if @job_spec.has_key?('source_root') and @job_spec['source_root'].empty?
|
130
|
+
|
131
|
+
# use web socket initiation ?
|
132
|
+
if @builder.process_param('wss_enabled',:get_value) and @options[:wss]
|
133
|
+
# by default use web socket session if available, unless removed by user
|
134
|
+
@builder.add_command_line_options(['--ws-connect'])
|
135
|
+
# TODO: option to give order ssh,ws (legacy http is implied bu ssh)
|
136
|
+
# quel bordel:
|
137
|
+
@job_spec['ssh_port']=@builder.process_param('wss_port',:get_value)
|
138
|
+
@job_spec.delete('fasp_port')
|
139
|
+
@job_spec.delete('EX_ssh_key_paths')
|
140
|
+
@job_spec.delete('sshfp')
|
141
|
+
else
|
142
|
+
# remove unused parameter (avoid warning)
|
143
|
+
@job_spec.delete('wss_port')
|
144
|
+
end
|
145
|
+
|
146
|
+
# process parameters as specified in table
|
147
|
+
@builder.process_params
|
148
|
+
|
149
|
+
# symbol must be index of Installation.paths
|
150
|
+
if @builder.process_param('use_ascp4',:get_value)
|
151
|
+
env_args[:ascp_version] = :ascp4
|
152
|
+
else
|
153
|
+
env_args[:ascp_version] = :ascp
|
154
|
+
# destination will be base64 encoded, put before path arguments
|
155
|
+
@builder.add_command_line_options(['--dest64'])
|
156
|
+
end
|
157
|
+
|
158
|
+
PARAM_DEFINITION['paths'][:mandatory]=!@job_spec.has_key?('keepalive')
|
159
|
+
paths_array=@builder.process_param('paths',:get_value)
|
160
|
+
unless paths_array.nil?
|
161
|
+
# use file list if there is storage defined for it.
|
162
|
+
if @@file_list_folder.nil?
|
163
|
+
# not safe for special characters ? (maybe not, depends on OS)
|
164
|
+
Log.log.debug("placing source file list on command line (no file list file)")
|
165
|
+
@builder.add_command_line_options(paths_array.map{|i|i['source']})
|
166
|
+
else
|
167
|
+
file_list_file=@builder.process_param('EX_file_list',:get_value)
|
168
|
+
if !file_list_file.nil?
|
169
|
+
option='--file-list'
|
170
|
+
else
|
171
|
+
file_list_file=@builder.process_param('EX_file_pair_list',:get_value)
|
172
|
+
if !file_list_file.nil?
|
173
|
+
option='--file-pair-list'
|
174
|
+
else
|
175
|
+
# safer option: file list
|
176
|
+
# if there is destination in paths, then use filepairlist
|
177
|
+
# TODO: well, we test only the first one, but anyway it shall be consistent
|
178
|
+
if paths_array.first.has_key?('destination')
|
179
|
+
option='--file-pair-list'
|
180
|
+
lines=paths_array.inject([]){|m,e|m.push(e['source'],e['destination']);m}
|
181
|
+
else
|
182
|
+
option='--file-list'
|
183
|
+
lines=paths_array.map{|i|i['source']}
|
184
|
+
end
|
185
|
+
file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(@@file_list_folder)
|
186
|
+
File.open(file_list_file, 'w+'){|f|f.puts(lines)}
|
187
|
+
Log.log.debug("#{option}=\n#{File.read(file_list_file)}".red)
|
188
|
+
end
|
189
|
+
end
|
190
|
+
@builder.add_command_line_options(["#{option}=#{file_list_file}"])
|
191
|
+
end
|
192
|
+
end
|
193
|
+
# optional args, at the end to override previous ones (to allow override)
|
194
|
+
@builder.add_command_line_options(@builder.process_param('EX_ascp_args',:get_value))
|
195
|
+
# process destination folder
|
196
|
+
destination_folder = @builder.process_param('destination_root',:get_value) || '/'
|
197
|
+
# ascp4 does not support base64 encoding of destination
|
198
|
+
destination_folder = Base64.strict_encode64(destination_folder) unless env_args[:ascp_version].eql?(:ascp4)
|
199
|
+
# destination MUST be last command line argument to ascp
|
200
|
+
@builder.add_command_line_options([destination_folder])
|
201
|
+
|
202
|
+
@builder.add_env_args(env_args[:env],env_args[:args])
|
203
|
+
|
204
|
+
return env_args
|
205
|
+
end
|
206
|
+
|
207
|
+
# temp file list files are created here
|
208
|
+
def self.file_list_folder=(v)
|
209
|
+
@@file_list_folder=v
|
210
|
+
unless @@file_list_folder.nil?
|
211
|
+
FileUtils.mkdir_p(@@file_list_folder)
|
212
|
+
# garbage collect undeleted files
|
213
|
+
Dir.entries(@@file_list_folder).each do |name|
|
214
|
+
file_path=File.join(@@file_list_folder,name)
|
215
|
+
age_sec=(Time.now - File.stat(file_path).mtime).to_i
|
216
|
+
# check age of file, delete too old
|
217
|
+
if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
|
218
|
+
Log.log.debug("garbage collecting #{name}")
|
219
|
+
File.delete(file_path)
|
220
|
+
end
|
221
|
+
end
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
# static methods
|
226
|
+
class << self
|
227
|
+
def file_list_folder; @@file_list_folder;end
|
228
|
+
|
229
|
+
def ts_to_env_args(transfer_spec,options)
|
230
|
+
return Parameters.new(transfer_spec,options).ascp_args()
|
231
|
+
end
|
232
|
+
end
|
233
|
+
end # Parameters
|
234
|
+
end
|
235
|
+
end
|
@@ -0,0 +1,76 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
require 'aspera/log'
|
3
|
+
|
4
|
+
module Aspera
|
5
|
+
module Fasp
|
6
|
+
# implements a simple resume policy
|
7
|
+
class ResumePolicy
|
8
|
+
|
9
|
+
# list of supported parameters and default values
|
10
|
+
DEFAULTS={
|
11
|
+
:iter_max => 7,
|
12
|
+
:sleep_initial => 2,
|
13
|
+
:sleep_factor => 2,
|
14
|
+
:sleep_max => 60
|
15
|
+
}
|
16
|
+
|
17
|
+
def initialize(params={})
|
18
|
+
@parameters=DEFAULTS.clone
|
19
|
+
return if params.nil?
|
20
|
+
raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
|
21
|
+
params.each do |k,v|
|
22
|
+
if DEFAULTS.has_key?(k)
|
23
|
+
@parameters[k]=v
|
24
|
+
else
|
25
|
+
raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
|
30
|
+
# calls block a number of times (resumes) until success or limit reached
|
31
|
+
# this is re-entrant, one resumer can handle multiple transfers in //
|
32
|
+
def process(&block)
|
33
|
+
# maximum of retry
|
34
|
+
remaining_resumes = @parameters[:iter_max]
|
35
|
+
sleep_seconds = @parameters[:sleep_initial]
|
36
|
+
Log.log.debug("retries=#{remaining_resumes}")
|
37
|
+
# try to send the file until ascp is succesful
|
38
|
+
loop do
|
39
|
+
Log.log.debug('transfer starting');
|
40
|
+
begin
|
41
|
+
block.call
|
42
|
+
break
|
43
|
+
rescue Fasp::Error => e
|
44
|
+
Log.log.warn("An error occured: #{e.message}" );
|
45
|
+
# failure in ascp
|
46
|
+
if e.retryable? then
|
47
|
+
# exit if we exceed the max number of retry
|
48
|
+
unless remaining_resumes > 0
|
49
|
+
Log.log.error "Maximum number of retry reached"
|
50
|
+
raise Fasp::Error,"max retry after: [#{status[:message]}]"
|
51
|
+
end
|
52
|
+
else
|
53
|
+
# give one chance only to non retryable errors
|
54
|
+
unless remaining_resumes.eql?(@parameters[:iter_max])
|
55
|
+
Log.log.error('non-retryable error')
|
56
|
+
raise e
|
57
|
+
end
|
58
|
+
end
|
59
|
+
end
|
60
|
+
|
61
|
+
# take this retry in account
|
62
|
+
remaining_resumes-=1
|
63
|
+
Log.log.warn( "resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})" );
|
64
|
+
|
65
|
+
# wait a bit before retrying, maybe network condition will be better
|
66
|
+
sleep(sleep_seconds)
|
67
|
+
|
68
|
+
# increase retry period
|
69
|
+
sleep_seconds *= @parameters[:sleep_factor]
|
70
|
+
# cap value
|
71
|
+
sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
|
72
|
+
end # loop
|
73
|
+
end
|
74
|
+
end
|
75
|
+
end
|
76
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
require 'aspera/log'
|
2
|
+
require 'aspera/command_line_builder'
|
3
|
+
|
4
|
+
module Aspera
|
5
|
+
module Fasp
|
6
|
+
# translates a "faspe:" URI (used in Faspex) into transfer spec hash
|
7
|
+
class Uri
|
8
|
+
def initialize(fasplink)
|
9
|
+
@fasp_uri=URI.parse(fasplink)
|
10
|
+
# TODO: check scheme is faspe
|
11
|
+
end
|
12
|
+
|
13
|
+
def transfer_spec
|
14
|
+
result_ts={}
|
15
|
+
result_ts['remote_host']=@fasp_uri.host
|
16
|
+
result_ts['remote_user']=@fasp_uri.user
|
17
|
+
result_ts['ssh_port']=@fasp_uri.port
|
18
|
+
result_ts['paths']=[{"source"=>URI.decode_www_form_component(@fasp_uri.path)}]
|
19
|
+
# faspex does not encode trailing base64 encoded tags, fix that
|
20
|
+
fixed_query = @fasp_uri.query.gsub(/(=+)$/){|x|'%3D'*x.length}
|
21
|
+
|
22
|
+
URI::decode_www_form(fixed_query).each do |i|
|
23
|
+
name=i[0]
|
24
|
+
value=i[1]
|
25
|
+
case name
|
26
|
+
when 'cookie'; result_ts['cookie']=value
|
27
|
+
when 'token'; result_ts['token']=value
|
28
|
+
when 'policy'; result_ts['rate_policy']=value
|
29
|
+
when 'httpport'; result_ts['http_fallback_port']=value.to_i
|
30
|
+
when 'targetrate'; result_ts['target_rate_kbps']=value.to_i
|
31
|
+
when 'minrate'; result_ts['min_rate_kbps']=value.to_i
|
32
|
+
when 'port'; result_ts['fasp_port']=value.to_i
|
33
|
+
when 'enc'; result_ts['cipher']=value.gsub('-','') # aes-128 -> aes128
|
34
|
+
when 'tags64'; result_ts['tags']=JSON.parse(Base64.strict_decode64(value))
|
35
|
+
when 'bwcap'; result_ts['target_rate_cap_kbps']=value.to_i
|
36
|
+
when 'createpath'; result_ts['create_dir']=CommandLineBuilder.yes_to_true(value)
|
37
|
+
when 'fallback'; result_ts['http_fallback']=CommandLineBuilder.yes_to_true(value)
|
38
|
+
when 'lockpolicy'; result_ts['lock_rate_policy']=CommandLineBuilder.yes_to_true(value)
|
39
|
+
when 'lockminrate'; result_ts['lock_min_rate']=CommandLineBuilder.yes_to_true(value)
|
40
|
+
when 'sshfp'; result_ts['sshfp']=value
|
41
|
+
when 'auth'; Log.log.debug("ignoring #{name}=#{value}") # TODO: translate into transfer spec ? yes/no
|
42
|
+
when 'v'; Log.log.debug("ignoring #{name}=#{value}") # TODO: translate into transfer spec ? 2
|
43
|
+
when 'protect'; Log.log.debug("ignoring #{name}=#{value}") # TODO: translate into transfer spec ?
|
44
|
+
else Log.log.error("non managed URI value: #{name} = #{value}")
|
45
|
+
end
|
46
|
+
end
|
47
|
+
return result_ts
|
48
|
+
end
|
49
|
+
end # Uri
|
50
|
+
end # Fasp
|
51
|
+
end # Aspera
|