aspera-cli 4.0.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +3592 -0
  3. data/bin/ascli +7 -0
  4. data/bin/asession +89 -0
  5. data/docs/Makefile +59 -0
  6. data/docs/README.erb.md +3012 -0
  7. data/docs/README.md +13 -0
  8. data/docs/diagrams.txt +49 -0
  9. data/docs/secrets.make +38 -0
  10. data/docs/test_env.conf +117 -0
  11. data/docs/transfer_spec.html +99 -0
  12. data/examples/aoc.rb +17 -0
  13. data/examples/proxy.pac +60 -0
  14. data/examples/transfer.rb +115 -0
  15. data/lib/aspera/api_detector.rb +60 -0
  16. data/lib/aspera/ascmd.rb +151 -0
  17. data/lib/aspera/ats_api.rb +43 -0
  18. data/lib/aspera/cli/basic_auth_plugin.rb +38 -0
  19. data/lib/aspera/cli/extended_value.rb +88 -0
  20. data/lib/aspera/cli/formater.rb +238 -0
  21. data/lib/aspera/cli/listener/line_dump.rb +17 -0
  22. data/lib/aspera/cli/listener/logger.rb +20 -0
  23. data/lib/aspera/cli/listener/progress.rb +52 -0
  24. data/lib/aspera/cli/listener/progress_multi.rb +91 -0
  25. data/lib/aspera/cli/main.rb +304 -0
  26. data/lib/aspera/cli/manager.rb +440 -0
  27. data/lib/aspera/cli/plugin.rb +90 -0
  28. data/lib/aspera/cli/plugins/alee.rb +24 -0
  29. data/lib/aspera/cli/plugins/ats.rb +231 -0
  30. data/lib/aspera/cli/plugins/bss.rb +71 -0
  31. data/lib/aspera/cli/plugins/config.rb +806 -0
  32. data/lib/aspera/cli/plugins/console.rb +62 -0
  33. data/lib/aspera/cli/plugins/cos.rb +106 -0
  34. data/lib/aspera/cli/plugins/faspex.rb +377 -0
  35. data/lib/aspera/cli/plugins/faspex5.rb +93 -0
  36. data/lib/aspera/cli/plugins/node.rb +438 -0
  37. data/lib/aspera/cli/plugins/oncloud.rb +937 -0
  38. data/lib/aspera/cli/plugins/orchestrator.rb +169 -0
  39. data/lib/aspera/cli/plugins/preview.rb +464 -0
  40. data/lib/aspera/cli/plugins/server.rb +216 -0
  41. data/lib/aspera/cli/plugins/shares.rb +63 -0
  42. data/lib/aspera/cli/plugins/shares2.rb +114 -0
  43. data/lib/aspera/cli/plugins/sync.rb +65 -0
  44. data/lib/aspera/cli/plugins/xnode.rb +115 -0
  45. data/lib/aspera/cli/transfer_agent.rb +251 -0
  46. data/lib/aspera/cli/version.rb +5 -0
  47. data/lib/aspera/colors.rb +39 -0
  48. data/lib/aspera/command_line_builder.rb +137 -0
  49. data/lib/aspera/fasp/aoc.rb +24 -0
  50. data/lib/aspera/fasp/connect.rb +99 -0
  51. data/lib/aspera/fasp/error.rb +21 -0
  52. data/lib/aspera/fasp/error_info.rb +60 -0
  53. data/lib/aspera/fasp/http_gw.rb +81 -0
  54. data/lib/aspera/fasp/installation.rb +240 -0
  55. data/lib/aspera/fasp/listener.rb +11 -0
  56. data/lib/aspera/fasp/local.rb +377 -0
  57. data/lib/aspera/fasp/manager.rb +69 -0
  58. data/lib/aspera/fasp/node.rb +88 -0
  59. data/lib/aspera/fasp/parameters.rb +235 -0
  60. data/lib/aspera/fasp/resume_policy.rb +76 -0
  61. data/lib/aspera/fasp/uri.rb +51 -0
  62. data/lib/aspera/faspex_gw.rb +196 -0
  63. data/lib/aspera/hash_ext.rb +28 -0
  64. data/lib/aspera/log.rb +80 -0
  65. data/lib/aspera/nagios.rb +71 -0
  66. data/lib/aspera/node.rb +14 -0
  67. data/lib/aspera/oauth.rb +319 -0
  68. data/lib/aspera/on_cloud.rb +421 -0
  69. data/lib/aspera/open_application.rb +72 -0
  70. data/lib/aspera/persistency_action_once.rb +42 -0
  71. data/lib/aspera/persistency_folder.rb +91 -0
  72. data/lib/aspera/preview/file_types.rb +300 -0
  73. data/lib/aspera/preview/generator.rb +258 -0
  74. data/lib/aspera/preview/image_error.png +0 -0
  75. data/lib/aspera/preview/options.rb +35 -0
  76. data/lib/aspera/preview/utils.rb +131 -0
  77. data/lib/aspera/preview/video_error.png +0 -0
  78. data/lib/aspera/proxy_auto_config.erb.js +287 -0
  79. data/lib/aspera/proxy_auto_config.rb +34 -0
  80. data/lib/aspera/rest.rb +296 -0
  81. data/lib/aspera/rest_call_error.rb +13 -0
  82. data/lib/aspera/rest_error_analyzer.rb +98 -0
  83. data/lib/aspera/rest_errors_aspera.rb +58 -0
  84. data/lib/aspera/ssh.rb +53 -0
  85. data/lib/aspera/sync.rb +82 -0
  86. data/lib/aspera/temp_file_manager.rb +37 -0
  87. data/lib/aspera/uri_reader.rb +25 -0
  88. metadata +288 -0
@@ -0,0 +1,88 @@
1
+ require 'aspera/fasp/manager'
2
+ require 'aspera/log'
3
+ require 'tty-spinner'
4
+
5
+ module Aspera
6
+ module Fasp
7
+ # this singleton class is used by the CLI to provide a common interface to start a transfer
8
+ # before using it, the use must set the `node_api` member.
9
+ class Node < Manager
10
+ def initialize(node_api)
11
+ super()
12
+ @node_api=node_api
13
+ # TODO: currently only supports one transfer. This is bad shortcut. but ok for CLI.
14
+ @transfer_id=nil
15
+ end
16
+
17
+ # used internally to ensure node api is set before using.
18
+ def node_api_
19
+ raise StandardError,'Before using this object, set the node_api attribute to a Aspera::Rest object' if @node_api.nil?
20
+ return @node_api
21
+ end
22
+ # use this to read the node_api end point.
23
+ attr_reader :node_api
24
+
25
+ # use this to set the node_api end point before using the class.
26
+ def node_api=(new_value)
27
+ if !@node_api.nil? and !new_value.nil?
28
+ Log.log.warn('overriding existing node api value')
29
+ end
30
+ @node_api=new_value
31
+ end
32
+
33
+ # generic method
34
+ def start_transfer(transfer_spec,options=nil)
35
+ if transfer_spec['tags'].is_a?(Hash) and transfer_spec['tags']['aspera'].is_a?(Hash)
36
+ transfer_spec['tags']['aspera']['xfer_retry']||=150
37
+ end
38
+ # optimisation in case of sending to the same node
39
+ if transfer_spec['remote_host'].eql?(URI.parse(node_api_.params[:base_url]).host)
40
+ transfer_spec['remote_host']='localhost'
41
+ end
42
+ resp=node_api_.create('ops/transfers',transfer_spec)[:data]
43
+ @transfer_id=resp['id']
44
+ Log.log.debug("tr_id=#{@transfer_id}")
45
+ return @transfer_id
46
+ end
47
+
48
+ # generic method
49
+ def wait_for_transfers_completion
50
+ started=false
51
+ spinner=nil
52
+ # lets emulate management events to display progress bar
53
+ loop do
54
+ # status is empty sometimes with status 200...
55
+ trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
56
+ case trdata['status']
57
+ when 'completed'
58
+ notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'DONE'})
59
+ break
60
+ when 'waiting','partially_completed','unknown','waiting(read error)'
61
+ if spinner.nil?
62
+ spinner = TTY::Spinner.new("[:spinner] :title", format: :classic)
63
+ spinner.start
64
+ end
65
+ spinner.update(title: trdata['status'])
66
+ spinner.spin
67
+ #puts trdata
68
+ when 'running'
69
+ #puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
70
+ if !started and trdata['precalc'].is_a?(Hash) and
71
+ trdata['precalc']['status'].eql?('ready')
72
+ notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'NOTIFICATION','PreTransferBytes'=>trdata['precalc']['bytes_expected']})
73
+ started=true
74
+ else
75
+ notify_listeners('emulated',{Manager::LISTENER_SESSION_ID_B=>@transfer_id,'Type'=>'STATS','Bytescont'=>trdata['bytes_transferred']})
76
+ end
77
+ else
78
+ Log.log.warn("trdata -> #{trdata}")
79
+ raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
80
+ end
81
+ sleep 1
82
+ end
83
+ #TODO get status of sessions
84
+ return []
85
+ end
86
+ end
87
+ end
88
+ end
@@ -0,0 +1,235 @@
1
+ require 'aspera/log'
2
+ require 'aspera/command_line_builder'
3
+ require 'aspera/temp_file_manager'
4
+ require 'securerandom'
5
+ require 'base64'
6
+ require 'json'
7
+ require 'securerandom'
8
+ require 'fileutils'
9
+
10
+ module Aspera
11
+ module Fasp
12
+ # translate transfer specification to ascp parameter list
13
+ class Parameters
14
+ private
15
+ # temp folder for file lists, must contain only file lists
16
+ # because of garbage collection takes any file there
17
+ # this could be refined, as , for instance, on macos, temp folder is already user specific
18
+ @@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
19
+ SEC_IN_DAY=86400
20
+ # assume no transfer last longer than this
21
+ # (garbage collect file list which were not deleted after transfer)
22
+ FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
23
+ PARAM_DEFINITION={
24
+ # parameters with env vars
25
+ 'remote_password' => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
26
+ 'token' => { :type => :envvar, :variable=>'ASPERA_SCP_TOKEN'},
27
+ 'cookie' => { :type => :envvar, :variable=>'ASPERA_SCP_COOKIE'},
28
+ 'ssh_private_key' => { :type => :envvar, :variable=>'ASPERA_SCP_KEY'},
29
+ 'EX_at_rest_password' => { :type => :envvar, :variable=>'ASPERA_SCP_FILEPASS'},
30
+ 'EX_proxy_password' => { :type => :envvar, :variable=>'ASPERA_PROXY_PASS'},
31
+ 'EX_license_text' => { :type => :envvar, :variable=>'ASPERA_SCP_LICENSE'},
32
+ # bool params
33
+ 'create_dir' => { :type => :opt_without_arg, :option_switch=>'-d'},
34
+ 'precalculate_job_size' => { :type => :opt_without_arg},
35
+ 'keepalive' => { :type => :opt_without_arg},
36
+ 'delete_before_transfer' => { :type => :opt_without_arg}, #TODO: doc readme
37
+ 'preserve_access_time' => { :type => :opt_without_arg}, #TODO: doc
38
+ 'preserve_creation_time' => { :type => :opt_without_arg}, #TODO: doc
39
+ 'preserve_times' => { :type => :opt_without_arg}, #TODO: doc
40
+ 'preserve_modification_time'=> { :type => :opt_without_arg}, #TODO: doc
41
+ 'remove_empty_directories'=> { :type => :opt_without_arg}, #TODO: doc
42
+ 'remove_after_transfer' => { :type => :opt_without_arg}, #TODO: doc
43
+ 'remove_empty_source_directory'=> { :type => :opt_without_arg}, #TODO: doc
44
+ # value params
45
+ 'cipher' => { :type => :opt_with_arg, :option_switch=>'-c',:accepted_types=>String,:encode=>lambda{|cipher|cipher.tr('-','')}},
46
+ 'resume_policy' => { :type => :opt_with_arg, :option_switch=>'-k',:accepted_types=>String,:default=>'sparse_csum',:translate_values=>{'none'=>0,'attrs'=>1,'sparse_csum'=>2,'full_csum'=>3}},
47
+ 'direction' => { :type => :opt_with_arg, :option_switch=>'--mode',:accepted_types=>String,:translate_values=>{'receive'=>'recv','send'=>'send'}},
48
+ 'remote_user' => { :type => :opt_with_arg, :option_switch=>'--user',:accepted_types=>String},
49
+ 'remote_host' => { :type => :opt_with_arg, :option_switch=>'--host',:accepted_types=>String},
50
+ 'ssh_port' => { :type => :opt_with_arg, :option_switch=>'-P',:accepted_types=>Integer},
51
+ 'fasp_port' => { :type => :opt_with_arg, :option_switch=>'-O',:accepted_types=>Integer},
52
+ 'dgram_size' => { :type => :opt_with_arg, :option_switch=>'-Z',:accepted_types=>Integer},
53
+ 'target_rate_kbps' => { :type => :opt_with_arg, :option_switch=>'-l',:accepted_types=>Integer},
54
+ 'min_rate_kbps' => { :type => :opt_with_arg, :option_switch=>'-m',:accepted_types=>Integer},
55
+ 'rate_policy' => { :type => :opt_with_arg, :option_switch=>'--policy',:accepted_types=>String},
56
+ 'http_fallback' => { :type => :opt_with_arg, :option_switch=>'-y',:accepted_types=>[String,*Aspera::CommandLineBuilder::BOOLEAN_CLASSES],:translate_values=>{'force'=>'F',true=>1,false=>0}},
57
+ 'http_fallback_port' => { :type => :opt_with_arg, :option_switch=>'-t',:accepted_types=>Integer},
58
+ 'source_root' => { :type => :opt_with_arg, :option_switch=>'--source-prefix64',:accepted_types=>String,:encode=>lambda{|prefix|Base64.strict_encode64(prefix)}},
59
+ 'sshfp' => { :type => :opt_with_arg, :option_switch=>'--check-sshfp',:accepted_types=>String},
60
+ 'symlink_policy' => { :type => :opt_with_arg, :option_switch=>'--symbolic-links',:accepted_types=>String},
61
+ 'overwrite' => { :type => :opt_with_arg, :accepted_types=>String},
62
+ 'exclude_newer_than' => { :type => :opt_with_arg, :accepted_types=>Integer},
63
+ 'exclude_older_than' => { :type => :opt_with_arg, :accepted_types=>Integer},
64
+ 'preserve_acls' => { :type => :opt_with_arg, :accepted_types=>String},
65
+ 'move_after_transfer' => { :type => :opt_with_arg, :accepted_types=>String},
66
+ 'multi_session_threshold' => { :type => :opt_with_arg, :accepted_types=>String},
67
+ # non standard parameters
68
+ 'EX_fasp_proxy_url' => { :type => :opt_with_arg, :option_switch=>'--proxy',:accepted_types=>String},
69
+ 'EX_http_proxy_url' => { :type => :opt_with_arg, :option_switch=>'-x',:accepted_types=>String},
70
+ 'EX_ssh_key_paths' => { :type => :opt_with_arg, :option_switch=>'-i',:accepted_types=>Array},
71
+ 'EX_http_transfer_jpeg' => { :type => :opt_with_arg, :option_switch=>'-j',:accepted_types=>Integer},
72
+ 'EX_multi_session_part' => { :type => :opt_with_arg, :option_switch=>'-C',:accepted_types=>String},
73
+ 'EX_no_read' => { :type => :opt_without_arg, :option_switch=>'--no-read'},
74
+ 'EX_no_write' => { :type => :opt_without_arg, :option_switch=>'--no-write'},
75
+ 'EX_apply_local_docroot' => { :type => :opt_without_arg, :option_switch=>'--apply-local-docroot'},
76
+ # TODO: manage those parameters, some are for connect only ? node api ?
77
+ 'target_rate_cap_kbps' => { :type => :ignore, :accepted_types=>Integer},
78
+ 'target_rate_percentage' => { :type => :ignore, :accepted_types=>String}, # -wf -l<rate>p
79
+ 'min_rate_cap_kbps' => { :type => :ignore, :accepted_types=>Integer},
80
+ 'rate_policy_allowed' => { :type => :ignore, :accepted_types=>String},
81
+ 'fasp_url' => { :type => :ignore, :accepted_types=>String},
82
+ 'lock_rate_policy' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
83
+ 'lock_min_rate' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
84
+ 'lock_target_rate' => { :type => :ignore, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
85
+ #'authentication' => { :type => :ignore, :accepted_types=>String}, # = token
86
+ 'https_fallback_port' => { :type => :ignore, :accepted_types=>Integer}, # same as http fallback, option -t ?
87
+ 'content_protection' => { :type => :ignore, :accepted_types=>String},
88
+ 'cipher_allowed' => { :type => :ignore, :accepted_types=>String},
89
+ 'multi_session' => { :type => :ignore, :accepted_types=>Integer}, # managed
90
+ # optional tags ( additional option to generate: {:space=>' ',:object_nl=>' ',:space_before=>'+',:array_nl=>'1'} )
91
+ 'tags' => { :type => :opt_with_arg, :option_switch=>'--tags64',:accepted_types=>Hash,:encode=>lambda{|tags|Base64.strict_encode64(JSON.generate(tags))}},
92
+ # special processing @builder.process_param( called individually
93
+ 'use_ascp4' => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
94
+ 'paths' => { :type => :defer, :accepted_types=>Array},
95
+ 'EX_file_list' => { :type => :defer, :option_switch=>'--file-list', :accepted_types=>String},
96
+ 'EX_file_pair_list' => { :type => :defer, :option_switch=>'--file-pair-list', :accepted_types=>String},
97
+ 'EX_ascp_args' => { :type => :defer, :accepted_types=>Array},
98
+ 'destination_root' => { :type => :defer, :accepted_types=>String},
99
+ 'wss_enabled' => { :type => :defer, :accepted_types=>Aspera::CommandLineBuilder::BOOLEAN_CLASSES},
100
+ 'wss_port' => { :type => :defer, :accepted_types=>Integer},
101
+ }
102
+
103
+ private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC,:PARAM_DEFINITION
104
+
105
+ def initialize(job_spec,options)
106
+ @job_spec=job_spec
107
+ @builder=Aspera::CommandLineBuilder.new(@job_spec,PARAM_DEFINITION)
108
+ @options=options
109
+ end
110
+
111
+ public
112
+
113
+ # translate transfer spec to env vars and command line arguments for ascp
114
+ # NOTE: parameters starting with "EX_" (extended) are not standard
115
+ def ascp_args()
116
+ env_args={
117
+ :args=>[],
118
+ :env=>{},
119
+ :ascp_version=>:ascp
120
+ }
121
+ # some ssh credentials are required to avoid interactive password input
122
+ if !@job_spec.has_key?('remote_password') and
123
+ !@job_spec.has_key?('ssh_private_key') and
124
+ !@job_spec.has_key?('EX_ssh_key_paths') then
125
+ raise Fasp::Error.new('required: password or ssh key (value or path)')
126
+ end
127
+
128
+ # special cases
129
+ @job_spec.delete('source_root') if @job_spec.has_key?('source_root') and @job_spec['source_root'].empty?
130
+
131
+ # use web socket initiation ?
132
+ if @builder.process_param('wss_enabled',:get_value) and @options[:wss]
133
+ # by default use web socket session if available, unless removed by user
134
+ @builder.add_command_line_options(['--ws-connect'])
135
+ # TODO: option to give order ssh,ws (legacy http is implied bu ssh)
136
+ # quel bordel:
137
+ @job_spec['ssh_port']=@builder.process_param('wss_port',:get_value)
138
+ @job_spec.delete('fasp_port')
139
+ @job_spec.delete('EX_ssh_key_paths')
140
+ @job_spec.delete('sshfp')
141
+ else
142
+ # remove unused parameter (avoid warning)
143
+ @job_spec.delete('wss_port')
144
+ end
145
+
146
+ # process parameters as specified in table
147
+ @builder.process_params
148
+
149
+ # symbol must be index of Installation.paths
150
+ if @builder.process_param('use_ascp4',:get_value)
151
+ env_args[:ascp_version] = :ascp4
152
+ else
153
+ env_args[:ascp_version] = :ascp
154
+ # destination will be base64 encoded, put before path arguments
155
+ @builder.add_command_line_options(['--dest64'])
156
+ end
157
+
158
+ PARAM_DEFINITION['paths'][:mandatory]=!@job_spec.has_key?('keepalive')
159
+ paths_array=@builder.process_param('paths',:get_value)
160
+ unless paths_array.nil?
161
+ # use file list if there is storage defined for it.
162
+ if @@file_list_folder.nil?
163
+ # not safe for special characters ? (maybe not, depends on OS)
164
+ Log.log.debug("placing source file list on command line (no file list file)")
165
+ @builder.add_command_line_options(paths_array.map{|i|i['source']})
166
+ else
167
+ file_list_file=@builder.process_param('EX_file_list',:get_value)
168
+ if !file_list_file.nil?
169
+ option='--file-list'
170
+ else
171
+ file_list_file=@builder.process_param('EX_file_pair_list',:get_value)
172
+ if !file_list_file.nil?
173
+ option='--file-pair-list'
174
+ else
175
+ # safer option: file list
176
+ # if there is destination in paths, then use filepairlist
177
+ # TODO: well, we test only the first one, but anyway it shall be consistent
178
+ if paths_array.first.has_key?('destination')
179
+ option='--file-pair-list'
180
+ lines=paths_array.inject([]){|m,e|m.push(e['source'],e['destination']);m}
181
+ else
182
+ option='--file-list'
183
+ lines=paths_array.map{|i|i['source']}
184
+ end
185
+ file_list_file=Aspera::TempFileManager.instance.new_file_path_in_folder(@@file_list_folder)
186
+ File.open(file_list_file, 'w+'){|f|f.puts(lines)}
187
+ Log.log.debug("#{option}=\n#{File.read(file_list_file)}".red)
188
+ end
189
+ end
190
+ @builder.add_command_line_options(["#{option}=#{file_list_file}"])
191
+ end
192
+ end
193
+ # optional args, at the end to override previous ones (to allow override)
194
+ @builder.add_command_line_options(@builder.process_param('EX_ascp_args',:get_value))
195
+ # process destination folder
196
+ destination_folder = @builder.process_param('destination_root',:get_value) || '/'
197
+ # ascp4 does not support base64 encoding of destination
198
+ destination_folder = Base64.strict_encode64(destination_folder) unless env_args[:ascp_version].eql?(:ascp4)
199
+ # destination MUST be last command line argument to ascp
200
+ @builder.add_command_line_options([destination_folder])
201
+
202
+ @builder.add_env_args(env_args[:env],env_args[:args])
203
+
204
+ return env_args
205
+ end
206
+
207
+ # temp file list files are created here
208
+ def self.file_list_folder=(v)
209
+ @@file_list_folder=v
210
+ unless @@file_list_folder.nil?
211
+ FileUtils.mkdir_p(@@file_list_folder)
212
+ # garbage collect undeleted files
213
+ Dir.entries(@@file_list_folder).each do |name|
214
+ file_path=File.join(@@file_list_folder,name)
215
+ age_sec=(Time.now - File.stat(file_path).mtime).to_i
216
+ # check age of file, delete too old
217
+ if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
218
+ Log.log.debug("garbage collecting #{name}")
219
+ File.delete(file_path)
220
+ end
221
+ end
222
+ end
223
+ end
224
+
225
+ # static methods
226
+ class << self
227
+ def file_list_folder; @@file_list_folder;end
228
+
229
+ def ts_to_env_args(transfer_spec,options)
230
+ return Parameters.new(transfer_spec,options).ascp_args()
231
+ end
232
+ end
233
+ end # Parameters
234
+ end
235
+ end
@@ -0,0 +1,76 @@
1
+ require 'singleton'
2
+ require 'aspera/log'
3
+
4
+ module Aspera
5
+ module Fasp
6
+ # implements a simple resume policy
7
+ class ResumePolicy
8
+
9
+ # list of supported parameters and default values
10
+ DEFAULTS={
11
+ :iter_max => 7,
12
+ :sleep_initial => 2,
13
+ :sleep_factor => 2,
14
+ :sleep_max => 60
15
+ }
16
+
17
+ def initialize(params={})
18
+ @parameters=DEFAULTS.clone
19
+ return if params.nil?
20
+ raise "expecting Hash (or nil), but have #{params.class}" unless params.is_a?(Hash)
21
+ params.each do |k,v|
22
+ if DEFAULTS.has_key?(k)
23
+ @parameters[k]=v
24
+ else
25
+ raise "unknown resume parameter: #{k}, expect one of #{DEFAULTS.keys.map{|i|i.to_s}.join(",")}"
26
+ end
27
+ end
28
+ end
29
+
30
+ # calls block a number of times (resumes) until success or limit reached
31
+ # this is re-entrant, one resumer can handle multiple transfers in //
32
+ def process(&block)
33
+ # maximum of retry
34
+ remaining_resumes = @parameters[:iter_max]
35
+ sleep_seconds = @parameters[:sleep_initial]
36
+ Log.log.debug("retries=#{remaining_resumes}")
37
+ # try to send the file until ascp is succesful
38
+ loop do
39
+ Log.log.debug('transfer starting');
40
+ begin
41
+ block.call
42
+ break
43
+ rescue Fasp::Error => e
44
+ Log.log.warn("An error occured: #{e.message}" );
45
+ # failure in ascp
46
+ if e.retryable? then
47
+ # exit if we exceed the max number of retry
48
+ unless remaining_resumes > 0
49
+ Log.log.error "Maximum number of retry reached"
50
+ raise Fasp::Error,"max retry after: [#{status[:message]}]"
51
+ end
52
+ else
53
+ # give one chance only to non retryable errors
54
+ unless remaining_resumes.eql?(@parameters[:iter_max])
55
+ Log.log.error('non-retryable error')
56
+ raise e
57
+ end
58
+ end
59
+ end
60
+
61
+ # take this retry in account
62
+ remaining_resumes-=1
63
+ Log.log.warn( "resuming in #{sleep_seconds} seconds (retry left:#{remaining_resumes})" );
64
+
65
+ # wait a bit before retrying, maybe network condition will be better
66
+ sleep(sleep_seconds)
67
+
68
+ # increase retry period
69
+ sleep_seconds *= @parameters[:sleep_factor]
70
+ # cap value
71
+ sleep_seconds = @parameters[:sleep_max] if sleep_seconds > @parameters[:sleep_max]
72
+ end # loop
73
+ end
74
+ end
75
+ end
76
+ end
@@ -0,0 +1,51 @@
1
+ require 'aspera/log'
2
+ require 'aspera/command_line_builder'
3
+
4
+ module Aspera
5
+ module Fasp
6
+ # translates a "faspe:" URI (used in Faspex) into transfer spec hash
7
+ class Uri
8
+ def initialize(fasplink)
9
+ @fasp_uri=URI.parse(fasplink)
10
+ # TODO: check scheme is faspe
11
+ end
12
+
13
+ def transfer_spec
14
+ result_ts={}
15
+ result_ts['remote_host']=@fasp_uri.host
16
+ result_ts['remote_user']=@fasp_uri.user
17
+ result_ts['ssh_port']=@fasp_uri.port
18
+ result_ts['paths']=[{"source"=>URI.decode_www_form_component(@fasp_uri.path)}]
19
+ # faspex does not encode trailing base64 encoded tags, fix that
20
+ fixed_query = @fasp_uri.query.gsub(/(=+)$/){|x|'%3D'*x.length}
21
+
22
+ URI::decode_www_form(fixed_query).each do |i|
23
+ name=i[0]
24
+ value=i[1]
25
+ case name
26
+ when 'cookie'; result_ts['cookie']=value
27
+ when 'token'; result_ts['token']=value
28
+ when 'policy'; result_ts['rate_policy']=value
29
+ when 'httpport'; result_ts['http_fallback_port']=value.to_i
30
+ when 'targetrate'; result_ts['target_rate_kbps']=value.to_i
31
+ when 'minrate'; result_ts['min_rate_kbps']=value.to_i
32
+ when 'port'; result_ts['fasp_port']=value.to_i
33
+ when 'enc'; result_ts['cipher']=value.gsub('-','') # aes-128 -> aes128
34
+ when 'tags64'; result_ts['tags']=JSON.parse(Base64.strict_decode64(value))
35
+ when 'bwcap'; result_ts['target_rate_cap_kbps']=value.to_i
36
+ when 'createpath'; result_ts['create_dir']=CommandLineBuilder.yes_to_true(value)
37
+ when 'fallback'; result_ts['http_fallback']=CommandLineBuilder.yes_to_true(value)
38
+ when 'lockpolicy'; result_ts['lock_rate_policy']=CommandLineBuilder.yes_to_true(value)
39
+ when 'lockminrate'; result_ts['lock_min_rate']=CommandLineBuilder.yes_to_true(value)
40
+ when 'sshfp'; result_ts['sshfp']=value
41
+ when 'auth'; Log.log.debug("ignoring #{name}=#{value}") # TODO: translate into transfer spec ? yes/no
42
+ when 'v'; Log.log.debug("ignoring #{name}=#{value}") # TODO: translate into transfer spec ? 2
43
+ when 'protect'; Log.log.debug("ignoring #{name}=#{value}") # TODO: translate into transfer spec ?
44
+ else Log.log.error("non managed URI value: #{name} = #{value}")
45
+ end
46
+ end
47
+ return result_ts
48
+ end
49
+ end # Uri
50
+ end # Fasp
51
+ end # Aspera