aspera-cli 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +518 -137
- data/bin/dascli +13 -0
- data/docs/README.erb.md +473 -105
- data/docs/test_env.conf +4 -1
- data/docs/transfer_spec.html +1 -1
- data/lib/aspera/aoc.rb +68 -86
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +27 -19
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +173 -136
- data/lib/aspera/cli/plugins/config.rb +80 -27
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +13 -6
- data/lib/aspera/cli/plugins/faspex5.rb +93 -37
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +25 -24
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +1 -1
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +28 -4
- data/lib/aspera/fasp/local.rb +24 -16
- data/lib/aspera/fasp/manager.rb +12 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +3 -16
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +48 -1
- data/lib/aspera/oauth.rb +24 -11
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +38 -18
- data/lib/aspera/temp_file_manager.rb +19 -0
- metadata +37 -20
@@ -88,7 +88,7 @@ module Aspera
|
|
88
88
|
raise StandardError,"expect: nil, String or Array"
|
89
89
|
end
|
90
90
|
|
91
|
-
SIMPLE_ACTIONS=[:
|
91
|
+
SIMPLE_ACTIONS=[:health,:events, :space, :info, :license, :mkdir, :mklink, :mkfile, :rename, :delete, :search ]
|
92
92
|
|
93
93
|
COMMON_ACTIONS=[:browse, :upload, :download, :api_details ].concat(SIMPLE_ACTIONS)
|
94
94
|
|
@@ -96,7 +96,7 @@ module Aspera
|
|
96
96
|
# prefix_path is used to list remote sources in Faspex
|
97
97
|
def execute_simple_common(command,prefix_path)
|
98
98
|
case command
|
99
|
-
when :
|
99
|
+
when :health
|
100
100
|
nagios=Nagios.new
|
101
101
|
begin
|
102
102
|
info=@api_node.read('info')[:data]
|
@@ -341,7 +341,7 @@ module Aspera
|
|
341
341
|
raise "error"
|
342
342
|
end
|
343
343
|
when :access_key
|
344
|
-
return self.entity_action(@api_node,'access_keys',
|
344
|
+
return self.entity_action(@api_node,'access_keys',nil,:id,'self')
|
345
345
|
when :service
|
346
346
|
command=self.options.get_next_command([ :list, :create, :delete])
|
347
347
|
if [:delete].include?(command)
|
@@ -56,9 +56,11 @@ module Aspera
|
|
56
56
|
self.options.add_opt_simple(:case,'basename of output for for test')
|
57
57
|
self.options.add_opt_simple(:scan_path,'subpath in folder id to start scan in (default=/)')
|
58
58
|
self.options.add_opt_simple(:scan_id,'forder id in storage to start scan in, default is access key main folder id')
|
59
|
+
self.options.add_opt_boolean(:mimemagic,'use Mime type detection of gem mimemagic')
|
59
60
|
self.options.add_opt_list(:overwrite,[:always,:never,:mtime],'when to overwrite result file')
|
60
61
|
self.options.add_opt_list(:file_access,[:local,:remote],'how to read and write files in repository')
|
61
62
|
self.options.set_option(:temp_folder,Dir.tmpdir)
|
63
|
+
self.options.set_option(:mimemagic,:false)
|
62
64
|
|
63
65
|
# add other options for generator (and set default values)
|
64
66
|
Aspera::Preview::Options::DESCRIPTIONS.each do |opt|
|
@@ -100,8 +102,6 @@ module Aspera
|
|
100
102
|
return @preview_formats_to_generate.map{|i|i.to_s}.join(',')
|
101
103
|
end
|
102
104
|
|
103
|
-
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
104
|
-
|
105
105
|
# /files/id/files is normally cached in redis, but we can discard the cache
|
106
106
|
# but /files/id is not cached
|
107
107
|
def get_folder_entries(file_id,request_args=nil)
|
@@ -112,14 +112,23 @@ module Aspera
|
|
112
112
|
end
|
113
113
|
|
114
114
|
# old version based on folders
|
115
|
-
def
|
115
|
+
def process_trevents(iteration_token)
|
116
116
|
events_filter={
|
117
117
|
'access_key'=>@access_key_self['id'],
|
118
118
|
'type'=>'download.ended'
|
119
119
|
}
|
120
120
|
# optionally by iteration token
|
121
121
|
events_filter['iteration_token']=iteration_token unless iteration_token.nil?
|
122
|
-
|
122
|
+
begin
|
123
|
+
events=@api_node.read('events',events_filter)[:data]
|
124
|
+
rescue RestCallError => e
|
125
|
+
if e.message.include?('Invalid iteration_token')
|
126
|
+
Log.log.warn("Retrying without iteration token: #{e}")
|
127
|
+
events_filter.delete('iteration_token')
|
128
|
+
retry
|
129
|
+
end
|
130
|
+
raise e
|
131
|
+
end
|
123
132
|
return if events.empty?
|
124
133
|
events.each do |event|
|
125
134
|
next unless event['data']['direction'].eql?('receive')
|
@@ -137,7 +146,7 @@ module Aspera
|
|
137
146
|
end
|
138
147
|
|
139
148
|
# requests recent events on node api and process newly modified folders
|
140
|
-
def
|
149
|
+
def process_events(iteration_token)
|
141
150
|
# get new file creation by access key (TODO: what if file already existed?)
|
142
151
|
events_filter={
|
143
152
|
'access_key'=>@access_key_self['id'],
|
@@ -276,7 +285,7 @@ module Aspera
|
|
276
285
|
end
|
277
286
|
end
|
278
287
|
# need generator for further checks
|
279
|
-
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type']
|
288
|
+
gen_info[:generator]=Aspera::Preview::Generator.new(@gen_options,gen_info[:src],gen_info[:dst],@tmp_folder,entry['content_type'])
|
280
289
|
# get conversion_type (if known) and check if supported
|
281
290
|
next false unless gen_info[:generator].supported?
|
282
291
|
# shall we skip it ?
|
@@ -360,6 +369,8 @@ module Aspera
|
|
360
369
|
end
|
361
370
|
end
|
362
371
|
|
372
|
+
ACTIONS=[:scan,:events,:trevents,:check,:test]
|
373
|
+
|
363
374
|
def execute_action
|
364
375
|
command=self.options.get_next_command(ACTIONS)
|
365
376
|
unless [:check,:test].include?(command)
|
@@ -402,6 +413,7 @@ module Aspera
|
|
402
413
|
end
|
403
414
|
end
|
404
415
|
end
|
416
|
+
Aspera::Preview::FileTypes.instance.use_mimemagic = self.options.get_option(:mimemagic,:mandatory)
|
405
417
|
case command
|
406
418
|
when :scan
|
407
419
|
scan_path=self.options.get_option(:scan_path,:optional)
|
@@ -417,30 +429,18 @@ module Aspera
|
|
417
429
|
end
|
418
430
|
scan_folder_files(folder_info,scan_path)
|
419
431
|
return Main.result_status('scan finished')
|
420
|
-
when :events
|
421
|
-
iteration_data=[]
|
422
|
-
iteration_persistency=nil
|
423
|
-
if self.options.get_option(:once_only,:mandatory)
|
424
|
-
iteration_persistency=PersistencyActionOnce.new(
|
425
|
-
manager: @agents[:persistency],
|
426
|
-
data: iteration_data,
|
427
|
-
ids: ['preview_iteration_events',self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
428
|
-
end
|
429
|
-
iteration_data[0]=process_file_events(iteration_data[0])
|
430
|
-
iteration_persistency.save unless iteration_persistency.nil?
|
431
|
-
return Main.result_status('events finished')
|
432
|
-
when :trevents
|
432
|
+
when :events,:trevents
|
433
433
|
iteration_data=[]
|
434
434
|
iteration_persistency=nil
|
435
435
|
if self.options.get_option(:once_only,:mandatory)
|
436
436
|
iteration_persistency=PersistencyActionOnce.new(
|
437
437
|
manager: @agents[:persistency],
|
438
438
|
data: iteration_data,
|
439
|
-
ids:
|
439
|
+
ids: ["preview_iteration_#{command}",self.options.get_option(:url,:mandatory),self.options.get_option(:username,:mandatory)])
|
440
440
|
end
|
441
|
-
iteration_data[0]=
|
441
|
+
iteration_data[0]=send("process_#{command}",iteration_data[0])
|
442
442
|
iteration_persistency.save unless iteration_persistency.nil?
|
443
|
-
return Main.result_status(
|
443
|
+
return Main.result_status("#{command} finished")
|
444
444
|
when :check
|
445
445
|
Aspera::Preview::Utils.check_tools(@skip_types)
|
446
446
|
return Main.result_status('tools validated')
|
@@ -448,8 +448,9 @@ module Aspera
|
|
448
448
|
format = self.options.get_next_argument('format',Aspera::Preview::Generator::PREVIEW_FORMATS)
|
449
449
|
source = self.options.get_next_argument('source file')
|
450
450
|
dest=preview_filename(format,self.options.get_option(:case,:optional))
|
451
|
-
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder)
|
452
|
-
raise "
|
451
|
+
g=Aspera::Preview::Generator.new(@gen_options,source,dest,@tmp_folder,nil)
|
452
|
+
raise "cannot find file type for #{source}" if g.conversion_type.nil?
|
453
|
+
raise "out format #{format} not supported" unless g.supported?
|
453
454
|
g.generate
|
454
455
|
return Main.result_status("generated: #{dest}")
|
455
456
|
else
|
@@ -62,7 +62,7 @@ module Aspera
|
|
62
62
|
end.select{|i|!i.nil?}
|
63
63
|
end
|
64
64
|
|
65
|
-
ACTIONS=[:
|
65
|
+
ACTIONS=[:health,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
|
66
66
|
|
67
67
|
def execute_action
|
68
68
|
server_uri=URI.parse(self.options.get_option(:url,:mandatory))
|
@@ -93,12 +93,14 @@ module Aspera
|
|
93
93
|
ssh_keys=[ssh_keys] if ssh_keys.is_a?(String)
|
94
94
|
ssh_keys.map!{|p|File.expand_path(p)}
|
95
95
|
Log.log.debug("ssh keys=#{ssh_keys}")
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
96
|
+
if !ssh_keys.empty?
|
97
|
+
ssh_options[:keys]=ssh_keys
|
98
|
+
server_transfer_spec['EX_ssh_key_paths']=ssh_keys
|
99
|
+
ssh_keys.each do |k|
|
100
|
+
Log.log.warn("no such key file: #{k}") unless File.exist?(k)
|
101
|
+
end
|
102
|
+
cred_set=true
|
100
103
|
end
|
101
|
-
cred_set=true
|
102
104
|
end
|
103
105
|
raise 'either password or key must be provided' if !cred_set
|
104
106
|
shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
|
@@ -114,9 +116,9 @@ module Aspera
|
|
114
116
|
command=:rm if command.eql?(:delete)
|
115
117
|
command=:mv if command.eql?(:rename)
|
116
118
|
case command
|
117
|
-
when :
|
119
|
+
when :health
|
118
120
|
nagios=Nagios.new
|
119
|
-
command_nagios=self.options.get_next_command([ :app_services, :transfer ])
|
121
|
+
command_nagios=self.options.get_next_command([ :app_services, :transfer, :asctlstatus ])
|
120
122
|
case command_nagios
|
121
123
|
when :app_services
|
122
124
|
# will not work with aspshell, requires Linux/bash
|
@@ -145,6 +147,19 @@ module Aspera
|
|
145
147
|
else
|
146
148
|
nagios.add_critical('transfer',statuses.select{|i|!i.eql?(:success)}.first.to_s)
|
147
149
|
end
|
150
|
+
when :asctlstatus
|
151
|
+
realcmd='asctl'
|
152
|
+
prefix=self.options.get_option(:cmd_prefix,:optional)
|
153
|
+
realcmd="#{prefix}#{realcmd} all:status" unless prefix.nil?
|
154
|
+
result=shell_executor.execute(realcmd.split(' '))
|
155
|
+
data=asctl_parse(result)
|
156
|
+
data.each do |i|
|
157
|
+
if i['state'].eql?('running')
|
158
|
+
nagios.add_ok(i['process'],i['state'])
|
159
|
+
else
|
160
|
+
nagios.add_critical(i['process'],i['state'])
|
161
|
+
end
|
162
|
+
end
|
148
163
|
else raise "ERROR"
|
149
164
|
end
|
150
165
|
return nagios.result
|
@@ -22,7 +22,7 @@ module Aspera
|
|
22
22
|
@opt_mgr=cli_objects[:options]
|
23
23
|
@config=cli_objects[:config]
|
24
24
|
# transfer spec overrides provided on command line
|
25
|
-
@transfer_spec_cmdline={}
|
25
|
+
@transfer_spec_cmdline={"create_dir"=>true}
|
26
26
|
# the currently selected transfer agent
|
27
27
|
@agent=nil
|
28
28
|
@progress_listener=Listener::ProgressMulti.new
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/fasp/connect.rb
CHANGED
@@ -12,7 +12,9 @@ module Aspera
|
|
12
12
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
13
13
|
def initialize
|
14
14
|
super
|
15
|
-
@
|
15
|
+
@connect_settings={
|
16
|
+
'app_id' => SecureRandom.uuid
|
17
|
+
}
|
16
18
|
# TODO: start here and create monitor
|
17
19
|
end
|
18
20
|
|
@@ -32,46 +34,51 @@ module Aspera
|
|
32
34
|
OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
|
33
35
|
raise StandardError,'Connect is not installed'
|
34
36
|
end
|
35
|
-
sleep
|
37
|
+
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
36
38
|
retry
|
37
39
|
end
|
38
40
|
if transfer_spec['direction'] == 'send'
|
39
41
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
40
42
|
transfer_spec.delete('paths')
|
41
|
-
resdata=@connect_api.create('windows/select-open-file-dialog/',{'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''
|
43
|
+
resdata=@connect_api.create('windows/select-open-file-dialog/',{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
|
42
44
|
transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
|
43
45
|
end
|
44
46
|
@request_id=SecureRandom.uuid
|
45
47
|
# if there is a token, we ask connect client to use well known ssh private keys
|
46
48
|
# instead of asking password
|
47
49
|
transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
|
50
|
+
connect_settings=
|
48
51
|
connect_transfer_args={
|
49
|
-
'
|
50
|
-
'
|
51
|
-
'
|
52
|
-
|
53
|
-
'
|
54
|
-
'
|
55
|
-
}
|
52
|
+
'aspera_connect_settings'=>@connect_settings.merge({
|
53
|
+
'request_id' =>@request_id,
|
54
|
+
'allow_dialogs' =>true,
|
55
|
+
}),
|
56
|
+
'transfer_specs' =>[{
|
57
|
+
'transfer_spec' =>transfer_spec,
|
58
|
+
}]}
|
56
59
|
# asynchronous anyway
|
57
|
-
|
60
|
+
res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
|
61
|
+
@xfer_id=res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
|
58
62
|
end
|
59
63
|
|
60
64
|
def wait_for_transfers_completion
|
61
|
-
connect_activity_args={'aspera_connect_settings'
|
65
|
+
connect_activity_args={'aspera_connect_settings'=>@connect_settings}
|
62
66
|
started=false
|
63
67
|
spinner=nil
|
64
68
|
loop do
|
65
|
-
|
66
|
-
if
|
67
|
-
trdata=
|
68
|
-
|
69
|
+
tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
|
70
|
+
if tr_info['transfer_info'].is_a?(Hash)
|
71
|
+
trdata=tr_info['transfer_info']
|
72
|
+
if trdata.nil?
|
73
|
+
Log.log.warn("no session in Connect")
|
74
|
+
break
|
75
|
+
end
|
69
76
|
# TODO: get session id
|
70
77
|
case trdata['status']
|
71
78
|
when 'completed'
|
72
|
-
|
79
|
+
notify_end(@connect_settings['app_id'])
|
73
80
|
break
|
74
|
-
when 'initiating'
|
81
|
+
when 'initiating','queued'
|
75
82
|
if spinner.nil?
|
76
83
|
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
77
84
|
spinner.start
|
@@ -81,13 +88,13 @@ module Aspera
|
|
81
88
|
when 'running'
|
82
89
|
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
83
90
|
if !started and trdata['bytes_expected'] != 0
|
84
|
-
|
91
|
+
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
85
92
|
started=true
|
86
93
|
else
|
87
|
-
|
94
|
+
notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
|
88
95
|
end
|
89
96
|
else
|
90
|
-
raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
|
97
|
+
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
91
98
|
end
|
92
99
|
end
|
93
100
|
sleep 1
|
data/lib/aspera/fasp/http_gw.rb
CHANGED
@@ -2,12 +2,147 @@
|
|
2
2
|
require 'aspera/fasp/manager'
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/rest'
|
5
|
+
require 'websocket-client-simple'
|
6
|
+
require 'securerandom'
|
7
|
+
require 'openssl'
|
8
|
+
require 'base64'
|
9
|
+
require 'json'
|
10
|
+
require 'uri'
|
5
11
|
|
6
12
|
# ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
|
7
13
|
module Aspera
|
8
14
|
module Fasp
|
9
15
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
10
16
|
class HttpGW < Manager
|
17
|
+
# message returned by HTTP GW in case of success
|
18
|
+
OK_MESSAGE='end upload'
|
19
|
+
# refresh rate for progress
|
20
|
+
UPLOAD_REFRESH_SEC=0.5
|
21
|
+
private_constant :OK_MESSAGE,:UPLOAD_REFRESH_SEC
|
22
|
+
# send message on http gw web socket
|
23
|
+
def ws_send(ws,type,data)
|
24
|
+
ws.send(JSON.generate({type => data}))
|
25
|
+
end
|
26
|
+
|
27
|
+
def upload(transfer_spec)
|
28
|
+
# precalculate size
|
29
|
+
total_size=0
|
30
|
+
# currently, files are sent flat
|
31
|
+
source_path=[]
|
32
|
+
transfer_spec['paths'].each do |item|
|
33
|
+
filepath=item['source']
|
34
|
+
item['source']=item['destination']=File.basename(filepath)
|
35
|
+
total_size+=item['file_size']=File.size(filepath)
|
36
|
+
source_path.push(filepath)
|
37
|
+
end
|
38
|
+
session_id=SecureRandom.uuid
|
39
|
+
ws=::WebSocket::Client::Simple::Client.new
|
40
|
+
error=nil
|
41
|
+
received=0
|
42
|
+
ws.on :message do |msg|
|
43
|
+
Log.log.info("ws: message: #{msg.data}")
|
44
|
+
message=msg.data
|
45
|
+
if message.eql?(OK_MESSAGE)
|
46
|
+
received+=1
|
47
|
+
else
|
48
|
+
message.chomp!
|
49
|
+
if message[0].eql?('"') and message[-1].eql?('"')
|
50
|
+
error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
51
|
+
else
|
52
|
+
error="expecting quotes in [#{message}]"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
ws.on :error do |e|
|
57
|
+
error=e
|
58
|
+
end
|
59
|
+
ws.on :open do
|
60
|
+
Log.log.info("ws: open")
|
61
|
+
end
|
62
|
+
ws.on :close do
|
63
|
+
Log.log.info("ws: close")
|
64
|
+
end
|
65
|
+
# open web socket to end point
|
66
|
+
ws.connect("#{@gw_api.params[:base_url]}/upload")
|
67
|
+
while !ws.open? and error.nil? do
|
68
|
+
Log.log.info("ws: wait")
|
69
|
+
sleep(0.2)
|
70
|
+
end
|
71
|
+
notify_begin(session_id,total_size)
|
72
|
+
ws_send(ws,:transfer_spec,transfer_spec)
|
73
|
+
# current file index
|
74
|
+
filenum=0
|
75
|
+
# aggregate size sent
|
76
|
+
sent_bytes=0
|
77
|
+
# last progress event
|
78
|
+
lastevent=Time.now-1
|
79
|
+
transfer_spec['paths'].each do |item|
|
80
|
+
# TODO: on destination write same path?
|
81
|
+
destination_path=item['source']
|
82
|
+
# TODO: get mime type?
|
83
|
+
file_mime_type=''
|
84
|
+
total=item['file_size']
|
85
|
+
# compute total number of slices
|
86
|
+
numslices=1+(total-1)/@upload_chunksize
|
87
|
+
# current slice index
|
88
|
+
slicenum=0
|
89
|
+
File.open(source_path[filenum]) do |file|
|
90
|
+
while !file.eof? do
|
91
|
+
data=file.read(@upload_chunksize)
|
92
|
+
slice_data={
|
93
|
+
name: destination_path,
|
94
|
+
type: file_mime_type,
|
95
|
+
size: total,
|
96
|
+
data: Base64.strict_encode64(data),
|
97
|
+
slice: slicenum,
|
98
|
+
total_slices: numslices,
|
99
|
+
fileIndex: filenum
|
100
|
+
}
|
101
|
+
ws_send(ws,:slice_upload, slice_data)
|
102
|
+
sent_bytes+=data.length
|
103
|
+
currenttime=Time.now
|
104
|
+
if (currenttime-lastevent)>UPLOAD_REFRESH_SEC
|
105
|
+
notify_progress(session_id,sent_bytes)
|
106
|
+
lastevent=currenttime
|
107
|
+
end
|
108
|
+
slicenum+=1
|
109
|
+
raise error unless error.nil?
|
110
|
+
end
|
111
|
+
end
|
112
|
+
filenum+=1
|
113
|
+
end
|
114
|
+
ws.close
|
115
|
+
notify_end(session_id)
|
116
|
+
end
|
117
|
+
|
118
|
+
def download(transfer_spec)
|
119
|
+
transfer_spec['zip_required']||=false
|
120
|
+
transfer_spec['source_root']||='/'
|
121
|
+
# is normally provided by application, like package name
|
122
|
+
if !transfer_spec.has_key?('download_name')
|
123
|
+
# by default it is the name of first file
|
124
|
+
dname=File.basename(transfer_spec['paths'].first['source'])
|
125
|
+
# we remove extension
|
126
|
+
dname=dname.gsub(/\.@gw_api.*$/,'')
|
127
|
+
# ands add indication of number of files if there is more than one
|
128
|
+
if transfer_spec['paths'].length > 1
|
129
|
+
dname=dname+" #{transfer_spec['paths'].length} Files"
|
130
|
+
end
|
131
|
+
transfer_spec['download_name']=dname
|
132
|
+
end
|
133
|
+
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
134
|
+
transfer_uuid=creation['url'].split('/').last
|
135
|
+
if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
|
136
|
+
# it is a zip file if zip is required or there is more than 1 file
|
137
|
+
file_dest=transfer_spec['download_name']+'.zip'
|
138
|
+
else
|
139
|
+
# it is a plain file if we don't require zip and there is only one file
|
140
|
+
file_dest=File.basename(transfer_spec['paths'].first['source'])
|
141
|
+
end
|
142
|
+
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
143
|
+
@gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
|
144
|
+
end
|
145
|
+
|
11
146
|
# start FASP transfer based on transfer spec (hash table)
|
12
147
|
# note that it is asynchronous
|
13
148
|
# HTTP download only supports file list
|
@@ -15,37 +150,13 @@ module Aspera
|
|
15
150
|
raise "GW URL must be set" unless !@gw_api.nil?
|
16
151
|
raise "option: must be hash (or nil)" unless options.is_a?(Hash)
|
17
152
|
raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
|
153
|
+
raise "on token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
|
154
|
+
transfer_spec['authentication']||='token'
|
18
155
|
case transfer_spec['direction']
|
19
156
|
when 'send'
|
20
|
-
|
21
|
-
raise "error, not implemented"
|
157
|
+
upload(transfer_spec)
|
22
158
|
when 'receive'
|
23
|
-
transfer_spec
|
24
|
-
transfer_spec['authentication']||='token'
|
25
|
-
transfer_spec['source_root']||='/'
|
26
|
-
# is normally provided by application, like package name
|
27
|
-
if !transfer_spec.has_key?('download_name')
|
28
|
-
# by default it is the name of first file
|
29
|
-
dname=File.basename(transfer_spec['paths'].first['source'])
|
30
|
-
# we remove extension
|
31
|
-
dname=dname.gsub(/\.@gw_api.*$/,'')
|
32
|
-
# ands add indication of number of files if there is more than one
|
33
|
-
if transfer_spec['paths'].length > 1
|
34
|
-
dname=dname+" #{transfer_spec['paths'].length} Files"
|
35
|
-
end
|
36
|
-
transfer_spec['download_name']=dname
|
37
|
-
end
|
38
|
-
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
39
|
-
transfer_uuid=creation['url'].split('/').last
|
40
|
-
if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
|
41
|
-
# it is a zip file if zip is required or there is more than 1 file
|
42
|
-
file_dest=transfer_spec['download_name']+'.zip'
|
43
|
-
else
|
44
|
-
# it is a plain file if we don't require zip and there is only one file
|
45
|
-
file_dest=File.basename(transfer_spec['paths'].first['source'])
|
46
|
-
end
|
47
|
-
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
48
|
-
@gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
|
159
|
+
download(transfer_spec)
|
49
160
|
else
|
50
161
|
raise "error"
|
51
162
|
end
|
@@ -74,6 +185,7 @@ module Aspera
|
|
74
185
|
@gw_api=Rest.new({:base_url => params[:url]})
|
75
186
|
api_info = @gw_api.read('info')[:data]
|
76
187
|
Log.log.info("#{api_info}")
|
188
|
+
@upload_chunksize=128000 # TODO: configurable ?
|
77
189
|
end
|
78
190
|
|
79
191
|
end # LocalHttp
|