aspera-cli 4.0.0.pre3 → 4.2.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +695 -205
- data/bin/dascli +13 -0
- data/docs/README.erb.md +615 -157
- data/docs/test_env.conf +23 -5
- data/docs/transfer_spec.html +1 -1
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +78 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +46 -34
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +207 -182
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +173 -117
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +51 -36
- data/lib/aspera/cli/plugins/faspex5.rb +82 -41
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +35 -25
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +7 -6
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +101 -53
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +6 -18
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +42 -22
@@ -62,7 +62,7 @@ module Aspera
|
|
62
62
|
end.select{|i|!i.nil?}
|
63
63
|
end
|
64
64
|
|
65
|
-
ACTIONS=[:
|
65
|
+
ACTIONS=[:health,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
|
66
66
|
|
67
67
|
def execute_action
|
68
68
|
server_uri=URI.parse(self.options.get_option(:url,:mandatory))
|
@@ -93,12 +93,14 @@ module Aspera
|
|
93
93
|
ssh_keys=[ssh_keys] if ssh_keys.is_a?(String)
|
94
94
|
ssh_keys.map!{|p|File.expand_path(p)}
|
95
95
|
Log.log.debug("ssh keys=#{ssh_keys}")
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
96
|
+
if !ssh_keys.empty?
|
97
|
+
ssh_options[:keys]=ssh_keys
|
98
|
+
server_transfer_spec['EX_ssh_key_paths']=ssh_keys
|
99
|
+
ssh_keys.each do |k|
|
100
|
+
Log.log.warn("no such key file: #{k}") unless File.exist?(k)
|
101
|
+
end
|
102
|
+
cred_set=true
|
100
103
|
end
|
101
|
-
cred_set=true
|
102
104
|
end
|
103
105
|
raise 'either password or key must be provided' if !cred_set
|
104
106
|
shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
|
@@ -114,9 +116,9 @@ module Aspera
|
|
114
116
|
command=:rm if command.eql?(:delete)
|
115
117
|
command=:mv if command.eql?(:rename)
|
116
118
|
case command
|
117
|
-
when :
|
119
|
+
when :health
|
118
120
|
nagios=Nagios.new
|
119
|
-
command_nagios=self.options.get_next_command([ :app_services, :transfer ])
|
121
|
+
command_nagios=self.options.get_next_command([ :app_services, :transfer, :asctlstatus ])
|
120
122
|
case command_nagios
|
121
123
|
when :app_services
|
122
124
|
# will not work with aspshell, requires Linux/bash
|
@@ -145,6 +147,19 @@ module Aspera
|
|
145
147
|
else
|
146
148
|
nagios.add_critical('transfer',statuses.select{|i|!i.eql?(:success)}.first.to_s)
|
147
149
|
end
|
150
|
+
when :asctlstatus
|
151
|
+
realcmd='asctl'
|
152
|
+
prefix=self.options.get_option(:cmd_prefix,:optional)
|
153
|
+
realcmd="#{prefix}#{realcmd} all:status" unless prefix.nil?
|
154
|
+
result=shell_executor.execute(realcmd.split(' '))
|
155
|
+
data=asctl_parse(result)
|
156
|
+
data.each do |i|
|
157
|
+
if i['state'].eql?('running')
|
158
|
+
nagios.add_ok(i['process'],i['state'])
|
159
|
+
else
|
160
|
+
nagios.add_critical(i['process'],i['state'])
|
161
|
+
end
|
162
|
+
end
|
148
163
|
else raise "ERROR"
|
149
164
|
end
|
150
165
|
return nagios.result
|
@@ -10,7 +10,7 @@ module Aspera
|
|
10
10
|
module Cli
|
11
11
|
# The Transfer agent is a common interface to start a transfer using
|
12
12
|
# one of the supported transfer agents
|
13
|
-
# provides CLI options to select one of the transfer agents (
|
13
|
+
# provides CLI options to select one of the transfer agents (FASP/ascp client)
|
14
14
|
class TransferAgent
|
15
15
|
# special value for --sources : read file list from arguments
|
16
16
|
FILE_LIST_FROM_ARGS='@args'
|
@@ -22,7 +22,7 @@ module Aspera
|
|
22
22
|
@opt_mgr=cli_objects[:options]
|
23
23
|
@config=cli_objects[:config]
|
24
24
|
# transfer spec overrides provided on command line
|
25
|
-
@transfer_spec_cmdline={}
|
25
|
+
@transfer_spec_cmdline={"create_dir"=>true}
|
26
26
|
# the currently selected transfer agent
|
27
27
|
@agent=nil
|
28
28
|
@progress_listener=Listener::ProgressMulti.new
|
@@ -45,6 +45,7 @@ module Aspera
|
|
45
45
|
|
46
46
|
def option_transfer_spec; @transfer_spec_cmdline; end
|
47
47
|
|
48
|
+
# multiple option are merged
|
48
49
|
def option_transfer_spec=(value); @transfer_spec_cmdline.merge!(value); end
|
49
50
|
|
50
51
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
|
@@ -75,7 +76,7 @@ module Aspera
|
|
75
76
|
when :connect
|
76
77
|
new_agent=Fasp::Connect.new
|
77
78
|
when :node
|
78
|
-
# way for code to setup alternate node api in
|
79
|
+
# way for code to setup alternate node api in advance
|
79
80
|
# support: @preset:<name>
|
80
81
|
# support extended values
|
81
82
|
node_config=@opt_mgr.get_option(:transfer_info,:optional)
|
@@ -118,7 +119,7 @@ module Aspera
|
|
118
119
|
aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
|
119
120
|
new_agent=Fasp::Aoc.new(aoc_config)
|
120
121
|
else
|
121
|
-
raise "
|
122
|
+
raise "Unexpected transfer agent type: #{agent_type}"
|
122
123
|
end
|
123
124
|
set_agent_instance(new_agent)
|
124
125
|
return nil
|
@@ -178,9 +179,9 @@ module Aspera
|
|
178
179
|
# when providing a list, just specify source
|
179
180
|
@transfer_paths=file_list.map{|i|{'source'=>i}}
|
180
181
|
when :pair
|
181
|
-
raise CliBadArgument,"
|
182
|
+
raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
|
182
183
|
@transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
|
183
|
-
else raise "
|
184
|
+
else raise "Unsupported src_type"
|
184
185
|
end
|
185
186
|
Log.log.debug("paths=#{@transfer_paths}")
|
186
187
|
return @transfer_paths
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/cos_node.rb
CHANGED
@@ -5,46 +5,51 @@ require 'xmlsimple'
|
|
5
5
|
module Aspera
|
6
6
|
class CosNode < Rest
|
7
7
|
attr_reader :add_ts
|
8
|
-
|
8
|
+
IBM_CLOUD_TOKEN_URL='https://iam.cloud.ibm.com/identity'
|
9
|
+
def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url=IBM_CLOUD_TOKEN_URL)
|
10
|
+
@auth_url=auth_url
|
11
|
+
@api_key=api_key
|
9
12
|
s3_api=Aspera::Rest.new({
|
10
|
-
:base_url
|
11
|
-
:not_auth_codes => ['401','403'],
|
12
|
-
:headers
|
13
|
-
:auth
|
14
|
-
:type
|
15
|
-
:base_url
|
16
|
-
:grant
|
17
|
-
:api_key
|
13
|
+
:base_url => storage_endpoint,
|
14
|
+
:not_auth_codes => ['401','403'], # error codes when not authorized
|
15
|
+
:headers => {'ibm-service-instance-id' => instance_id},
|
16
|
+
:auth => {
|
17
|
+
:type => :oauth2,
|
18
|
+
:base_url => @auth_url,
|
19
|
+
:grant => :ibm_apikey,
|
20
|
+
:api_key => @api_key
|
18
21
|
}})
|
19
22
|
# read FASP connection information for bucket
|
20
23
|
xml_result_text=s3_api.call({:operation=>'GET',:subpath=>bucket_name,:headers=>{'Accept'=>'application/xml'},:url_params=>{'faspConnectionInfo'=>nil}})[:http].body
|
21
24
|
ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
22
25
|
Aspera::Log.dump('ats_info',ats_info)
|
23
|
-
# get delegated token
|
24
|
-
delegated_oauth=Oauth.new({
|
25
|
-
:type => :oauth2,
|
26
|
-
:base_url => auth_url,
|
27
|
-
:grant => :delegated_refresh,
|
28
|
-
:api_key => api_key,
|
29
|
-
:token_field=> 'delegated_refresh_token'
|
30
|
-
})
|
31
|
-
# to be placed in rest call header and in transfer tags
|
32
|
-
aspera_storage_credentials={
|
33
|
-
'type' => 'token',
|
34
|
-
'token' => {'delegated_refresh_token'=>delegated_oauth.get_authorization().gsub(/^Bearer /,'')}
|
35
|
-
}
|
36
|
-
# transfer spec addition
|
37
|
-
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>aspera_storage_credentials}}}}
|
38
|
-
# set a general addon to transfer spec
|
39
|
-
# here we choose to use the add_request_param
|
40
|
-
#self.transfer.option_transfer_spec_deep_merge(@add_ts)
|
41
26
|
super({
|
42
27
|
:base_url => ats_info['ATSEndpoint'],
|
43
|
-
:headers => {'X-Aspera-Storage-Credentials'=>JSON.generate(aspera_storage_credentials)},
|
44
28
|
:auth => {
|
45
29
|
:type => :basic,
|
46
30
|
:username => ats_info['AccessKey']['Id'],
|
47
31
|
:password => ats_info['AccessKey']['Secret']}})
|
32
|
+
# prepare transfer spec addition
|
33
|
+
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
|
34
|
+
'type' => 'token',
|
35
|
+
'token' => {'delegated_refresh_token'=>nil}
|
36
|
+
}}}}}
|
37
|
+
generate_token
|
38
|
+
end
|
39
|
+
|
40
|
+
# potentially call this if delegated token is expired
|
41
|
+
def generate_token
|
42
|
+
# OAuth API to get delegated token
|
43
|
+
delegated_oauth=Oauth.new({
|
44
|
+
:type => :oauth2,
|
45
|
+
:base_url => @auth_url,
|
46
|
+
:grant => :delegated_refresh,
|
47
|
+
:api_key => @api_key,
|
48
|
+
:token_field=> 'delegated_refresh_token'
|
49
|
+
})
|
50
|
+
# get delagated token to be placed in rest call header and in transfer tags
|
51
|
+
@add_ts['tags']['aspera']['node']['storage_credentials']['token']['delegated_refresh_token']=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
|
52
|
+
@params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
|
48
53
|
end
|
49
54
|
end
|
50
55
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -2,7 +2,7 @@ require 'aspera/log'
|
|
2
2
|
require 'rbconfig'
|
3
3
|
|
4
4
|
module Aspera
|
5
|
-
#
|
5
|
+
# detect OS, architecture, and OS specific stuff
|
6
6
|
class Environment
|
7
7
|
OS_WINDOWS = :windows
|
8
8
|
OS_X = :osx
|
@@ -53,7 +53,7 @@ module Aspera
|
|
53
53
|
return ''
|
54
54
|
end
|
55
55
|
|
56
|
-
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably
|
56
|
+
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably than %HOMEDRIVE%%HOMEPATH%
|
57
57
|
def self.fix_home
|
58
58
|
if os.eql?(OS_WINDOWS)
|
59
59
|
if ENV.has_key?('USERPROFILE') and Dir.exist?(ENV['USERPROFILE'])
|
data/lib/aspera/fasp/connect.rb
CHANGED
@@ -12,7 +12,9 @@ module Aspera
|
|
12
12
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
13
13
|
def initialize
|
14
14
|
super
|
15
|
-
@
|
15
|
+
@connect_settings={
|
16
|
+
'app_id' => SecureRandom.uuid
|
17
|
+
}
|
16
18
|
# TODO: start here and create monitor
|
17
19
|
end
|
18
20
|
|
@@ -32,46 +34,51 @@ module Aspera
|
|
32
34
|
OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
|
33
35
|
raise StandardError,'Connect is not installed'
|
34
36
|
end
|
35
|
-
sleep
|
37
|
+
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
36
38
|
retry
|
37
39
|
end
|
38
40
|
if transfer_spec['direction'] == 'send'
|
39
41
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
40
42
|
transfer_spec.delete('paths')
|
41
|
-
resdata=@connect_api.create('windows/select-open-file-dialog/',{'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''
|
43
|
+
resdata=@connect_api.create('windows/select-open-file-dialog/',{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
|
42
44
|
transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
|
43
45
|
end
|
44
46
|
@request_id=SecureRandom.uuid
|
45
47
|
# if there is a token, we ask connect client to use well known ssh private keys
|
46
48
|
# instead of asking password
|
47
49
|
transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
|
50
|
+
connect_settings=
|
48
51
|
connect_transfer_args={
|
49
|
-
'
|
50
|
-
'
|
51
|
-
'
|
52
|
-
|
53
|
-
'
|
54
|
-
'
|
55
|
-
}
|
52
|
+
'aspera_connect_settings'=>@connect_settings.merge({
|
53
|
+
'request_id' =>@request_id,
|
54
|
+
'allow_dialogs' =>true,
|
55
|
+
}),
|
56
|
+
'transfer_specs' =>[{
|
57
|
+
'transfer_spec' =>transfer_spec,
|
58
|
+
}]}
|
56
59
|
# asynchronous anyway
|
57
|
-
|
60
|
+
res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
|
61
|
+
@xfer_id=res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
|
58
62
|
end
|
59
63
|
|
60
64
|
def wait_for_transfers_completion
|
61
|
-
connect_activity_args={'aspera_connect_settings'
|
65
|
+
connect_activity_args={'aspera_connect_settings'=>@connect_settings}
|
62
66
|
started=false
|
63
67
|
spinner=nil
|
64
68
|
loop do
|
65
|
-
|
66
|
-
if
|
67
|
-
trdata=
|
68
|
-
|
69
|
+
tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
|
70
|
+
if tr_info['transfer_info'].is_a?(Hash)
|
71
|
+
trdata=tr_info['transfer_info']
|
72
|
+
if trdata.nil?
|
73
|
+
Log.log.warn("no session in Connect")
|
74
|
+
break
|
75
|
+
end
|
69
76
|
# TODO: get session id
|
70
77
|
case trdata['status']
|
71
78
|
when 'completed'
|
72
|
-
|
79
|
+
notify_end(@connect_settings['app_id'])
|
73
80
|
break
|
74
|
-
when 'initiating'
|
81
|
+
when 'initiating','queued'
|
75
82
|
if spinner.nil?
|
76
83
|
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
77
84
|
spinner.start
|
@@ -81,13 +88,13 @@ module Aspera
|
|
81
88
|
when 'running'
|
82
89
|
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
83
90
|
if !started and trdata['bytes_expected'] != 0
|
84
|
-
|
91
|
+
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
85
92
|
started=true
|
86
93
|
else
|
87
|
-
|
94
|
+
notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
|
88
95
|
end
|
89
96
|
else
|
90
|
-
raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
|
97
|
+
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
91
98
|
end
|
92
99
|
end
|
93
100
|
sleep 1
|
data/lib/aspera/fasp/http_gw.rb
CHANGED
@@ -2,12 +2,147 @@
|
|
2
2
|
require 'aspera/fasp/manager'
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/rest'
|
5
|
+
require 'websocket-client-simple'
|
6
|
+
require 'securerandom'
|
7
|
+
require 'openssl'
|
8
|
+
require 'base64'
|
9
|
+
require 'json'
|
10
|
+
require 'uri'
|
5
11
|
|
6
12
|
# ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
|
7
13
|
module Aspera
|
8
14
|
module Fasp
|
9
15
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
10
16
|
class HttpGW < Manager
|
17
|
+
# message returned by HTTP GW in case of success
|
18
|
+
OK_MESSAGE='end upload'
|
19
|
+
# refresh rate for progress
|
20
|
+
UPLOAD_REFRESH_SEC=0.5
|
21
|
+
private_constant :OK_MESSAGE,:UPLOAD_REFRESH_SEC
|
22
|
+
# send message on http gw web socket
|
23
|
+
def ws_send(ws,type,data)
|
24
|
+
ws.send(JSON.generate({type => data}))
|
25
|
+
end
|
26
|
+
|
27
|
+
def upload(transfer_spec)
|
28
|
+
# precalculate size
|
29
|
+
total_size=0
|
30
|
+
# currently, files are sent flat
|
31
|
+
source_path=[]
|
32
|
+
transfer_spec['paths'].each do |item|
|
33
|
+
filepath=item['source']
|
34
|
+
item['source']=item['destination']=File.basename(filepath)
|
35
|
+
total_size+=item['file_size']=File.size(filepath)
|
36
|
+
source_path.push(filepath)
|
37
|
+
end
|
38
|
+
session_id=SecureRandom.uuid
|
39
|
+
ws=::WebSocket::Client::Simple::Client.new
|
40
|
+
error=nil
|
41
|
+
received=0
|
42
|
+
ws.on :message do |msg|
|
43
|
+
Log.log.info("ws: message: #{msg.data}")
|
44
|
+
message=msg.data
|
45
|
+
if message.eql?(OK_MESSAGE)
|
46
|
+
received+=1
|
47
|
+
else
|
48
|
+
message.chomp!
|
49
|
+
if message[0].eql?('"') and message[-1].eql?('"')
|
50
|
+
error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
51
|
+
else
|
52
|
+
error="expecting quotes in [#{message}]"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
ws.on :error do |e|
|
57
|
+
error=e
|
58
|
+
end
|
59
|
+
ws.on :open do
|
60
|
+
Log.log.info("ws: open")
|
61
|
+
end
|
62
|
+
ws.on :close do
|
63
|
+
Log.log.info("ws: close")
|
64
|
+
end
|
65
|
+
# open web socket to end point
|
66
|
+
ws.connect("#{@gw_api.params[:base_url]}/upload")
|
67
|
+
while !ws.open? and error.nil? do
|
68
|
+
Log.log.info("ws: wait")
|
69
|
+
sleep(0.2)
|
70
|
+
end
|
71
|
+
notify_begin(session_id,total_size)
|
72
|
+
ws_send(ws,:transfer_spec,transfer_spec)
|
73
|
+
# current file index
|
74
|
+
filenum=0
|
75
|
+
# aggregate size sent
|
76
|
+
sent_bytes=0
|
77
|
+
# last progress event
|
78
|
+
lastevent=Time.now-1
|
79
|
+
transfer_spec['paths'].each do |item|
|
80
|
+
# TODO: on destination write same path?
|
81
|
+
destination_path=item['source']
|
82
|
+
# TODO: get mime type?
|
83
|
+
file_mime_type=''
|
84
|
+
total=item['file_size']
|
85
|
+
# compute total number of slices
|
86
|
+
numslices=1+(total-1)/@upload_chunksize
|
87
|
+
# current slice index
|
88
|
+
slicenum=0
|
89
|
+
File.open(source_path[filenum]) do |file|
|
90
|
+
while !file.eof? do
|
91
|
+
data=file.read(@upload_chunksize)
|
92
|
+
slice_data={
|
93
|
+
name: destination_path,
|
94
|
+
type: file_mime_type,
|
95
|
+
size: total,
|
96
|
+
data: Base64.strict_encode64(data),
|
97
|
+
slice: slicenum,
|
98
|
+
total_slices: numslices,
|
99
|
+
fileIndex: filenum
|
100
|
+
}
|
101
|
+
ws_send(ws,:slice_upload, slice_data)
|
102
|
+
sent_bytes+=data.length
|
103
|
+
currenttime=Time.now
|
104
|
+
if (currenttime-lastevent)>UPLOAD_REFRESH_SEC
|
105
|
+
notify_progress(session_id,sent_bytes)
|
106
|
+
lastevent=currenttime
|
107
|
+
end
|
108
|
+
slicenum+=1
|
109
|
+
raise error unless error.nil?
|
110
|
+
end
|
111
|
+
end
|
112
|
+
filenum+=1
|
113
|
+
end
|
114
|
+
ws.close
|
115
|
+
notify_end(session_id)
|
116
|
+
end
|
117
|
+
|
118
|
+
def download(transfer_spec)
|
119
|
+
transfer_spec['zip_required']||=false
|
120
|
+
transfer_spec['source_root']||='/'
|
121
|
+
# is normally provided by application, like package name
|
122
|
+
if !transfer_spec.has_key?('download_name')
|
123
|
+
# by default it is the name of first file
|
124
|
+
dname=File.basename(transfer_spec['paths'].first['source'])
|
125
|
+
# we remove extension
|
126
|
+
dname=dname.gsub(/\.@gw_api.*$/,'')
|
127
|
+
# ands add indication of number of files if there is more than one
|
128
|
+
if transfer_spec['paths'].length > 1
|
129
|
+
dname=dname+" #{transfer_spec['paths'].length} Files"
|
130
|
+
end
|
131
|
+
transfer_spec['download_name']=dname
|
132
|
+
end
|
133
|
+
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
134
|
+
transfer_uuid=creation['url'].split('/').last
|
135
|
+
if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
|
136
|
+
# it is a zip file if zip is required or there is more than 1 file
|
137
|
+
file_dest=transfer_spec['download_name']+'.zip'
|
138
|
+
else
|
139
|
+
# it is a plain file if we don't require zip and there is only one file
|
140
|
+
file_dest=File.basename(transfer_spec['paths'].first['source'])
|
141
|
+
end
|
142
|
+
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
143
|
+
@gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
|
144
|
+
end
|
145
|
+
|
11
146
|
# start FASP transfer based on transfer spec (hash table)
|
12
147
|
# note that it is asynchronous
|
13
148
|
# HTTP download only supports file list
|
@@ -15,37 +150,13 @@ module Aspera
|
|
15
150
|
raise "GW URL must be set" unless !@gw_api.nil?
|
16
151
|
raise "option: must be hash (or nil)" unless options.is_a?(Hash)
|
17
152
|
raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
|
153
|
+
raise "on token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
|
154
|
+
transfer_spec['authentication']||='token'
|
18
155
|
case transfer_spec['direction']
|
19
156
|
when 'send'
|
20
|
-
|
21
|
-
raise "error, not implemented"
|
157
|
+
upload(transfer_spec)
|
22
158
|
when 'receive'
|
23
|
-
transfer_spec
|
24
|
-
transfer_spec['authentication']||='token'
|
25
|
-
transfer_spec['source_root']||='/'
|
26
|
-
# is normally provided by application, like package name
|
27
|
-
if !transfer_spec.has_key?('download_name')
|
28
|
-
# by default it is the name of first file
|
29
|
-
dname=File.basename(transfer_spec['paths'].first['source'])
|
30
|
-
# we remove extension
|
31
|
-
dname=dname.gsub(/\.@gw_api.*$/,'')
|
32
|
-
# ands add indication of number of files if there is more than one
|
33
|
-
if transfer_spec['paths'].length > 1
|
34
|
-
dname=dname+" #{transfer_spec['paths'].length} Files"
|
35
|
-
end
|
36
|
-
transfer_spec['download_name']=dname
|
37
|
-
end
|
38
|
-
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
39
|
-
transfer_uuid=creation['url'].split('/').last
|
40
|
-
if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
|
41
|
-
# it is a zip file if zip is required or there is more than 1 file
|
42
|
-
file_dest=transfer_spec['download_name']+'.zip'
|
43
|
-
else
|
44
|
-
# it is a plain file if we don't require zip and there is only one file
|
45
|
-
file_dest=File.basename(transfer_spec['paths'].first['source'])
|
46
|
-
end
|
47
|
-
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
48
|
-
@gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
|
159
|
+
download(transfer_spec)
|
49
160
|
else
|
50
161
|
raise "error"
|
51
162
|
end
|
@@ -74,6 +185,7 @@ module Aspera
|
|
74
185
|
@gw_api=Rest.new({:base_url => params[:url]})
|
75
186
|
api_info = @gw_api.read('info')[:data]
|
76
187
|
Log.log.info("#{api_info}")
|
188
|
+
@upload_chunksize=128000 # TODO: configurable ?
|
77
189
|
end
|
78
190
|
|
79
191
|
end # LocalHttp
|