aspera-cli 4.0.0.pre3 → 4.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +695 -205
- data/bin/dascli +13 -0
- data/docs/README.erb.md +615 -157
- data/docs/test_env.conf +23 -5
- data/docs/transfer_spec.html +1 -1
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +78 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +46 -34
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +207 -182
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +173 -117
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +51 -36
- data/lib/aspera/cli/plugins/faspex5.rb +82 -41
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +35 -25
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +7 -6
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +101 -53
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +6 -18
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +42 -22
@@ -62,7 +62,7 @@ module Aspera
|
|
62
62
|
end.select{|i|!i.nil?}
|
63
63
|
end
|
64
64
|
|
65
|
-
ACTIONS=[:
|
65
|
+
ACTIONS=[:health,:nodeadmin,:userdata,:configurator,:ctl,:download,:upload,:browse,:delete,:rename].concat(Aspera::AsCmd::OPERATIONS)
|
66
66
|
|
67
67
|
def execute_action
|
68
68
|
server_uri=URI.parse(self.options.get_option(:url,:mandatory))
|
@@ -93,12 +93,14 @@ module Aspera
|
|
93
93
|
ssh_keys=[ssh_keys] if ssh_keys.is_a?(String)
|
94
94
|
ssh_keys.map!{|p|File.expand_path(p)}
|
95
95
|
Log.log.debug("ssh keys=#{ssh_keys}")
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
96
|
+
if !ssh_keys.empty?
|
97
|
+
ssh_options[:keys]=ssh_keys
|
98
|
+
server_transfer_spec['EX_ssh_key_paths']=ssh_keys
|
99
|
+
ssh_keys.each do |k|
|
100
|
+
Log.log.warn("no such key file: #{k}") unless File.exist?(k)
|
101
|
+
end
|
102
|
+
cred_set=true
|
100
103
|
end
|
101
|
-
cred_set=true
|
102
104
|
end
|
103
105
|
raise 'either password or key must be provided' if !cred_set
|
104
106
|
shell_executor=Ssh.new(server_transfer_spec['remote_host'],server_transfer_spec['remote_user'],ssh_options)
|
@@ -114,9 +116,9 @@ module Aspera
|
|
114
116
|
command=:rm if command.eql?(:delete)
|
115
117
|
command=:mv if command.eql?(:rename)
|
116
118
|
case command
|
117
|
-
when :
|
119
|
+
when :health
|
118
120
|
nagios=Nagios.new
|
119
|
-
command_nagios=self.options.get_next_command([ :app_services, :transfer ])
|
121
|
+
command_nagios=self.options.get_next_command([ :app_services, :transfer, :asctlstatus ])
|
120
122
|
case command_nagios
|
121
123
|
when :app_services
|
122
124
|
# will not work with aspshell, requires Linux/bash
|
@@ -145,6 +147,19 @@ module Aspera
|
|
145
147
|
else
|
146
148
|
nagios.add_critical('transfer',statuses.select{|i|!i.eql?(:success)}.first.to_s)
|
147
149
|
end
|
150
|
+
when :asctlstatus
|
151
|
+
realcmd='asctl'
|
152
|
+
prefix=self.options.get_option(:cmd_prefix,:optional)
|
153
|
+
realcmd="#{prefix}#{realcmd} all:status" unless prefix.nil?
|
154
|
+
result=shell_executor.execute(realcmd.split(' '))
|
155
|
+
data=asctl_parse(result)
|
156
|
+
data.each do |i|
|
157
|
+
if i['state'].eql?('running')
|
158
|
+
nagios.add_ok(i['process'],i['state'])
|
159
|
+
else
|
160
|
+
nagios.add_critical(i['process'],i['state'])
|
161
|
+
end
|
162
|
+
end
|
148
163
|
else raise "ERROR"
|
149
164
|
end
|
150
165
|
return nagios.result
|
@@ -10,7 +10,7 @@ module Aspera
|
|
10
10
|
module Cli
|
11
11
|
# The Transfer agent is a common interface to start a transfer using
|
12
12
|
# one of the supported transfer agents
|
13
|
-
# provides CLI options to select one of the transfer agents (
|
13
|
+
# provides CLI options to select one of the transfer agents (FASP/ascp client)
|
14
14
|
class TransferAgent
|
15
15
|
# special value for --sources : read file list from arguments
|
16
16
|
FILE_LIST_FROM_ARGS='@args'
|
@@ -22,7 +22,7 @@ module Aspera
|
|
22
22
|
@opt_mgr=cli_objects[:options]
|
23
23
|
@config=cli_objects[:config]
|
24
24
|
# transfer spec overrides provided on command line
|
25
|
-
@transfer_spec_cmdline={}
|
25
|
+
@transfer_spec_cmdline={"create_dir"=>true}
|
26
26
|
# the currently selected transfer agent
|
27
27
|
@agent=nil
|
28
28
|
@progress_listener=Listener::ProgressMulti.new
|
@@ -45,6 +45,7 @@ module Aspera
|
|
45
45
|
|
46
46
|
def option_transfer_spec; @transfer_spec_cmdline; end
|
47
47
|
|
48
|
+
# multiple option are merged
|
48
49
|
def option_transfer_spec=(value); @transfer_spec_cmdline.merge!(value); end
|
49
50
|
|
50
51
|
def option_transfer_spec_deep_merge(ts); @transfer_spec_cmdline.deep_merge!(ts); end
|
@@ -75,7 +76,7 @@ module Aspera
|
|
75
76
|
when :connect
|
76
77
|
new_agent=Fasp::Connect.new
|
77
78
|
when :node
|
78
|
-
# way for code to setup alternate node api in
|
79
|
+
# way for code to setup alternate node api in advance
|
79
80
|
# support: @preset:<name>
|
80
81
|
# support extended values
|
81
82
|
node_config=@opt_mgr.get_option(:transfer_info,:optional)
|
@@ -118,7 +119,7 @@ module Aspera
|
|
118
119
|
aoc_config[:private_key]=ExtendedValue.instance.evaluate(aoc_config[:private_key])
|
119
120
|
new_agent=Fasp::Aoc.new(aoc_config)
|
120
121
|
else
|
121
|
-
raise "
|
122
|
+
raise "Unexpected transfer agent type: #{agent_type}"
|
122
123
|
end
|
123
124
|
set_agent_instance(new_agent)
|
124
125
|
return nil
|
@@ -178,9 +179,9 @@ module Aspera
|
|
178
179
|
# when providing a list, just specify source
|
179
180
|
@transfer_paths=file_list.map{|i|{'source'=>i}}
|
180
181
|
when :pair
|
181
|
-
raise CliBadArgument,"
|
182
|
+
raise CliBadArgument,"When using pair, provide an even number of paths: #{file_list.length}" unless file_list.length.even?
|
182
183
|
@transfer_paths=file_list.each_slice(2).to_a.map{|s,d|{'source'=>s,'destination'=>d}}
|
183
|
-
else raise "
|
184
|
+
else raise "Unsupported src_type"
|
184
185
|
end
|
185
186
|
Log.log.debug("paths=#{@transfer_paths}")
|
186
187
|
return @transfer_paths
|
data/lib/aspera/cli/version.rb
CHANGED
data/lib/aspera/cos_node.rb
CHANGED
@@ -5,46 +5,51 @@ require 'xmlsimple'
|
|
5
5
|
module Aspera
|
6
6
|
class CosNode < Rest
|
7
7
|
attr_reader :add_ts
|
8
|
-
|
8
|
+
IBM_CLOUD_TOKEN_URL='https://iam.cloud.ibm.com/identity'
|
9
|
+
def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url=IBM_CLOUD_TOKEN_URL)
|
10
|
+
@auth_url=auth_url
|
11
|
+
@api_key=api_key
|
9
12
|
s3_api=Aspera::Rest.new({
|
10
|
-
:base_url
|
11
|
-
:not_auth_codes => ['401','403'],
|
12
|
-
:headers
|
13
|
-
:auth
|
14
|
-
:type
|
15
|
-
:base_url
|
16
|
-
:grant
|
17
|
-
:api_key
|
13
|
+
:base_url => storage_endpoint,
|
14
|
+
:not_auth_codes => ['401','403'], # error codes when not authorized
|
15
|
+
:headers => {'ibm-service-instance-id' => instance_id},
|
16
|
+
:auth => {
|
17
|
+
:type => :oauth2,
|
18
|
+
:base_url => @auth_url,
|
19
|
+
:grant => :ibm_apikey,
|
20
|
+
:api_key => @api_key
|
18
21
|
}})
|
19
22
|
# read FASP connection information for bucket
|
20
23
|
xml_result_text=s3_api.call({:operation=>'GET',:subpath=>bucket_name,:headers=>{'Accept'=>'application/xml'},:url_params=>{'faspConnectionInfo'=>nil}})[:http].body
|
21
24
|
ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
22
25
|
Aspera::Log.dump('ats_info',ats_info)
|
23
|
-
# get delegated token
|
24
|
-
delegated_oauth=Oauth.new({
|
25
|
-
:type => :oauth2,
|
26
|
-
:base_url => auth_url,
|
27
|
-
:grant => :delegated_refresh,
|
28
|
-
:api_key => api_key,
|
29
|
-
:token_field=> 'delegated_refresh_token'
|
30
|
-
})
|
31
|
-
# to be placed in rest call header and in transfer tags
|
32
|
-
aspera_storage_credentials={
|
33
|
-
'type' => 'token',
|
34
|
-
'token' => {'delegated_refresh_token'=>delegated_oauth.get_authorization().gsub(/^Bearer /,'')}
|
35
|
-
}
|
36
|
-
# transfer spec addition
|
37
|
-
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>aspera_storage_credentials}}}}
|
38
|
-
# set a general addon to transfer spec
|
39
|
-
# here we choose to use the add_request_param
|
40
|
-
#self.transfer.option_transfer_spec_deep_merge(@add_ts)
|
41
26
|
super({
|
42
27
|
:base_url => ats_info['ATSEndpoint'],
|
43
|
-
:headers => {'X-Aspera-Storage-Credentials'=>JSON.generate(aspera_storage_credentials)},
|
44
28
|
:auth => {
|
45
29
|
:type => :basic,
|
46
30
|
:username => ats_info['AccessKey']['Id'],
|
47
31
|
:password => ats_info['AccessKey']['Secret']}})
|
32
|
+
# prepare transfer spec addition
|
33
|
+
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
|
34
|
+
'type' => 'token',
|
35
|
+
'token' => {'delegated_refresh_token'=>nil}
|
36
|
+
}}}}}
|
37
|
+
generate_token
|
38
|
+
end
|
39
|
+
|
40
|
+
# potentially call this if delegated token is expired
|
41
|
+
def generate_token
|
42
|
+
# OAuth API to get delegated token
|
43
|
+
delegated_oauth=Oauth.new({
|
44
|
+
:type => :oauth2,
|
45
|
+
:base_url => @auth_url,
|
46
|
+
:grant => :delegated_refresh,
|
47
|
+
:api_key => @api_key,
|
48
|
+
:token_field=> 'delegated_refresh_token'
|
49
|
+
})
|
50
|
+
# get delagated token to be placed in rest call header and in transfer tags
|
51
|
+
@add_ts['tags']['aspera']['node']['storage_credentials']['token']['delegated_refresh_token']=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
|
52
|
+
@params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
|
48
53
|
end
|
49
54
|
end
|
50
55
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -2,7 +2,7 @@ require 'aspera/log'
|
|
2
2
|
require 'rbconfig'
|
3
3
|
|
4
4
|
module Aspera
|
5
|
-
#
|
5
|
+
# detect OS, architecture, and OS specific stuff
|
6
6
|
class Environment
|
7
7
|
OS_WINDOWS = :windows
|
8
8
|
OS_X = :osx
|
@@ -53,7 +53,7 @@ module Aspera
|
|
53
53
|
return ''
|
54
54
|
end
|
55
55
|
|
56
|
-
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably
|
56
|
+
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably than %HOMEDRIVE%%HOMEPATH%
|
57
57
|
def self.fix_home
|
58
58
|
if os.eql?(OS_WINDOWS)
|
59
59
|
if ENV.has_key?('USERPROFILE') and Dir.exist?(ENV['USERPROFILE'])
|
data/lib/aspera/fasp/connect.rb
CHANGED
@@ -12,7 +12,9 @@ module Aspera
|
|
12
12
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
13
13
|
def initialize
|
14
14
|
super
|
15
|
-
@
|
15
|
+
@connect_settings={
|
16
|
+
'app_id' => SecureRandom.uuid
|
17
|
+
}
|
16
18
|
# TODO: start here and create monitor
|
17
19
|
end
|
18
20
|
|
@@ -32,46 +34,51 @@ module Aspera
|
|
32
34
|
OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
|
33
35
|
raise StandardError,'Connect is not installed'
|
34
36
|
end
|
35
|
-
sleep
|
37
|
+
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
36
38
|
retry
|
37
39
|
end
|
38
40
|
if transfer_spec['direction'] == 'send'
|
39
41
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
40
42
|
transfer_spec.delete('paths')
|
41
|
-
resdata=@connect_api.create('windows/select-open-file-dialog/',{'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''
|
43
|
+
resdata=@connect_api.create('windows/select-open-file-dialog/',{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
|
42
44
|
transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
|
43
45
|
end
|
44
46
|
@request_id=SecureRandom.uuid
|
45
47
|
# if there is a token, we ask connect client to use well known ssh private keys
|
46
48
|
# instead of asking password
|
47
49
|
transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
|
50
|
+
connect_settings=
|
48
51
|
connect_transfer_args={
|
49
|
-
'
|
50
|
-
'
|
51
|
-
'
|
52
|
-
|
53
|
-
'
|
54
|
-
'
|
55
|
-
}
|
52
|
+
'aspera_connect_settings'=>@connect_settings.merge({
|
53
|
+
'request_id' =>@request_id,
|
54
|
+
'allow_dialogs' =>true,
|
55
|
+
}),
|
56
|
+
'transfer_specs' =>[{
|
57
|
+
'transfer_spec' =>transfer_spec,
|
58
|
+
}]}
|
56
59
|
# asynchronous anyway
|
57
|
-
|
60
|
+
res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
|
61
|
+
@xfer_id=res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
|
58
62
|
end
|
59
63
|
|
60
64
|
def wait_for_transfers_completion
|
61
|
-
connect_activity_args={'aspera_connect_settings'
|
65
|
+
connect_activity_args={'aspera_connect_settings'=>@connect_settings}
|
62
66
|
started=false
|
63
67
|
spinner=nil
|
64
68
|
loop do
|
65
|
-
|
66
|
-
if
|
67
|
-
trdata=
|
68
|
-
|
69
|
+
tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
|
70
|
+
if tr_info['transfer_info'].is_a?(Hash)
|
71
|
+
trdata=tr_info['transfer_info']
|
72
|
+
if trdata.nil?
|
73
|
+
Log.log.warn("no session in Connect")
|
74
|
+
break
|
75
|
+
end
|
69
76
|
# TODO: get session id
|
70
77
|
case trdata['status']
|
71
78
|
when 'completed'
|
72
|
-
|
79
|
+
notify_end(@connect_settings['app_id'])
|
73
80
|
break
|
74
|
-
when 'initiating'
|
81
|
+
when 'initiating','queued'
|
75
82
|
if spinner.nil?
|
76
83
|
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
77
84
|
spinner.start
|
@@ -81,13 +88,13 @@ module Aspera
|
|
81
88
|
when 'running'
|
82
89
|
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
83
90
|
if !started and trdata['bytes_expected'] != 0
|
84
|
-
|
91
|
+
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
85
92
|
started=true
|
86
93
|
else
|
87
|
-
|
94
|
+
notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
|
88
95
|
end
|
89
96
|
else
|
90
|
-
raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
|
97
|
+
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
91
98
|
end
|
92
99
|
end
|
93
100
|
sleep 1
|
data/lib/aspera/fasp/http_gw.rb
CHANGED
@@ -2,12 +2,147 @@
|
|
2
2
|
require 'aspera/fasp/manager'
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/rest'
|
5
|
+
require 'websocket-client-simple'
|
6
|
+
require 'securerandom'
|
7
|
+
require 'openssl'
|
8
|
+
require 'base64'
|
9
|
+
require 'json'
|
10
|
+
require 'uri'
|
5
11
|
|
6
12
|
# ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
|
7
13
|
module Aspera
|
8
14
|
module Fasp
|
9
15
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
10
16
|
class HttpGW < Manager
|
17
|
+
# message returned by HTTP GW in case of success
|
18
|
+
OK_MESSAGE='end upload'
|
19
|
+
# refresh rate for progress
|
20
|
+
UPLOAD_REFRESH_SEC=0.5
|
21
|
+
private_constant :OK_MESSAGE,:UPLOAD_REFRESH_SEC
|
22
|
+
# send message on http gw web socket
|
23
|
+
def ws_send(ws,type,data)
|
24
|
+
ws.send(JSON.generate({type => data}))
|
25
|
+
end
|
26
|
+
|
27
|
+
def upload(transfer_spec)
|
28
|
+
# precalculate size
|
29
|
+
total_size=0
|
30
|
+
# currently, files are sent flat
|
31
|
+
source_path=[]
|
32
|
+
transfer_spec['paths'].each do |item|
|
33
|
+
filepath=item['source']
|
34
|
+
item['source']=item['destination']=File.basename(filepath)
|
35
|
+
total_size+=item['file_size']=File.size(filepath)
|
36
|
+
source_path.push(filepath)
|
37
|
+
end
|
38
|
+
session_id=SecureRandom.uuid
|
39
|
+
ws=::WebSocket::Client::Simple::Client.new
|
40
|
+
error=nil
|
41
|
+
received=0
|
42
|
+
ws.on :message do |msg|
|
43
|
+
Log.log.info("ws: message: #{msg.data}")
|
44
|
+
message=msg.data
|
45
|
+
if message.eql?(OK_MESSAGE)
|
46
|
+
received+=1
|
47
|
+
else
|
48
|
+
message.chomp!
|
49
|
+
if message[0].eql?('"') and message[-1].eql?('"')
|
50
|
+
error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
51
|
+
else
|
52
|
+
error="expecting quotes in [#{message}]"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
ws.on :error do |e|
|
57
|
+
error=e
|
58
|
+
end
|
59
|
+
ws.on :open do
|
60
|
+
Log.log.info("ws: open")
|
61
|
+
end
|
62
|
+
ws.on :close do
|
63
|
+
Log.log.info("ws: close")
|
64
|
+
end
|
65
|
+
# open web socket to end point
|
66
|
+
ws.connect("#{@gw_api.params[:base_url]}/upload")
|
67
|
+
while !ws.open? and error.nil? do
|
68
|
+
Log.log.info("ws: wait")
|
69
|
+
sleep(0.2)
|
70
|
+
end
|
71
|
+
notify_begin(session_id,total_size)
|
72
|
+
ws_send(ws,:transfer_spec,transfer_spec)
|
73
|
+
# current file index
|
74
|
+
filenum=0
|
75
|
+
# aggregate size sent
|
76
|
+
sent_bytes=0
|
77
|
+
# last progress event
|
78
|
+
lastevent=Time.now-1
|
79
|
+
transfer_spec['paths'].each do |item|
|
80
|
+
# TODO: on destination write same path?
|
81
|
+
destination_path=item['source']
|
82
|
+
# TODO: get mime type?
|
83
|
+
file_mime_type=''
|
84
|
+
total=item['file_size']
|
85
|
+
# compute total number of slices
|
86
|
+
numslices=1+(total-1)/@upload_chunksize
|
87
|
+
# current slice index
|
88
|
+
slicenum=0
|
89
|
+
File.open(source_path[filenum]) do |file|
|
90
|
+
while !file.eof? do
|
91
|
+
data=file.read(@upload_chunksize)
|
92
|
+
slice_data={
|
93
|
+
name: destination_path,
|
94
|
+
type: file_mime_type,
|
95
|
+
size: total,
|
96
|
+
data: Base64.strict_encode64(data),
|
97
|
+
slice: slicenum,
|
98
|
+
total_slices: numslices,
|
99
|
+
fileIndex: filenum
|
100
|
+
}
|
101
|
+
ws_send(ws,:slice_upload, slice_data)
|
102
|
+
sent_bytes+=data.length
|
103
|
+
currenttime=Time.now
|
104
|
+
if (currenttime-lastevent)>UPLOAD_REFRESH_SEC
|
105
|
+
notify_progress(session_id,sent_bytes)
|
106
|
+
lastevent=currenttime
|
107
|
+
end
|
108
|
+
slicenum+=1
|
109
|
+
raise error unless error.nil?
|
110
|
+
end
|
111
|
+
end
|
112
|
+
filenum+=1
|
113
|
+
end
|
114
|
+
ws.close
|
115
|
+
notify_end(session_id)
|
116
|
+
end
|
117
|
+
|
118
|
+
def download(transfer_spec)
|
119
|
+
transfer_spec['zip_required']||=false
|
120
|
+
transfer_spec['source_root']||='/'
|
121
|
+
# is normally provided by application, like package name
|
122
|
+
if !transfer_spec.has_key?('download_name')
|
123
|
+
# by default it is the name of first file
|
124
|
+
dname=File.basename(transfer_spec['paths'].first['source'])
|
125
|
+
# we remove extension
|
126
|
+
dname=dname.gsub(/\.@gw_api.*$/,'')
|
127
|
+
# ands add indication of number of files if there is more than one
|
128
|
+
if transfer_spec['paths'].length > 1
|
129
|
+
dname=dname+" #{transfer_spec['paths'].length} Files"
|
130
|
+
end
|
131
|
+
transfer_spec['download_name']=dname
|
132
|
+
end
|
133
|
+
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
134
|
+
transfer_uuid=creation['url'].split('/').last
|
135
|
+
if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
|
136
|
+
# it is a zip file if zip is required or there is more than 1 file
|
137
|
+
file_dest=transfer_spec['download_name']+'.zip'
|
138
|
+
else
|
139
|
+
# it is a plain file if we don't require zip and there is only one file
|
140
|
+
file_dest=File.basename(transfer_spec['paths'].first['source'])
|
141
|
+
end
|
142
|
+
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
143
|
+
@gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
|
144
|
+
end
|
145
|
+
|
11
146
|
# start FASP transfer based on transfer spec (hash table)
|
12
147
|
# note that it is asynchronous
|
13
148
|
# HTTP download only supports file list
|
@@ -15,37 +150,13 @@ module Aspera
|
|
15
150
|
raise "GW URL must be set" unless !@gw_api.nil?
|
16
151
|
raise "option: must be hash (or nil)" unless options.is_a?(Hash)
|
17
152
|
raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
|
153
|
+
raise "on token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
|
154
|
+
transfer_spec['authentication']||='token'
|
18
155
|
case transfer_spec['direction']
|
19
156
|
when 'send'
|
20
|
-
|
21
|
-
raise "error, not implemented"
|
157
|
+
upload(transfer_spec)
|
22
158
|
when 'receive'
|
23
|
-
transfer_spec
|
24
|
-
transfer_spec['authentication']||='token'
|
25
|
-
transfer_spec['source_root']||='/'
|
26
|
-
# is normally provided by application, like package name
|
27
|
-
if !transfer_spec.has_key?('download_name')
|
28
|
-
# by default it is the name of first file
|
29
|
-
dname=File.basename(transfer_spec['paths'].first['source'])
|
30
|
-
# we remove extension
|
31
|
-
dname=dname.gsub(/\.@gw_api.*$/,'')
|
32
|
-
# ands add indication of number of files if there is more than one
|
33
|
-
if transfer_spec['paths'].length > 1
|
34
|
-
dname=dname+" #{transfer_spec['paths'].length} Files"
|
35
|
-
end
|
36
|
-
transfer_spec['download_name']=dname
|
37
|
-
end
|
38
|
-
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
39
|
-
transfer_uuid=creation['url'].split('/').last
|
40
|
-
if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
|
41
|
-
# it is a zip file if zip is required or there is more than 1 file
|
42
|
-
file_dest=transfer_spec['download_name']+'.zip'
|
43
|
-
else
|
44
|
-
# it is a plain file if we don't require zip and there is only one file
|
45
|
-
file_dest=File.basename(transfer_spec['paths'].first['source'])
|
46
|
-
end
|
47
|
-
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
48
|
-
@gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
|
159
|
+
download(transfer_spec)
|
49
160
|
else
|
50
161
|
raise "error"
|
51
162
|
end
|
@@ -74,6 +185,7 @@ module Aspera
|
|
74
185
|
@gw_api=Rest.new({:base_url => params[:url]})
|
75
186
|
api_info = @gw_api.read('info')[:data]
|
76
187
|
Log.log.info("#{api_info}")
|
188
|
+
@upload_chunksize=128000 # TODO: configurable ?
|
77
189
|
end
|
78
190
|
|
79
191
|
end # LocalHttp
|