aspera-cli 4.0.0 → 4.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +843 -304
- data/bin/dascli +13 -0
- data/docs/Makefile +4 -4
- data/docs/README.erb.md +805 -172
- data/docs/test_env.conf +22 -3
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +89 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +89 -49
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +188 -173
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +218 -145
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +114 -61
- data/lib/aspera/cli/plugins/faspex5.rb +85 -43
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +59 -45
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +77 -49
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +49 -31
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +93 -46
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +59 -101
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +54 -20
- data/docs/transfer_spec.html +0 -99
@@ -4,21 +4,46 @@ module Aspera
|
|
4
4
|
# process_param is called repeatedly with all known parameters
|
5
5
|
# add_env_args is called to get resulting param list and env var (also checks that all params were used)
|
6
6
|
class CommandLineBuilder
|
7
|
+
# transform yes/no to trye/false
|
8
|
+
def self.yes_to_true(value)
|
9
|
+
case value
|
10
|
+
when 'yes'; return true
|
11
|
+
when 'no'; return false
|
12
|
+
end
|
13
|
+
raise "unsupported value: #{value}"
|
14
|
+
end
|
7
15
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
16
|
+
# Called by provider of definition before constructor of this class so that params_definition has all mandatory fields
|
17
|
+
def self.normalize_description(d)
|
18
|
+
d.each do |param_name,options|
|
19
|
+
Log.log.debug("def: #{param_name}")
|
20
|
+
raise "Expecting Hash, but have #{options.class} in #{param_name}" unless options.is_a?(Hash)
|
21
|
+
#options[:accepted_types]=:bool if options[:cltype].eql?(:envvar) and !options.has_key?(:accepted_types)
|
22
|
+
# by default : not mandatory
|
23
|
+
options[:mandatory]||=false
|
24
|
+
options[:desc]||=''
|
25
|
+
# by default : string, unless it's without arg
|
26
|
+
if ! options.has_key?(:accepted_types)
|
27
|
+
options[:accepted_types]=options[:cltype].eql?(:opt_without_arg) ? :bool : :string
|
28
|
+
end
|
29
|
+
# single type is placed in array
|
30
|
+
options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
|
31
|
+
if !options.has_key?(:option_switch) and options.has_key?(:cltype) and [:opt_without_arg,:opt_with_arg].include?(options[:cltype])
|
32
|
+
options[:option_switch]='--'+param_name.to_s.gsub('_','-')
|
33
|
+
end
|
34
|
+
end
|
13
35
|
end
|
14
36
|
|
37
|
+
private
|
38
|
+
|
39
|
+
# clvarname : command line variable name
|
15
40
|
def env_name(param_name,options)
|
16
|
-
return options[:
|
41
|
+
return options[:clvarname]
|
17
42
|
end
|
18
43
|
|
19
44
|
public
|
20
45
|
|
21
|
-
|
46
|
+
attr_reader :params_definition
|
22
47
|
|
23
48
|
# @param param_hash
|
24
49
|
def initialize(param_hash,params_definition)
|
@@ -44,15 +69,6 @@ module Aspera
|
|
44
69
|
return nil
|
45
70
|
end
|
46
71
|
|
47
|
-
# transform yes/no to trye/false
|
48
|
-
def self.yes_to_true(value)
|
49
|
-
case value
|
50
|
-
when 'yes'; return true
|
51
|
-
when 'no'; return false
|
52
|
-
end
|
53
|
-
raise "unsupported value: #{value}"
|
54
|
-
end
|
55
|
-
|
56
72
|
# add options directly to ascp command line
|
57
73
|
def add_command_line_options(options)
|
58
74
|
return if options.nil?
|
@@ -71,24 +87,25 @@ module Aspera
|
|
71
87
|
# @param options : options for type
|
72
88
|
def process_param(param_name,action=nil)
|
73
89
|
options=@params_definition[param_name]
|
74
|
-
action=options[:
|
90
|
+
action=options[:cltype] if action.nil?
|
75
91
|
# should not happen
|
76
92
|
raise "Internal error: ask processing of param #{param_name}" if options.nil?
|
77
|
-
# by default : not mandatory
|
78
|
-
options[:mandatory]||=false
|
79
|
-
if options.has_key?(:accepted_types)
|
80
|
-
# single type is placed in array
|
81
|
-
options[:accepted_types]=[options[:accepted_types]] unless options[:accepted_types].is_a?(Array)
|
82
|
-
else
|
83
|
-
# by default : string, unless it's without arg
|
84
|
-
options[:accepted_types]=action.eql?(:opt_without_arg) ? BOOLEAN_CLASSES : [String]
|
85
|
-
end
|
86
93
|
# check mandatory parameter (nil is valid value)
|
87
94
|
raise Fasp::Error.new("mandatory parameter: #{param_name}") if options[:mandatory] and !@param_hash.has_key?(param_name)
|
88
95
|
parameter_value=@param_hash[param_name]
|
89
|
-
parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
|
96
|
+
#parameter_value=options[:default] if parameter_value.nil? and options.has_key?(:default)
|
97
|
+
expected_classes=options[:accepted_types].map do |s|
|
98
|
+
case s
|
99
|
+
when :string; String
|
100
|
+
when :array; Array
|
101
|
+
when :hash; Hash
|
102
|
+
when :int; Integer
|
103
|
+
when :bool; [TrueClass,FalseClass]
|
104
|
+
else raise "INTERNAL: unexpected value: #{s}"
|
105
|
+
end
|
106
|
+
end.flatten
|
90
107
|
# check provided type
|
91
|
-
raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or
|
108
|
+
raise Fasp::Error.new("#{param_name} is : #{parameter_value.class} (#{parameter_value}), shall be #{options[:accepted_types]}, ") unless parameter_value.nil? or expected_classes.include?(parameter_value.class)
|
92
109
|
@used_param_names.push(param_name) unless action.eql?(:defer)
|
93
110
|
|
94
111
|
# process only non-nil values
|
@@ -102,7 +119,8 @@ module Aspera
|
|
102
119
|
end
|
103
120
|
raise "unsupported value: #{parameter_value}" unless options[:accepted_values].nil? or options[:accepted_values].include?(parameter_value)
|
104
121
|
if options[:encode]
|
105
|
-
|
122
|
+
# :encode has name of class with encoding method
|
123
|
+
newvalue=Kernel.const_get(options[:encode]).send("encode_#{param_name}",parameter_value)
|
106
124
|
raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}") if newvalue.nil?
|
107
125
|
parameter_value=newvalue
|
108
126
|
end
|
@@ -123,12 +141,12 @@ module Aspera
|
|
123
141
|
else raise Fasp::Error.new("unsupported #{param_name}: #{parameter_value}")
|
124
142
|
end
|
125
143
|
add_param=!add_param if options[:add_on_false]
|
126
|
-
add_command_line_options([
|
144
|
+
add_command_line_options([options[:option_switch]]) if add_param
|
127
145
|
when :opt_with_arg # transform into command line option with value
|
128
146
|
#parameter_value=parameter_value.to_s if parameter_value.is_a?(Integer)
|
129
147
|
parameter_value=[parameter_value] unless parameter_value.is_a?(Array)
|
130
148
|
# if transfer_spec value is an array, applies option many times
|
131
|
-
parameter_value.each{|v|add_command_line_options([
|
149
|
+
parameter_value.each{|v|add_command_line_options([options[:option_switch],v])}
|
132
150
|
else
|
133
151
|
raise "Error"
|
134
152
|
end
|
data/lib/aspera/cos_node.rb
CHANGED
@@ -5,46 +5,51 @@ require 'xmlsimple'
|
|
5
5
|
module Aspera
|
6
6
|
class CosNode < Rest
|
7
7
|
attr_reader :add_ts
|
8
|
-
|
8
|
+
IBM_CLOUD_TOKEN_URL='https://iam.cloud.ibm.com/identity'
|
9
|
+
def initialize(bucket_name,storage_endpoint,instance_id,api_key,auth_url=IBM_CLOUD_TOKEN_URL)
|
10
|
+
@auth_url=auth_url
|
11
|
+
@api_key=api_key
|
9
12
|
s3_api=Aspera::Rest.new({
|
10
|
-
:base_url
|
11
|
-
:not_auth_codes => ['401','403'],
|
12
|
-
:headers
|
13
|
-
:auth
|
14
|
-
:type
|
15
|
-
:base_url
|
16
|
-
:grant
|
17
|
-
:api_key
|
13
|
+
:base_url => storage_endpoint,
|
14
|
+
:not_auth_codes => ['401','403'], # error codes when not authorized
|
15
|
+
:headers => {'ibm-service-instance-id' => instance_id},
|
16
|
+
:auth => {
|
17
|
+
:type => :oauth2,
|
18
|
+
:base_url => @auth_url,
|
19
|
+
:grant => :ibm_apikey,
|
20
|
+
:api_key => @api_key
|
18
21
|
}})
|
19
22
|
# read FASP connection information for bucket
|
20
23
|
xml_result_text=s3_api.call({:operation=>'GET',:subpath=>bucket_name,:headers=>{'Accept'=>'application/xml'},:url_params=>{'faspConnectionInfo'=>nil}})[:http].body
|
21
24
|
ats_info=XmlSimple.xml_in(xml_result_text, {'ForceArray' => false})
|
22
25
|
Aspera::Log.dump('ats_info',ats_info)
|
23
|
-
# get delegated token
|
24
|
-
delegated_oauth=Oauth.new({
|
25
|
-
:type => :oauth2,
|
26
|
-
:base_url => auth_url,
|
27
|
-
:grant => :delegated_refresh,
|
28
|
-
:api_key => api_key,
|
29
|
-
:token_field=> 'delegated_refresh_token'
|
30
|
-
})
|
31
|
-
# to be placed in rest call header and in transfer tags
|
32
|
-
aspera_storage_credentials={
|
33
|
-
'type' => 'token',
|
34
|
-
'token' => {'delegated_refresh_token'=>delegated_oauth.get_authorization().gsub(/^Bearer /,'')}
|
35
|
-
}
|
36
|
-
# transfer spec addition
|
37
|
-
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>aspera_storage_credentials}}}}
|
38
|
-
# set a general addon to transfer spec
|
39
|
-
# here we choose to use the add_request_param
|
40
|
-
#self.transfer.option_transfer_spec_deep_merge(@add_ts)
|
41
26
|
super({
|
42
27
|
:base_url => ats_info['ATSEndpoint'],
|
43
|
-
:headers => {'X-Aspera-Storage-Credentials'=>JSON.generate(aspera_storage_credentials)},
|
44
28
|
:auth => {
|
45
29
|
:type => :basic,
|
46
30
|
:username => ats_info['AccessKey']['Id'],
|
47
31
|
:password => ats_info['AccessKey']['Secret']}})
|
32
|
+
# prepare transfer spec addition
|
33
|
+
@add_ts={'tags'=>{'aspera'=>{'node'=>{'storage_credentials'=>{
|
34
|
+
'type' => 'token',
|
35
|
+
'token' => {'delegated_refresh_token'=>nil}
|
36
|
+
}}}}}
|
37
|
+
generate_token
|
38
|
+
end
|
39
|
+
|
40
|
+
# potentially call this if delegated token is expired
|
41
|
+
def generate_token
|
42
|
+
# OAuth API to get delegated token
|
43
|
+
delegated_oauth=Oauth.new({
|
44
|
+
:type => :oauth2,
|
45
|
+
:base_url => @auth_url,
|
46
|
+
:grant => :delegated_refresh,
|
47
|
+
:api_key => @api_key,
|
48
|
+
:token_field=> 'delegated_refresh_token'
|
49
|
+
})
|
50
|
+
# get delagated token to be placed in rest call header and in transfer tags
|
51
|
+
@add_ts['tags']['aspera']['node']['storage_credentials']['token']['delegated_refresh_token']=delegated_oauth.get_authorization().gsub(/^Bearer /,'')
|
52
|
+
@params[:headers]={'X-Aspera-Storage-Credentials'=>JSON.generate(@add_ts['tags']['aspera']['node']['storage_credentials'])}
|
48
53
|
end
|
49
54
|
end
|
50
55
|
end
|
data/lib/aspera/environment.rb
CHANGED
@@ -2,7 +2,7 @@ require 'aspera/log'
|
|
2
2
|
require 'rbconfig'
|
3
3
|
|
4
4
|
module Aspera
|
5
|
-
#
|
5
|
+
# detect OS, architecture, and OS specific stuff
|
6
6
|
class Environment
|
7
7
|
OS_WINDOWS = :windows
|
8
8
|
OS_X = :osx
|
@@ -53,7 +53,7 @@ module Aspera
|
|
53
53
|
return ''
|
54
54
|
end
|
55
55
|
|
56
|
-
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably
|
56
|
+
# on Windows, the env var %USERPROFILE% provides the path to user's home more reliably than %HOMEDRIVE%%HOMEPATH%
|
57
57
|
def self.fix_home
|
58
58
|
if os.eql?(OS_WINDOWS)
|
59
59
|
if ENV.has_key?('USERPROFILE') and Dir.exist?(ENV['USERPROFILE'])
|
data/lib/aspera/fasp/connect.rb
CHANGED
@@ -12,7 +12,9 @@ module Aspera
|
|
12
12
|
private_constant :MAX_CONNECT_START_RETRY,:SLEEP_SEC_BETWEEN_RETRY
|
13
13
|
def initialize
|
14
14
|
super
|
15
|
-
@
|
15
|
+
@connect_settings={
|
16
|
+
'app_id' => SecureRandom.uuid
|
17
|
+
}
|
16
18
|
# TODO: start here and create monitor
|
17
19
|
end
|
18
20
|
|
@@ -32,46 +34,51 @@ module Aspera
|
|
32
34
|
OpenApplication.uri_graphical('https://downloads.asperasoft.com/connect2/')
|
33
35
|
raise StandardError,'Connect is not installed'
|
34
36
|
end
|
35
|
-
sleep
|
37
|
+
sleep(SLEEP_SEC_BETWEEN_RETRY)
|
36
38
|
retry
|
37
39
|
end
|
38
40
|
if transfer_spec['direction'] == 'send'
|
39
41
|
Log.log.warn("Connect requires upload selection using GUI, ignoring #{transfer_spec['paths']}".red)
|
40
42
|
transfer_spec.delete('paths')
|
41
|
-
resdata=@connect_api.create('windows/select-open-file-dialog/',{'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''
|
43
|
+
resdata=@connect_api.create('windows/select-open-file-dialog/',{'aspera_connect_settings'=>@connect_settings,'title'=>'Select Files','suggestedName'=>'','allowMultipleSelection'=>true,'allowedFileTypes'=>''})[:data]
|
42
44
|
transfer_spec['paths']=resdata['dataTransfer']['files'].map { |i| {'source'=>i['name']}}
|
43
45
|
end
|
44
46
|
@request_id=SecureRandom.uuid
|
45
47
|
# if there is a token, we ask connect client to use well known ssh private keys
|
46
48
|
# instead of asking password
|
47
49
|
transfer_spec['authentication']='token' if transfer_spec.has_key?('token')
|
50
|
+
connect_settings=
|
48
51
|
connect_transfer_args={
|
49
|
-
'
|
50
|
-
'
|
51
|
-
'
|
52
|
-
|
53
|
-
'
|
54
|
-
'
|
55
|
-
}
|
52
|
+
'aspera_connect_settings'=>@connect_settings.merge({
|
53
|
+
'request_id' =>@request_id,
|
54
|
+
'allow_dialogs' =>true,
|
55
|
+
}),
|
56
|
+
'transfer_specs' =>[{
|
57
|
+
'transfer_spec' =>transfer_spec,
|
58
|
+
}]}
|
56
59
|
# asynchronous anyway
|
57
|
-
|
60
|
+
res=@connect_api.create('transfers/start',connect_transfer_args)[:data]
|
61
|
+
@xfer_id=res['transfer_specs'].first['transfer_spec']['tags']['aspera']['xfer_id']
|
58
62
|
end
|
59
63
|
|
60
64
|
def wait_for_transfers_completion
|
61
|
-
connect_activity_args={'aspera_connect_settings'
|
65
|
+
connect_activity_args={'aspera_connect_settings'=>@connect_settings}
|
62
66
|
started=false
|
63
67
|
spinner=nil
|
64
68
|
loop do
|
65
|
-
|
66
|
-
if
|
67
|
-
trdata=
|
68
|
-
|
69
|
+
tr_info=@connect_api.create("transfers/info/#{@xfer_id}",connect_activity_args)[:data]
|
70
|
+
if tr_info['transfer_info'].is_a?(Hash)
|
71
|
+
trdata=tr_info['transfer_info']
|
72
|
+
if trdata.nil?
|
73
|
+
Log.log.warn("no session in Connect")
|
74
|
+
break
|
75
|
+
end
|
69
76
|
# TODO: get session id
|
70
77
|
case trdata['status']
|
71
78
|
when 'completed'
|
72
|
-
|
79
|
+
notify_end(@connect_settings['app_id'])
|
73
80
|
break
|
74
|
-
when 'initiating'
|
81
|
+
when 'initiating','queued'
|
75
82
|
if spinner.nil?
|
76
83
|
spinner = TTY::Spinner.new('[:spinner] :title', format: :classic)
|
77
84
|
spinner.start
|
@@ -81,13 +88,13 @@ module Aspera
|
|
81
88
|
when 'running'
|
82
89
|
#puts "running: sessions:#{trdata['sessions'].length}, #{trdata['sessions'].map{|i| i['bytes_transferred']}.join(',')}"
|
83
90
|
if !started and trdata['bytes_expected'] != 0
|
84
|
-
|
91
|
+
notify_begin(@connect_settings['app_id'],trdata['bytes_expected'])
|
85
92
|
started=true
|
86
93
|
else
|
87
|
-
|
94
|
+
notify_progress(@connect_settings['app_id'],trdata['bytes_written'])
|
88
95
|
end
|
89
96
|
else
|
90
|
-
raise Fasp::Error.new("#{trdata['status']}: #{trdata['error_desc']}")
|
97
|
+
raise Fasp::Error.new("unknown status: #{trdata['status']}: #{trdata['error_desc']}")
|
91
98
|
end
|
92
99
|
end
|
93
100
|
sleep 1
|
data/lib/aspera/fasp/http_gw.rb
CHANGED
@@ -2,12 +2,147 @@
|
|
2
2
|
require 'aspera/fasp/manager'
|
3
3
|
require 'aspera/log'
|
4
4
|
require 'aspera/rest'
|
5
|
+
require 'websocket-client-simple'
|
6
|
+
require 'securerandom'
|
7
|
+
require 'openssl'
|
8
|
+
require 'base64'
|
9
|
+
require 'json'
|
10
|
+
require 'uri'
|
5
11
|
|
6
12
|
# ref: https://api.ibm.com/explorer/catalog/aspera/product/ibm-aspera/api/http-gateway-api/doc/guides-toc
|
7
13
|
module Aspera
|
8
14
|
module Fasp
|
9
15
|
# executes a local "ascp", connects mgt port, equivalent of "Fasp Manager"
|
10
16
|
class HttpGW < Manager
|
17
|
+
# message returned by HTTP GW in case of success
|
18
|
+
OK_MESSAGE='end upload'
|
19
|
+
# refresh rate for progress
|
20
|
+
UPLOAD_REFRESH_SEC=0.5
|
21
|
+
private_constant :OK_MESSAGE,:UPLOAD_REFRESH_SEC
|
22
|
+
# send message on http gw web socket
|
23
|
+
def ws_send(ws,type,data)
|
24
|
+
ws.send(JSON.generate({type => data}))
|
25
|
+
end
|
26
|
+
|
27
|
+
def upload(transfer_spec)
|
28
|
+
# precalculate size
|
29
|
+
total_size=0
|
30
|
+
# currently, files are sent flat
|
31
|
+
source_path=[]
|
32
|
+
transfer_spec['paths'].each do |item|
|
33
|
+
filepath=item['source']
|
34
|
+
item['source']=item['destination']=File.basename(filepath)
|
35
|
+
total_size+=item['file_size']=File.size(filepath)
|
36
|
+
source_path.push(filepath)
|
37
|
+
end
|
38
|
+
session_id=SecureRandom.uuid
|
39
|
+
ws=::WebSocket::Client::Simple::Client.new
|
40
|
+
error=nil
|
41
|
+
received=0
|
42
|
+
ws.on :message do |msg|
|
43
|
+
Log.log.info("ws: message: #{msg.data}")
|
44
|
+
message=msg.data
|
45
|
+
if message.eql?(OK_MESSAGE)
|
46
|
+
received+=1
|
47
|
+
else
|
48
|
+
message.chomp!
|
49
|
+
if message[0].eql?('"') and message[-1].eql?('"')
|
50
|
+
error=JSON.parse(Base64.strict_decode64(message.chomp[1..-2]))['message']
|
51
|
+
else
|
52
|
+
error="expecting quotes in [#{message}]"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
ws.on :error do |e|
|
57
|
+
error=e
|
58
|
+
end
|
59
|
+
ws.on :open do
|
60
|
+
Log.log.info("ws: open")
|
61
|
+
end
|
62
|
+
ws.on :close do
|
63
|
+
Log.log.info("ws: close")
|
64
|
+
end
|
65
|
+
# open web socket to end point
|
66
|
+
ws.connect("#{@gw_api.params[:base_url]}/upload")
|
67
|
+
while !ws.open? and error.nil? do
|
68
|
+
Log.log.info("ws: wait")
|
69
|
+
sleep(0.2)
|
70
|
+
end
|
71
|
+
notify_begin(session_id,total_size)
|
72
|
+
ws_send(ws,:transfer_spec,transfer_spec)
|
73
|
+
# current file index
|
74
|
+
filenum=0
|
75
|
+
# aggregate size sent
|
76
|
+
sent_bytes=0
|
77
|
+
# last progress event
|
78
|
+
lastevent=Time.now-1
|
79
|
+
transfer_spec['paths'].each do |item|
|
80
|
+
# TODO: on destination write same path?
|
81
|
+
destination_path=item['source']
|
82
|
+
# TODO: get mime type?
|
83
|
+
file_mime_type=''
|
84
|
+
total=item['file_size']
|
85
|
+
# compute total number of slices
|
86
|
+
numslices=1+(total-1)/@upload_chunksize
|
87
|
+
# current slice index
|
88
|
+
slicenum=0
|
89
|
+
File.open(source_path[filenum]) do |file|
|
90
|
+
while !file.eof? do
|
91
|
+
data=file.read(@upload_chunksize)
|
92
|
+
slice_data={
|
93
|
+
name: destination_path,
|
94
|
+
type: file_mime_type,
|
95
|
+
size: total,
|
96
|
+
data: Base64.strict_encode64(data),
|
97
|
+
slice: slicenum,
|
98
|
+
total_slices: numslices,
|
99
|
+
fileIndex: filenum
|
100
|
+
}
|
101
|
+
ws_send(ws,:slice_upload, slice_data)
|
102
|
+
sent_bytes+=data.length
|
103
|
+
currenttime=Time.now
|
104
|
+
if (currenttime-lastevent)>UPLOAD_REFRESH_SEC
|
105
|
+
notify_progress(session_id,sent_bytes)
|
106
|
+
lastevent=currenttime
|
107
|
+
end
|
108
|
+
slicenum+=1
|
109
|
+
raise error unless error.nil?
|
110
|
+
end
|
111
|
+
end
|
112
|
+
filenum+=1
|
113
|
+
end
|
114
|
+
ws.close
|
115
|
+
notify_end(session_id)
|
116
|
+
end
|
117
|
+
|
118
|
+
def download(transfer_spec)
|
119
|
+
transfer_spec['zip_required']||=false
|
120
|
+
transfer_spec['source_root']||='/'
|
121
|
+
# is normally provided by application, like package name
|
122
|
+
if !transfer_spec.has_key?('download_name')
|
123
|
+
# by default it is the name of first file
|
124
|
+
dname=File.basename(transfer_spec['paths'].first['source'])
|
125
|
+
# we remove extension
|
126
|
+
dname=dname.gsub(/\.@gw_api.*$/,'')
|
127
|
+
# ands add indication of number of files if there is more than one
|
128
|
+
if transfer_spec['paths'].length > 1
|
129
|
+
dname=dname+" #{transfer_spec['paths'].length} Files"
|
130
|
+
end
|
131
|
+
transfer_spec['download_name']=dname
|
132
|
+
end
|
133
|
+
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
134
|
+
transfer_uuid=creation['url'].split('/').last
|
135
|
+
if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
|
136
|
+
# it is a zip file if zip is required or there is more than 1 file
|
137
|
+
file_dest=transfer_spec['download_name']+'.zip'
|
138
|
+
else
|
139
|
+
# it is a plain file if we don't require zip and there is only one file
|
140
|
+
file_dest=File.basename(transfer_spec['paths'].first['source'])
|
141
|
+
end
|
142
|
+
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
143
|
+
@gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
|
144
|
+
end
|
145
|
+
|
11
146
|
# start FASP transfer based on transfer spec (hash table)
|
12
147
|
# note that it is asynchronous
|
13
148
|
# HTTP download only supports file list
|
@@ -15,37 +150,13 @@ module Aspera
|
|
15
150
|
raise "GW URL must be set" unless !@gw_api.nil?
|
16
151
|
raise "option: must be hash (or nil)" unless options.is_a?(Hash)
|
17
152
|
raise "paths: must be Array" unless transfer_spec['paths'].is_a?(Array)
|
153
|
+
raise "on token based transfer is supported in GW" unless transfer_spec['token'].is_a?(String)
|
154
|
+
transfer_spec['authentication']||='token'
|
18
155
|
case transfer_spec['direction']
|
19
156
|
when 'send'
|
20
|
-
|
21
|
-
raise "error, not implemented"
|
157
|
+
upload(transfer_spec)
|
22
158
|
when 'receive'
|
23
|
-
transfer_spec
|
24
|
-
transfer_spec['authentication']||='token'
|
25
|
-
transfer_spec['source_root']||='/'
|
26
|
-
# is normally provided by application, like package name
|
27
|
-
if !transfer_spec.has_key?('download_name')
|
28
|
-
# by default it is the name of first file
|
29
|
-
dname=File.basename(transfer_spec['paths'].first['source'])
|
30
|
-
# we remove extension
|
31
|
-
dname=dname.gsub(/\.@gw_api.*$/,'')
|
32
|
-
# ands add indication of number of files if there is more than one
|
33
|
-
if transfer_spec['paths'].length > 1
|
34
|
-
dname=dname+" #{transfer_spec['paths'].length} Files"
|
35
|
-
end
|
36
|
-
transfer_spec['download_name']=dname
|
37
|
-
end
|
38
|
-
creation=@gw_api.create('download',{'transfer_spec'=>transfer_spec})[:data]
|
39
|
-
transfer_uuid=creation['url'].split('/').last
|
40
|
-
if transfer_spec['zip_required'] or transfer_spec['paths'].length > 1
|
41
|
-
# it is a zip file if zip is required or there is more than 1 file
|
42
|
-
file_dest=transfer_spec['download_name']+'.zip'
|
43
|
-
else
|
44
|
-
# it is a plain file if we don't require zip and there is only one file
|
45
|
-
file_dest=File.basename(transfer_spec['paths'].first['source'])
|
46
|
-
end
|
47
|
-
file_dest=File.join(transfer_spec['destination_root'],file_dest)
|
48
|
-
@gw_api.call({:operation=>'GET',:subpath=>"download/#{transfer_uuid}",:save_to_file=>file_dest})
|
159
|
+
download(transfer_spec)
|
49
160
|
else
|
50
161
|
raise "error"
|
51
162
|
end
|
@@ -74,6 +185,7 @@ module Aspera
|
|
74
185
|
@gw_api=Rest.new({:base_url => params[:url]})
|
75
186
|
api_info = @gw_api.read('info')[:data]
|
76
187
|
Log.log.info("#{api_info}")
|
188
|
+
@upload_chunksize=128000 # TODO: configurable ?
|
77
189
|
end
|
78
190
|
|
79
191
|
end # LocalHttp
|