aspera-cli 4.0.0 → 4.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +843 -304
- data/bin/dascli +13 -0
- data/docs/Makefile +4 -4
- data/docs/README.erb.md +805 -172
- data/docs/test_env.conf +22 -3
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +89 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +89 -49
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +188 -173
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +218 -145
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +114 -61
- data/lib/aspera/cli/plugins/faspex5.rb +85 -43
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +59 -45
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +77 -49
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +49 -31
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +93 -46
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +59 -101
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +54 -20
- data/docs/transfer_spec.html +0 -99
data/lib/aspera/oauth.rb
CHANGED
@@ -1,22 +1,26 @@
|
|
1
1
|
require 'aspera/open_application'
|
2
|
+
require 'aspera/web_auth'
|
2
3
|
require 'base64'
|
3
4
|
require 'date'
|
4
5
|
require 'socket'
|
5
6
|
require 'securerandom'
|
6
7
|
|
7
8
|
module Aspera
|
8
|
-
#
|
9
|
+
# Implement OAuth 2 for the REST client and generate a bearer token
|
9
10
|
# call get_authorization() to get a token.
|
10
11
|
# bearer tokens are kept in memory and also in a file cache for later re-use
|
11
12
|
# if a token is expired (api returns 4xx), call again get_authorization({:refresh=>true})
|
12
13
|
class Oauth
|
13
14
|
private
|
14
15
|
# remove 5 minutes to account for time offset (TODO: configurable?)
|
15
|
-
|
16
|
+
JWT_NOTBEFORE_OFFSET_SEC=300
|
16
17
|
# one hour validity (TODO: configurable?)
|
17
|
-
|
18
|
+
JWT_EXPIRY_OFFSET_SEC=3600
|
19
|
+
# tokens older than 30 minutes will be discarded from cache
|
20
|
+
TOKEN_CACHE_EXPIRY_SEC=1800
|
21
|
+
# a prefix for persistency of tokens (garbage collect)
|
18
22
|
PERSIST_CATEGORY_TOKEN='token'
|
19
|
-
private_constant :
|
23
|
+
private_constant :JWT_NOTBEFORE_OFFSET_SEC,:JWT_EXPIRY_OFFSET_SEC,:PERSIST_CATEGORY_TOKEN,:TOKEN_CACHE_EXPIRY_SEC
|
20
24
|
class << self
|
21
25
|
# OAuth methods supported
|
22
26
|
def auth_types
|
@@ -28,12 +32,18 @@ module Aspera
|
|
28
32
|
end
|
29
33
|
|
30
34
|
def persist_mgr
|
31
|
-
|
35
|
+
if @persist.nil?
|
36
|
+
Log.log.warn('Not using persistency (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)')
|
37
|
+
# create NULL persistency class
|
38
|
+
@persist=Class.new do
|
39
|
+
def get(x);nil;end;def delete(x);nil;end;def put(x,y);nil;end;def garbage_collect(x,y);nil;end
|
40
|
+
end.new
|
41
|
+
end
|
32
42
|
return @persist
|
33
43
|
end
|
34
44
|
|
35
45
|
def flush_tokens
|
36
|
-
persist_mgr.
|
46
|
+
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
|
37
47
|
end
|
38
48
|
end
|
39
49
|
|
@@ -79,13 +89,19 @@ module Aspera
|
|
79
89
|
raise "redirect_uri must have a port" if uri.port.nil?
|
80
90
|
# we could check that host is localhost or local address
|
81
91
|
end
|
92
|
+
# cleanup expired tokens
|
93
|
+
self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
|
82
94
|
end
|
83
95
|
|
84
|
-
THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
|
85
|
-
|
86
96
|
# open the login page, wait for code and check_code, then return code
|
87
97
|
def goto_page_and_get_code(login_page_url,check_code)
|
88
|
-
|
98
|
+
Log.log.info("login_page_url=#{login_page_url}".bg_red.gray)
|
99
|
+
# start a web server to receive request code
|
100
|
+
webserver=WebAuth.new(@params[:redirect_uri])
|
101
|
+
# start browser on login page
|
102
|
+
OpenApplication.instance.uri(login_page_url)
|
103
|
+
# wait for code in request
|
104
|
+
request_params=webserver.get_request
|
89
105
|
Log.log.error("state does not match") if !check_code.eql?(request_params['state'])
|
90
106
|
code=request_params['code']
|
91
107
|
return code
|
@@ -117,6 +133,9 @@ module Aspera
|
|
117
133
|
|
118
134
|
public
|
119
135
|
|
136
|
+
# used to change parameter, such as scope
|
137
|
+
attr_reader :params
|
138
|
+
|
120
139
|
# @param options : :scope and :refresh
|
121
140
|
def get_authorization(options={})
|
122
141
|
# api scope can be overriden to get auth for other scope
|
@@ -184,14 +203,13 @@ module Aspera
|
|
184
203
|
when :web
|
185
204
|
# AoC Web based Auth
|
186
205
|
check_code=SecureRandom.uuid
|
187
|
-
|
188
|
-
"#{@params[:base_url]}/#{@params[:path_authorize]}",
|
189
|
-
p_client_id_and_scope.merge({
|
206
|
+
auth_params=p_client_id_and_scope.merge({
|
190
207
|
:response_type => 'code',
|
191
208
|
:redirect_uri => @params[:redirect_uri],
|
192
|
-
:client_secret => @params[:client_secret],
|
193
209
|
:state => check_code
|
194
|
-
})
|
210
|
+
})
|
211
|
+
auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
|
212
|
+
login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
|
195
213
|
# here, we need a human to authorize on a web page
|
196
214
|
code=goto_page_and_get_code(login_page_url,check_code)
|
197
215
|
# exchange code for token
|
@@ -211,9 +229,19 @@ module Aspera
|
|
211
229
|
:iss => @params[:client_id], # issuer
|
212
230
|
:sub => @params[:jwt_subject], # subject
|
213
231
|
:aud => @params[:jwt_audience], # audience
|
214
|
-
:nbf => seconds_since_epoch-
|
215
|
-
:exp => seconds_since_epoch+
|
232
|
+
:nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET_SEC, # not before
|
233
|
+
:exp => seconds_since_epoch+JWT_EXPIRY_OFFSET_SEC # expiration
|
216
234
|
}
|
235
|
+
# Hum.. compliant ? TODO: remove when Faspex5 API is clarified
|
236
|
+
if @params.has_key?(:f5_username)
|
237
|
+
payload[:jti] = SecureRandom.uuid # JWT id
|
238
|
+
payload[:iat] = seconds_since_epoch # issued at
|
239
|
+
payload.delete(:nbf)
|
240
|
+
p_scope[:redirect_uri]="https://127.0.0.1:5000/token"
|
241
|
+
p_scope[:state]=SecureRandom.uuid
|
242
|
+
p_scope[:client_id]=@params[:client_id]
|
243
|
+
@token_auth_api.params[:auth]={type: :basic,username: @params[:f5_username], password: @params[:f5_password]}
|
244
|
+
end
|
217
245
|
|
218
246
|
# non standard, only for global ids
|
219
247
|
payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
|
@@ -222,8 +250,8 @@ module Aspera
|
|
222
250
|
|
223
251
|
Log.log.debug("private=[#{rsa_private}]")
|
224
252
|
|
225
|
-
Log.log.debug("JWT
|
226
|
-
assertion = JWT.encode(payload, rsa_private, 'RS256')
|
253
|
+
Log.log.debug("JWT payload=[#{payload}]")
|
254
|
+
assertion = JWT.encode(payload, rsa_private, 'RS256',@params[:jwt_headers]||{})
|
227
255
|
|
228
256
|
Log.log.debug("assertion=[#{assertion}]")
|
229
257
|
|
@@ -233,8 +261,10 @@ module Aspera
|
|
233
261
|
}))
|
234
262
|
when :url_token
|
235
263
|
# AoC Public Link
|
264
|
+
params={:url_token=>@params[:url_token]}
|
265
|
+
params[:password]=@params[:password] if @params.has_key?(:password)
|
236
266
|
resp=create_token_advanced({
|
237
|
-
:json_params =>
|
267
|
+
:json_params => params,
|
238
268
|
:url_params => p_scope.merge({
|
239
269
|
:grant_type => 'url_token'
|
240
270
|
})})
|
@@ -288,32 +318,5 @@ module Aspera
|
|
288
318
|
return 'Bearer '+token_data[@params[:token_field]]
|
289
319
|
end
|
290
320
|
|
291
|
-
# open the login page, wait for code and return parameters
|
292
|
-
def self.goto_page_and_get_request(redirect_uri,login_page_url,html_page=THANK_YOU_HTML)
|
293
|
-
Log.log.info "login_page_url=#{login_page_url}".bg_red().gray()
|
294
|
-
# browser start is not blocking, we hope here that starting is slower than opening port
|
295
|
-
OpenApplication.instance.uri(login_page_url)
|
296
|
-
port=URI.parse(redirect_uri).port
|
297
|
-
Log.log.info "listening on port #{port}"
|
298
|
-
request_params=nil
|
299
|
-
TCPServer.open('127.0.0.1', port) { |webserver|
|
300
|
-
Log.log.info "server=#{webserver}"
|
301
|
-
websession = webserver.accept
|
302
|
-
sleep 1 # TODO: sometimes: returns nil ? use webrick ?
|
303
|
-
line = websession.gets.chomp
|
304
|
-
Log.log.info "line=#{line}"
|
305
|
-
if ! line.start_with?('GET /?') then
|
306
|
-
raise "unexpected request"
|
307
|
-
end
|
308
|
-
request = line.partition('?').last.partition(' ').first
|
309
|
-
data=URI.decode_www_form(request)
|
310
|
-
request_params=data.to_h
|
311
|
-
Log.log.debug "request_params=#{request_params}"
|
312
|
-
websession.print "HTTP/1.1 200/OK\r\nContent-type:text/html\r\n\r\n#{html_page}"
|
313
|
-
websession.close
|
314
|
-
}
|
315
|
-
return request_params
|
316
|
-
end
|
317
|
-
|
318
321
|
end # OAuth
|
319
322
|
end # Aspera
|
@@ -55,12 +55,17 @@ module Aspera
|
|
55
55
|
@cache.delete(object_id)
|
56
56
|
end
|
57
57
|
|
58
|
-
def
|
59
|
-
|
60
|
-
|
58
|
+
def garbage_collect(persist_category,max_age_seconds=nil)
|
59
|
+
garbage_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
|
60
|
+
if !max_age_seconds.nil?
|
61
|
+
current_time = Time.now
|
62
|
+
garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
|
63
|
+
end
|
64
|
+
garbage_files.each do |filepath|
|
61
65
|
File.delete(filepath)
|
66
|
+
Log.log.debug("Deleted expired: #{filepath}")
|
62
67
|
end
|
63
|
-
return
|
68
|
+
return garbage_files
|
64
69
|
end
|
65
70
|
|
66
71
|
private
|
@@ -1,9 +1,11 @@
|
|
1
|
-
require '
|
2
|
-
require '
|
1
|
+
require 'aspera/log'
|
2
|
+
require 'singleton'
|
3
3
|
|
4
4
|
module Aspera
|
5
5
|
module Preview
|
6
|
+
# function conversion_type returns one of the types: CONVERSION_TYPES
|
6
7
|
class FileTypes
|
8
|
+
include Singleton
|
7
9
|
# values for conversion_type : input format
|
8
10
|
CONVERSION_TYPES=[
|
9
11
|
:image,
|
@@ -148,6 +150,7 @@ module Aspera
|
|
148
150
|
'dif' => :office,
|
149
151
|
'divx' => :video,
|
150
152
|
'dng' => :image,
|
153
|
+
'docx' => :office,
|
151
154
|
'dpx' => :image,
|
152
155
|
'epdf' => :image,
|
153
156
|
'epi' => :image,
|
@@ -204,6 +207,7 @@ module Aspera
|
|
204
207
|
'pam' => :image,
|
205
208
|
'pcd' => :image,
|
206
209
|
'pcds' => :image,
|
210
|
+
'pdf' => :pdf,
|
207
211
|
'pef' => :image,
|
208
212
|
'picon' => :image,
|
209
213
|
'pict' => :image,
|
@@ -261,39 +265,67 @@ module Aspera
|
|
261
265
|
'x3f' => :image,
|
262
266
|
'xcf' => :image,
|
263
267
|
'xlk' => :office,
|
268
|
+
'xlsx' => :office,
|
269
|
+
'xls' => :office,
|
264
270
|
'ycbcr' => :image,
|
265
271
|
'ycbcra' => :image,
|
266
272
|
'yuv' => :image,
|
267
273
|
'zabw' => :office}
|
268
274
|
|
269
|
-
|
275
|
+
private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
|
270
276
|
|
271
|
-
|
277
|
+
# @attr use_mimemagic [bool] true to use mimemagic to determine real mime type based on file content
|
278
|
+
attr_accessor :use_mimemagic
|
279
|
+
|
280
|
+
def initialize
|
281
|
+
@use_mimemagic=false
|
282
|
+
end
|
283
|
+
|
284
|
+
# use mime magic to find mime type based on file content (magic numbers)
|
285
|
+
def mime_from_file(filepath)
|
286
|
+
# moved here, as mimemagic can cause installation issues
|
287
|
+
require 'mimemagic'
|
288
|
+
require 'mimemagic/version'
|
289
|
+
require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
|
290
|
+
# check magic number inside file (empty string if not found)
|
272
291
|
detected_mime=MimeMagic.by_magic(File.open(filepath)).to_s
|
273
|
-
|
292
|
+
# check extension only
|
293
|
+
if !SUPPORTED_MIME_TYPES.has_key?(detected_mime)
|
294
|
+
Log.log.debug("no conversion for #{detected_mime}, trying extension")
|
295
|
+
detected_mime=MimeMagic.by_extension(File.extname(filepath)).to_s
|
296
|
+
end
|
274
297
|
detected_mime=nil if detected_mime.empty?
|
298
|
+
Log.log.debug("mimemagic: #{detected_mime.class.name} [#{detected_mime}]")
|
275
299
|
return detected_mime
|
276
300
|
end
|
277
301
|
|
278
|
-
|
279
|
-
|
280
|
-
|
302
|
+
# return file type, one of enum CONVERSION_TYPES
|
303
|
+
# @param filepath [String] full path to file
|
304
|
+
# @param mimetype [String] provided by node api
|
305
|
+
def conversion_type(filepath,mimetype)
|
306
|
+
Log.log.debug("conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})")
|
307
|
+
# 1- get type from provided mime type, using local mapping
|
308
|
+
conv_type=SUPPORTED_MIME_TYPES[mimetype] if ! mimetype.nil?
|
309
|
+
# 2- else, from computed mime type (if available)
|
310
|
+
if conv_type.nil? and @use_mimemagic
|
281
311
|
detected_mime=mime_from_file(filepath)
|
282
|
-
if
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
312
|
+
if ! detected_mime.nil?
|
313
|
+
conv_type=SUPPORTED_MIME_TYPES[detected_mime]
|
314
|
+
if ! mimetype.nil?
|
315
|
+
if mimetype.eql?(detected_mime)
|
316
|
+
Log.log.debug("matching mime type per magic number")
|
317
|
+
else
|
318
|
+
# note: detected can be nil
|
319
|
+
Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
|
320
|
+
end
|
321
|
+
end
|
287
322
|
end
|
288
323
|
end
|
289
|
-
#
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
# 3- else, from local extensions
|
295
|
-
result||=FileTypes::SUPPORTED_EXTENSIONS[extension]
|
296
|
-
return result
|
324
|
+
# 3- else, from extensions, using local mapping
|
325
|
+
extension = File.extname(filepath.downcase)[1..-1]
|
326
|
+
conv_type=SUPPORTED_EXTENSIONS[extension] if conv_type.nil?
|
327
|
+
Log.log.debug("conversion_type(#{extension}): #{conv_type.class.name} [#{conv_type}]")
|
328
|
+
return conv_type
|
297
329
|
end
|
298
330
|
end
|
299
331
|
end
|
@@ -15,7 +15,7 @@ module Aspera
|
|
15
15
|
|
16
16
|
# @param src source file path
|
17
17
|
# @param dst destination file path
|
18
|
-
# @param api_mime_type optional mime type as provided by node api
|
18
|
+
# @param api_mime_type optional mime type as provided by node api (or nil)
|
19
19
|
# node API mime types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
|
20
20
|
# supported preview type is one of Preview::PREVIEW_FORMATS
|
21
21
|
# the resulting preview file type is taken from destination file extension.
|
@@ -25,14 +25,14 @@ module Aspera
|
|
25
25
|
# -> preview_format is one of Generator::PREVIEW_FORMATS
|
26
26
|
# the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
|
27
27
|
# -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
|
28
|
-
def initialize(options,src,dst,main_temp_dir,api_mime_type
|
28
|
+
def initialize(options,src,dst,main_temp_dir,api_mime_type)
|
29
29
|
@options=options
|
30
30
|
@source_file_path=src
|
31
31
|
@destination_file_path=dst
|
32
32
|
@temp_folder=File.join(main_temp_dir,@source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
|
33
33
|
# extract preview format from extension of target file
|
34
34
|
@preview_format_symb=File.extname(@destination_file_path).gsub(/^\./,'').to_sym
|
35
|
-
@conversion_type=FileTypes.conversion_type(@source_file_path,api_mime_type
|
35
|
+
@conversion_type=FileTypes.instance.conversion_type(@source_file_path,api_mime_type)
|
36
36
|
end
|
37
37
|
|
38
38
|
# name of processing method in this object
|
data/lib/aspera/rest.rb
CHANGED
@@ -7,6 +7,7 @@ require 'net/http'
|
|
7
7
|
require 'net/https'
|
8
8
|
require 'json'
|
9
9
|
require 'base64'
|
10
|
+
require 'cgi'
|
10
11
|
require 'ruby-progressbar'
|
11
12
|
|
12
13
|
# add cancel method to http
|
@@ -33,6 +34,7 @@ module Aspera
|
|
33
34
|
# true if https ignore certificate
|
34
35
|
@@insecure=false
|
35
36
|
@@user_agent='Ruby'
|
37
|
+
@@download_partial_suffix='.http_partial'
|
36
38
|
|
37
39
|
public
|
38
40
|
def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
|
@@ -98,9 +100,10 @@ module Aspera
|
|
98
100
|
public
|
99
101
|
|
100
102
|
attr_reader :params
|
103
|
+
attr_reader :oauth
|
101
104
|
|
102
|
-
# @param a_rest_params default call parameters and authentication (:auth) :
|
103
|
-
# :type (:basic, :oauth2, :url)
|
105
|
+
# @param a_rest_params default call parameters (merged at call) and (+) authentication (:auth) :
|
106
|
+
# :type (:basic, :oauth2, :url, :none)
|
104
107
|
# :username [:basic]
|
105
108
|
# :password [:basic]
|
106
109
|
# :url_creds [:url]
|
@@ -117,16 +120,6 @@ module Aspera
|
|
117
120
|
# default is no auth
|
118
121
|
@params[:auth]||={:type=>:none}
|
119
122
|
@params[:not_auth_codes]||=['401']
|
120
|
-
# translate old auth parameters, remove prefix, place in auth
|
121
|
-
[:auth,:basic,:oauth].each do |p_sym|
|
122
|
-
p_str=p_sym.to_s+'_'
|
123
|
-
@params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
|
124
|
-
name=k_sym.to_s[p_str.length..-1]
|
125
|
-
name='grant' if k_sym.eql?(:oauth_type)
|
126
|
-
@params[:auth][name.to_sym]=@params[k_sym]
|
127
|
-
@params.delete(k_sym)
|
128
|
-
end
|
129
|
-
end
|
130
123
|
@oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
|
131
124
|
Log.dump('REST params(2)',@params)
|
132
125
|
end
|
@@ -153,6 +146,7 @@ module Aspera
|
|
153
146
|
Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
|
154
147
|
call_data[:headers]||={}
|
155
148
|
call_data[:headers]['User-Agent'] ||= @@user_agent
|
149
|
+
# defaults from @params are overriden by call dataz
|
156
150
|
call_data=@params.deep_merge(call_data)
|
157
151
|
case call_data[:auth][:type]
|
158
152
|
when :none
|
@@ -206,10 +200,13 @@ module Aspera
|
|
206
200
|
|
207
201
|
Log.log.debug("call_data = #{call_data}")
|
208
202
|
result={:http=>nil}
|
203
|
+
# start a block to be able to retry the actual HTTP request
|
209
204
|
begin
|
210
205
|
# we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
|
211
206
|
oauth_tries ||= 2
|
207
|
+
tries_remain_redirect||=4
|
212
208
|
Log.log.debug("send request")
|
209
|
+
# make http request (pipelined)
|
213
210
|
http_session.request(req) do |response|
|
214
211
|
result[:http] = response
|
215
212
|
if call_data.has_key?(:save_to_file)
|
@@ -223,12 +220,12 @@ module Aspera
|
|
223
220
|
target_file=call_data[:save_to_file]
|
224
221
|
# override user's path to path in header
|
225
222
|
if !response['Content-Disposition'].nil? and m=response['Content-Disposition'].match(/filename="([^"]+)"/)
|
226
|
-
target_file=m[1]
|
223
|
+
target_file=File.join(File.dirname(target_file),m[1])
|
227
224
|
end
|
228
225
|
# download with temp filename
|
229
|
-
target_file_tmp=target_file
|
226
|
+
target_file_tmp="#{target_file}#{@@download_partial_suffix}"
|
230
227
|
Log.log.debug("saving to: #{target_file}")
|
231
|
-
File.open(target_file_tmp,
|
228
|
+
File.open(target_file_tmp, 'wb') do |file|
|
232
229
|
result[:http].read_body do |fragment|
|
233
230
|
file.write(fragment)
|
234
231
|
new_process=progress.progress+fragment.length
|
@@ -239,7 +236,7 @@ module Aspera
|
|
239
236
|
# rename at the end
|
240
237
|
File.rename(target_file_tmp, target_file)
|
241
238
|
progress=nil
|
242
|
-
end
|
239
|
+
end # save_to_file
|
243
240
|
end
|
244
241
|
# sometimes there is a ITF8 char (e.g. (c) )
|
245
242
|
result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
|
@@ -267,10 +264,24 @@ module Aspera
|
|
267
264
|
end
|
268
265
|
Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
|
269
266
|
retry unless (oauth_tries -= 1).zero?
|
270
|
-
end # if
|
267
|
+
end # if oauth
|
268
|
+
# moved ?
|
269
|
+
if e.response.is_a?(Net::HTTPRedirection)
|
270
|
+
if tries_remain_redirect > 0
|
271
|
+
tries_remain_redirect-=1
|
272
|
+
Log.log.info("URL is moved: #{e.response['location']}")
|
273
|
+
raise e
|
274
|
+
# TODO: rebuild request with new location
|
275
|
+
#retry
|
276
|
+
else
|
277
|
+
raise "too many redirect"
|
278
|
+
end
|
279
|
+
else
|
280
|
+
raise e
|
281
|
+
end
|
271
282
|
# raise exception if could not retry and not return error in result
|
272
283
|
raise e unless call_data[:return_error]
|
273
|
-
end
|
284
|
+
end # begin request
|
274
285
|
Log.log.debug("result=#{result}")
|
275
286
|
return result
|
276
287
|
|
@@ -0,0 +1,20 @@
|
|
1
|
+
module Aspera
|
2
|
+
# Manage secrets in CLI using secure way (encryption, wallet, etc...)
|
3
|
+
class Secrets
|
4
|
+
attr_accessor :default_secret,:all_secrets
|
5
|
+
def initialize()
|
6
|
+
@default_secret=nil
|
7
|
+
@all_secrets={}
|
8
|
+
end
|
9
|
+
|
10
|
+
def get_secret(id=nil,mandatory=true)
|
11
|
+
secret=@default_secret || @all_secrets[id]
|
12
|
+
raise "please provide secret for #{id}" if secret.nil? and mandatory
|
13
|
+
return secret
|
14
|
+
end
|
15
|
+
|
16
|
+
def get_secrets
|
17
|
+
return @all_secrets
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
data/lib/aspera/sync.rb
CHANGED
@@ -1,48 +1,53 @@
|
|
1
|
+
require 'aspera/command_line_builder'
|
2
|
+
|
1
3
|
module Aspera
|
2
4
|
# builds command line arg for async
|
3
5
|
class Sync
|
4
|
-
private
|
5
6
|
INSTANCE_PARAMS=
|
6
7
|
{
|
7
|
-
'alt_logdir' => { :
|
8
|
-
'watchd' => { :
|
9
|
-
'apply_local_docroot' => { :
|
10
|
-
'quiet' => { :
|
8
|
+
'alt_logdir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
9
|
+
'watchd' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
10
|
+
'apply_local_docroot' => { :cltype => :opt_without_arg},
|
11
|
+
'quiet' => { :cltype => :opt_without_arg},
|
11
12
|
}
|
12
13
|
SESSION_PARAMS=
|
13
14
|
{
|
14
|
-
'name' => { :
|
15
|
-
'local_dir' => { :
|
16
|
-
'remote_dir' => { :
|
17
|
-
'local_db_dir' => { :
|
18
|
-
'remote_db_dir' => { :
|
19
|
-
'host' => { :
|
20
|
-
'user' => { :
|
21
|
-
'private_key_path' => { :
|
22
|
-
'direction' => { :
|
23
|
-
'checksum' => { :
|
24
|
-
'tcp_port' => { :
|
25
|
-
'rate_policy' => { :
|
26
|
-
'target_rate' => { :
|
27
|
-
'cooloff' => { :
|
28
|
-
'pending_max' => { :
|
29
|
-
'scan_intensity' => { :
|
30
|
-
'cipher' => { :
|
31
|
-
'transfer_threads' => { :
|
32
|
-
'preserve_time' => { :
|
33
|
-
'preserve_access_time' => { :
|
34
|
-
'preserve_modification_time' => { :
|
35
|
-
'preserve_uid' => { :
|
36
|
-
'preserve_gid' => { :
|
37
|
-
'create_dir' => { :
|
38
|
-
'reset' => { :
|
15
|
+
'name' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
16
|
+
'local_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
17
|
+
'remote_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
18
|
+
'local_db_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
19
|
+
'remote_db_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
20
|
+
'host' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
21
|
+
'user' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
22
|
+
'private_key_path' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
23
|
+
'direction' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
24
|
+
'checksum' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
25
|
+
'tcp_port' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
26
|
+
'rate_policy' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
27
|
+
'target_rate' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
28
|
+
'cooloff' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
29
|
+
'pending_max' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
30
|
+
'scan_intensity' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
31
|
+
'cipher' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
32
|
+
'transfer_threads' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
33
|
+
'preserve_time' => { :cltype => :opt_without_arg},
|
34
|
+
'preserve_access_time' => { :cltype => :opt_without_arg},
|
35
|
+
'preserve_modification_time' => { :cltype => :opt_without_arg},
|
36
|
+
'preserve_uid' => { :cltype => :opt_without_arg},
|
37
|
+
'preserve_gid' => { :cltype => :opt_without_arg},
|
38
|
+
'create_dir' => { :cltype => :opt_without_arg},
|
39
|
+
'reset' => { :cltype => :opt_without_arg},
|
39
40
|
# note: only one env var, but multiple sessions... may be a problem
|
40
|
-
'remote_password' => { :
|
41
|
-
'cookie' => { :
|
42
|
-
'token' => { :
|
43
|
-
'license' => { :
|
41
|
+
'remote_password' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_PASS'},
|
42
|
+
'cookie' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_COOKIE'},
|
43
|
+
'token' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_TOKEN'},
|
44
|
+
'license' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_LICENSE'},
|
44
45
|
}
|
45
|
-
|
46
|
+
|
47
|
+
Aspera::CommandLineBuilder.normalize_description(INSTANCE_PARAMS)
|
48
|
+
Aspera::CommandLineBuilder.normalize_description(SESSION_PARAMS)
|
49
|
+
|
50
|
+
private_constant :INSTANCE_PARAMS,:SESSION_PARAMS
|
46
51
|
|
47
52
|
def initialize(sync_params)
|
48
53
|
@sync_params=sync_params
|
@@ -6,6 +6,11 @@ module Aspera
|
|
6
6
|
# create a temp file name for a given folder
|
7
7
|
# files can be deleted on process exit by calling cleanup
|
8
8
|
class TempFileManager
|
9
|
+
SEC_IN_DAY=86400
|
10
|
+
# assume no transfer last longer than this
|
11
|
+
# (garbage collect file list which were not deleted after transfer)
|
12
|
+
FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
|
13
|
+
private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC
|
9
14
|
include Singleton
|
10
15
|
def initialize
|
11
16
|
@created_files=[]
|
@@ -33,5 +38,19 @@ module Aspera
|
|
33
38
|
username = Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user' rescue 'unknown_user'
|
34
39
|
return new_file_path_in_folder(Etc.systmpdir,base_name+'_'+username+'_')
|
35
40
|
end
|
41
|
+
|
42
|
+
def cleanup_expired(temp_folder)
|
43
|
+
# garbage collect undeleted files
|
44
|
+
Dir.entries(temp_folder).each do |name|
|
45
|
+
file_path=File.join(temp_folder,name)
|
46
|
+
age_sec=(Time.now - File.stat(file_path).mtime).to_i
|
47
|
+
# check age of file, delete too old
|
48
|
+
if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
|
49
|
+
Log.log.debug("garbage collecting #{name}")
|
50
|
+
File.delete(file_path)
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
36
55
|
end
|
37
56
|
end
|