aspera-cli 4.0.0 → 4.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +843 -304
- data/bin/dascli +13 -0
- data/docs/Makefile +4 -4
- data/docs/README.erb.md +805 -172
- data/docs/test_env.conf +22 -3
- data/examples/aoc.rb +14 -3
- data/examples/faspex4.rb +89 -0
- data/lib/aspera/aoc.rb +87 -108
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +89 -49
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +188 -173
- data/lib/aspera/cli/plugins/ats.rb +2 -2
- data/lib/aspera/cli/plugins/config.rb +218 -145
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +114 -61
- data/lib/aspera/cli/plugins/faspex5.rb +85 -43
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +59 -45
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +77 -49
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/command_line_builder.rb +49 -31
- data/lib/aspera/cos_node.rb +33 -28
- data/lib/aspera/environment.rb +2 -2
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +93 -46
- data/lib/aspera/fasp/local.rb +88 -45
- data/lib/aspera/fasp/manager.rb +15 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +59 -101
- data/lib/aspera/fasp/parameters.yaml +531 -0
- data/lib/aspera/fasp/resume_policy.rb +13 -12
- data/lib/aspera/fasp/uri.rb +1 -1
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +61 -1
- data/lib/aspera/oauth.rb +49 -46
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +29 -18
- data/lib/aspera/secrets.rb +20 -0
- data/lib/aspera/sync.rb +40 -35
- data/lib/aspera/temp_file_manager.rb +19 -0
- data/lib/aspera/web_auth.rb +105 -0
- metadata +54 -20
- data/docs/transfer_spec.html +0 -99
data/lib/aspera/oauth.rb
CHANGED
|
@@ -1,22 +1,26 @@
|
|
|
1
1
|
require 'aspera/open_application'
|
|
2
|
+
require 'aspera/web_auth'
|
|
2
3
|
require 'base64'
|
|
3
4
|
require 'date'
|
|
4
5
|
require 'socket'
|
|
5
6
|
require 'securerandom'
|
|
6
7
|
|
|
7
8
|
module Aspera
|
|
8
|
-
#
|
|
9
|
+
# Implement OAuth 2 for the REST client and generate a bearer token
|
|
9
10
|
# call get_authorization() to get a token.
|
|
10
11
|
# bearer tokens are kept in memory and also in a file cache for later re-use
|
|
11
12
|
# if a token is expired (api returns 4xx), call again get_authorization({:refresh=>true})
|
|
12
13
|
class Oauth
|
|
13
14
|
private
|
|
14
15
|
# remove 5 minutes to account for time offset (TODO: configurable?)
|
|
15
|
-
|
|
16
|
+
JWT_NOTBEFORE_OFFSET_SEC=300
|
|
16
17
|
# one hour validity (TODO: configurable?)
|
|
17
|
-
|
|
18
|
+
JWT_EXPIRY_OFFSET_SEC=3600
|
|
19
|
+
# tokens older than 30 minutes will be discarded from cache
|
|
20
|
+
TOKEN_CACHE_EXPIRY_SEC=1800
|
|
21
|
+
# a prefix for persistency of tokens (garbage collect)
|
|
18
22
|
PERSIST_CATEGORY_TOKEN='token'
|
|
19
|
-
private_constant :
|
|
23
|
+
private_constant :JWT_NOTBEFORE_OFFSET_SEC,:JWT_EXPIRY_OFFSET_SEC,:PERSIST_CATEGORY_TOKEN,:TOKEN_CACHE_EXPIRY_SEC
|
|
20
24
|
class << self
|
|
21
25
|
# OAuth methods supported
|
|
22
26
|
def auth_types
|
|
@@ -28,12 +32,18 @@ module Aspera
|
|
|
28
32
|
end
|
|
29
33
|
|
|
30
34
|
def persist_mgr
|
|
31
|
-
|
|
35
|
+
if @persist.nil?
|
|
36
|
+
Log.log.warn('Not using persistency (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)')
|
|
37
|
+
# create NULL persistency class
|
|
38
|
+
@persist=Class.new do
|
|
39
|
+
def get(x);nil;end;def delete(x);nil;end;def put(x,y);nil;end;def garbage_collect(x,y);nil;end
|
|
40
|
+
end.new
|
|
41
|
+
end
|
|
32
42
|
return @persist
|
|
33
43
|
end
|
|
34
44
|
|
|
35
45
|
def flush_tokens
|
|
36
|
-
persist_mgr.
|
|
46
|
+
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
|
|
37
47
|
end
|
|
38
48
|
end
|
|
39
49
|
|
|
@@ -79,13 +89,19 @@ module Aspera
|
|
|
79
89
|
raise "redirect_uri must have a port" if uri.port.nil?
|
|
80
90
|
# we could check that host is localhost or local address
|
|
81
91
|
end
|
|
92
|
+
# cleanup expired tokens
|
|
93
|
+
self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_CACHE_EXPIRY_SEC)
|
|
82
94
|
end
|
|
83
95
|
|
|
84
|
-
THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
|
|
85
|
-
|
|
86
96
|
# open the login page, wait for code and check_code, then return code
|
|
87
97
|
def goto_page_and_get_code(login_page_url,check_code)
|
|
88
|
-
|
|
98
|
+
Log.log.info("login_page_url=#{login_page_url}".bg_red.gray)
|
|
99
|
+
# start a web server to receive request code
|
|
100
|
+
webserver=WebAuth.new(@params[:redirect_uri])
|
|
101
|
+
# start browser on login page
|
|
102
|
+
OpenApplication.instance.uri(login_page_url)
|
|
103
|
+
# wait for code in request
|
|
104
|
+
request_params=webserver.get_request
|
|
89
105
|
Log.log.error("state does not match") if !check_code.eql?(request_params['state'])
|
|
90
106
|
code=request_params['code']
|
|
91
107
|
return code
|
|
@@ -117,6 +133,9 @@ module Aspera
|
|
|
117
133
|
|
|
118
134
|
public
|
|
119
135
|
|
|
136
|
+
# used to change parameter, such as scope
|
|
137
|
+
attr_reader :params
|
|
138
|
+
|
|
120
139
|
# @param options : :scope and :refresh
|
|
121
140
|
def get_authorization(options={})
|
|
122
141
|
# api scope can be overriden to get auth for other scope
|
|
@@ -184,14 +203,13 @@ module Aspera
|
|
|
184
203
|
when :web
|
|
185
204
|
# AoC Web based Auth
|
|
186
205
|
check_code=SecureRandom.uuid
|
|
187
|
-
|
|
188
|
-
"#{@params[:base_url]}/#{@params[:path_authorize]}",
|
|
189
|
-
p_client_id_and_scope.merge({
|
|
206
|
+
auth_params=p_client_id_and_scope.merge({
|
|
190
207
|
:response_type => 'code',
|
|
191
208
|
:redirect_uri => @params[:redirect_uri],
|
|
192
|
-
:client_secret => @params[:client_secret],
|
|
193
209
|
:state => check_code
|
|
194
|
-
})
|
|
210
|
+
})
|
|
211
|
+
auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
|
|
212
|
+
login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
|
|
195
213
|
# here, we need a human to authorize on a web page
|
|
196
214
|
code=goto_page_and_get_code(login_page_url,check_code)
|
|
197
215
|
# exchange code for token
|
|
@@ -211,9 +229,19 @@ module Aspera
|
|
|
211
229
|
:iss => @params[:client_id], # issuer
|
|
212
230
|
:sub => @params[:jwt_subject], # subject
|
|
213
231
|
:aud => @params[:jwt_audience], # audience
|
|
214
|
-
:nbf => seconds_since_epoch-
|
|
215
|
-
:exp => seconds_since_epoch+
|
|
232
|
+
:nbf => seconds_since_epoch-JWT_NOTBEFORE_OFFSET_SEC, # not before
|
|
233
|
+
:exp => seconds_since_epoch+JWT_EXPIRY_OFFSET_SEC # expiration
|
|
216
234
|
}
|
|
235
|
+
# Hum.. compliant ? TODO: remove when Faspex5 API is clarified
|
|
236
|
+
if @params.has_key?(:f5_username)
|
|
237
|
+
payload[:jti] = SecureRandom.uuid # JWT id
|
|
238
|
+
payload[:iat] = seconds_since_epoch # issued at
|
|
239
|
+
payload.delete(:nbf)
|
|
240
|
+
p_scope[:redirect_uri]="https://127.0.0.1:5000/token"
|
|
241
|
+
p_scope[:state]=SecureRandom.uuid
|
|
242
|
+
p_scope[:client_id]=@params[:client_id]
|
|
243
|
+
@token_auth_api.params[:auth]={type: :basic,username: @params[:f5_username], password: @params[:f5_password]}
|
|
244
|
+
end
|
|
217
245
|
|
|
218
246
|
# non standard, only for global ids
|
|
219
247
|
payload.merge!(@params[:jwt_add]) if @params.has_key?(:jwt_add)
|
|
@@ -222,8 +250,8 @@ module Aspera
|
|
|
222
250
|
|
|
223
251
|
Log.log.debug("private=[#{rsa_private}]")
|
|
224
252
|
|
|
225
|
-
Log.log.debug("JWT
|
|
226
|
-
assertion = JWT.encode(payload, rsa_private, 'RS256')
|
|
253
|
+
Log.log.debug("JWT payload=[#{payload}]")
|
|
254
|
+
assertion = JWT.encode(payload, rsa_private, 'RS256',@params[:jwt_headers]||{})
|
|
227
255
|
|
|
228
256
|
Log.log.debug("assertion=[#{assertion}]")
|
|
229
257
|
|
|
@@ -233,8 +261,10 @@ module Aspera
|
|
|
233
261
|
}))
|
|
234
262
|
when :url_token
|
|
235
263
|
# AoC Public Link
|
|
264
|
+
params={:url_token=>@params[:url_token]}
|
|
265
|
+
params[:password]=@params[:password] if @params.has_key?(:password)
|
|
236
266
|
resp=create_token_advanced({
|
|
237
|
-
:json_params =>
|
|
267
|
+
:json_params => params,
|
|
238
268
|
:url_params => p_scope.merge({
|
|
239
269
|
:grant_type => 'url_token'
|
|
240
270
|
})})
|
|
@@ -288,32 +318,5 @@ module Aspera
|
|
|
288
318
|
return 'Bearer '+token_data[@params[:token_field]]
|
|
289
319
|
end
|
|
290
320
|
|
|
291
|
-
# open the login page, wait for code and return parameters
|
|
292
|
-
def self.goto_page_and_get_request(redirect_uri,login_page_url,html_page=THANK_YOU_HTML)
|
|
293
|
-
Log.log.info "login_page_url=#{login_page_url}".bg_red().gray()
|
|
294
|
-
# browser start is not blocking, we hope here that starting is slower than opening port
|
|
295
|
-
OpenApplication.instance.uri(login_page_url)
|
|
296
|
-
port=URI.parse(redirect_uri).port
|
|
297
|
-
Log.log.info "listening on port #{port}"
|
|
298
|
-
request_params=nil
|
|
299
|
-
TCPServer.open('127.0.0.1', port) { |webserver|
|
|
300
|
-
Log.log.info "server=#{webserver}"
|
|
301
|
-
websession = webserver.accept
|
|
302
|
-
sleep 1 # TODO: sometimes: returns nil ? use webrick ?
|
|
303
|
-
line = websession.gets.chomp
|
|
304
|
-
Log.log.info "line=#{line}"
|
|
305
|
-
if ! line.start_with?('GET /?') then
|
|
306
|
-
raise "unexpected request"
|
|
307
|
-
end
|
|
308
|
-
request = line.partition('?').last.partition(' ').first
|
|
309
|
-
data=URI.decode_www_form(request)
|
|
310
|
-
request_params=data.to_h
|
|
311
|
-
Log.log.debug "request_params=#{request_params}"
|
|
312
|
-
websession.print "HTTP/1.1 200/OK\r\nContent-type:text/html\r\n\r\n#{html_page}"
|
|
313
|
-
websession.close
|
|
314
|
-
}
|
|
315
|
-
return request_params
|
|
316
|
-
end
|
|
317
|
-
|
|
318
321
|
end # OAuth
|
|
319
322
|
end # Aspera
|
|
@@ -55,12 +55,17 @@ module Aspera
|
|
|
55
55
|
@cache.delete(object_id)
|
|
56
56
|
end
|
|
57
57
|
|
|
58
|
-
def
|
|
59
|
-
|
|
60
|
-
|
|
58
|
+
def garbage_collect(persist_category,max_age_seconds=nil)
|
|
59
|
+
garbage_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
|
|
60
|
+
if !max_age_seconds.nil?
|
|
61
|
+
current_time = Time.now
|
|
62
|
+
garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
|
|
63
|
+
end
|
|
64
|
+
garbage_files.each do |filepath|
|
|
61
65
|
File.delete(filepath)
|
|
66
|
+
Log.log.debug("Deleted expired: #{filepath}")
|
|
62
67
|
end
|
|
63
|
-
return
|
|
68
|
+
return garbage_files
|
|
64
69
|
end
|
|
65
70
|
|
|
66
71
|
private
|
|
@@ -1,9 +1,11 @@
|
|
|
1
|
-
require '
|
|
2
|
-
require '
|
|
1
|
+
require 'aspera/log'
|
|
2
|
+
require 'singleton'
|
|
3
3
|
|
|
4
4
|
module Aspera
|
|
5
5
|
module Preview
|
|
6
|
+
# function conversion_type returns one of the types: CONVERSION_TYPES
|
|
6
7
|
class FileTypes
|
|
8
|
+
include Singleton
|
|
7
9
|
# values for conversion_type : input format
|
|
8
10
|
CONVERSION_TYPES=[
|
|
9
11
|
:image,
|
|
@@ -148,6 +150,7 @@ module Aspera
|
|
|
148
150
|
'dif' => :office,
|
|
149
151
|
'divx' => :video,
|
|
150
152
|
'dng' => :image,
|
|
153
|
+
'docx' => :office,
|
|
151
154
|
'dpx' => :image,
|
|
152
155
|
'epdf' => :image,
|
|
153
156
|
'epi' => :image,
|
|
@@ -204,6 +207,7 @@ module Aspera
|
|
|
204
207
|
'pam' => :image,
|
|
205
208
|
'pcd' => :image,
|
|
206
209
|
'pcds' => :image,
|
|
210
|
+
'pdf' => :pdf,
|
|
207
211
|
'pef' => :image,
|
|
208
212
|
'picon' => :image,
|
|
209
213
|
'pict' => :image,
|
|
@@ -261,39 +265,67 @@ module Aspera
|
|
|
261
265
|
'x3f' => :image,
|
|
262
266
|
'xcf' => :image,
|
|
263
267
|
'xlk' => :office,
|
|
268
|
+
'xlsx' => :office,
|
|
269
|
+
'xls' => :office,
|
|
264
270
|
'ycbcr' => :image,
|
|
265
271
|
'ycbcra' => :image,
|
|
266
272
|
'yuv' => :image,
|
|
267
273
|
'zabw' => :office}
|
|
268
274
|
|
|
269
|
-
|
|
275
|
+
private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
|
|
270
276
|
|
|
271
|
-
|
|
277
|
+
# @attr use_mimemagic [bool] true to use mimemagic to determine real mime type based on file content
|
|
278
|
+
attr_accessor :use_mimemagic
|
|
279
|
+
|
|
280
|
+
def initialize
|
|
281
|
+
@use_mimemagic=false
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
# use mime magic to find mime type based on file content (magic numbers)
|
|
285
|
+
def mime_from_file(filepath)
|
|
286
|
+
# moved here, as mimemagic can cause installation issues
|
|
287
|
+
require 'mimemagic'
|
|
288
|
+
require 'mimemagic/version'
|
|
289
|
+
require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
|
|
290
|
+
# check magic number inside file (empty string if not found)
|
|
272
291
|
detected_mime=MimeMagic.by_magic(File.open(filepath)).to_s
|
|
273
|
-
|
|
292
|
+
# check extension only
|
|
293
|
+
if !SUPPORTED_MIME_TYPES.has_key?(detected_mime)
|
|
294
|
+
Log.log.debug("no conversion for #{detected_mime}, trying extension")
|
|
295
|
+
detected_mime=MimeMagic.by_extension(File.extname(filepath)).to_s
|
|
296
|
+
end
|
|
274
297
|
detected_mime=nil if detected_mime.empty?
|
|
298
|
+
Log.log.debug("mimemagic: #{detected_mime.class.name} [#{detected_mime}]")
|
|
275
299
|
return detected_mime
|
|
276
300
|
end
|
|
277
301
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
302
|
+
# return file type, one of enum CONVERSION_TYPES
|
|
303
|
+
# @param filepath [String] full path to file
|
|
304
|
+
# @param mimetype [String] provided by node api
|
|
305
|
+
def conversion_type(filepath,mimetype)
|
|
306
|
+
Log.log.debug("conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})")
|
|
307
|
+
# 1- get type from provided mime type, using local mapping
|
|
308
|
+
conv_type=SUPPORTED_MIME_TYPES[mimetype] if ! mimetype.nil?
|
|
309
|
+
# 2- else, from computed mime type (if available)
|
|
310
|
+
if conv_type.nil? and @use_mimemagic
|
|
281
311
|
detected_mime=mime_from_file(filepath)
|
|
282
|
-
if
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
312
|
+
if ! detected_mime.nil?
|
|
313
|
+
conv_type=SUPPORTED_MIME_TYPES[detected_mime]
|
|
314
|
+
if ! mimetype.nil?
|
|
315
|
+
if mimetype.eql?(detected_mime)
|
|
316
|
+
Log.log.debug("matching mime type per magic number")
|
|
317
|
+
else
|
|
318
|
+
# note: detected can be nil
|
|
319
|
+
Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
|
|
320
|
+
end
|
|
321
|
+
end
|
|
287
322
|
end
|
|
288
323
|
end
|
|
289
|
-
#
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
# 3- else, from local extensions
|
|
295
|
-
result||=FileTypes::SUPPORTED_EXTENSIONS[extension]
|
|
296
|
-
return result
|
|
324
|
+
# 3- else, from extensions, using local mapping
|
|
325
|
+
extension = File.extname(filepath.downcase)[1..-1]
|
|
326
|
+
conv_type=SUPPORTED_EXTENSIONS[extension] if conv_type.nil?
|
|
327
|
+
Log.log.debug("conversion_type(#{extension}): #{conv_type.class.name} [#{conv_type}]")
|
|
328
|
+
return conv_type
|
|
297
329
|
end
|
|
298
330
|
end
|
|
299
331
|
end
|
|
@@ -15,7 +15,7 @@ module Aspera
|
|
|
15
15
|
|
|
16
16
|
# @param src source file path
|
|
17
17
|
# @param dst destination file path
|
|
18
|
-
# @param api_mime_type optional mime type as provided by node api
|
|
18
|
+
# @param api_mime_type optional mime type as provided by node api (or nil)
|
|
19
19
|
# node API mime types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
|
|
20
20
|
# supported preview type is one of Preview::PREVIEW_FORMATS
|
|
21
21
|
# the resulting preview file type is taken from destination file extension.
|
|
@@ -25,14 +25,14 @@ module Aspera
|
|
|
25
25
|
# -> preview_format is one of Generator::PREVIEW_FORMATS
|
|
26
26
|
# the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
|
|
27
27
|
# -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
|
|
28
|
-
def initialize(options,src,dst,main_temp_dir,api_mime_type
|
|
28
|
+
def initialize(options,src,dst,main_temp_dir,api_mime_type)
|
|
29
29
|
@options=options
|
|
30
30
|
@source_file_path=src
|
|
31
31
|
@destination_file_path=dst
|
|
32
32
|
@temp_folder=File.join(main_temp_dir,@source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
|
|
33
33
|
# extract preview format from extension of target file
|
|
34
34
|
@preview_format_symb=File.extname(@destination_file_path).gsub(/^\./,'').to_sym
|
|
35
|
-
@conversion_type=FileTypes.conversion_type(@source_file_path,api_mime_type
|
|
35
|
+
@conversion_type=FileTypes.instance.conversion_type(@source_file_path,api_mime_type)
|
|
36
36
|
end
|
|
37
37
|
|
|
38
38
|
# name of processing method in this object
|
data/lib/aspera/rest.rb
CHANGED
|
@@ -7,6 +7,7 @@ require 'net/http'
|
|
|
7
7
|
require 'net/https'
|
|
8
8
|
require 'json'
|
|
9
9
|
require 'base64'
|
|
10
|
+
require 'cgi'
|
|
10
11
|
require 'ruby-progressbar'
|
|
11
12
|
|
|
12
13
|
# add cancel method to http
|
|
@@ -33,6 +34,7 @@ module Aspera
|
|
|
33
34
|
# true if https ignore certificate
|
|
34
35
|
@@insecure=false
|
|
35
36
|
@@user_agent='Ruby'
|
|
37
|
+
@@download_partial_suffix='.http_partial'
|
|
36
38
|
|
|
37
39
|
public
|
|
38
40
|
def self.insecure=(v); @@insecure=v;Log.log.debug("insecure => #{@@insecure}".red);end
|
|
@@ -98,9 +100,10 @@ module Aspera
|
|
|
98
100
|
public
|
|
99
101
|
|
|
100
102
|
attr_reader :params
|
|
103
|
+
attr_reader :oauth
|
|
101
104
|
|
|
102
|
-
# @param a_rest_params default call parameters and authentication (:auth) :
|
|
103
|
-
# :type (:basic, :oauth2, :url)
|
|
105
|
+
# @param a_rest_params default call parameters (merged at call) and (+) authentication (:auth) :
|
|
106
|
+
# :type (:basic, :oauth2, :url, :none)
|
|
104
107
|
# :username [:basic]
|
|
105
108
|
# :password [:basic]
|
|
106
109
|
# :url_creds [:url]
|
|
@@ -117,16 +120,6 @@ module Aspera
|
|
|
117
120
|
# default is no auth
|
|
118
121
|
@params[:auth]||={:type=>:none}
|
|
119
122
|
@params[:not_auth_codes]||=['401']
|
|
120
|
-
# translate old auth parameters, remove prefix, place in auth
|
|
121
|
-
[:auth,:basic,:oauth].each do |p_sym|
|
|
122
|
-
p_str=p_sym.to_s+'_'
|
|
123
|
-
@params.keys.select{|k|k.to_s.start_with?(p_str)}.each do |k_sym|
|
|
124
|
-
name=k_sym.to_s[p_str.length..-1]
|
|
125
|
-
name='grant' if k_sym.eql?(:oauth_type)
|
|
126
|
-
@params[:auth][name.to_sym]=@params[k_sym]
|
|
127
|
-
@params.delete(k_sym)
|
|
128
|
-
end
|
|
129
|
-
end
|
|
130
123
|
@oauth=Oauth.new(@params[:auth]) if @params[:auth][:type].eql?(:oauth2)
|
|
131
124
|
Log.dump('REST params(2)',@params)
|
|
132
125
|
end
|
|
@@ -153,6 +146,7 @@ module Aspera
|
|
|
153
146
|
Log.log.debug("accessing #{call_data[:subpath]}".red.bold.bg_green)
|
|
154
147
|
call_data[:headers]||={}
|
|
155
148
|
call_data[:headers]['User-Agent'] ||= @@user_agent
|
|
149
|
+
# defaults from @params are overriden by call dataz
|
|
156
150
|
call_data=@params.deep_merge(call_data)
|
|
157
151
|
case call_data[:auth][:type]
|
|
158
152
|
when :none
|
|
@@ -206,10 +200,13 @@ module Aspera
|
|
|
206
200
|
|
|
207
201
|
Log.log.debug("call_data = #{call_data}")
|
|
208
202
|
result={:http=>nil}
|
|
203
|
+
# start a block to be able to retry the actual HTTP request
|
|
209
204
|
begin
|
|
210
205
|
# we try the call, and will retry only if oauth, as we can, first with refresh, and then re-auth if refresh is bad
|
|
211
206
|
oauth_tries ||= 2
|
|
207
|
+
tries_remain_redirect||=4
|
|
212
208
|
Log.log.debug("send request")
|
|
209
|
+
# make http request (pipelined)
|
|
213
210
|
http_session.request(req) do |response|
|
|
214
211
|
result[:http] = response
|
|
215
212
|
if call_data.has_key?(:save_to_file)
|
|
@@ -223,12 +220,12 @@ module Aspera
|
|
|
223
220
|
target_file=call_data[:save_to_file]
|
|
224
221
|
# override user's path to path in header
|
|
225
222
|
if !response['Content-Disposition'].nil? and m=response['Content-Disposition'].match(/filename="([^"]+)"/)
|
|
226
|
-
target_file=m[1]
|
|
223
|
+
target_file=File.join(File.dirname(target_file),m[1])
|
|
227
224
|
end
|
|
228
225
|
# download with temp filename
|
|
229
|
-
target_file_tmp=target_file
|
|
226
|
+
target_file_tmp="#{target_file}#{@@download_partial_suffix}"
|
|
230
227
|
Log.log.debug("saving to: #{target_file}")
|
|
231
|
-
File.open(target_file_tmp,
|
|
228
|
+
File.open(target_file_tmp, 'wb') do |file|
|
|
232
229
|
result[:http].read_body do |fragment|
|
|
233
230
|
file.write(fragment)
|
|
234
231
|
new_process=progress.progress+fragment.length
|
|
@@ -239,7 +236,7 @@ module Aspera
|
|
|
239
236
|
# rename at the end
|
|
240
237
|
File.rename(target_file_tmp, target_file)
|
|
241
238
|
progress=nil
|
|
242
|
-
end
|
|
239
|
+
end # save_to_file
|
|
243
240
|
end
|
|
244
241
|
# sometimes there is a ITF8 char (e.g. (c) )
|
|
245
242
|
result[:http].body.force_encoding("UTF-8") if result[:http].body.is_a?(String)
|
|
@@ -267,10 +264,24 @@ module Aspera
|
|
|
267
264
|
end
|
|
268
265
|
Log.log.debug("using new token=#{call_data[:headers]['Authorization']}")
|
|
269
266
|
retry unless (oauth_tries -= 1).zero?
|
|
270
|
-
end # if
|
|
267
|
+
end # if oauth
|
|
268
|
+
# moved ?
|
|
269
|
+
if e.response.is_a?(Net::HTTPRedirection)
|
|
270
|
+
if tries_remain_redirect > 0
|
|
271
|
+
tries_remain_redirect-=1
|
|
272
|
+
Log.log.info("URL is moved: #{e.response['location']}")
|
|
273
|
+
raise e
|
|
274
|
+
# TODO: rebuild request with new location
|
|
275
|
+
#retry
|
|
276
|
+
else
|
|
277
|
+
raise "too many redirect"
|
|
278
|
+
end
|
|
279
|
+
else
|
|
280
|
+
raise e
|
|
281
|
+
end
|
|
271
282
|
# raise exception if could not retry and not return error in result
|
|
272
283
|
raise e unless call_data[:return_error]
|
|
273
|
-
end
|
|
284
|
+
end # begin request
|
|
274
285
|
Log.log.debug("result=#{result}")
|
|
275
286
|
return result
|
|
276
287
|
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
module Aspera
|
|
2
|
+
# Manage secrets in CLI using secure way (encryption, wallet, etc...)
|
|
3
|
+
class Secrets
|
|
4
|
+
attr_accessor :default_secret,:all_secrets
|
|
5
|
+
def initialize()
|
|
6
|
+
@default_secret=nil
|
|
7
|
+
@all_secrets={}
|
|
8
|
+
end
|
|
9
|
+
|
|
10
|
+
def get_secret(id=nil,mandatory=true)
|
|
11
|
+
secret=@default_secret || @all_secrets[id]
|
|
12
|
+
raise "please provide secret for #{id}" if secret.nil? and mandatory
|
|
13
|
+
return secret
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def get_secrets
|
|
17
|
+
return @all_secrets
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
data/lib/aspera/sync.rb
CHANGED
|
@@ -1,48 +1,53 @@
|
|
|
1
|
+
require 'aspera/command_line_builder'
|
|
2
|
+
|
|
1
3
|
module Aspera
|
|
2
4
|
# builds command line arg for async
|
|
3
5
|
class Sync
|
|
4
|
-
private
|
|
5
6
|
INSTANCE_PARAMS=
|
|
6
7
|
{
|
|
7
|
-
'alt_logdir' => { :
|
|
8
|
-
'watchd' => { :
|
|
9
|
-
'apply_local_docroot' => { :
|
|
10
|
-
'quiet' => { :
|
|
8
|
+
'alt_logdir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
9
|
+
'watchd' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
10
|
+
'apply_local_docroot' => { :cltype => :opt_without_arg},
|
|
11
|
+
'quiet' => { :cltype => :opt_without_arg},
|
|
11
12
|
}
|
|
12
13
|
SESSION_PARAMS=
|
|
13
14
|
{
|
|
14
|
-
'name' => { :
|
|
15
|
-
'local_dir' => { :
|
|
16
|
-
'remote_dir' => { :
|
|
17
|
-
'local_db_dir' => { :
|
|
18
|
-
'remote_db_dir' => { :
|
|
19
|
-
'host' => { :
|
|
20
|
-
'user' => { :
|
|
21
|
-
'private_key_path' => { :
|
|
22
|
-
'direction' => { :
|
|
23
|
-
'checksum' => { :
|
|
24
|
-
'tcp_port' => { :
|
|
25
|
-
'rate_policy' => { :
|
|
26
|
-
'target_rate' => { :
|
|
27
|
-
'cooloff' => { :
|
|
28
|
-
'pending_max' => { :
|
|
29
|
-
'scan_intensity' => { :
|
|
30
|
-
'cipher' => { :
|
|
31
|
-
'transfer_threads' => { :
|
|
32
|
-
'preserve_time' => { :
|
|
33
|
-
'preserve_access_time' => { :
|
|
34
|
-
'preserve_modification_time' => { :
|
|
35
|
-
'preserve_uid' => { :
|
|
36
|
-
'preserve_gid' => { :
|
|
37
|
-
'create_dir' => { :
|
|
38
|
-
'reset' => { :
|
|
15
|
+
'name' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
16
|
+
'local_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
17
|
+
'remote_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
18
|
+
'local_db_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
19
|
+
'remote_db_dir' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
20
|
+
'host' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
21
|
+
'user' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
22
|
+
'private_key_path' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
23
|
+
'direction' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
24
|
+
'checksum' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
25
|
+
'tcp_port' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
|
26
|
+
'rate_policy' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
27
|
+
'target_rate' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
28
|
+
'cooloff' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
|
29
|
+
'pending_max' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
|
30
|
+
'scan_intensity' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
31
|
+
'cipher' => { :cltype => :opt_with_arg, :accepted_types=>:string},
|
|
32
|
+
'transfer_threads' => { :cltype => :opt_with_arg, :accepted_types=>:int},
|
|
33
|
+
'preserve_time' => { :cltype => :opt_without_arg},
|
|
34
|
+
'preserve_access_time' => { :cltype => :opt_without_arg},
|
|
35
|
+
'preserve_modification_time' => { :cltype => :opt_without_arg},
|
|
36
|
+
'preserve_uid' => { :cltype => :opt_without_arg},
|
|
37
|
+
'preserve_gid' => { :cltype => :opt_without_arg},
|
|
38
|
+
'create_dir' => { :cltype => :opt_without_arg},
|
|
39
|
+
'reset' => { :cltype => :opt_without_arg},
|
|
39
40
|
# note: only one env var, but multiple sessions... may be a problem
|
|
40
|
-
'remote_password' => { :
|
|
41
|
-
'cookie' => { :
|
|
42
|
-
'token' => { :
|
|
43
|
-
'license' => { :
|
|
41
|
+
'remote_password' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_PASS'},
|
|
42
|
+
'cookie' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_COOKIE'},
|
|
43
|
+
'token' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_TOKEN'},
|
|
44
|
+
'license' => { :cltype => :envvar, :clvarname=>'ASPERA_SCP_LICENSE'},
|
|
44
45
|
}
|
|
45
|
-
|
|
46
|
+
|
|
47
|
+
Aspera::CommandLineBuilder.normalize_description(INSTANCE_PARAMS)
|
|
48
|
+
Aspera::CommandLineBuilder.normalize_description(SESSION_PARAMS)
|
|
49
|
+
|
|
50
|
+
private_constant :INSTANCE_PARAMS,:SESSION_PARAMS
|
|
46
51
|
|
|
47
52
|
def initialize(sync_params)
|
|
48
53
|
@sync_params=sync_params
|
|
@@ -6,6 +6,11 @@ module Aspera
|
|
|
6
6
|
# create a temp file name for a given folder
|
|
7
7
|
# files can be deleted on process exit by calling cleanup
|
|
8
8
|
class TempFileManager
|
|
9
|
+
SEC_IN_DAY=86400
|
|
10
|
+
# assume no transfer last longer than this
|
|
11
|
+
# (garbage collect file list which were not deleted after transfer)
|
|
12
|
+
FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
|
|
13
|
+
private_constant :SEC_IN_DAY,:FILE_LIST_AGE_MAX_SEC
|
|
9
14
|
include Singleton
|
|
10
15
|
def initialize
|
|
11
16
|
@created_files=[]
|
|
@@ -33,5 +38,19 @@ module Aspera
|
|
|
33
38
|
username = Etc.getlogin || Etc.getpwuid(Process.uid).name || 'unknown_user' rescue 'unknown_user'
|
|
34
39
|
return new_file_path_in_folder(Etc.systmpdir,base_name+'_'+username+'_')
|
|
35
40
|
end
|
|
41
|
+
|
|
42
|
+
def cleanup_expired(temp_folder)
|
|
43
|
+
# garbage collect undeleted files
|
|
44
|
+
Dir.entries(temp_folder).each do |name|
|
|
45
|
+
file_path=File.join(temp_folder,name)
|
|
46
|
+
age_sec=(Time.now - File.stat(file_path).mtime).to_i
|
|
47
|
+
# check age of file, delete too old
|
|
48
|
+
if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
|
|
49
|
+
Log.log.debug("garbage collecting #{name}")
|
|
50
|
+
File.delete(file_path)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
end
|
|
36
55
|
end
|
|
37
56
|
end
|