aspera-cli 4.0.0 → 4.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +518 -137
- data/bin/dascli +13 -0
- data/docs/README.erb.md +473 -105
- data/docs/test_env.conf +4 -1
- data/docs/transfer_spec.html +1 -1
- data/lib/aspera/aoc.rb +68 -86
- data/lib/aspera/cli/formater.rb +2 -0
- data/lib/aspera/cli/main.rb +27 -19
- data/lib/aspera/cli/plugin.rb +9 -4
- data/lib/aspera/cli/plugins/alee.rb +1 -1
- data/lib/aspera/cli/plugins/aoc.rb +173 -136
- data/lib/aspera/cli/plugins/config.rb +80 -27
- data/lib/aspera/cli/plugins/console.rb +2 -2
- data/lib/aspera/cli/plugins/faspex.rb +13 -6
- data/lib/aspera/cli/plugins/faspex5.rb +93 -37
- data/lib/aspera/cli/plugins/node.rb +3 -3
- data/lib/aspera/cli/plugins/preview.rb +25 -24
- data/lib/aspera/cli/plugins/server.rb +23 -8
- data/lib/aspera/cli/transfer_agent.rb +1 -1
- data/lib/aspera/cli/version.rb +1 -1
- data/lib/aspera/fasp/connect.rb +28 -21
- data/lib/aspera/fasp/http_gw.rb +140 -28
- data/lib/aspera/fasp/installation.rb +28 -4
- data/lib/aspera/fasp/local.rb +24 -16
- data/lib/aspera/fasp/manager.rb +12 -0
- data/lib/aspera/fasp/node.rb +4 -4
- data/lib/aspera/fasp/parameters.rb +3 -16
- data/lib/aspera/log.rb +1 -1
- data/lib/aspera/node.rb +48 -1
- data/lib/aspera/oauth.rb +24 -11
- data/lib/aspera/persistency_folder.rb +9 -4
- data/lib/aspera/preview/file_types.rb +53 -21
- data/lib/aspera/preview/generator.rb +3 -3
- data/lib/aspera/rest.rb +38 -18
- data/lib/aspera/temp_file_manager.rb +19 -0
- metadata +37 -20
@@ -176,10 +176,34 @@ module Aspera
|
|
176
176
|
return [:ssh_bypass_key_dsa,:ssh_bypass_key_rsa].map{|i|Installation.instance.path(i)}
|
177
177
|
end
|
178
178
|
|
179
|
-
|
179
|
+
# download aspera SDK or use local file
|
180
|
+
# extract only ascp binary for current system architecture
|
181
|
+
def install_sdk(sdk_url)
|
180
182
|
require 'zip'
|
181
183
|
sdk_zip_path=File.join(Dir.tmpdir,'sdk.zip')
|
182
|
-
|
184
|
+
if sdk_url.start_with?('file:')
|
185
|
+
# require specific file scheme: the path part is "relative", or absolute if there are 4 slash
|
186
|
+
raise 'use format: file:///<path>' unless sdk_url.start_with?('file:///')
|
187
|
+
sdk_zip_path=sdk_url.gsub(%r{^file:///},'')
|
188
|
+
else
|
189
|
+
redirect_remain=2
|
190
|
+
begin
|
191
|
+
Aspera::Rest.new(base_url: sdk_url).call(operation: 'GET',save_to_file: sdk_zip_path)
|
192
|
+
rescue Aspera::RestCallError => e
|
193
|
+
if e.response.is_a?(Net::HTTPRedirection)
|
194
|
+
if redirect_remain > 0
|
195
|
+
redirect_remain-=1
|
196
|
+
sdk_url=e.response['location']
|
197
|
+
retry
|
198
|
+
else
|
199
|
+
raise "too meny redirect"
|
200
|
+
end
|
201
|
+
else
|
202
|
+
raise e
|
203
|
+
end
|
204
|
+
end
|
205
|
+
end
|
206
|
+
# SDK is organized by architecture
|
183
207
|
filter="/#{Environment.architecture}/"
|
184
208
|
ascp_path=nil
|
185
209
|
# first ensure license file is here so that ascp invokation for version works
|
@@ -187,6 +211,7 @@ module Aspera
|
|
187
211
|
self.path(:aspera_conf)
|
188
212
|
Zip::File.open(sdk_zip_path) do |zip_file|
|
189
213
|
zip_file.each do |entry|
|
214
|
+
# get only specified arch, but not folder, only files
|
190
215
|
if entry.name.include?(filter) and !entry.name.end_with?('/')
|
191
216
|
archive_file=File.join(folder_path,File.basename(entry.name))
|
192
217
|
File.open(archive_file, 'wb') do |output_stream|
|
@@ -220,9 +245,8 @@ module Aspera
|
|
220
245
|
PRODUCT_INFO='product-info.mf'
|
221
246
|
# policy for product selection
|
222
247
|
FIRST_FOUND='FIRST'
|
223
|
-
SDK_URL='https://eudemo.asperademo.com/aspera/faspex/sdk.zip'
|
224
248
|
|
225
|
-
private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
|
249
|
+
private_constant :BIN_SUBFOLDER,:ETC_SUBFOLDER,:VARRUN_SUBFOLDER,:PRODUCT_INFO
|
226
250
|
|
227
251
|
# get some specific folder from specific applications: Connect or CLI
|
228
252
|
def get_product_folders(name)
|
data/lib/aspera/fasp/local.rb
CHANGED
@@ -232,23 +232,26 @@ module Aspera
|
|
232
232
|
end # case
|
233
233
|
end # loop (process mgt port lines)
|
234
234
|
# check that last status was received before process exit
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
240
|
-
|
241
|
-
|
242
|
-
|
243
|
-
Log.log.error("
|
244
|
-
if
|
245
|
-
|
246
|
-
|
235
|
+
if last_status_event.nil?
|
236
|
+
Log.log.warn("no status read from ascp mgt port")
|
237
|
+
else
|
238
|
+
case last_status_event['Type']
|
239
|
+
when 'DONE'
|
240
|
+
# return method (or just don't do anything)
|
241
|
+
return
|
242
|
+
when 'ERROR'
|
243
|
+
Log.log.error("code: #{last_status_event['Code']}")
|
244
|
+
if last_status_event['Description'] =~ /bearer token/i
|
245
|
+
Log.log.error("need to regenerate token".red)
|
246
|
+
if session[:options].is_a?(Hash) and session[:options].has_key?(:regenerate_token)
|
247
|
+
# regenerate token here, expired, or error on it
|
248
|
+
env_args[:env]['ASPERA_SCP_TOKEN']=session[:options][:regenerate_token].call(true)
|
249
|
+
end
|
247
250
|
end
|
251
|
+
raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
|
252
|
+
else
|
253
|
+
raise "unexpected last event type: #{last_status_event['Type']}"
|
248
254
|
end
|
249
|
-
raise Fasp::Error.new(last_status_event['Description'],last_status_event['Code'].to_i)
|
250
|
-
else
|
251
|
-
raise "unexpected last event type: #{last_status_event['Type']}"
|
252
255
|
end
|
253
256
|
rescue SystemCallError => e
|
254
257
|
# Process.spawn
|
@@ -262,8 +265,12 @@ module Aspera
|
|
262
265
|
unless ascp_pid.nil?
|
263
266
|
# "wait" for process to avoid zombie
|
264
267
|
Process.wait(ascp_pid)
|
268
|
+
status=$?
|
265
269
|
ascp_pid=nil
|
266
270
|
session.delete(:io)
|
271
|
+
if !status.success?
|
272
|
+
raise Fasp::Error.new("ascp failed with code #{status.exitstatus}")
|
273
|
+
end
|
267
274
|
end
|
268
275
|
end # begin-ensure
|
269
276
|
end # start_transfer_with_args_env
|
@@ -301,8 +308,9 @@ module Aspera
|
|
301
308
|
@jobs={}
|
302
309
|
# mutex protects global data accessed by threads
|
303
310
|
@mutex=Mutex.new
|
304
|
-
@resume_policy=ResumePolicy.new(agent_options)
|
305
311
|
@enable_wss = agent_options[:wss] || false
|
312
|
+
agent_options.delete(:wss)
|
313
|
+
@resume_policy=ResumePolicy.new(agent_options)
|
306
314
|
end
|
307
315
|
|
308
316
|
# transfer thread entry
|
data/lib/aspera/fasp/manager.rb
CHANGED
@@ -45,6 +45,18 @@ module Aspera
|
|
45
45
|
end
|
46
46
|
end # notify_listeners
|
47
47
|
|
48
|
+
def notify_begin(id,size)
|
49
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'NOTIFICATION','PreTransferBytes'=>size})
|
50
|
+
end
|
51
|
+
|
52
|
+
def notify_progress(id,size)
|
53
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'STATS','Bytescont'=>size})
|
54
|
+
end
|
55
|
+
|
56
|
+
def notify_end(id)
|
57
|
+
notify_listeners('emulated',{LISTENER_SESSION_ID_B=>id,'Type'=>'DONE'})
|
58
|
+
end
|
59
|
+
|
48
60
|
public
|
49
61
|
LISTENER_SESSION_ID_B='ListenerSessionId'
|
50
62
|
LISTENER_SESSION_ID_S='listener_session_id'
|
data/lib/aspera/fasp/node.rb
CHANGED
@@ -55,7 +55,7 @@ module Aspera
|
|
55
55
|
trdata=node_api_.read("ops/transfers/#{@transfer_id}")[:data] || {"status"=>"unknown"} rescue {"status"=>"waiting(read error)"}
|
56
56
|
case trdata['status']
|
57
57
|
when 'completed'
|
58
|
-
|
58
|
+
notify_end(@transfer_id)
|
59
59
|
break
|
60
60
|
when 'waiting','partially_completed','unknown','waiting(read error)'
|
61
61
|
if spinner.nil?
|
@@ -69,10 +69,10 @@ module Aspera
|
|
69
69
|
#puts "running: sessions:#{trdata["sessions"].length}, #{trdata["sessions"].map{|i| i['bytes_transferred']}.join(',')}"
|
70
70
|
if !started and trdata['precalc'].is_a?(Hash) and
|
71
71
|
trdata['precalc']['status'].eql?('ready')
|
72
|
-
|
72
|
+
notify_begin(@transfer_id,trdata['precalc']['bytes_expected'])
|
73
73
|
started=true
|
74
74
|
else
|
75
|
-
|
75
|
+
notify_progress(@transfer_id,trdata['bytes_transferred'])
|
76
76
|
end
|
77
77
|
else
|
78
78
|
Log.log.warn("trdata -> #{trdata}")
|
@@ -81,7 +81,7 @@ module Aspera
|
|
81
81
|
sleep 1
|
82
82
|
end
|
83
83
|
#TODO get status of sessions
|
84
|
-
return []
|
84
|
+
return []
|
85
85
|
end
|
86
86
|
end
|
87
87
|
end
|
@@ -16,10 +16,6 @@ module Aspera
|
|
16
16
|
# because of garbage collection takes any file there
|
17
17
|
# this could be refined, as , for instance, on macos, temp folder is already user specific
|
18
18
|
@@file_list_folder=TempFileManager.instance.new_file_path_global('asession_filelists')
|
19
|
-
SEC_IN_DAY=86400
|
20
|
-
# assume no transfer last longer than this
|
21
|
-
# (garbage collect file list which were not deleted after transfer)
|
22
|
-
FILE_LIST_AGE_MAX_SEC=5*SEC_IN_DAY
|
23
19
|
PARAM_DEFINITION={
|
24
20
|
# parameters with env vars
|
25
21
|
'remote_password' => { :type => :envvar, :variable=>'ASPERA_SCP_PASS'},
|
@@ -100,7 +96,7 @@ module Aspera
|
|
100
96
|
'wss_port' => { :type => :defer, :accepted_types=>Integer},
|
101
97
|
}
|
102
98
|
|
103
|
-
private_constant :
|
99
|
+
private_constant :PARAM_DEFINITION
|
104
100
|
|
105
101
|
def initialize(job_spec,options)
|
106
102
|
@job_spec=job_spec
|
@@ -207,18 +203,9 @@ module Aspera
|
|
207
203
|
# temp file list files are created here
|
208
204
|
def self.file_list_folder=(v)
|
209
205
|
@@file_list_folder=v
|
210
|
-
|
206
|
+
if !@@file_list_folder.nil?
|
211
207
|
FileUtils.mkdir_p(@@file_list_folder)
|
212
|
-
|
213
|
-
Dir.entries(@@file_list_folder).each do |name|
|
214
|
-
file_path=File.join(@@file_list_folder,name)
|
215
|
-
age_sec=(Time.now - File.stat(file_path).mtime).to_i
|
216
|
-
# check age of file, delete too old
|
217
|
-
if File.file?(file_path) and age_sec > FILE_LIST_AGE_MAX_SEC
|
218
|
-
Log.log.debug("garbage collecting #{name}")
|
219
|
-
File.delete(file_path)
|
220
|
-
end
|
221
|
-
end
|
208
|
+
TempFileManager.instance.cleanup_expired(@@file_list_folder)
|
222
209
|
end
|
223
210
|
end
|
224
211
|
|
data/lib/aspera/log.rb
CHANGED
@@ -14,7 +14,7 @@ module Aspera
|
|
14
14
|
attr_reader :logger
|
15
15
|
attr_reader :logger_type
|
16
16
|
# levels are :debug,:info,:warn,:error,fatal,:unknown
|
17
|
-
def self.levels; Logger::Severity.constants.map{|c|
|
17
|
+
def self.levels; Logger::Severity.constants.sort{|a,b|Logger::Severity.const_get(a)<=>Logger::Severity.const_get(b)}.map{|c|c.downcase.to_sym};end
|
18
18
|
|
19
19
|
# where logs are sent to
|
20
20
|
def self.logtypes; [:stderr,:stdout,:syslog];end
|
data/lib/aspera/node.rb
CHANGED
@@ -1,14 +1,61 @@
|
|
1
|
+
require 'aspera/rest'
|
2
|
+
require 'aspera/log'
|
1
3
|
require 'zlib'
|
2
4
|
require 'base64'
|
5
|
+
|
3
6
|
module Aspera
|
4
7
|
# Provides additional functions using node API.
|
5
8
|
class Node < Rest
|
9
|
+
# permissions
|
10
|
+
ACCESS_LEVELS=['delete','list','mkdir','preview','read','rename','write']
|
11
|
+
|
12
|
+
# for information only
|
6
13
|
def self.decode_bearer_token(token)
|
7
14
|
return JSON.parse(Zlib::Inflate.inflate(Base64.decode64(token)).partition('==SIGNATURE==').first)
|
8
15
|
end
|
16
|
+
|
9
17
|
def initialize(rest_params)
|
10
18
|
super(rest_params)
|
11
|
-
|
19
|
+
end
|
20
|
+
|
21
|
+
# recursively crawl in a folder.
|
22
|
+
# subfolders a processed if the processing method returns true
|
23
|
+
# @param processor must provide a method to process each entry
|
24
|
+
# @param opt options
|
25
|
+
# - top_file_id file id to start at (default = access key root file id)
|
26
|
+
# - top_file_path path of top folder (default = /)
|
27
|
+
# - method processing method (default= process_entry)
|
28
|
+
def crawl(processor,opt={})
|
29
|
+
Log.log.debug("crawl1 #{opt}")
|
30
|
+
# not possible with bearer token
|
31
|
+
opt[:top_file_id] ||= read('access_keys/self')[:data]['root_file_id']
|
32
|
+
opt[:top_file_path] ||= '/'
|
33
|
+
opt[:method] ||= :process_entry
|
34
|
+
raise "processor must have #{opt[:method]}" unless processor.respond_to?(opt[:method])
|
35
|
+
Log.log.debug("crawl #{opt}")
|
36
|
+
#top_info=read("files/#{opt[:top_file_id]}")[:data]
|
37
|
+
folders_to_explore=[{id: opt[:top_file_id], relpath: opt[:top_file_path]}]
|
38
|
+
Log.dump(:folders_to_explore,folders_to_explore)
|
39
|
+
while !folders_to_explore.empty? do
|
40
|
+
current_item = folders_to_explore.shift
|
41
|
+
Log.log.debug("searching #{current_item[:relpath]}".bg_green)
|
42
|
+
# get folder content
|
43
|
+
folder_contents = begin
|
44
|
+
read("files/#{current_item[:id]}/files")[:data]
|
45
|
+
rescue => e
|
46
|
+
Log.log.warn("#{current_item[:relpath]}: #{e.class} #{e.message}")
|
47
|
+
[]
|
48
|
+
end
|
49
|
+
Log.dump(:folder_contents,folder_contents)
|
50
|
+
folder_contents.each do |entry|
|
51
|
+
relative_path=File.join(current_item[:relpath],entry['name'])
|
52
|
+
Log.log.debug("looking #{relative_path}".bg_green)
|
53
|
+
# entry type is file, folder or link
|
54
|
+
if processor.send(opt[:method],entry,relative_path) and entry['type'].eql?('folder')
|
55
|
+
folders_to_explore.push({:id=>entry['id'],:relpath=>relative_path})
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
12
59
|
end
|
13
60
|
end
|
14
61
|
end
|
data/lib/aspera/oauth.rb
CHANGED
@@ -16,7 +16,10 @@ module Aspera
|
|
16
16
|
# one hour validity (TODO: configurable?)
|
17
17
|
JWT_EXPIRY_OFFSET=3600
|
18
18
|
PERSIST_CATEGORY_TOKEN='token'
|
19
|
-
|
19
|
+
# tokens older than 30 minutes will be discarded
|
20
|
+
TOKEN_EXPIRY_SECONDS=1800
|
21
|
+
THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
|
22
|
+
private_constant :JWT_NOTBEFORE_OFFSET,:JWT_EXPIRY_OFFSET,:PERSIST_CATEGORY_TOKEN,:TOKEN_EXPIRY_SECONDS,:THANK_YOU_HTML
|
20
23
|
class << self
|
21
24
|
# OAuth methods supported
|
22
25
|
def auth_types
|
@@ -28,12 +31,18 @@ module Aspera
|
|
28
31
|
end
|
29
32
|
|
30
33
|
def persist_mgr
|
31
|
-
|
34
|
+
if @persist.nil?
|
35
|
+
Log.log.warn('Not using persistency (use Aspera::Oauth.persist_mgr=Aspera::PersistencyFolder.new)')
|
36
|
+
# create NULL persistency class
|
37
|
+
@persist=Class.new do
|
38
|
+
def get(x);nil;end;def delete(x);nil;end;def put(x,y);nil;end;def garbage_collect(x,y);nil;end
|
39
|
+
end.new
|
40
|
+
end
|
32
41
|
return @persist
|
33
42
|
end
|
34
43
|
|
35
44
|
def flush_tokens
|
36
|
-
persist_mgr.
|
45
|
+
persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,nil)
|
37
46
|
end
|
38
47
|
end
|
39
48
|
|
@@ -79,10 +88,10 @@ module Aspera
|
|
79
88
|
raise "redirect_uri must have a port" if uri.port.nil?
|
80
89
|
# we could check that host is localhost or local address
|
81
90
|
end
|
91
|
+
# cleanup expired tokens
|
92
|
+
self.class.persist_mgr.garbage_collect(PERSIST_CATEGORY_TOKEN,TOKEN_EXPIRY_SECONDS)
|
82
93
|
end
|
83
94
|
|
84
|
-
THANK_YOU_HTML = "<html><head><title>Ok</title></head><body><h1>Thank you !</h1><p>You can close this window.</p></body></html>"
|
85
|
-
|
86
95
|
# open the login page, wait for code and check_code, then return code
|
87
96
|
def goto_page_and_get_code(login_page_url,check_code)
|
88
97
|
request_params=self.class.goto_page_and_get_request(@params[:redirect_uri],login_page_url)
|
@@ -117,6 +126,9 @@ module Aspera
|
|
117
126
|
|
118
127
|
public
|
119
128
|
|
129
|
+
# used to change parameter, such as scope
|
130
|
+
attr_reader :params
|
131
|
+
|
120
132
|
# @param options : :scope and :refresh
|
121
133
|
def get_authorization(options={})
|
122
134
|
# api scope can be overriden to get auth for other scope
|
@@ -184,14 +196,13 @@ module Aspera
|
|
184
196
|
when :web
|
185
197
|
# AoC Web based Auth
|
186
198
|
check_code=SecureRandom.uuid
|
187
|
-
|
188
|
-
"#{@params[:base_url]}/#{@params[:path_authorize]}",
|
189
|
-
p_client_id_and_scope.merge({
|
199
|
+
auth_params=p_client_id_and_scope.merge({
|
190
200
|
:response_type => 'code',
|
191
201
|
:redirect_uri => @params[:redirect_uri],
|
192
|
-
:client_secret => @params[:client_secret],
|
193
202
|
:state => check_code
|
194
|
-
})
|
203
|
+
})
|
204
|
+
auth_params[:client_secret]=@params[:client_secret] if @params.has_key?(:client_secret)
|
205
|
+
login_page_url=Rest.build_uri("#{@params[:base_url]}/#{@params[:path_authorize]}",auth_params)
|
195
206
|
# here, we need a human to authorize on a web page
|
196
207
|
code=goto_page_and_get_code(login_page_url,check_code)
|
197
208
|
# exchange code for token
|
@@ -233,8 +244,10 @@ module Aspera
|
|
233
244
|
}))
|
234
245
|
when :url_token
|
235
246
|
# AoC Public Link
|
247
|
+
params={:url_token=>@params[:url_token]}
|
248
|
+
params[:password]=@params[:password] if @params.has_key?(:password)
|
236
249
|
resp=create_token_advanced({
|
237
|
-
:json_params =>
|
250
|
+
:json_params => params,
|
238
251
|
:url_params => p_scope.merge({
|
239
252
|
:grant_type => 'url_token'
|
240
253
|
})})
|
@@ -55,12 +55,17 @@ module Aspera
|
|
55
55
|
@cache.delete(object_id)
|
56
56
|
end
|
57
57
|
|
58
|
-
def
|
59
|
-
|
60
|
-
|
58
|
+
def garbage_collect(persist_category,max_age_seconds=nil)
|
59
|
+
garbage_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
|
60
|
+
if !max_age_seconds.nil?
|
61
|
+
current_time = Time.now
|
62
|
+
garbage_files.select! { |filepath| (current_time - File.stat(filepath).mtime).to_i > max_age_seconds}
|
63
|
+
end
|
64
|
+
garbage_files.each do |filepath|
|
61
65
|
File.delete(filepath)
|
66
|
+
Log.log.debug("Deleted expired: #{filepath}")
|
62
67
|
end
|
63
|
-
return
|
68
|
+
return garbage_files
|
64
69
|
end
|
65
70
|
|
66
71
|
private
|
@@ -1,9 +1,11 @@
|
|
1
|
-
require '
|
2
|
-
require '
|
1
|
+
require 'aspera/log'
|
2
|
+
require 'singleton'
|
3
3
|
|
4
4
|
module Aspera
|
5
5
|
module Preview
|
6
|
+
# function conversion_type returns one of the types: CONVERSION_TYPES
|
6
7
|
class FileTypes
|
8
|
+
include Singleton
|
7
9
|
# values for conversion_type : input format
|
8
10
|
CONVERSION_TYPES=[
|
9
11
|
:image,
|
@@ -148,6 +150,7 @@ module Aspera
|
|
148
150
|
'dif' => :office,
|
149
151
|
'divx' => :video,
|
150
152
|
'dng' => :image,
|
153
|
+
'docx' => :office,
|
151
154
|
'dpx' => :image,
|
152
155
|
'epdf' => :image,
|
153
156
|
'epi' => :image,
|
@@ -204,6 +207,7 @@ module Aspera
|
|
204
207
|
'pam' => :image,
|
205
208
|
'pcd' => :image,
|
206
209
|
'pcds' => :image,
|
210
|
+
'pdf' => :pdf,
|
207
211
|
'pef' => :image,
|
208
212
|
'picon' => :image,
|
209
213
|
'pict' => :image,
|
@@ -261,39 +265,67 @@ module Aspera
|
|
261
265
|
'x3f' => :image,
|
262
266
|
'xcf' => :image,
|
263
267
|
'xlk' => :office,
|
268
|
+
'xlsx' => :office,
|
269
|
+
'xls' => :office,
|
264
270
|
'ycbcr' => :image,
|
265
271
|
'ycbcra' => :image,
|
266
272
|
'yuv' => :image,
|
267
273
|
'zabw' => :office}
|
268
274
|
|
269
|
-
|
275
|
+
private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
|
270
276
|
|
271
|
-
|
277
|
+
# @attr use_mimemagic [bool] true to use mimemagic to determine real mime type based on file content
|
278
|
+
attr_accessor :use_mimemagic
|
279
|
+
|
280
|
+
def initialize
|
281
|
+
@use_mimemagic=false
|
282
|
+
end
|
283
|
+
|
284
|
+
# use mime magic to find mime type based on file content (magic numbers)
|
285
|
+
def mime_from_file(filepath)
|
286
|
+
# moved here, as mimemagic can cause installation issues
|
287
|
+
require 'mimemagic'
|
288
|
+
require 'mimemagic/version'
|
289
|
+
require 'mimemagic/overlay' if MimeMagic::VERSION.start_with?('0.3.')
|
290
|
+
# check magic number inside file (empty string if not found)
|
272
291
|
detected_mime=MimeMagic.by_magic(File.open(filepath)).to_s
|
273
|
-
|
292
|
+
# check extension only
|
293
|
+
if !SUPPORTED_MIME_TYPES.has_key?(detected_mime)
|
294
|
+
Log.log.debug("no conversion for #{detected_mime}, trying extension")
|
295
|
+
detected_mime=MimeMagic.by_extension(File.extname(filepath)).to_s
|
296
|
+
end
|
274
297
|
detected_mime=nil if detected_mime.empty?
|
298
|
+
Log.log.debug("mimemagic: #{detected_mime.class.name} [#{detected_mime}]")
|
275
299
|
return detected_mime
|
276
300
|
end
|
277
301
|
|
278
|
-
|
279
|
-
|
280
|
-
|
302
|
+
# return file type, one of enum CONVERSION_TYPES
|
303
|
+
# @param filepath [String] full path to file
|
304
|
+
# @param mimetype [String] provided by node api
|
305
|
+
def conversion_type(filepath,mimetype)
|
306
|
+
Log.log.debug("conversion_type(#{filepath},m=#{mimetype},t=#{@use_mimemagic})")
|
307
|
+
# 1- get type from provided mime type, using local mapping
|
308
|
+
conv_type=SUPPORTED_MIME_TYPES[mimetype] if ! mimetype.nil?
|
309
|
+
# 2- else, from computed mime type (if available)
|
310
|
+
if conv_type.nil? and @use_mimemagic
|
281
311
|
detected_mime=mime_from_file(filepath)
|
282
|
-
if
|
283
|
-
|
284
|
-
|
285
|
-
|
286
|
-
|
312
|
+
if ! detected_mime.nil?
|
313
|
+
conv_type=SUPPORTED_MIME_TYPES[detected_mime]
|
314
|
+
if ! mimetype.nil?
|
315
|
+
if mimetype.eql?(detected_mime)
|
316
|
+
Log.log.debug("matching mime type per magic number")
|
317
|
+
else
|
318
|
+
# note: detected can be nil
|
319
|
+
Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
|
320
|
+
end
|
321
|
+
end
|
287
322
|
end
|
288
323
|
end
|
289
|
-
#
|
290
|
-
|
291
|
-
|
292
|
-
|
293
|
-
|
294
|
-
# 3- else, from local extensions
|
295
|
-
result||=FileTypes::SUPPORTED_EXTENSIONS[extension]
|
296
|
-
return result
|
324
|
+
# 3- else, from extensions, using local mapping
|
325
|
+
extension = File.extname(filepath.downcase)[1..-1]
|
326
|
+
conv_type=SUPPORTED_EXTENSIONS[extension] if conv_type.nil?
|
327
|
+
Log.log.debug("conversion_type(#{extension}): #{conv_type.class.name} [#{conv_type}]")
|
328
|
+
return conv_type
|
297
329
|
end
|
298
330
|
end
|
299
331
|
end
|