aspera-cli 4.0.0.pre1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/README.md +3592 -0
- data/bin/ascli +7 -0
- data/bin/asession +89 -0
- data/docs/Makefile +59 -0
- data/docs/README.erb.md +3012 -0
- data/docs/README.md +13 -0
- data/docs/diagrams.txt +49 -0
- data/docs/secrets.make +38 -0
- data/docs/test_env.conf +117 -0
- data/docs/transfer_spec.html +99 -0
- data/examples/aoc.rb +17 -0
- data/examples/proxy.pac +60 -0
- data/examples/transfer.rb +115 -0
- data/lib/aspera/api_detector.rb +60 -0
- data/lib/aspera/ascmd.rb +151 -0
- data/lib/aspera/ats_api.rb +43 -0
- data/lib/aspera/cli/basic_auth_plugin.rb +38 -0
- data/lib/aspera/cli/extended_value.rb +88 -0
- data/lib/aspera/cli/formater.rb +238 -0
- data/lib/aspera/cli/listener/line_dump.rb +17 -0
- data/lib/aspera/cli/listener/logger.rb +20 -0
- data/lib/aspera/cli/listener/progress.rb +52 -0
- data/lib/aspera/cli/listener/progress_multi.rb +91 -0
- data/lib/aspera/cli/main.rb +304 -0
- data/lib/aspera/cli/manager.rb +440 -0
- data/lib/aspera/cli/plugin.rb +90 -0
- data/lib/aspera/cli/plugins/alee.rb +24 -0
- data/lib/aspera/cli/plugins/ats.rb +231 -0
- data/lib/aspera/cli/plugins/bss.rb +71 -0
- data/lib/aspera/cli/plugins/config.rb +806 -0
- data/lib/aspera/cli/plugins/console.rb +62 -0
- data/lib/aspera/cli/plugins/cos.rb +106 -0
- data/lib/aspera/cli/plugins/faspex.rb +377 -0
- data/lib/aspera/cli/plugins/faspex5.rb +93 -0
- data/lib/aspera/cli/plugins/node.rb +438 -0
- data/lib/aspera/cli/plugins/oncloud.rb +937 -0
- data/lib/aspera/cli/plugins/orchestrator.rb +169 -0
- data/lib/aspera/cli/plugins/preview.rb +464 -0
- data/lib/aspera/cli/plugins/server.rb +216 -0
- data/lib/aspera/cli/plugins/shares.rb +63 -0
- data/lib/aspera/cli/plugins/shares2.rb +114 -0
- data/lib/aspera/cli/plugins/sync.rb +65 -0
- data/lib/aspera/cli/plugins/xnode.rb +115 -0
- data/lib/aspera/cli/transfer_agent.rb +251 -0
- data/lib/aspera/cli/version.rb +5 -0
- data/lib/aspera/colors.rb +39 -0
- data/lib/aspera/command_line_builder.rb +137 -0
- data/lib/aspera/fasp/aoc.rb +24 -0
- data/lib/aspera/fasp/connect.rb +99 -0
- data/lib/aspera/fasp/error.rb +21 -0
- data/lib/aspera/fasp/error_info.rb +60 -0
- data/lib/aspera/fasp/http_gw.rb +81 -0
- data/lib/aspera/fasp/installation.rb +240 -0
- data/lib/aspera/fasp/listener.rb +11 -0
- data/lib/aspera/fasp/local.rb +377 -0
- data/lib/aspera/fasp/manager.rb +69 -0
- data/lib/aspera/fasp/node.rb +88 -0
- data/lib/aspera/fasp/parameters.rb +235 -0
- data/lib/aspera/fasp/resume_policy.rb +76 -0
- data/lib/aspera/fasp/uri.rb +51 -0
- data/lib/aspera/faspex_gw.rb +196 -0
- data/lib/aspera/hash_ext.rb +28 -0
- data/lib/aspera/log.rb +80 -0
- data/lib/aspera/nagios.rb +71 -0
- data/lib/aspera/node.rb +14 -0
- data/lib/aspera/oauth.rb +319 -0
- data/lib/aspera/on_cloud.rb +421 -0
- data/lib/aspera/open_application.rb +72 -0
- data/lib/aspera/persistency_action_once.rb +42 -0
- data/lib/aspera/persistency_folder.rb +91 -0
- data/lib/aspera/preview/file_types.rb +300 -0
- data/lib/aspera/preview/generator.rb +258 -0
- data/lib/aspera/preview/image_error.png +0 -0
- data/lib/aspera/preview/options.rb +35 -0
- data/lib/aspera/preview/utils.rb +131 -0
- data/lib/aspera/preview/video_error.png +0 -0
- data/lib/aspera/proxy_auto_config.erb.js +287 -0
- data/lib/aspera/proxy_auto_config.rb +34 -0
- data/lib/aspera/rest.rb +296 -0
- data/lib/aspera/rest_call_error.rb +13 -0
- data/lib/aspera/rest_error_analyzer.rb +98 -0
- data/lib/aspera/rest_errors_aspera.rb +58 -0
- data/lib/aspera/ssh.rb +53 -0
- data/lib/aspera/sync.rb +82 -0
- data/lib/aspera/temp_file_manager.rb +37 -0
- data/lib/aspera/uri_reader.rb +25 -0
- metadata +288 -0
@@ -0,0 +1,42 @@
|
|
1
|
+
require 'json'
|
2
|
+
require 'aspera/log'
|
3
|
+
|
4
|
+
module Aspera
|
5
|
+
# Persist data on file system
|
6
|
+
class PersistencyActionOnce
|
7
|
+
# @param :manager Mandatory Database
|
8
|
+
# @param :data Mandatory object to persist, must be same object from begin to end (assume array by default)
|
9
|
+
# @param :ids Mandatory identifiers
|
10
|
+
# @param :delete Optional delete persistency condition
|
11
|
+
# @param :parse Optional parse method (default to JSON)
|
12
|
+
# @param :format Optional dump method (default to JSON)
|
13
|
+
# @param :merge Optional merge data from file to current data
|
14
|
+
def initialize(options)
|
15
|
+
Log.log.debug("persistency: #{options}")
|
16
|
+
raise "options shall be Hash" unless options.is_a?(Hash)
|
17
|
+
raise "mandatory :manager" if options[:manager].nil?
|
18
|
+
raise "mandatory :data" if options[:data].nil?
|
19
|
+
raise "mandatory :ids (Array)" unless options[:ids].is_a?(Array)
|
20
|
+
raise "mandatory 1 element in :ids" unless options[:ids].length >= 1
|
21
|
+
@manager=options[:manager]
|
22
|
+
@persisted_object=options[:data]
|
23
|
+
@object_ids=options[:ids]
|
24
|
+
# by default , at save time, file is deleted if data is nil
|
25
|
+
@delete_condition=options[:delete] || lambda{|d|d.empty?}
|
26
|
+
@persist_format=options[:format] || lambda {|h| JSON.generate(h)}
|
27
|
+
persist_parse=options[:parse] || lambda {|t| JSON.parse(t)}
|
28
|
+
persist_merge=options[:merge] || lambda {|current,file| current.concat(file).uniq rescue current}
|
29
|
+
value=@manager.get(@object_ids)
|
30
|
+
persist_merge.call(@persisted_object,persist_parse.call(value)) unless value.nil?
|
31
|
+
end
|
32
|
+
|
33
|
+
def save
|
34
|
+
if @delete_condition.call(@persisted_object)
|
35
|
+
@manager.delete(@object_ids)
|
36
|
+
else
|
37
|
+
@manager.put(@object_ids,@persist_format.call(@persisted_object))
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
end
|
@@ -0,0 +1,91 @@
|
|
1
|
+
require 'fileutils'
|
2
|
+
require 'aspera/log'
|
3
|
+
|
4
|
+
# search: persistency_folder PersistencyFolder
|
5
|
+
|
6
|
+
module Aspera
|
7
|
+
# Persist data on file system
|
8
|
+
class PersistencyFolder
|
9
|
+
WINDOWS_PROTECTED_CHAR=%r{[/:"<>\\\*\?]}
|
10
|
+
PROTECTED_CHAR_REPLACE='_'
|
11
|
+
ID_SEPARATOR='_'
|
12
|
+
FILE_SUFFIX='.txt'
|
13
|
+
private_constant :PROTECTED_CHAR_REPLACE,:FILE_SUFFIX,:WINDOWS_PROTECTED_CHAR,:ID_SEPARATOR
|
14
|
+
def initialize(folder)
|
15
|
+
@cache={}
|
16
|
+
set_folder(folder)
|
17
|
+
end
|
18
|
+
|
19
|
+
def set_folder(folder)
|
20
|
+
@folder=folder
|
21
|
+
Log.log.debug("persistency folder: #{@folder}")
|
22
|
+
end
|
23
|
+
|
24
|
+
# @return String or nil string on existing persist, else nil
|
25
|
+
def get(object_id)
|
26
|
+
object_id=marshalled_id(object_id)
|
27
|
+
Log.log.debug("persistency get: #{object_id}")
|
28
|
+
if @cache.has_key?(object_id)
|
29
|
+
Log.log.debug("got from memory cache")
|
30
|
+
else
|
31
|
+
persist_filepath=id_to_filepath(object_id)
|
32
|
+
Log.log.debug("persistency = #{persist_filepath}")
|
33
|
+
if File.exist?(persist_filepath)
|
34
|
+
Log.log.debug("got from file cache")
|
35
|
+
@cache[object_id]=File.read(persist_filepath)
|
36
|
+
end
|
37
|
+
end
|
38
|
+
return @cache[object_id]
|
39
|
+
end
|
40
|
+
|
41
|
+
def put(object_id,value)
|
42
|
+
raise "only String supported" unless value.is_a?(String)
|
43
|
+
object_id=marshalled_id(object_id)
|
44
|
+
persist_filepath=id_to_filepath(object_id)
|
45
|
+
Log.log.debug("saving: #{persist_filepath}")
|
46
|
+
File.write(persist_filepath,value)
|
47
|
+
@cache[object_id]=value
|
48
|
+
end
|
49
|
+
|
50
|
+
def delete(object_id)
|
51
|
+
object_id=marshalled_id(object_id)
|
52
|
+
persist_filepath=id_to_filepath(object_id)
|
53
|
+
Log.log.debug("empty data, deleting: #{persist_filepath}")
|
54
|
+
File.delete(persist_filepath) if File.exist?(persist_filepath)
|
55
|
+
@cache.delete(object_id)
|
56
|
+
end
|
57
|
+
|
58
|
+
def flush_by_prefix(persist_category)
|
59
|
+
persist_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
|
60
|
+
persist_files.each do |filepath|
|
61
|
+
File.delete(filepath)
|
62
|
+
end
|
63
|
+
return persist_files
|
64
|
+
end
|
65
|
+
|
66
|
+
private
|
67
|
+
|
68
|
+
# @param object_id String or Array
|
69
|
+
def id_to_filepath(object_id)
|
70
|
+
FileUtils.mkdir_p(@folder)
|
71
|
+
return File.join(@folder,"#{object_id}#{FILE_SUFFIX}")
|
72
|
+
#.gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
|
73
|
+
end
|
74
|
+
|
75
|
+
def marshalled_id(object_id)
|
76
|
+
if object_id.is_a?(Array)
|
77
|
+
# special case, url in second position: TODO: check any position
|
78
|
+
if object_id[1].is_a?(String) and object_id[1] =~ URI::ABS_URI
|
79
|
+
object_id=object_id.clone
|
80
|
+
object_id[1]=URI.parse(object_id[1]).host
|
81
|
+
end
|
82
|
+
object_id=object_id.join(ID_SEPARATOR)
|
83
|
+
end
|
84
|
+
raise "id must be a String" unless object_id.is_a?(String)
|
85
|
+
return object_id.
|
86
|
+
gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
|
87
|
+
gsub('.',PROTECTED_CHAR_REPLACE). # keep dot for extension only (nicer)
|
88
|
+
downcase
|
89
|
+
end
|
90
|
+
end # PersistencyFolder
|
91
|
+
end # Aspera
|
@@ -0,0 +1,300 @@
|
|
1
|
+
require 'mimemagic'
|
2
|
+
require 'mimemagic/overlay'
|
3
|
+
|
4
|
+
module Aspera
|
5
|
+
module Preview
|
6
|
+
class FileTypes
|
7
|
+
# values for conversion_type : input format
|
8
|
+
CONVERSION_TYPES=[
|
9
|
+
:image,
|
10
|
+
:office,
|
11
|
+
:pdf,
|
12
|
+
:plaintext,
|
13
|
+
:video
|
14
|
+
]
|
15
|
+
|
16
|
+
# define how files are processed based on mime type
|
17
|
+
SUPPORTED_MIME_TYPES={
|
18
|
+
'application/json' => :plaintext,
|
19
|
+
'application/mac-binhex40' => :office,
|
20
|
+
'application/msword' => :office,
|
21
|
+
'application/pdf' => :pdf,
|
22
|
+
'application/postscript' => :image,
|
23
|
+
'application/rtf' => :office,
|
24
|
+
'application/vnd.3gpp.pic-bw-small' => :image,
|
25
|
+
'application/vnd.hp-hpgl' => :image,
|
26
|
+
'application/vnd.hp-pcl' => :image,
|
27
|
+
'application/vnd.lotus-wordpro' => :office,
|
28
|
+
'application/vnd.mobius.msl' => :image,
|
29
|
+
'application/vnd.mophun.certificate' => :image,
|
30
|
+
'application/vnd.ms-excel' => :office,
|
31
|
+
'application/vnd.ms-excel.sheet.binary.macroenabled.12' => :office,
|
32
|
+
'application/vnd.ms-excel.sheet.macroenabled.12' => :office,
|
33
|
+
'application/vnd.ms-excel.template.macroenabled.12' => :office,
|
34
|
+
'application/vnd.ms-powerpoint' => :office,
|
35
|
+
'application/vnd.ms-powerpoint.presentation.macroenabled.12' => :office,
|
36
|
+
'application/vnd.ms-powerpoint.template.macroenabled.12' => :office,
|
37
|
+
'application/vnd.ms-word.document.macroenabled.12' => :office,
|
38
|
+
'application/vnd.ms-word.template.macroenabled.12' => :office,
|
39
|
+
'application/vnd.ms-works' => :office,
|
40
|
+
'application/vnd.oasis.opendocument.chart' => :office,
|
41
|
+
'application/vnd.oasis.opendocument.formula' => :office,
|
42
|
+
'application/vnd.oasis.opendocument.graphics' => :office,
|
43
|
+
'application/vnd.oasis.opendocument.graphics-template' => :office,
|
44
|
+
'application/vnd.oasis.opendocument.presentation' => :office,
|
45
|
+
'application/vnd.oasis.opendocument.presentation-template' => :office,
|
46
|
+
'application/vnd.oasis.opendocument.spreadsheet' => :office,
|
47
|
+
'application/vnd.oasis.opendocument.spreadsheet-template' => :office,
|
48
|
+
'application/vnd.oasis.opendocument.text' => :office,
|
49
|
+
'application/vnd.oasis.opendocument.text-template' => :office,
|
50
|
+
'application/vnd.openxmlformats-officedocument.presentationml.presentation' => :office,
|
51
|
+
'application/vnd.openxmlformats-officedocument.presentationml.slideshow' => :office,
|
52
|
+
'application/vnd.openxmlformats-officedocument.presentationml.template' => :office,
|
53
|
+
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => :office,
|
54
|
+
'application/vnd.openxmlformats-officedocument.spreadsheetml.template' => :office,
|
55
|
+
'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => :office,
|
56
|
+
'application/vnd.openxmlformats-officedocument.wordprocessingml.template' => :office,
|
57
|
+
'application/vnd.palm' => :office,
|
58
|
+
'application/vnd.sun.xml.calc' => :office,
|
59
|
+
'application/vnd.sun.xml.calc.template' => :office,
|
60
|
+
'application/vnd.sun.xml.draw' => :office,
|
61
|
+
'application/vnd.sun.xml.draw.template' => :office,
|
62
|
+
'application/vnd.sun.xml.impress' => :office,
|
63
|
+
'application/vnd.sun.xml.impress.template' => :office,
|
64
|
+
'application/vnd.sun.xml.math' => :office,
|
65
|
+
'application/vnd.sun.xml.writer' => :office,
|
66
|
+
'application/vnd.sun.xml.writer.template' => :office,
|
67
|
+
'application/vnd.wordperfect' => :office,
|
68
|
+
'application/x-abiword' => :office,
|
69
|
+
'application/x-director' => :image,
|
70
|
+
'application/x-font-type1' => :image,
|
71
|
+
'application/x-msmetafile' => :image,
|
72
|
+
'application/x-mspublisher' => :office,
|
73
|
+
'application/x-xfig' => :image,
|
74
|
+
'audio/ogg' => :video,
|
75
|
+
'font/ttf' => :image,
|
76
|
+
'image/bmp' => :image,
|
77
|
+
'image/cgm' => :image,
|
78
|
+
'image/gif' => :image,
|
79
|
+
'image/jpeg' => :image,
|
80
|
+
'image/png' => :image,
|
81
|
+
'image/sgi' => :image,
|
82
|
+
'image/svg+xml' => :image,
|
83
|
+
'image/tiff' => :image,
|
84
|
+
'image/vnd.adobe.photoshop' => :image,
|
85
|
+
'image/vnd.djvu' => :image,
|
86
|
+
'image/vnd.dxf' => :office,
|
87
|
+
'image/vnd.fpx' => :image,
|
88
|
+
'image/vnd.ms-photo' => :image,
|
89
|
+
'image/vnd.wap.wbmp' => :image,
|
90
|
+
'image/webp' => :image,
|
91
|
+
'image/x-cmx' => :office,
|
92
|
+
'image/x-freehand' => :office,
|
93
|
+
'image/x-icon' => :image,
|
94
|
+
'image/x-mrsid-image' => :image,
|
95
|
+
'image/x-pcx' => :image,
|
96
|
+
'image/x-pict' => :office,
|
97
|
+
'image/x-portable-anymap' => :image,
|
98
|
+
'image/x-portable-bitmap' => :image,
|
99
|
+
'image/x-portable-graymap' => :image,
|
100
|
+
'image/x-portable-pixmap' => :image,
|
101
|
+
'image/x-rgb' => :image,
|
102
|
+
'image/x-tga' => :image,
|
103
|
+
'image/x-xbitmap' => :image,
|
104
|
+
'image/x-xpixmap' => :image,
|
105
|
+
'image/x-xwindowdump' => :image,
|
106
|
+
'text/csv' => :office,
|
107
|
+
'text/html' => :office,
|
108
|
+
'text/plain' => :plaintext,
|
109
|
+
'text/troff' => :image,
|
110
|
+
'video/h261' => :video,
|
111
|
+
'video/h263' => :video,
|
112
|
+
'video/h264' => :video,
|
113
|
+
'video/mp4' => :video,
|
114
|
+
'video/mpeg' => :video,
|
115
|
+
'video/quicktime' => :video,
|
116
|
+
'video/x-flv' => :video,
|
117
|
+
'video/x-m4v' => :video,
|
118
|
+
'video/x-matroska' => :video,
|
119
|
+
'video/x-mng' => :image,
|
120
|
+
'video/x-ms-wmv' => :video,
|
121
|
+
'video/x-msvideo' => :video}
|
122
|
+
|
123
|
+
# this is a way to add support for extensions that are otherwise not known by node api (mime type)
|
124
|
+
SUPPORTED_EXTENSIONS={
|
125
|
+
'aai' => :image,
|
126
|
+
'art' => :image,
|
127
|
+
'arw' => :image,
|
128
|
+
'avs' => :image,
|
129
|
+
'bmp2' => :image,
|
130
|
+
'bmp3' => :image,
|
131
|
+
'bpg' => :image,
|
132
|
+
'cals' => :image,
|
133
|
+
'cdr' => :office,
|
134
|
+
'cin' => :image,
|
135
|
+
'clipboard' => :image,
|
136
|
+
'cmyk' => :image,
|
137
|
+
'cmyka' => :image,
|
138
|
+
'cr2' => :image,
|
139
|
+
'crw' => :image,
|
140
|
+
'cur' => :image,
|
141
|
+
'cut' => :image,
|
142
|
+
'cwk' => :office,
|
143
|
+
'dbf' => :office,
|
144
|
+
'dcm' => :image,
|
145
|
+
'dcx' => :image,
|
146
|
+
'dds' => :image,
|
147
|
+
'dib' => :image,
|
148
|
+
'dif' => :office,
|
149
|
+
'divx' => :video,
|
150
|
+
'dng' => :image,
|
151
|
+
'dpx' => :image,
|
152
|
+
'epdf' => :image,
|
153
|
+
'epi' => :image,
|
154
|
+
'eps2' => :image,
|
155
|
+
'eps3' => :image,
|
156
|
+
'epsf' => :image,
|
157
|
+
'epsi' => :image,
|
158
|
+
'ept' => :image,
|
159
|
+
'exr' => :image,
|
160
|
+
'fax' => :image,
|
161
|
+
'fb2' => :office,
|
162
|
+
'fits' => :image,
|
163
|
+
'fodg' => :office,
|
164
|
+
'fodp' => :office,
|
165
|
+
'fods' => :office,
|
166
|
+
'fodt' => :office,
|
167
|
+
'gplt' => :image,
|
168
|
+
'gray' => :image,
|
169
|
+
'hdr' => :image,
|
170
|
+
'hpw' => :office,
|
171
|
+
'hrz' => :image,
|
172
|
+
'info' => :image,
|
173
|
+
'inline' => :image,
|
174
|
+
'j2c' => :image,
|
175
|
+
'j2k' => :image,
|
176
|
+
'jbig' => :image,
|
177
|
+
'jng' => :image,
|
178
|
+
'jp2' => :image,
|
179
|
+
'jpt' => :image,
|
180
|
+
'jxr' => :image,
|
181
|
+
'key' => :office,
|
182
|
+
'log' => :plaintext,
|
183
|
+
'mat' => :image,
|
184
|
+
'mcw' => :office,
|
185
|
+
'met' => :office,
|
186
|
+
'miff' => :image,
|
187
|
+
'mml' => :office,
|
188
|
+
'mono' => :image,
|
189
|
+
'mpr' => :image,
|
190
|
+
'mrsid' => :image,
|
191
|
+
'mrw' => :image,
|
192
|
+
'mtv' => :image,
|
193
|
+
'mvg' => :image,
|
194
|
+
'mw' => :office,
|
195
|
+
'mwd' => :office,
|
196
|
+
'mxf' => :video,
|
197
|
+
'nef' => :image,
|
198
|
+
'numbers' => :office,
|
199
|
+
'orf' => :image,
|
200
|
+
'otb' => :image,
|
201
|
+
'p7' => :image,
|
202
|
+
'pages' => :office,
|
203
|
+
'palm' => :image,
|
204
|
+
'pam' => :image,
|
205
|
+
'pcd' => :image,
|
206
|
+
'pcds' => :image,
|
207
|
+
'pef' => :image,
|
208
|
+
'picon' => :image,
|
209
|
+
'pict' => :image,
|
210
|
+
'pix' => :image,
|
211
|
+
'pm' => :office,
|
212
|
+
'pm6' => :office,
|
213
|
+
'pmd' => :office,
|
214
|
+
'png00' => :image,
|
215
|
+
'png24' => :image,
|
216
|
+
'png32' => :image,
|
217
|
+
'png48' => :image,
|
218
|
+
'png64' => :image,
|
219
|
+
'png8' => :image,
|
220
|
+
'ps2' => :image,
|
221
|
+
'ps3' => :image,
|
222
|
+
'ptif' => :image,
|
223
|
+
'pwp' => :image,
|
224
|
+
'rad' => :image,
|
225
|
+
'raf' => :image,
|
226
|
+
'rfg' => :image,
|
227
|
+
'rgba' => :image,
|
228
|
+
'rla' => :image,
|
229
|
+
'rle' => :image,
|
230
|
+
'sct' => :image,
|
231
|
+
'sfw' => :image,
|
232
|
+
'sgf' => :office,
|
233
|
+
'sgv' => :office,
|
234
|
+
'slk' => :office,
|
235
|
+
'sparse-color' => :image,
|
236
|
+
'sun' => :image,
|
237
|
+
'svm' => :office,
|
238
|
+
'sylk' => :office,
|
239
|
+
'tim' => :image,
|
240
|
+
'txt' => :plaintext,
|
241
|
+
'uil' => :image,
|
242
|
+
'uof' => :office,
|
243
|
+
'uop' => :office,
|
244
|
+
'uos' => :office,
|
245
|
+
'uot' => :office,
|
246
|
+
'uyvy' => :image,
|
247
|
+
'vds' => :office,
|
248
|
+
'vdx' => :office,
|
249
|
+
'vicar' => :image,
|
250
|
+
'viff' => :image,
|
251
|
+
'vsdx' => :office,
|
252
|
+
'webm' => :video,
|
253
|
+
'wb2' => :office,
|
254
|
+
'wk1' => :office,
|
255
|
+
'wk3' => :office,
|
256
|
+
'wn' => :office,
|
257
|
+
'wpg' => :image,
|
258
|
+
'wq1' => :office,
|
259
|
+
'wq2' => :office,
|
260
|
+
'x' => :image,
|
261
|
+
'x3f' => :image,
|
262
|
+
'xcf' => :image,
|
263
|
+
'xlk' => :office,
|
264
|
+
'ycbcr' => :image,
|
265
|
+
'ycbcra' => :image,
|
266
|
+
'yuv' => :image,
|
267
|
+
'zabw' => :office}
|
268
|
+
|
269
|
+
#private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
|
270
|
+
|
271
|
+
def self.mime_from_file(filepath)
|
272
|
+
detected_mime=MimeMagic.by_magic(File.open(filepath)).to_s
|
273
|
+
detected_mime=MimeMagic.by_path(filepath).to_s if detected_mime.empty?
|
274
|
+
detected_mime=nil if detected_mime.empty?
|
275
|
+
return detected_mime
|
276
|
+
end
|
277
|
+
|
278
|
+
def self.conversion_type(filepath,mimetype,try_local_mime)
|
279
|
+
detected_mime=nil
|
280
|
+
if try_local_mime
|
281
|
+
detected_mime=mime_from_file(filepath)
|
282
|
+
if mimetype.eql?(detected_mime)
|
283
|
+
Log.log.debug("matching mime type per magic number")
|
284
|
+
else
|
285
|
+
# note: detected can be nil
|
286
|
+
Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
|
287
|
+
end
|
288
|
+
end
|
289
|
+
# 1- get type from provided mime type
|
290
|
+
result=FileTypes::SUPPORTED_MIME_TYPES[mimetype] unless mimetype.nil?
|
291
|
+
# 2- else, from computed mime type
|
292
|
+
result||=FileTypes::SUPPORTED_MIME_TYPES[detected_mime] unless detected_mime.nil?
|
293
|
+
extension = File.extname(filepath).downcase.gsub(/^\./,'')
|
294
|
+
# 3- else, from local extensions
|
295
|
+
result||=FileTypes::SUPPORTED_EXTENSIONS[extension]
|
296
|
+
return result
|
297
|
+
end
|
298
|
+
end
|
299
|
+
end
|
300
|
+
end
|
@@ -0,0 +1,258 @@
|
|
1
|
+
require 'open3'
|
2
|
+
require 'aspera/preview/options'
|
3
|
+
require 'aspera/preview/utils'
|
4
|
+
require 'aspera/preview/file_types'
|
5
|
+
|
6
|
+
module Aspera
|
7
|
+
module Preview
|
8
|
+
# generate one preview file for one format for one file at a time
|
9
|
+
class Generator
|
10
|
+
# values for preview_format : output format
|
11
|
+
PREVIEW_FORMATS=[:png,:mp4]
|
12
|
+
|
13
|
+
# CLI needs to know conversion type to know if need skip it
|
14
|
+
attr_reader :conversion_type
|
15
|
+
|
16
|
+
# @param src source file path
|
17
|
+
# @param dst destination file path
|
18
|
+
# @param api_mime_type optional mime type as provided by node api
|
19
|
+
# node API mime types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
|
20
|
+
# supported preview type is one of Preview::PREVIEW_FORMATS
|
21
|
+
# the resulting preview file type is taken from destination file extension.
|
22
|
+
# conversion methods are provided by private methods: convert_<conversion_type>_to_<preview_format>
|
23
|
+
# (combi = combination of source file type and destination format)
|
24
|
+
# -> conversion_type is one of FileTypes::CONVERSION_TYPES
|
25
|
+
# -> preview_format is one of Generator::PREVIEW_FORMATS
|
26
|
+
# the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
|
27
|
+
# -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
|
28
|
+
def initialize(options,src,dst,main_temp_dir,api_mime_type=nil,try_local_mime=true)
|
29
|
+
@options=options
|
30
|
+
@source_file_path=src
|
31
|
+
@destination_file_path=dst
|
32
|
+
@temp_folder=File.join(main_temp_dir,@source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
|
33
|
+
# extract preview format from extension of target file
|
34
|
+
@preview_format_symb=File.extname(@destination_file_path).gsub(/^\./,'').to_sym
|
35
|
+
@conversion_type=FileTypes.conversion_type(@source_file_path,api_mime_type,try_local_mime)
|
36
|
+
end
|
37
|
+
|
38
|
+
# name of processing method in this object
|
39
|
+
# combination of: conversion type and output format (and video_conversion for video)
|
40
|
+
def processing_method_symb
|
41
|
+
name="convert_#{@conversion_type}_to_#{@preview_format_symb}"
|
42
|
+
if @conversion_type.eql?(:video)
|
43
|
+
case @preview_format_symb
|
44
|
+
when :mp4
|
45
|
+
name="#{name}_using_#{@options.video_conversion}"
|
46
|
+
when :png
|
47
|
+
name="#{name}_using_#{@options.video_png_conv}"
|
48
|
+
end
|
49
|
+
end
|
50
|
+
Log.log.debug("method: #{name}")
|
51
|
+
return name.to_sym
|
52
|
+
end
|
53
|
+
|
54
|
+
# @return true if conversion is supported.
|
55
|
+
# for instance, plaintext to mp4 is not supported
|
56
|
+
def supported?
|
57
|
+
return false if @conversion_type.nil?
|
58
|
+
return respond_to?(processing_method_symb,true)
|
59
|
+
end
|
60
|
+
|
61
|
+
# create preview as specified in constructor
|
62
|
+
def generate
|
63
|
+
raise "could not detect type of file" if @conversion_type.nil?
|
64
|
+
method_symb=processing_method_symb
|
65
|
+
Log.log.info("#{@source_file_path}->#{@destination_file_path} (#{method_symb})")
|
66
|
+
begin
|
67
|
+
self.send(method_symb)
|
68
|
+
# check that generated size does not exceed maximum
|
69
|
+
result_size=File.size(@destination_file_path)
|
70
|
+
if result_size > @options.max_size
|
71
|
+
Log.log.warn("preview size exceeds maximum #{result_size} > #{@options.max_size}")
|
72
|
+
end
|
73
|
+
rescue => e
|
74
|
+
Log.log.error("#{e.message}")
|
75
|
+
Log.log.debug(e.backtrace.join("\n").red)
|
76
|
+
FileUtils.cp(File.expand_path(@preview_format_symb.eql?(:mp4)?'video_error.png':'image_error.png',File.dirname(__FILE__)),@destination_file_path)
|
77
|
+
ensure
|
78
|
+
FileUtils.rm_rf(@temp_folder)
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
# creates a unique temp folder for file
|
85
|
+
def this_tmpdir
|
86
|
+
FileUtils.mkdir_p(@temp_folder)
|
87
|
+
return @temp_folder
|
88
|
+
end
|
89
|
+
|
90
|
+
# @return offset in seconds suitable for ffmpeg -ss option
|
91
|
+
# @param duration of video
|
92
|
+
# @param start_offset of parts
|
93
|
+
# @param total_count of parts
|
94
|
+
# @param index of part (start at 1)
|
95
|
+
def get_offset(duration, start_offset, total_count, index)
|
96
|
+
return start_offset + (index-1)*(1.0*duration - start_offset) / total_count
|
97
|
+
end
|
98
|
+
|
99
|
+
def convert_video_to_mp4_using_blend()
|
100
|
+
p_duration = Utils.video_get_duration(@source_file_path)
|
101
|
+
p_start_offset = @options.video_start_sec.to_i
|
102
|
+
p_keyframecount = @options.blend_keyframes.to_i
|
103
|
+
last_keyframe = nil
|
104
|
+
current_index = 1
|
105
|
+
1.upto(p_keyframecount) do |i|
|
106
|
+
offset_seconds=get_offset(p_duration,p_start_offset,p_keyframecount,i)
|
107
|
+
Utils.video_dump_frame(@source_file_path, offset_seconds, @options.video_scale, this_tmpdir, current_index)
|
108
|
+
Utils.video_dupe_frame(this_tmpdir, current_index, @options.blend_pauseframes)
|
109
|
+
Utils.video_blend_frames(this_tmpdir,last_keyframe, current_index) unless last_keyframe.nil?
|
110
|
+
# go to last dupe frame
|
111
|
+
last_keyframe = current_index + @options.blend_pauseframes
|
112
|
+
# go after last dupe frame and keep space to blend
|
113
|
+
current_index = last_keyframe + 1 + @options.blend_transframes
|
114
|
+
end
|
115
|
+
Utils.ffmpeg(
|
116
|
+
in_f: Utils.ffmpeg_fmt(this_tmpdir),
|
117
|
+
in_p: ['-framerate',@options.blend_fps],
|
118
|
+
out_f: @destination_file_path,
|
119
|
+
out_p: [
|
120
|
+
'-filter:v',"scale='trunc(iw/2)*2:trunc(ih/2)*2'",
|
121
|
+
'-codec:v','libx264',
|
122
|
+
'-r',30,
|
123
|
+
'-pix_fmt','yuv420p'])
|
124
|
+
end
|
125
|
+
|
126
|
+
# generate n clips starting at offset
|
127
|
+
def convert_video_to_mp4_using_clips()
|
128
|
+
p_duration = Utils.video_get_duration(@source_file_path)
|
129
|
+
filelist = File.join(this_tmpdir,'clip_files.txt')
|
130
|
+
File.open(filelist, 'w+') do |f|
|
131
|
+
1.upto(@options.clips_count.to_i) do |i|
|
132
|
+
offset_seconds=get_offset(p_duration,@options.video_start_sec.to_i,@options.clips_count.to_i,i)
|
133
|
+
tmpfilename=sprintf("clip%04d.mp4",i)
|
134
|
+
Utils.ffmpeg(
|
135
|
+
in_f: @source_file_path,
|
136
|
+
in_p: ['-ss',0.9*offset_seconds],
|
137
|
+
out_f: File.join(this_tmpdir,tmpfilename),
|
138
|
+
out_p: [
|
139
|
+
'-ss',0.1*offset_seconds,
|
140
|
+
'-t',@options.clips_length,
|
141
|
+
'-filter:v',"scale=#{@options.video_scale}",
|
142
|
+
'-codec:a','libmp3lame'])
|
143
|
+
f.puts("file '#{tmpfilename}'")
|
144
|
+
end
|
145
|
+
end
|
146
|
+
# concat clips
|
147
|
+
Utils.ffmpeg(
|
148
|
+
in_f: filelist,
|
149
|
+
in_p: ['-f','concat'],
|
150
|
+
out_f: @destination_file_path,
|
151
|
+
out_p: ['-codec','copy'])
|
152
|
+
end
|
153
|
+
|
154
|
+
# do a simple reencoding
|
155
|
+
def convert_video_to_mp4_using_reencode()
|
156
|
+
Utils.ffmpeg(
|
157
|
+
in_f: @source_file_path,
|
158
|
+
out_f: @destination_file_path,
|
159
|
+
out_p: [
|
160
|
+
'-t','60',
|
161
|
+
'-codec:v','libx264',
|
162
|
+
'-profile:v','high',
|
163
|
+
'-pix_fmt','yuv420p',
|
164
|
+
'-preset','slow',
|
165
|
+
'-b:v','500k',
|
166
|
+
'-maxrate','500k',
|
167
|
+
'-bufsize','1000k',
|
168
|
+
'-filter:v',"scale=#{@options.video_scale}",
|
169
|
+
'-threads','0',
|
170
|
+
'-codec:a','libmp3lame',
|
171
|
+
'-ac','2',
|
172
|
+
'-b:a','128k',
|
173
|
+
'-movflags','faststart'])
|
174
|
+
end
|
175
|
+
|
176
|
+
def convert_video_to_png_using_fixed()
|
177
|
+
Utils.video_dump_frame(
|
178
|
+
@source_file_path,
|
179
|
+
Utils.video_get_duration(@source_file_path)*@options.thumb_vid_fraction,
|
180
|
+
@options.thumb_vid_scale,
|
181
|
+
@destination_file_path)
|
182
|
+
end
|
183
|
+
|
184
|
+
# https://trac.ffmpeg.org/wiki/SponsoringPrograms/GSoC/2015#AnimatedPortableNetworkGraphicsAPNG
|
185
|
+
# ffmpeg -h muxer=apng
|
186
|
+
# thumb is 32x32
|
187
|
+
# ffmpeg output.png
|
188
|
+
def convert_video_to_png_using_animated()
|
189
|
+
Utils.ffmpeg(
|
190
|
+
in_f: @source_file_path,
|
191
|
+
in_p: [
|
192
|
+
'-ss',10, # seek to input position
|
193
|
+
'-t',20, # max seconds
|
194
|
+
],
|
195
|
+
out_f: @destination_file_path,
|
196
|
+
out_p: [
|
197
|
+
'-vf',"fps=5,scale=120:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse",
|
198
|
+
'-loop',0,
|
199
|
+
'-f','gif'
|
200
|
+
])
|
201
|
+
end
|
202
|
+
|
203
|
+
def convert_office_to_png()
|
204
|
+
tmp_pdf_file=File.join(this_tmpdir,File.basename(@source_file_path,File.extname(@source_file_path))+'.pdf')
|
205
|
+
Utils.external_command(:unoconv,[
|
206
|
+
|
207
|
+
'-f','pdf',
|
208
|
+
'-o',tmp_pdf_file,
|
209
|
+
@source_file_path])
|
210
|
+
convert_pdf_to_png(tmp_pdf_file)
|
211
|
+
end
|
212
|
+
|
213
|
+
def convert_pdf_to_png(source_file_path=nil)
|
214
|
+
source_file_path||=@source_file_path
|
215
|
+
Utils.external_command(:convert,[
|
216
|
+
|
217
|
+
'-size',"x#{@options.thumb_img_size}",
|
218
|
+
'-background','white',
|
219
|
+
'-flatten',
|
220
|
+
"#{source_file_path}[0]",
|
221
|
+
@destination_file_path])
|
222
|
+
end
|
223
|
+
|
224
|
+
def convert_image_to_png()
|
225
|
+
Utils.external_command(:convert,[
|
226
|
+
|
227
|
+
'-auto-orient',
|
228
|
+
'-thumbnail',"#{@options.thumb_img_size}x#{@options.thumb_img_size}>",
|
229
|
+
'-quality',95,
|
230
|
+
'+dither',
|
231
|
+
'-posterize',40,
|
232
|
+
"#{@source_file_path}[0]",
|
233
|
+
@destination_file_path])
|
234
|
+
Utils.external_command(:optipng,[@destination_file_path])
|
235
|
+
end
|
236
|
+
|
237
|
+
# text to png
|
238
|
+
def convert_plaintext_to_png()
|
239
|
+
# get 100 first lines of text file
|
240
|
+
first_lines=File.open(@source_file_path){|f|100.times.map{f.readline rescue ''}}.join
|
241
|
+
Utils.external_command(:convert,[
|
242
|
+
|
243
|
+
'-size',"#{@options.thumb_img_size}x#{@options.thumb_img_size}",
|
244
|
+
'xc:white',
|
245
|
+
'-font','Courier',
|
246
|
+
'-pointsize',12,
|
247
|
+
'-fill','black',
|
248
|
+
'-annotate','+15+15',first_lines,
|
249
|
+
'-trim',
|
250
|
+
'-bordercolor','#FFF',
|
251
|
+
'-border',10,
|
252
|
+
'+repage',
|
253
|
+
@destination_file_path])
|
254
|
+
end
|
255
|
+
|
256
|
+
end # Generator
|
257
|
+
end # Preview
|
258
|
+
end # Aspera
|