aspera-cli 4.0.0.pre1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +7 -0
  2. data/README.md +3592 -0
  3. data/bin/ascli +7 -0
  4. data/bin/asession +89 -0
  5. data/docs/Makefile +59 -0
  6. data/docs/README.erb.md +3012 -0
  7. data/docs/README.md +13 -0
  8. data/docs/diagrams.txt +49 -0
  9. data/docs/secrets.make +38 -0
  10. data/docs/test_env.conf +117 -0
  11. data/docs/transfer_spec.html +99 -0
  12. data/examples/aoc.rb +17 -0
  13. data/examples/proxy.pac +60 -0
  14. data/examples/transfer.rb +115 -0
  15. data/lib/aspera/api_detector.rb +60 -0
  16. data/lib/aspera/ascmd.rb +151 -0
  17. data/lib/aspera/ats_api.rb +43 -0
  18. data/lib/aspera/cli/basic_auth_plugin.rb +38 -0
  19. data/lib/aspera/cli/extended_value.rb +88 -0
  20. data/lib/aspera/cli/formater.rb +238 -0
  21. data/lib/aspera/cli/listener/line_dump.rb +17 -0
  22. data/lib/aspera/cli/listener/logger.rb +20 -0
  23. data/lib/aspera/cli/listener/progress.rb +52 -0
  24. data/lib/aspera/cli/listener/progress_multi.rb +91 -0
  25. data/lib/aspera/cli/main.rb +304 -0
  26. data/lib/aspera/cli/manager.rb +440 -0
  27. data/lib/aspera/cli/plugin.rb +90 -0
  28. data/lib/aspera/cli/plugins/alee.rb +24 -0
  29. data/lib/aspera/cli/plugins/ats.rb +231 -0
  30. data/lib/aspera/cli/plugins/bss.rb +71 -0
  31. data/lib/aspera/cli/plugins/config.rb +806 -0
  32. data/lib/aspera/cli/plugins/console.rb +62 -0
  33. data/lib/aspera/cli/plugins/cos.rb +106 -0
  34. data/lib/aspera/cli/plugins/faspex.rb +377 -0
  35. data/lib/aspera/cli/plugins/faspex5.rb +93 -0
  36. data/lib/aspera/cli/plugins/node.rb +438 -0
  37. data/lib/aspera/cli/plugins/oncloud.rb +937 -0
  38. data/lib/aspera/cli/plugins/orchestrator.rb +169 -0
  39. data/lib/aspera/cli/plugins/preview.rb +464 -0
  40. data/lib/aspera/cli/plugins/server.rb +216 -0
  41. data/lib/aspera/cli/plugins/shares.rb +63 -0
  42. data/lib/aspera/cli/plugins/shares2.rb +114 -0
  43. data/lib/aspera/cli/plugins/sync.rb +65 -0
  44. data/lib/aspera/cli/plugins/xnode.rb +115 -0
  45. data/lib/aspera/cli/transfer_agent.rb +251 -0
  46. data/lib/aspera/cli/version.rb +5 -0
  47. data/lib/aspera/colors.rb +39 -0
  48. data/lib/aspera/command_line_builder.rb +137 -0
  49. data/lib/aspera/fasp/aoc.rb +24 -0
  50. data/lib/aspera/fasp/connect.rb +99 -0
  51. data/lib/aspera/fasp/error.rb +21 -0
  52. data/lib/aspera/fasp/error_info.rb +60 -0
  53. data/lib/aspera/fasp/http_gw.rb +81 -0
  54. data/lib/aspera/fasp/installation.rb +240 -0
  55. data/lib/aspera/fasp/listener.rb +11 -0
  56. data/lib/aspera/fasp/local.rb +377 -0
  57. data/lib/aspera/fasp/manager.rb +69 -0
  58. data/lib/aspera/fasp/node.rb +88 -0
  59. data/lib/aspera/fasp/parameters.rb +235 -0
  60. data/lib/aspera/fasp/resume_policy.rb +76 -0
  61. data/lib/aspera/fasp/uri.rb +51 -0
  62. data/lib/aspera/faspex_gw.rb +196 -0
  63. data/lib/aspera/hash_ext.rb +28 -0
  64. data/lib/aspera/log.rb +80 -0
  65. data/lib/aspera/nagios.rb +71 -0
  66. data/lib/aspera/node.rb +14 -0
  67. data/lib/aspera/oauth.rb +319 -0
  68. data/lib/aspera/on_cloud.rb +421 -0
  69. data/lib/aspera/open_application.rb +72 -0
  70. data/lib/aspera/persistency_action_once.rb +42 -0
  71. data/lib/aspera/persistency_folder.rb +91 -0
  72. data/lib/aspera/preview/file_types.rb +300 -0
  73. data/lib/aspera/preview/generator.rb +258 -0
  74. data/lib/aspera/preview/image_error.png +0 -0
  75. data/lib/aspera/preview/options.rb +35 -0
  76. data/lib/aspera/preview/utils.rb +131 -0
  77. data/lib/aspera/preview/video_error.png +0 -0
  78. data/lib/aspera/proxy_auto_config.erb.js +287 -0
  79. data/lib/aspera/proxy_auto_config.rb +34 -0
  80. data/lib/aspera/rest.rb +296 -0
  81. data/lib/aspera/rest_call_error.rb +13 -0
  82. data/lib/aspera/rest_error_analyzer.rb +98 -0
  83. data/lib/aspera/rest_errors_aspera.rb +58 -0
  84. data/lib/aspera/ssh.rb +53 -0
  85. data/lib/aspera/sync.rb +82 -0
  86. data/lib/aspera/temp_file_manager.rb +37 -0
  87. data/lib/aspera/uri_reader.rb +25 -0
  88. metadata +288 -0
@@ -0,0 +1,42 @@
1
+ require 'json'
2
+ require 'aspera/log'
3
+
4
+ module Aspera
5
+ # Persist data on file system
6
+ class PersistencyActionOnce
7
+ # @param :manager Mandatory Database
8
+ # @param :data Mandatory object to persist, must be same object from begin to end (assume array by default)
9
+ # @param :ids Mandatory identifiers
10
+ # @param :delete Optional delete persistency condition
11
+ # @param :parse Optional parse method (default to JSON)
12
+ # @param :format Optional dump method (default to JSON)
13
+ # @param :merge Optional merge data from file to current data
14
+ def initialize(options)
15
+ Log.log.debug("persistency: #{options}")
16
+ raise "options shall be Hash" unless options.is_a?(Hash)
17
+ raise "mandatory :manager" if options[:manager].nil?
18
+ raise "mandatory :data" if options[:data].nil?
19
+ raise "mandatory :ids (Array)" unless options[:ids].is_a?(Array)
20
+ raise "mandatory 1 element in :ids" unless options[:ids].length >= 1
21
+ @manager=options[:manager]
22
+ @persisted_object=options[:data]
23
+ @object_ids=options[:ids]
24
+ # by default , at save time, file is deleted if data is nil
25
+ @delete_condition=options[:delete] || lambda{|d|d.empty?}
26
+ @persist_format=options[:format] || lambda {|h| JSON.generate(h)}
27
+ persist_parse=options[:parse] || lambda {|t| JSON.parse(t)}
28
+ persist_merge=options[:merge] || lambda {|current,file| current.concat(file).uniq rescue current}
29
+ value=@manager.get(@object_ids)
30
+ persist_merge.call(@persisted_object,persist_parse.call(value)) unless value.nil?
31
+ end
32
+
33
+ def save
34
+ if @delete_condition.call(@persisted_object)
35
+ @manager.delete(@object_ids)
36
+ else
37
+ @manager.put(@object_ids,@persist_format.call(@persisted_object))
38
+ end
39
+ end
40
+
41
+ end
42
+ end
@@ -0,0 +1,91 @@
1
+ require 'fileutils'
2
+ require 'aspera/log'
3
+
4
+ # search: persistency_folder PersistencyFolder
5
+
6
+ module Aspera
7
+ # Persist data on file system
8
+ class PersistencyFolder
9
+ WINDOWS_PROTECTED_CHAR=%r{[/:"<>\\\*\?]}
10
+ PROTECTED_CHAR_REPLACE='_'
11
+ ID_SEPARATOR='_'
12
+ FILE_SUFFIX='.txt'
13
+ private_constant :PROTECTED_CHAR_REPLACE,:FILE_SUFFIX,:WINDOWS_PROTECTED_CHAR,:ID_SEPARATOR
14
+ def initialize(folder)
15
+ @cache={}
16
+ set_folder(folder)
17
+ end
18
+
19
+ def set_folder(folder)
20
+ @folder=folder
21
+ Log.log.debug("persistency folder: #{@folder}")
22
+ end
23
+
24
+ # @return String or nil string on existing persist, else nil
25
+ def get(object_id)
26
+ object_id=marshalled_id(object_id)
27
+ Log.log.debug("persistency get: #{object_id}")
28
+ if @cache.has_key?(object_id)
29
+ Log.log.debug("got from memory cache")
30
+ else
31
+ persist_filepath=id_to_filepath(object_id)
32
+ Log.log.debug("persistency = #{persist_filepath}")
33
+ if File.exist?(persist_filepath)
34
+ Log.log.debug("got from file cache")
35
+ @cache[object_id]=File.read(persist_filepath)
36
+ end
37
+ end
38
+ return @cache[object_id]
39
+ end
40
+
41
+ def put(object_id,value)
42
+ raise "only String supported" unless value.is_a?(String)
43
+ object_id=marshalled_id(object_id)
44
+ persist_filepath=id_to_filepath(object_id)
45
+ Log.log.debug("saving: #{persist_filepath}")
46
+ File.write(persist_filepath,value)
47
+ @cache[object_id]=value
48
+ end
49
+
50
+ def delete(object_id)
51
+ object_id=marshalled_id(object_id)
52
+ persist_filepath=id_to_filepath(object_id)
53
+ Log.log.debug("empty data, deleting: #{persist_filepath}")
54
+ File.delete(persist_filepath) if File.exist?(persist_filepath)
55
+ @cache.delete(object_id)
56
+ end
57
+
58
+ def flush_by_prefix(persist_category)
59
+ persist_files=Dir[File.join(@folder,persist_category+'*'+FILE_SUFFIX)]
60
+ persist_files.each do |filepath|
61
+ File.delete(filepath)
62
+ end
63
+ return persist_files
64
+ end
65
+
66
+ private
67
+
68
+ # @param object_id String or Array
69
+ def id_to_filepath(object_id)
70
+ FileUtils.mkdir_p(@folder)
71
+ return File.join(@folder,"#{object_id}#{FILE_SUFFIX}")
72
+ #.gsub(/[^a-z]+/,FILE_FIELD_SEPARATOR)
73
+ end
74
+
75
+ def marshalled_id(object_id)
76
+ if object_id.is_a?(Array)
77
+ # special case, url in second position: TODO: check any position
78
+ if object_id[1].is_a?(String) and object_id[1] =~ URI::ABS_URI
79
+ object_id=object_id.clone
80
+ object_id[1]=URI.parse(object_id[1]).host
81
+ end
82
+ object_id=object_id.join(ID_SEPARATOR)
83
+ end
84
+ raise "id must be a String" unless object_id.is_a?(String)
85
+ return object_id.
86
+ gsub(WINDOWS_PROTECTED_CHAR,PROTECTED_CHAR_REPLACE). # remove windows forbidden chars
87
+ gsub('.',PROTECTED_CHAR_REPLACE). # keep dot for extension only (nicer)
88
+ downcase
89
+ end
90
+ end # PersistencyFolder
91
+ end # Aspera
@@ -0,0 +1,300 @@
1
+ require 'mimemagic'
2
+ require 'mimemagic/overlay'
3
+
4
+ module Aspera
5
+ module Preview
6
+ class FileTypes
7
+ # values for conversion_type : input format
8
+ CONVERSION_TYPES=[
9
+ :image,
10
+ :office,
11
+ :pdf,
12
+ :plaintext,
13
+ :video
14
+ ]
15
+
16
+ # define how files are processed based on mime type
17
+ SUPPORTED_MIME_TYPES={
18
+ 'application/json' => :plaintext,
19
+ 'application/mac-binhex40' => :office,
20
+ 'application/msword' => :office,
21
+ 'application/pdf' => :pdf,
22
+ 'application/postscript' => :image,
23
+ 'application/rtf' => :office,
24
+ 'application/vnd.3gpp.pic-bw-small' => :image,
25
+ 'application/vnd.hp-hpgl' => :image,
26
+ 'application/vnd.hp-pcl' => :image,
27
+ 'application/vnd.lotus-wordpro' => :office,
28
+ 'application/vnd.mobius.msl' => :image,
29
+ 'application/vnd.mophun.certificate' => :image,
30
+ 'application/vnd.ms-excel' => :office,
31
+ 'application/vnd.ms-excel.sheet.binary.macroenabled.12' => :office,
32
+ 'application/vnd.ms-excel.sheet.macroenabled.12' => :office,
33
+ 'application/vnd.ms-excel.template.macroenabled.12' => :office,
34
+ 'application/vnd.ms-powerpoint' => :office,
35
+ 'application/vnd.ms-powerpoint.presentation.macroenabled.12' => :office,
36
+ 'application/vnd.ms-powerpoint.template.macroenabled.12' => :office,
37
+ 'application/vnd.ms-word.document.macroenabled.12' => :office,
38
+ 'application/vnd.ms-word.template.macroenabled.12' => :office,
39
+ 'application/vnd.ms-works' => :office,
40
+ 'application/vnd.oasis.opendocument.chart' => :office,
41
+ 'application/vnd.oasis.opendocument.formula' => :office,
42
+ 'application/vnd.oasis.opendocument.graphics' => :office,
43
+ 'application/vnd.oasis.opendocument.graphics-template' => :office,
44
+ 'application/vnd.oasis.opendocument.presentation' => :office,
45
+ 'application/vnd.oasis.opendocument.presentation-template' => :office,
46
+ 'application/vnd.oasis.opendocument.spreadsheet' => :office,
47
+ 'application/vnd.oasis.opendocument.spreadsheet-template' => :office,
48
+ 'application/vnd.oasis.opendocument.text' => :office,
49
+ 'application/vnd.oasis.opendocument.text-template' => :office,
50
+ 'application/vnd.openxmlformats-officedocument.presentationml.presentation' => :office,
51
+ 'application/vnd.openxmlformats-officedocument.presentationml.slideshow' => :office,
52
+ 'application/vnd.openxmlformats-officedocument.presentationml.template' => :office,
53
+ 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' => :office,
54
+ 'application/vnd.openxmlformats-officedocument.spreadsheetml.template' => :office,
55
+ 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' => :office,
56
+ 'application/vnd.openxmlformats-officedocument.wordprocessingml.template' => :office,
57
+ 'application/vnd.palm' => :office,
58
+ 'application/vnd.sun.xml.calc' => :office,
59
+ 'application/vnd.sun.xml.calc.template' => :office,
60
+ 'application/vnd.sun.xml.draw' => :office,
61
+ 'application/vnd.sun.xml.draw.template' => :office,
62
+ 'application/vnd.sun.xml.impress' => :office,
63
+ 'application/vnd.sun.xml.impress.template' => :office,
64
+ 'application/vnd.sun.xml.math' => :office,
65
+ 'application/vnd.sun.xml.writer' => :office,
66
+ 'application/vnd.sun.xml.writer.template' => :office,
67
+ 'application/vnd.wordperfect' => :office,
68
+ 'application/x-abiword' => :office,
69
+ 'application/x-director' => :image,
70
+ 'application/x-font-type1' => :image,
71
+ 'application/x-msmetafile' => :image,
72
+ 'application/x-mspublisher' => :office,
73
+ 'application/x-xfig' => :image,
74
+ 'audio/ogg' => :video,
75
+ 'font/ttf' => :image,
76
+ 'image/bmp' => :image,
77
+ 'image/cgm' => :image,
78
+ 'image/gif' => :image,
79
+ 'image/jpeg' => :image,
80
+ 'image/png' => :image,
81
+ 'image/sgi' => :image,
82
+ 'image/svg+xml' => :image,
83
+ 'image/tiff' => :image,
84
+ 'image/vnd.adobe.photoshop' => :image,
85
+ 'image/vnd.djvu' => :image,
86
+ 'image/vnd.dxf' => :office,
87
+ 'image/vnd.fpx' => :image,
88
+ 'image/vnd.ms-photo' => :image,
89
+ 'image/vnd.wap.wbmp' => :image,
90
+ 'image/webp' => :image,
91
+ 'image/x-cmx' => :office,
92
+ 'image/x-freehand' => :office,
93
+ 'image/x-icon' => :image,
94
+ 'image/x-mrsid-image' => :image,
95
+ 'image/x-pcx' => :image,
96
+ 'image/x-pict' => :office,
97
+ 'image/x-portable-anymap' => :image,
98
+ 'image/x-portable-bitmap' => :image,
99
+ 'image/x-portable-graymap' => :image,
100
+ 'image/x-portable-pixmap' => :image,
101
+ 'image/x-rgb' => :image,
102
+ 'image/x-tga' => :image,
103
+ 'image/x-xbitmap' => :image,
104
+ 'image/x-xpixmap' => :image,
105
+ 'image/x-xwindowdump' => :image,
106
+ 'text/csv' => :office,
107
+ 'text/html' => :office,
108
+ 'text/plain' => :plaintext,
109
+ 'text/troff' => :image,
110
+ 'video/h261' => :video,
111
+ 'video/h263' => :video,
112
+ 'video/h264' => :video,
113
+ 'video/mp4' => :video,
114
+ 'video/mpeg' => :video,
115
+ 'video/quicktime' => :video,
116
+ 'video/x-flv' => :video,
117
+ 'video/x-m4v' => :video,
118
+ 'video/x-matroska' => :video,
119
+ 'video/x-mng' => :image,
120
+ 'video/x-ms-wmv' => :video,
121
+ 'video/x-msvideo' => :video}
122
+
123
+ # this is a way to add support for extensions that are otherwise not known by node api (mime type)
124
+ SUPPORTED_EXTENSIONS={
125
+ 'aai' => :image,
126
+ 'art' => :image,
127
+ 'arw' => :image,
128
+ 'avs' => :image,
129
+ 'bmp2' => :image,
130
+ 'bmp3' => :image,
131
+ 'bpg' => :image,
132
+ 'cals' => :image,
133
+ 'cdr' => :office,
134
+ 'cin' => :image,
135
+ 'clipboard' => :image,
136
+ 'cmyk' => :image,
137
+ 'cmyka' => :image,
138
+ 'cr2' => :image,
139
+ 'crw' => :image,
140
+ 'cur' => :image,
141
+ 'cut' => :image,
142
+ 'cwk' => :office,
143
+ 'dbf' => :office,
144
+ 'dcm' => :image,
145
+ 'dcx' => :image,
146
+ 'dds' => :image,
147
+ 'dib' => :image,
148
+ 'dif' => :office,
149
+ 'divx' => :video,
150
+ 'dng' => :image,
151
+ 'dpx' => :image,
152
+ 'epdf' => :image,
153
+ 'epi' => :image,
154
+ 'eps2' => :image,
155
+ 'eps3' => :image,
156
+ 'epsf' => :image,
157
+ 'epsi' => :image,
158
+ 'ept' => :image,
159
+ 'exr' => :image,
160
+ 'fax' => :image,
161
+ 'fb2' => :office,
162
+ 'fits' => :image,
163
+ 'fodg' => :office,
164
+ 'fodp' => :office,
165
+ 'fods' => :office,
166
+ 'fodt' => :office,
167
+ 'gplt' => :image,
168
+ 'gray' => :image,
169
+ 'hdr' => :image,
170
+ 'hpw' => :office,
171
+ 'hrz' => :image,
172
+ 'info' => :image,
173
+ 'inline' => :image,
174
+ 'j2c' => :image,
175
+ 'j2k' => :image,
176
+ 'jbig' => :image,
177
+ 'jng' => :image,
178
+ 'jp2' => :image,
179
+ 'jpt' => :image,
180
+ 'jxr' => :image,
181
+ 'key' => :office,
182
+ 'log' => :plaintext,
183
+ 'mat' => :image,
184
+ 'mcw' => :office,
185
+ 'met' => :office,
186
+ 'miff' => :image,
187
+ 'mml' => :office,
188
+ 'mono' => :image,
189
+ 'mpr' => :image,
190
+ 'mrsid' => :image,
191
+ 'mrw' => :image,
192
+ 'mtv' => :image,
193
+ 'mvg' => :image,
194
+ 'mw' => :office,
195
+ 'mwd' => :office,
196
+ 'mxf' => :video,
197
+ 'nef' => :image,
198
+ 'numbers' => :office,
199
+ 'orf' => :image,
200
+ 'otb' => :image,
201
+ 'p7' => :image,
202
+ 'pages' => :office,
203
+ 'palm' => :image,
204
+ 'pam' => :image,
205
+ 'pcd' => :image,
206
+ 'pcds' => :image,
207
+ 'pef' => :image,
208
+ 'picon' => :image,
209
+ 'pict' => :image,
210
+ 'pix' => :image,
211
+ 'pm' => :office,
212
+ 'pm6' => :office,
213
+ 'pmd' => :office,
214
+ 'png00' => :image,
215
+ 'png24' => :image,
216
+ 'png32' => :image,
217
+ 'png48' => :image,
218
+ 'png64' => :image,
219
+ 'png8' => :image,
220
+ 'ps2' => :image,
221
+ 'ps3' => :image,
222
+ 'ptif' => :image,
223
+ 'pwp' => :image,
224
+ 'rad' => :image,
225
+ 'raf' => :image,
226
+ 'rfg' => :image,
227
+ 'rgba' => :image,
228
+ 'rla' => :image,
229
+ 'rle' => :image,
230
+ 'sct' => :image,
231
+ 'sfw' => :image,
232
+ 'sgf' => :office,
233
+ 'sgv' => :office,
234
+ 'slk' => :office,
235
+ 'sparse-color' => :image,
236
+ 'sun' => :image,
237
+ 'svm' => :office,
238
+ 'sylk' => :office,
239
+ 'tim' => :image,
240
+ 'txt' => :plaintext,
241
+ 'uil' => :image,
242
+ 'uof' => :office,
243
+ 'uop' => :office,
244
+ 'uos' => :office,
245
+ 'uot' => :office,
246
+ 'uyvy' => :image,
247
+ 'vds' => :office,
248
+ 'vdx' => :office,
249
+ 'vicar' => :image,
250
+ 'viff' => :image,
251
+ 'vsdx' => :office,
252
+ 'webm' => :video,
253
+ 'wb2' => :office,
254
+ 'wk1' => :office,
255
+ 'wk3' => :office,
256
+ 'wn' => :office,
257
+ 'wpg' => :image,
258
+ 'wq1' => :office,
259
+ 'wq2' => :office,
260
+ 'x' => :image,
261
+ 'x3f' => :image,
262
+ 'xcf' => :image,
263
+ 'xlk' => :office,
264
+ 'ycbcr' => :image,
265
+ 'ycbcra' => :image,
266
+ 'yuv' => :image,
267
+ 'zabw' => :office}
268
+
269
+ #private_constant :SUPPORTED_MIME_TYPES, :SUPPORTED_EXTENSIONS
270
+
271
+ def self.mime_from_file(filepath)
272
+ detected_mime=MimeMagic.by_magic(File.open(filepath)).to_s
273
+ detected_mime=MimeMagic.by_path(filepath).to_s if detected_mime.empty?
274
+ detected_mime=nil if detected_mime.empty?
275
+ return detected_mime
276
+ end
277
+
278
+ def self.conversion_type(filepath,mimetype,try_local_mime)
279
+ detected_mime=nil
280
+ if try_local_mime
281
+ detected_mime=mime_from_file(filepath)
282
+ if mimetype.eql?(detected_mime)
283
+ Log.log.debug("matching mime type per magic number")
284
+ else
285
+ # note: detected can be nil
286
+ Log.log.debug("non matching mime types: node=[#{mimetype}], magic=[#{detected_mime}]")
287
+ end
288
+ end
289
+ # 1- get type from provided mime type
290
+ result=FileTypes::SUPPORTED_MIME_TYPES[mimetype] unless mimetype.nil?
291
+ # 2- else, from computed mime type
292
+ result||=FileTypes::SUPPORTED_MIME_TYPES[detected_mime] unless detected_mime.nil?
293
+ extension = File.extname(filepath).downcase.gsub(/^\./,'')
294
+ # 3- else, from local extensions
295
+ result||=FileTypes::SUPPORTED_EXTENSIONS[extension]
296
+ return result
297
+ end
298
+ end
299
+ end
300
+ end
@@ -0,0 +1,258 @@
1
+ require 'open3'
2
+ require 'aspera/preview/options'
3
+ require 'aspera/preview/utils'
4
+ require 'aspera/preview/file_types'
5
+
6
+ module Aspera
7
+ module Preview
8
+ # generate one preview file for one format for one file at a time
9
+ class Generator
10
+ # values for preview_format : output format
11
+ PREVIEW_FORMATS=[:png,:mp4]
12
+
13
+ # CLI needs to know conversion type to know if need skip it
14
+ attr_reader :conversion_type
15
+
16
+ # @param src source file path
17
+ # @param dst destination file path
18
+ # @param api_mime_type optional mime type as provided by node api
19
+ # node API mime types are from: http://svn.apache.org/repos/asf/httpd/httpd/trunk/docs/conf/mime.types
20
+ # supported preview type is one of Preview::PREVIEW_FORMATS
21
+ # the resulting preview file type is taken from destination file extension.
22
+ # conversion methods are provided by private methods: convert_<conversion_type>_to_<preview_format>
23
+ # (combi = combination of source file type and destination format)
24
+ # -> conversion_type is one of FileTypes::CONVERSION_TYPES
25
+ # -> preview_format is one of Generator::PREVIEW_FORMATS
26
+ # the conversion video->mp4 is implemented in methods: convert_video_to_mp4_using_<video_conversion>
27
+ # -> conversion method is one of Generator::VIDEO_CONVERSION_METHODS
28
+ def initialize(options,src,dst,main_temp_dir,api_mime_type=nil,try_local_mime=true)
29
+ @options=options
30
+ @source_file_path=src
31
+ @destination_file_path=dst
32
+ @temp_folder=File.join(main_temp_dir,@source_file_path.split('/').last.gsub(/\s/, '_').gsub(/\W/, ''))
33
+ # extract preview format from extension of target file
34
+ @preview_format_symb=File.extname(@destination_file_path).gsub(/^\./,'').to_sym
35
+ @conversion_type=FileTypes.conversion_type(@source_file_path,api_mime_type,try_local_mime)
36
+ end
37
+
38
+ # name of processing method in this object
39
+ # combination of: conversion type and output format (and video_conversion for video)
40
+ def processing_method_symb
41
+ name="convert_#{@conversion_type}_to_#{@preview_format_symb}"
42
+ if @conversion_type.eql?(:video)
43
+ case @preview_format_symb
44
+ when :mp4
45
+ name="#{name}_using_#{@options.video_conversion}"
46
+ when :png
47
+ name="#{name}_using_#{@options.video_png_conv}"
48
+ end
49
+ end
50
+ Log.log.debug("method: #{name}")
51
+ return name.to_sym
52
+ end
53
+
54
+ # @return true if conversion is supported.
55
+ # for instance, plaintext to mp4 is not supported
56
+ def supported?
57
+ return false if @conversion_type.nil?
58
+ return respond_to?(processing_method_symb,true)
59
+ end
60
+
61
+ # create preview as specified in constructor
62
+ def generate
63
+ raise "could not detect type of file" if @conversion_type.nil?
64
+ method_symb=processing_method_symb
65
+ Log.log.info("#{@source_file_path}->#{@destination_file_path} (#{method_symb})")
66
+ begin
67
+ self.send(method_symb)
68
+ # check that generated size does not exceed maximum
69
+ result_size=File.size(@destination_file_path)
70
+ if result_size > @options.max_size
71
+ Log.log.warn("preview size exceeds maximum #{result_size} > #{@options.max_size}")
72
+ end
73
+ rescue => e
74
+ Log.log.error("#{e.message}")
75
+ Log.log.debug(e.backtrace.join("\n").red)
76
+ FileUtils.cp(File.expand_path(@preview_format_symb.eql?(:mp4)?'video_error.png':'image_error.png',File.dirname(__FILE__)),@destination_file_path)
77
+ ensure
78
+ FileUtils.rm_rf(@temp_folder)
79
+ end
80
+ end
81
+
82
+ private
83
+
84
+ # creates a unique temp folder for file
85
+ def this_tmpdir
86
+ FileUtils.mkdir_p(@temp_folder)
87
+ return @temp_folder
88
+ end
89
+
90
+ # @return offset in seconds suitable for ffmpeg -ss option
91
+ # @param duration of video
92
+ # @param start_offset of parts
93
+ # @param total_count of parts
94
+ # @param index of part (start at 1)
95
+ def get_offset(duration, start_offset, total_count, index)
96
+ return start_offset + (index-1)*(1.0*duration - start_offset) / total_count
97
+ end
98
+
99
+ def convert_video_to_mp4_using_blend()
100
+ p_duration = Utils.video_get_duration(@source_file_path)
101
+ p_start_offset = @options.video_start_sec.to_i
102
+ p_keyframecount = @options.blend_keyframes.to_i
103
+ last_keyframe = nil
104
+ current_index = 1
105
+ 1.upto(p_keyframecount) do |i|
106
+ offset_seconds=get_offset(p_duration,p_start_offset,p_keyframecount,i)
107
+ Utils.video_dump_frame(@source_file_path, offset_seconds, @options.video_scale, this_tmpdir, current_index)
108
+ Utils.video_dupe_frame(this_tmpdir, current_index, @options.blend_pauseframes)
109
+ Utils.video_blend_frames(this_tmpdir,last_keyframe, current_index) unless last_keyframe.nil?
110
+ # go to last dupe frame
111
+ last_keyframe = current_index + @options.blend_pauseframes
112
+ # go after last dupe frame and keep space to blend
113
+ current_index = last_keyframe + 1 + @options.blend_transframes
114
+ end
115
+ Utils.ffmpeg(
116
+ in_f: Utils.ffmpeg_fmt(this_tmpdir),
117
+ in_p: ['-framerate',@options.blend_fps],
118
+ out_f: @destination_file_path,
119
+ out_p: [
120
+ '-filter:v',"scale='trunc(iw/2)*2:trunc(ih/2)*2'",
121
+ '-codec:v','libx264',
122
+ '-r',30,
123
+ '-pix_fmt','yuv420p'])
124
+ end
125
+
126
+ # generate n clips starting at offset
127
+ def convert_video_to_mp4_using_clips()
128
+ p_duration = Utils.video_get_duration(@source_file_path)
129
+ filelist = File.join(this_tmpdir,'clip_files.txt')
130
+ File.open(filelist, 'w+') do |f|
131
+ 1.upto(@options.clips_count.to_i) do |i|
132
+ offset_seconds=get_offset(p_duration,@options.video_start_sec.to_i,@options.clips_count.to_i,i)
133
+ tmpfilename=sprintf("clip%04d.mp4",i)
134
+ Utils.ffmpeg(
135
+ in_f: @source_file_path,
136
+ in_p: ['-ss',0.9*offset_seconds],
137
+ out_f: File.join(this_tmpdir,tmpfilename),
138
+ out_p: [
139
+ '-ss',0.1*offset_seconds,
140
+ '-t',@options.clips_length,
141
+ '-filter:v',"scale=#{@options.video_scale}",
142
+ '-codec:a','libmp3lame'])
143
+ f.puts("file '#{tmpfilename}'")
144
+ end
145
+ end
146
+ # concat clips
147
+ Utils.ffmpeg(
148
+ in_f: filelist,
149
+ in_p: ['-f','concat'],
150
+ out_f: @destination_file_path,
151
+ out_p: ['-codec','copy'])
152
+ end
153
+
154
+ # do a simple reencoding
155
+ def convert_video_to_mp4_using_reencode()
156
+ Utils.ffmpeg(
157
+ in_f: @source_file_path,
158
+ out_f: @destination_file_path,
159
+ out_p: [
160
+ '-t','60',
161
+ '-codec:v','libx264',
162
+ '-profile:v','high',
163
+ '-pix_fmt','yuv420p',
164
+ '-preset','slow',
165
+ '-b:v','500k',
166
+ '-maxrate','500k',
167
+ '-bufsize','1000k',
168
+ '-filter:v',"scale=#{@options.video_scale}",
169
+ '-threads','0',
170
+ '-codec:a','libmp3lame',
171
+ '-ac','2',
172
+ '-b:a','128k',
173
+ '-movflags','faststart'])
174
+ end
175
+
176
+ def convert_video_to_png_using_fixed()
177
+ Utils.video_dump_frame(
178
+ @source_file_path,
179
+ Utils.video_get_duration(@source_file_path)*@options.thumb_vid_fraction,
180
+ @options.thumb_vid_scale,
181
+ @destination_file_path)
182
+ end
183
+
184
+ # https://trac.ffmpeg.org/wiki/SponsoringPrograms/GSoC/2015#AnimatedPortableNetworkGraphicsAPNG
185
+ # ffmpeg -h muxer=apng
186
+ # thumb is 32x32
187
+ # ffmpeg output.png
188
+ def convert_video_to_png_using_animated()
189
+ Utils.ffmpeg(
190
+ in_f: @source_file_path,
191
+ in_p: [
192
+ '-ss',10, # seek to input position
193
+ '-t',20, # max seconds
194
+ ],
195
+ out_f: @destination_file_path,
196
+ out_p: [
197
+ '-vf',"fps=5,scale=120:-1:flags=lanczos,split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse",
198
+ '-loop',0,
199
+ '-f','gif'
200
+ ])
201
+ end
202
+
203
+ def convert_office_to_png()
204
+ tmp_pdf_file=File.join(this_tmpdir,File.basename(@source_file_path,File.extname(@source_file_path))+'.pdf')
205
+ Utils.external_command(:unoconv,[
206
+
207
+ '-f','pdf',
208
+ '-o',tmp_pdf_file,
209
+ @source_file_path])
210
+ convert_pdf_to_png(tmp_pdf_file)
211
+ end
212
+
213
+ def convert_pdf_to_png(source_file_path=nil)
214
+ source_file_path||=@source_file_path
215
+ Utils.external_command(:convert,[
216
+
217
+ '-size',"x#{@options.thumb_img_size}",
218
+ '-background','white',
219
+ '-flatten',
220
+ "#{source_file_path}[0]",
221
+ @destination_file_path])
222
+ end
223
+
224
+ def convert_image_to_png()
225
+ Utils.external_command(:convert,[
226
+
227
+ '-auto-orient',
228
+ '-thumbnail',"#{@options.thumb_img_size}x#{@options.thumb_img_size}>",
229
+ '-quality',95,
230
+ '+dither',
231
+ '-posterize',40,
232
+ "#{@source_file_path}[0]",
233
+ @destination_file_path])
234
+ Utils.external_command(:optipng,[@destination_file_path])
235
+ end
236
+
237
+ # text to png
238
+ def convert_plaintext_to_png()
239
+ # get 100 first lines of text file
240
+ first_lines=File.open(@source_file_path){|f|100.times.map{f.readline rescue ''}}.join
241
+ Utils.external_command(:convert,[
242
+
243
+ '-size',"#{@options.thumb_img_size}x#{@options.thumb_img_size}",
244
+ 'xc:white',
245
+ '-font','Courier',
246
+ '-pointsize',12,
247
+ '-fill','black',
248
+ '-annotate','+15+15',first_lines,
249
+ '-trim',
250
+ '-bordercolor','#FFF',
251
+ '-border',10,
252
+ '+repage',
253
+ @destination_file_path])
254
+ end
255
+
256
+ end # Generator
257
+ end # Preview
258
+ end # Aspera