tootsie 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +15 -0
- data/Gemfile +2 -0
- data/License +7 -0
- data/README.md +256 -0
- data/Rakefile +1 -0
- data/Tootsie.gemspec +36 -0
- data/bin/tootsie_task_manager +82 -0
- data/config.ru +22 -0
- data/config/development-sample.yml +4 -0
- data/lib/tootsie.rb +21 -0
- data/lib/tootsie/application.rb +48 -0
- data/lib/tootsie/client.rb +12 -0
- data/lib/tootsie/command_runner.rb +58 -0
- data/lib/tootsie/configuration.rb +29 -0
- data/lib/tootsie/daemon.rb +282 -0
- data/lib/tootsie/ffmpeg_adapter.rb +132 -0
- data/lib/tootsie/image_metadata_extractor.rb +64 -0
- data/lib/tootsie/input.rb +55 -0
- data/lib/tootsie/output.rb +67 -0
- data/lib/tootsie/processors/image_processor.rb +181 -0
- data/lib/tootsie/processors/video_processor.rb +85 -0
- data/lib/tootsie/queues/file_system_queue.rb +65 -0
- data/lib/tootsie/queues/sqs_queue.rb +93 -0
- data/lib/tootsie/s3_utilities.rb +24 -0
- data/lib/tootsie/spawner.rb +99 -0
- data/lib/tootsie/task_manager.rb +51 -0
- data/lib/tootsie/tasks/job_task.rb +111 -0
- data/lib/tootsie/tasks/notify_task.rb +27 -0
- data/lib/tootsie/version.rb +3 -0
- data/lib/tootsie/web_service.rb +37 -0
- data/spec/spec_helper.rb +21 -0
- data/spec/test_files/BF 0622 1820.tif +0 -0
- data/spec/tootsie/command_runner_spec.rb +29 -0
- data/spec/tootsie/image_metadata_extracter_spec.rb +39 -0
- data/spec/tootsie/s3_utilities_spec.rb +40 -0
- metadata +337 -0
@@ -0,0 +1,64 @@
|
|
1
|
+
require 'iconv'
|
2
|
+
require 'time'
|
3
|
+
|
4
|
+
module Tootsie
|
5
|
+
|
6
|
+
class ImageMetadataExtractor
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
@metadata = {}
|
10
|
+
end
|
11
|
+
|
12
|
+
def extract_from_file(file_name)
|
13
|
+
run_exiv2("-pt :file", :file => file_name) do |line|
|
14
|
+
parse_exiv2_line(line)
|
15
|
+
end
|
16
|
+
run_exiv2("-pi :file", :file => file_name) do |line|
|
17
|
+
parse_exiv2_line(line)
|
18
|
+
end
|
19
|
+
run_exiv2("-px :file", :file => file_name) do |line|
|
20
|
+
parse_exiv2_line(line)
|
21
|
+
end
|
22
|
+
@metadata = Hash[*@metadata.entries.map { |key, values|
|
23
|
+
[key, values.length > 1 ? values : values.first]
|
24
|
+
}.flatten(1)]
|
25
|
+
@metadata
|
26
|
+
end
|
27
|
+
|
28
|
+
attr_reader :metadata
|
29
|
+
|
30
|
+
private
|
31
|
+
|
32
|
+
def run_exiv2(args, params, &block)
|
33
|
+
CommandRunner.new("exiv2 #{args}",
|
34
|
+
:output_encoding => 'binary',
|
35
|
+
:ignore_exit_code => true
|
36
|
+
).run(params, &block)
|
37
|
+
end
|
38
|
+
|
39
|
+
def parse_exiv2_line(line)
|
40
|
+
if line =~ /^([^\s]+)\s+([^\s]+)\s+\d+ (.*)$/
|
41
|
+
key, type, value = $1, $2, $3
|
42
|
+
unless value.blank?
|
43
|
+
case type
|
44
|
+
when 'Short', 'Long'
|
45
|
+
value = value.to_i
|
46
|
+
when 'Date'
|
47
|
+
value = Time.parse(value)
|
48
|
+
else
|
49
|
+
begin
|
50
|
+
Iconv.iconv("utf-8", "utf-8", value)
|
51
|
+
rescue Iconv::IllegalSequence, Iconv::InvalidCharacter
|
52
|
+
value = Iconv.iconv("utf-8", "iso-8859-1", value)[0]
|
53
|
+
else
|
54
|
+
value.force_encoding 'utf-8'
|
55
|
+
end
|
56
|
+
end
|
57
|
+
entry = {:value => value, :type => type.underscore}
|
58
|
+
(@metadata[key] ||= []) << entry
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
63
|
+
end
|
64
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
require 'httpclient'
|
2
|
+
require 'tempfile'
|
3
|
+
require 's3'
|
4
|
+
|
5
|
+
module Tootsie
|
6
|
+
|
7
|
+
class InputNotFound < Exception; end
|
8
|
+
|
9
|
+
class Input
|
10
|
+
|
11
|
+
def initialize(url)
|
12
|
+
@url = url
|
13
|
+
@temp_file = Tempfile.new('tootsie')
|
14
|
+
@temp_file.close
|
15
|
+
@file_name = @temp_file.path
|
16
|
+
@logger = Application.get.logger
|
17
|
+
end
|
18
|
+
|
19
|
+
def get!
|
20
|
+
@logger.info("Fetching #{@url} as #{@temp_file.path}")
|
21
|
+
case @url
|
22
|
+
when /^file:(.*)/
|
23
|
+
@file_name = $1
|
24
|
+
raise InputNotFound, @url unless File.exist?(@file_name)
|
25
|
+
when /^s3:.*$/
|
26
|
+
s3_options = S3Utilities.parse_uri(@url)
|
27
|
+
bucket_name, path = s3_options[:bucket], s3_options[:key]
|
28
|
+
s3_service = Tootsie::Application.get.s3_service
|
29
|
+
begin
|
30
|
+
File.open(@temp_file.path, 'wb') do |f|
|
31
|
+
f << s3_service.buckets.find(bucket_name).objects.find(path).content
|
32
|
+
end
|
33
|
+
rescue ::S3::Error::NoSuchBucket, ::S3::Error::NoSuchKey
|
34
|
+
raise InputNotFound, @url
|
35
|
+
end
|
36
|
+
when /http(s?):\/\//
|
37
|
+
response = HTTPClient.new.get(@url)
|
38
|
+
File.open(@temp_file.path, 'wb') do |f|
|
39
|
+
f << response.body
|
40
|
+
end
|
41
|
+
else
|
42
|
+
raise ArgumentError, "Don't know to handle URL: #{@url}"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
|
46
|
+
def close
|
47
|
+
@temp_file.unlink
|
48
|
+
end
|
49
|
+
|
50
|
+
attr_reader :url
|
51
|
+
attr_reader :file_name
|
52
|
+
|
53
|
+
end
|
54
|
+
|
55
|
+
end
|
@@ -0,0 +1,67 @@
|
|
1
|
+
require 'httpclient'
|
2
|
+
require 's3'
|
3
|
+
|
4
|
+
module Tootsie
|
5
|
+
|
6
|
+
class IncompatibleOutputError < Exception; end
|
7
|
+
|
8
|
+
class Output
|
9
|
+
|
10
|
+
def initialize(url)
|
11
|
+
@url = url
|
12
|
+
@temp_file = Tempfile.new('tootsie')
|
13
|
+
@temp_file.close
|
14
|
+
@file_name = @temp_file.path
|
15
|
+
@logger = Application.get.logger
|
16
|
+
end
|
17
|
+
|
18
|
+
# Put data into the output. Options:
|
19
|
+
#
|
20
|
+
# * +:content_type+ - content type of the stored data.
|
21
|
+
#
|
22
|
+
def put!(options = {})
|
23
|
+
@logger.info("Storing #{@url}")
|
24
|
+
case @url
|
25
|
+
when /^file:(.*)/
|
26
|
+
FileUtils.cp(@temp_file.path, $1)
|
27
|
+
when /^s3:.*/
|
28
|
+
s3_options = S3Utilities.parse_uri(@url)
|
29
|
+
bucket_name, path = s3_options[:bucket], s3_options[:key]
|
30
|
+
File.open(@temp_file.path, 'r') do |file|
|
31
|
+
s3_service = Tootsie::Application.get.s3_service
|
32
|
+
begin
|
33
|
+
object = s3_service.buckets.find(bucket_name).objects.build(path)
|
34
|
+
object.acl = s3_options[:acl] || :private
|
35
|
+
object.content_type = s3_options[:content_type]
|
36
|
+
object.content_type ||= @content_type if @content_type
|
37
|
+
object.storage_class = s3_options[:storage_class] || :standard
|
38
|
+
object.content = file
|
39
|
+
object.save
|
40
|
+
@result_url = object.url
|
41
|
+
rescue ::S3::Error::NoSuchBucket
|
42
|
+
raise IncompatibleOutputError, "Bucket #{bucket_name} not found"
|
43
|
+
end
|
44
|
+
end
|
45
|
+
when /^http(s?):\/\//
|
46
|
+
File.open(@temp_file.path, 'wb') do |file|
|
47
|
+
HTTPClient.new.get_content(@url) do |chunk|
|
48
|
+
file << chunk
|
49
|
+
end
|
50
|
+
end
|
51
|
+
else
|
52
|
+
raise IncompatibleOutputError, "Don't know to store output URL: #{@url}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
def close
|
57
|
+
@temp_file.unlink
|
58
|
+
end
|
59
|
+
|
60
|
+
attr_reader :url
|
61
|
+
attr_reader :result_url
|
62
|
+
attr_reader :file_name
|
63
|
+
attr_accessor :content_type
|
64
|
+
|
65
|
+
end
|
66
|
+
|
67
|
+
end
|
@@ -0,0 +1,181 @@
|
|
1
|
+
module Tootsie
|
2
|
+
module Processors
|
3
|
+
|
4
|
+
class ImageProcessor
|
5
|
+
|
6
|
+
def initialize(params = {})
|
7
|
+
@input_url = params[:input_url]
|
8
|
+
@versions = [params[:versions] || {}].flatten
|
9
|
+
@logger = Application.get.logger
|
10
|
+
end
|
11
|
+
|
12
|
+
def valid?
|
13
|
+
return @input_url && !@versions.blank?
|
14
|
+
end
|
15
|
+
|
16
|
+
def params
|
17
|
+
return {
|
18
|
+
:input_url => @input_url,
|
19
|
+
:versions => @versions
|
20
|
+
}
|
21
|
+
end
|
22
|
+
|
23
|
+
def execute!(&block)
|
24
|
+
result = {:outputs => []}
|
25
|
+
input, output = Input.new(@input_url), nil
|
26
|
+
begin
|
27
|
+
input.get!
|
28
|
+
begin
|
29
|
+
versions.each_with_index do |version_options, version_index|
|
30
|
+
version_options = version_options.with_indifferent_access
|
31
|
+
@logger.info("Handling version: #{version_options.inspect}")
|
32
|
+
|
33
|
+
output = Output.new(version_options[:target_url])
|
34
|
+
begin
|
35
|
+
result[:metadata] ||= ImageMetadataExtractor.new.extract_from_file(input.file_name)
|
36
|
+
|
37
|
+
original_depth = nil
|
38
|
+
original_width = nil
|
39
|
+
original_height = nil
|
40
|
+
original_type = nil
|
41
|
+
original_format = nil
|
42
|
+
original_orientation = nil
|
43
|
+
CommandRunner.new("identify -format '%z %w %h %r %m %[EXIF:Orientation]' :file").
|
44
|
+
run(:file => input.file_name) do |line|
|
45
|
+
if line =~ /(\d+) (\d+) (\d+) ([^\s]+) ([^\s]+) (\d+)?/
|
46
|
+
original_depth, original_width, original_height = $~[1, 3].map(&:to_i)
|
47
|
+
original_type = $4
|
48
|
+
original_format = $5.downcase
|
49
|
+
original_orientation = $6.try(:to_i)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
unless original_width and original_height
|
53
|
+
raise "Unable to determine dimensions of input image"
|
54
|
+
end
|
55
|
+
|
56
|
+
# Correct for EXIF orientation
|
57
|
+
if [5, 6, 7, 8].include?(original_orientation)
|
58
|
+
original_width, original_height = original_height, original_width
|
59
|
+
end
|
60
|
+
|
61
|
+
original_aspect = original_height / original_width.to_f
|
62
|
+
|
63
|
+
result[:width] = original_width
|
64
|
+
result[:height] = original_height
|
65
|
+
result[:depth] = original_depth
|
66
|
+
|
67
|
+
medium = version_options[:medium]
|
68
|
+
medium &&= medium.to_sym
|
69
|
+
|
70
|
+
new_width, new_height = version_options[:width], version_options[:height]
|
71
|
+
if new_width
|
72
|
+
new_height ||= (new_width * original_aspect).ceil
|
73
|
+
elsif new_height
|
74
|
+
new_width ||= (new_height / original_aspect).ceil
|
75
|
+
else
|
76
|
+
new_width, new_height = original_width, original_height
|
77
|
+
end
|
78
|
+
|
79
|
+
scale_width, scale_height = new_width, new_height
|
80
|
+
scale = (version_options[:scale] || 'down').to_sym
|
81
|
+
case scale
|
82
|
+
when :up, :none
|
83
|
+
# Do nothing
|
84
|
+
when :down
|
85
|
+
if scale_width > original_width
|
86
|
+
scale_width = original_width
|
87
|
+
scale_height = (scale_width * original_aspect).ceil
|
88
|
+
elsif scale_height > original_height
|
89
|
+
scale_height = original_height
|
90
|
+
scale_width = (scale_height / original_aspect).ceil
|
91
|
+
end
|
92
|
+
when :fit
|
93
|
+
if (scale_width * original_aspect).ceil < new_height
|
94
|
+
scale_height = new_height
|
95
|
+
scale_width = (new_height / original_aspect).ceil
|
96
|
+
elsif (scale_height / original_aspect).ceil < new_width
|
97
|
+
scale_width = new_width
|
98
|
+
scale_height = (scale_width * original_aspect).ceil
|
99
|
+
end
|
100
|
+
end
|
101
|
+
|
102
|
+
convert_command = "convert"
|
103
|
+
convert_options = {:input_file => input.file_name}
|
104
|
+
case version_options[:format]
|
105
|
+
when 'png', 'jpeg', 'gif'
|
106
|
+
convert_options[:output_file] = "#{version_options[:format]}:#{output.file_name}"
|
107
|
+
else
|
108
|
+
convert_options[:output_file] = output.file_name
|
109
|
+
end
|
110
|
+
if scale != :none
|
111
|
+
convert_command << " -resize :resize"
|
112
|
+
convert_options[:resize] = "#{scale_width}x#{scale_height}"
|
113
|
+
end
|
114
|
+
if version_options[:crop]
|
115
|
+
convert_command << " -gravity center -crop :crop"
|
116
|
+
convert_options[:crop] = "#{new_width}x#{new_height}+0+0"
|
117
|
+
end
|
118
|
+
if version_options[:strip_metadata]
|
119
|
+
convert_command << " +profile :remove_profiles -set comment ''"
|
120
|
+
convert_options[:remove_profiles] = "8bim,iptc,xmp,exif"
|
121
|
+
end
|
122
|
+
|
123
|
+
convert_command << " -quality #{((version_options[:quality] || 1.0) * 100).ceil}%"
|
124
|
+
|
125
|
+
# Work around a problem with ImageMagick being too clever and "optimizing"
|
126
|
+
# the bit depth of RGB images that contain a single grayscale channel.
|
127
|
+
# Coincidentally, this avoids ImageMagick rewriting the ICC data and
|
128
|
+
# corrupting it in the process.
|
129
|
+
if original_type =~ /(?:Gray|RGB)(Matte)?$/ and original_format != 'png'
|
130
|
+
convert_command << " -type TrueColor#{$1}"
|
131
|
+
end
|
132
|
+
|
133
|
+
# Fix CMYK images
|
134
|
+
if medium == :web and original_type =~ /CMYK$/
|
135
|
+
convert_command << " -colorspace rgb"
|
136
|
+
end
|
137
|
+
|
138
|
+
# Auto-orient images when web or we're stripping EXIF
|
139
|
+
if medium == :web or version_options[:strip_metadata]
|
140
|
+
convert_command << " -auto-orient"
|
141
|
+
end
|
142
|
+
|
143
|
+
convert_command << " :input_file :output_file"
|
144
|
+
CommandRunner.new(convert_command).run(convert_options)
|
145
|
+
|
146
|
+
if version_options[:format] == 'png'
|
147
|
+
Tempfile.open('tootsie') do |file|
|
148
|
+
# TODO: Make less sloppy
|
149
|
+
file.write(File.read(output.file_name))
|
150
|
+
file.close
|
151
|
+
CommandRunner.new('pngcrush :input_file :output_file').run(
|
152
|
+
:input_file => file.path, :output_file => output.file_name)
|
153
|
+
end
|
154
|
+
end
|
155
|
+
|
156
|
+
output.content_type = version_options[:content_type] if version_options[:content_type]
|
157
|
+
output.content_type ||= case version_options[:format]
|
158
|
+
when 'jpeg' then 'image/jpeg'
|
159
|
+
when 'png' then 'image/png'
|
160
|
+
when 'gif' then 'image/gif'
|
161
|
+
end
|
162
|
+
output.put!
|
163
|
+
result[:outputs] << {:url => output.result_url}
|
164
|
+
ensure
|
165
|
+
output.close
|
166
|
+
end
|
167
|
+
end
|
168
|
+
end
|
169
|
+
ensure
|
170
|
+
input.close
|
171
|
+
end
|
172
|
+
result
|
173
|
+
end
|
174
|
+
|
175
|
+
attr_accessor :input_url
|
176
|
+
attr_accessor :versions
|
177
|
+
|
178
|
+
end
|
179
|
+
|
180
|
+
end
|
181
|
+
end
|
@@ -0,0 +1,85 @@
|
|
1
|
+
require 'json'
|
2
|
+
|
3
|
+
module Tootsie
|
4
|
+
module Processors
|
5
|
+
|
6
|
+
class VideoProcessor
|
7
|
+
|
8
|
+
def initialize(params = {})
|
9
|
+
@input_url = params[:input_url]
|
10
|
+
@thumbnail_options = (params[:thumbnail] || {}).with_indifferent_access
|
11
|
+
@versions = [params[:versions] || {}].flatten
|
12
|
+
@thread_count = Application.get.configuration.ffmpeg_thread_count
|
13
|
+
end
|
14
|
+
|
15
|
+
def valid?
|
16
|
+
return @input_url && !@versions.blank?
|
17
|
+
end
|
18
|
+
|
19
|
+
def params
|
20
|
+
return {
|
21
|
+
:input_url => @input_url,
|
22
|
+
:thumbnail => @thumbnail_options,
|
23
|
+
:versions => @versions
|
24
|
+
}
|
25
|
+
end
|
26
|
+
|
27
|
+
def execute!(&block)
|
28
|
+
result = {:urls => []}
|
29
|
+
input, output, thumbnail_output = Input.new(@input_url), nil, nil
|
30
|
+
begin
|
31
|
+
input.get!
|
32
|
+
begin
|
33
|
+
versions.each_with_index do |version_options, version_index|
|
34
|
+
version_options = version_options.with_indifferent_access
|
35
|
+
|
36
|
+
if version_index == 0 and @thumbnail_options[:target_url]
|
37
|
+
thumbnail_output = Output.new(@thumbnail_options[:target_url])
|
38
|
+
else
|
39
|
+
thumbnail_output = nil
|
40
|
+
end
|
41
|
+
begin
|
42
|
+
output = Output.new(version_options[:target_url])
|
43
|
+
begin
|
44
|
+
adapter_options = version_options.dup
|
45
|
+
adapter_options.delete(:target_url)
|
46
|
+
adapter_options[:thumbnail] = @thumbnail_options.merge(:filename => thumbnail_output.file_name) if thumbnail_output
|
47
|
+
|
48
|
+
adapter = Tootsie::FfmpegAdapter.new(:thread_count => @thread_count)
|
49
|
+
if block
|
50
|
+
adapter.progress = lambda { |seconds, total_seconds|
|
51
|
+
yield(:progress => (seconds + (total_seconds * version_index)) / (total_seconds * versions.length).to_f)
|
52
|
+
}
|
53
|
+
end
|
54
|
+
adapter.transcode(input.file_name, output.file_name, adapter_options)
|
55
|
+
|
56
|
+
output.content_type = version_options[:content_type] if version_options[:content_type]
|
57
|
+
output.put!
|
58
|
+
|
59
|
+
result[:urls].push output.result_url
|
60
|
+
ensure
|
61
|
+
output.close
|
62
|
+
end
|
63
|
+
if thumbnail_output
|
64
|
+
thumbnail_output.put!
|
65
|
+
result[:thumbnail_url] = thumbnail_output.result_url
|
66
|
+
end
|
67
|
+
ensure
|
68
|
+
thumbnail_output.close if thumbnail_output
|
69
|
+
end
|
70
|
+
end
|
71
|
+
end
|
72
|
+
ensure
|
73
|
+
input.close
|
74
|
+
end
|
75
|
+
result
|
76
|
+
end
|
77
|
+
|
78
|
+
attr_accessor :input_url
|
79
|
+
attr_accessor :versions
|
80
|
+
attr_accessor :thumbnail_options
|
81
|
+
|
82
|
+
end
|
83
|
+
|
84
|
+
end
|
85
|
+
end
|