httpimagestore 0.5.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +10 -12
- data/Gemfile.lock +57 -55
- data/README.md +829 -0
- data/VERSION +1 -1
- data/bin/httpimagestore +114 -180
- data/features/cache-control.feature +26 -90
- data/features/compatibility.feature +129 -0
- data/features/error-reporting.feature +207 -0
- data/features/health-check.feature +30 -0
- data/features/s3-store-and-thumbnail.feature +65 -0
- data/features/step_definitions/httpimagestore_steps.rb +66 -26
- data/features/support/env.rb +32 -5
- data/features/support/test.empty +0 -0
- data/httpimagestore.gemspec +60 -47
- data/lib/httpimagestore/aws_sdk_regions_hack.rb +23 -0
- data/lib/httpimagestore/configuration/file.rb +120 -0
- data/lib/httpimagestore/configuration/handler.rb +239 -0
- data/lib/httpimagestore/configuration/output.rb +119 -0
- data/lib/httpimagestore/configuration/path.rb +77 -0
- data/lib/httpimagestore/configuration/s3.rb +194 -0
- data/lib/httpimagestore/configuration/thumbnailer.rb +244 -0
- data/lib/httpimagestore/configuration.rb +126 -29
- data/lib/httpimagestore/error_reporter.rb +36 -0
- data/lib/httpimagestore/ruby_string_template.rb +26 -0
- data/load_test/load_test.1k.23a022f6e.m1.small-comp.csv +3 -0
- data/load_test/load_test.1k.ec9bde794.m1.small.csv +4 -0
- data/load_test/load_test.jmx +344 -0
- data/load_test/thumbnail_specs.csv +11 -0
- data/spec/configuration_file_spec.rb +309 -0
- data/spec/configuration_handler_spec.rb +124 -0
- data/spec/configuration_output_spec.rb +338 -0
- data/spec/configuration_path_spec.rb +92 -0
- data/spec/configuration_s3_spec.rb +571 -0
- data/spec/configuration_spec.rb +80 -105
- data/spec/configuration_thumbnailer_spec.rb +417 -0
- data/spec/ruby_string_template_spec.rb +43 -0
- data/spec/spec_helper.rb +61 -0
- data/spec/support/compute.jpg +0 -0
- data/spec/support/cuba_response_env.rb +40 -0
- data/spec/support/full.cfg +49 -0
- metadata +138 -84
- data/README.rdoc +0 -23
- data/features/httpimagestore.feature +0 -167
- data/lib/httpimagestore/image_path.rb +0 -54
- data/lib/httpimagestore/s3_service.rb +0 -37
- data/lib/httpimagestore/thumbnail_class.rb +0 -13
- data/spec/image_path_spec.rb +0 -72
- data/spec/test.cfg +0 -8
@@ -0,0 +1,77 @@
|
|
1
|
+
require 'httpimagestore/ruby_string_template'
|
2
|
+
require 'digest/sha2'
|
3
|
+
|
4
|
+
module Configuration
|
5
|
+
class PathNotDefinedError < ConfigurationError
|
6
|
+
def initialize(path_name)
|
7
|
+
super "path '#{path_name}' not defined"
|
8
|
+
end
|
9
|
+
end
|
10
|
+
|
11
|
+
class PathRenderingError < ConfigurationError
|
12
|
+
def initialize(path_name, template, message)
|
13
|
+
super "cannot generate path '#{path_name}' from template '#{template}': #{message}"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
class NoValueForPathTemplatePlaceholerError < PathRenderingError
|
18
|
+
def initialize(path_name, template, value_name)
|
19
|
+
super path_name, template, "no value for '\#{#{value_name}}'"
|
20
|
+
end
|
21
|
+
end
|
22
|
+
|
23
|
+
class NoMetaValueForPathTemplatePlaceholerError < PathRenderingError
|
24
|
+
def initialize(path_name, template, value_name, meta_value)
|
25
|
+
super path_name, template, "need '#{value_name}' to generate value for '\#{#{meta_value}}'"
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
class Path < RubyStringTemplate
|
30
|
+
def self.match(node)
|
31
|
+
node.name == 'path'
|
32
|
+
end
|
33
|
+
|
34
|
+
def self.pre(configuration)
|
35
|
+
configuration.paths ||= Hash.new{|hash, path_name| raise PathNotDefinedError.new(path_name)}
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.parse(configuration, node)
|
39
|
+
nodes = []
|
40
|
+
nodes << node unless node.values.empty?
|
41
|
+
nodes |= node.children
|
42
|
+
|
43
|
+
nodes.empty? and raise NoValueError.new(node, 'path name')
|
44
|
+
nodes.each do |node|
|
45
|
+
path_name, template = *node.grab_values('path name', 'path template')
|
46
|
+
configuration.paths[path_name] = Path.new(path_name, template)
|
47
|
+
end
|
48
|
+
end
|
49
|
+
|
50
|
+
def initialize(path_name, template)
|
51
|
+
super(template) do |locals, name|
|
52
|
+
case name
|
53
|
+
when :basename
|
54
|
+
path = locals[:path] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :path, name)
|
55
|
+
path = Pathname.new(path)
|
56
|
+
path.basename(path.extname).to_s
|
57
|
+
when :dirname
|
58
|
+
path = locals[:path] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :path, name)
|
59
|
+
Pathname.new(path).dirname.to_s
|
60
|
+
when :extension
|
61
|
+
path = locals[:path] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :path, name)
|
62
|
+
Pathname.new(path).extname.delete('.')
|
63
|
+
when :digest
|
64
|
+
return locals[:_digest] if locals.include? :_digest
|
65
|
+
data = locals[:body] or raise NoMetaValueForPathTemplatePlaceholerError.new(path_name, template, :body, name)
|
66
|
+
digest = Digest::SHA2.new.update(data).to_s[0,16]
|
67
|
+
# cache digest in request locals
|
68
|
+
locals[:_digest] = digest
|
69
|
+
else
|
70
|
+
locals[name] or raise NoValueForPathTemplatePlaceholerError.new(path_name, template, name)
|
71
|
+
end
|
72
|
+
end
|
73
|
+
end
|
74
|
+
end
|
75
|
+
Global.register_node_parser Path
|
76
|
+
end
|
77
|
+
|
@@ -0,0 +1,194 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
require 'httpimagestore/aws_sdk_regions_hack'
|
3
|
+
require 'httpimagestore/configuration/path'
|
4
|
+
require 'httpimagestore/configuration/handler'
|
5
|
+
|
6
|
+
module Configuration
|
7
|
+
class S3NotConfiguredError < ConfigurationError
|
8
|
+
def initialize
|
9
|
+
super "S3 client not configured"
|
10
|
+
end
|
11
|
+
end
|
12
|
+
|
13
|
+
class S3NoSuchBucketError < ConfigurationError
|
14
|
+
def initialize(bucket)
|
15
|
+
super "S3 bucket '#{bucket}' does not exist"
|
16
|
+
end
|
17
|
+
end
|
18
|
+
|
19
|
+
class S3NoSuchKeyError < ConfigurationError
|
20
|
+
def initialize(bucket, path)
|
21
|
+
super "S3 bucket '#{bucket}' does not contain key '#{path}'"
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
class S3AccessDenied < ConfigurationError
|
26
|
+
def initialize(bucket, path)
|
27
|
+
super "access to S3 bucket '#{bucket}' or key '#{path}' was denied"
|
28
|
+
end
|
29
|
+
end
|
30
|
+
|
31
|
+
class S3
|
32
|
+
include ClassLogging
|
33
|
+
|
34
|
+
def self.match(node)
|
35
|
+
node.name == 's3'
|
36
|
+
end
|
37
|
+
|
38
|
+
def self.parse(configuration, node)
|
39
|
+
configuration.s3 and raise StatementCollisionError.new(node, 's3')
|
40
|
+
|
41
|
+
node.grab_values
|
42
|
+
node.required_attributes('key', 'secret')
|
43
|
+
node.valid_attribute_values('ssl', true, false, nil)
|
44
|
+
|
45
|
+
key, secret, ssl = node.grab_attributes('key', 'secret', 'ssl')
|
46
|
+
ssl = true if ssl.nil?
|
47
|
+
|
48
|
+
configuration.s3 = AWS::S3.new(
|
49
|
+
access_key_id: key,
|
50
|
+
secret_access_key: secret,
|
51
|
+
logger: logger_for(AWS::S3),
|
52
|
+
log_level: :debug,
|
53
|
+
use_ssl: ssl
|
54
|
+
)
|
55
|
+
|
56
|
+
log.info "S3 client using '#{key}' key and #{ssl ? 'HTTPS' : 'HTTP'} connections"
|
57
|
+
end
|
58
|
+
end
|
59
|
+
Global.register_node_parser S3
|
60
|
+
|
61
|
+
class S3SourceStoreBase < SourceStoreBase
|
62
|
+
include ClassLogging
|
63
|
+
|
64
|
+
extend Stats
|
65
|
+
def_stats(
|
66
|
+
:total_s3_store,
|
67
|
+
:total_s3_store_bytes,
|
68
|
+
:total_s3_source,
|
69
|
+
:total_s3_source_bytes
|
70
|
+
)
|
71
|
+
|
72
|
+
def self.parse(configuration, node)
|
73
|
+
image_name = node.grab_values('image name').first
|
74
|
+
|
75
|
+
node.required_attributes('bucket', 'path')
|
76
|
+
node.valid_attribute_values('public_access', true, false, nil)
|
77
|
+
|
78
|
+
bucket, path_spec, cache_control, public_access, if_image_name_on =
|
79
|
+
*node.grab_attributes('bucket', 'path', 'cache-control', 'public', 'if-image-name-on')
|
80
|
+
public_access = false if public_access.nil?
|
81
|
+
|
82
|
+
self.new(
|
83
|
+
configuration.global,
|
84
|
+
image_name,
|
85
|
+
InclusionMatcher.new(image_name, if_image_name_on),
|
86
|
+
bucket,
|
87
|
+
path_spec,
|
88
|
+
public_access,
|
89
|
+
cache_control
|
90
|
+
)
|
91
|
+
end
|
92
|
+
|
93
|
+
def initialize(global, image_name, matcher, bucket, path_spec, public_access, cache_control)
|
94
|
+
super global, image_name, matcher
|
95
|
+
@bucket = bucket
|
96
|
+
@path_spec = path_spec
|
97
|
+
@public_access = public_access
|
98
|
+
@cache_control = cache_control
|
99
|
+
local :bucket, @bucket
|
100
|
+
end
|
101
|
+
|
102
|
+
def client
|
103
|
+
@global.s3 or raise S3NotConfiguredError
|
104
|
+
end
|
105
|
+
|
106
|
+
def url(object)
|
107
|
+
if @public_access
|
108
|
+
object.public_url.to_s
|
109
|
+
else
|
110
|
+
object.url_for(:read, expires: 60 * 60 * 24 * 365 * 20).to_s # expire in 20 years
|
111
|
+
end
|
112
|
+
end
|
113
|
+
|
114
|
+
def object(path)
|
115
|
+
begin
|
116
|
+
bucket = client.buckets[@bucket]
|
117
|
+
yield bucket.objects[path]
|
118
|
+
rescue AWS::S3::Errors::AccessDenied
|
119
|
+
raise S3AccessDenied.new(@bucket, path)
|
120
|
+
rescue AWS::S3::Errors::NoSuchBucket
|
121
|
+
raise S3NoSuchBucketError.new(@bucket)
|
122
|
+
rescue AWS::S3::Errors::NoSuchKey
|
123
|
+
raise S3NoSuchKeyError.new(@bucket, path)
|
124
|
+
end
|
125
|
+
end
|
126
|
+
end
|
127
|
+
|
128
|
+
class S3Source < S3SourceStoreBase
|
129
|
+
def self.match(node)
|
130
|
+
node.name == 'source_s3'
|
131
|
+
end
|
132
|
+
|
133
|
+
def self.parse(configuration, node)
|
134
|
+
configuration.image_sources << super
|
135
|
+
end
|
136
|
+
|
137
|
+
def realize(request_state)
|
138
|
+
put_sourced_named_image(request_state) do |image_name, rendered_path|
|
139
|
+
log.info "sourcing '#{image_name}' image from S3 '#{@bucket}' bucket under '#{rendered_path}' key"
|
140
|
+
|
141
|
+
object(rendered_path) do |object|
|
142
|
+
data = request_state.memory_limit.get do |limit|
|
143
|
+
object.read range: 0..(limit + 1)
|
144
|
+
end
|
145
|
+
S3SourceStoreBase.stats.incr_total_s3_source
|
146
|
+
S3SourceStoreBase.stats.incr_total_s3_source_bytes(data.bytesize)
|
147
|
+
|
148
|
+
image = Image.new(data, object.head[:content_type])
|
149
|
+
image.source_url = url(object)
|
150
|
+
image
|
151
|
+
end
|
152
|
+
end
|
153
|
+
end
|
154
|
+
end
|
155
|
+
Handler::register_node_parser S3Source
|
156
|
+
|
157
|
+
class S3Store < S3SourceStoreBase
|
158
|
+
def self.match(node)
|
159
|
+
node.name == 'store_s3'
|
160
|
+
end
|
161
|
+
|
162
|
+
def self.parse(configuration, node)
|
163
|
+
configuration.stores << super
|
164
|
+
end
|
165
|
+
|
166
|
+
def realize(request_state)
|
167
|
+
get_named_image_for_storage(request_state) do |image_name, image, rendered_path|
|
168
|
+
acl = @public_access ? :public_read : :private
|
169
|
+
|
170
|
+
log.info "storing '#{image_name}' image in S3 '#{@bucket}' bucket under '#{rendered_path}' key with #{acl} access"
|
171
|
+
|
172
|
+
object(rendered_path) do |object|
|
173
|
+
image.mime_type or log.warn "storing '#{image_name}' in S3 '#{@bucket}' bucket under '#{rendered_path}' key with unknown mime type"
|
174
|
+
|
175
|
+
options = {}
|
176
|
+
options[:single_request] = true
|
177
|
+
options[:content_type] = image.mime_type
|
178
|
+
options[:acl] = acl
|
179
|
+
options[:cache_control] = @cache_control if @cache_control
|
180
|
+
|
181
|
+
object.write(image.data, options)
|
182
|
+
S3SourceStoreBase.stats.incr_total_s3_store
|
183
|
+
S3SourceStoreBase.stats.incr_total_s3_store_bytes(image.data.bytesize)
|
184
|
+
|
185
|
+
image.store_url = url(object)
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
end
|
190
|
+
Handler::register_node_parser S3Store
|
191
|
+
StatsReporter << S3SourceStoreBase.stats
|
192
|
+
end
|
193
|
+
|
194
|
+
|
@@ -0,0 +1,244 @@
|
|
1
|
+
require 'unicorn-cuba-base'
|
2
|
+
require 'httpthumbnailer-client'
|
3
|
+
require 'httpimagestore/ruby_string_template'
|
4
|
+
require 'httpimagestore/configuration/handler'
|
5
|
+
|
6
|
+
module Configuration
|
7
|
+
class Thumnailer
|
8
|
+
include ClassLogging
|
9
|
+
|
10
|
+
def self.match(node)
|
11
|
+
node.name == 'thumbnailer'
|
12
|
+
end
|
13
|
+
|
14
|
+
def self.parse(configuration, node)
|
15
|
+
configuration.thumbnailer and raise StatementCollisionError.new(node, 'thumbnailer')
|
16
|
+
node.required_attributes('url')
|
17
|
+
configuration.thumbnailer = HTTPThumbnailerClient.new(node.grab_attributes('url').first)
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.post(configuration)
|
21
|
+
if not configuration.thumbnailer
|
22
|
+
configuration.thumbnailer = HTTPThumbnailerClient.new(configuration.defaults[:thumbnailer_url] || 'http://localhost:3100')
|
23
|
+
end
|
24
|
+
log.info "using thumbnailer at #{configuration.thumbnailer.server_url}"
|
25
|
+
end
|
26
|
+
end
|
27
|
+
Global.register_node_parser Thumnailer
|
28
|
+
|
29
|
+
class NoValueForSpecTemplatePlaceholerError < ConfigurationError
|
30
|
+
def initialize(image_name, spec_name, value_name, template)
|
31
|
+
super "cannot generate specification for thumbnail '#{image_name}': cannot generate value for attribute '#{spec_name}' from template '#{template}': no value for \#{#{value_name}}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
class Thumbnail
|
36
|
+
include ClassLogging
|
37
|
+
|
38
|
+
extend Stats
|
39
|
+
def_stats(
|
40
|
+
:total_thumbnail_requests,
|
41
|
+
:total_thumbnail_requests_bytes,
|
42
|
+
:total_thumbnail_thumbnails,
|
43
|
+
:total_thumbnail_thumbnails_bytes
|
44
|
+
)
|
45
|
+
|
46
|
+
class ThumbnailingError < RuntimeError
|
47
|
+
def initialize(input_image_name, output_image_name, remote_error)
|
48
|
+
@remote_error = remote_error
|
49
|
+
if output_image_name
|
50
|
+
super "thumbnailing of '#{input_image_name}' into '#{output_image_name}' failed: #{remote_error.message}"
|
51
|
+
else
|
52
|
+
super "thumbnailing of '#{input_image_name}' failed: #{remote_error.message}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
attr_reader :remote_error
|
57
|
+
end
|
58
|
+
|
59
|
+
class ThumbnailSpec
|
60
|
+
class Spec < RubyStringTemplate
|
61
|
+
def initialize(image_name, sepc_name, template)
|
62
|
+
super(template) do |locals, name|
|
63
|
+
locals[name] or raise NoValueForSpecTemplatePlaceholerError.new(image_name, sepc_name, name, template)
|
64
|
+
end
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
include ConditionalInclusion
|
69
|
+
|
70
|
+
def initialize(image_name, method, width, height, format, options = {}, matcher = nil)
|
71
|
+
@image_name = image_name
|
72
|
+
@method = Spec.new(image_name, 'method', method)
|
73
|
+
@width = Spec.new(image_name, 'width', width)
|
74
|
+
@height = Spec.new(image_name, 'height', height)
|
75
|
+
@format = Spec.new(image_name, 'format', format)
|
76
|
+
@options = options.inject({}){|h, v| h[v.first] = Spec.new(image_name, v.first, v.last); h}
|
77
|
+
inclusion_matcher matcher if matcher
|
78
|
+
end
|
79
|
+
|
80
|
+
attr_reader :image_name
|
81
|
+
|
82
|
+
def render(locals = {})
|
83
|
+
options = @options.inject({}){|h, v| h[v.first] = v.last.render(locals); h}
|
84
|
+
nested_options = options['options'] ? Hash[options.delete('options').to_s.split(',').map{|pair| pair.split(':', 2)}] : {}
|
85
|
+
{
|
86
|
+
@image_name =>
|
87
|
+
[
|
88
|
+
@method.render(locals),
|
89
|
+
@width.render(locals),
|
90
|
+
@height.render(locals),
|
91
|
+
@format.render(locals),
|
92
|
+
nested_options.merge(options)
|
93
|
+
]
|
94
|
+
}
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
include ConditionalInclusion
|
99
|
+
|
100
|
+
def self.match(node)
|
101
|
+
node.name == 'thumbnail'
|
102
|
+
end
|
103
|
+
|
104
|
+
def self.parse(configuration, node)
|
105
|
+
use_multipart_api = node.values.length == 1 ? true : false
|
106
|
+
|
107
|
+
nodes = use_multipart_api ? node.children : [node]
|
108
|
+
source_image_name = use_multipart_api ? node.grab_values('source image name').first : nil # parsed later
|
109
|
+
|
110
|
+
nodes.empty? and raise NoValueError.new(node, 'thumbnail image name')
|
111
|
+
matcher = nil
|
112
|
+
|
113
|
+
specs = nodes.map do |node|
|
114
|
+
if use_multipart_api
|
115
|
+
image_name = node.grab_values('thumbnail image name').first
|
116
|
+
else
|
117
|
+
source_image_name, image_name = *node.grab_values('source image name', 'thumbnail image name')
|
118
|
+
end
|
119
|
+
|
120
|
+
operation, width, height, format, if_image_name_on, remaining = *node.grab_attributes_with_remaining('operation', 'width', 'height', 'format', 'if-image-name-on')
|
121
|
+
|
122
|
+
matcher = InclusionMatcher.new(image_name, if_image_name_on) if if_image_name_on
|
123
|
+
|
124
|
+
ThumbnailSpec.new(
|
125
|
+
image_name,
|
126
|
+
operation || 'fit',
|
127
|
+
width || 'input',
|
128
|
+
height || 'input',
|
129
|
+
format || 'jpeg',
|
130
|
+
remaining || {},
|
131
|
+
matcher
|
132
|
+
)
|
133
|
+
end
|
134
|
+
|
135
|
+
matcher = InclusionMatcher.new(source_image_name, node.grab_attributes('if-image-name-on').first) if use_multipart_api
|
136
|
+
|
137
|
+
configuration.image_sources << self.new(
|
138
|
+
configuration.global,
|
139
|
+
source_image_name,
|
140
|
+
specs,
|
141
|
+
use_multipart_api,
|
142
|
+
matcher
|
143
|
+
)
|
144
|
+
end
|
145
|
+
|
146
|
+
def initialize(global, source_image_name, specs, use_multipart_api, matcher)
|
147
|
+
@global = global
|
148
|
+
@source_image_name = source_image_name
|
149
|
+
@specs = specs
|
150
|
+
@use_multipart_api = use_multipart_api
|
151
|
+
inclusion_matcher matcher
|
152
|
+
end
|
153
|
+
|
154
|
+
def realize(request_state)
|
155
|
+
client = @global.thumbnailer or fail 'thumbnailer configuration'
|
156
|
+
|
157
|
+
rendered_specs = {}
|
158
|
+
@specs.select do |spec|
|
159
|
+
spec.included?(request_state)
|
160
|
+
end.each do |spec|
|
161
|
+
rendered_specs.merge! spec.render(request_state.locals)
|
162
|
+
end
|
163
|
+
source_image = request_state.images[@source_image_name]
|
164
|
+
|
165
|
+
thumbnails = {}
|
166
|
+
input_mime_type = nil
|
167
|
+
|
168
|
+
Thumbnail.stats.incr_total_thumbnail_requests
|
169
|
+
Thumbnail.stats.incr_total_thumbnail_requests_bytes source_image.data.bytesize
|
170
|
+
|
171
|
+
if @use_multipart_api
|
172
|
+
log.info "thumbnailing '#{@source_image_name}' to multiple specs: #{rendered_specs}"
|
173
|
+
|
174
|
+
# need to reference to local so they are available within thumbnail() block context
|
175
|
+
source_image_name = @source_image_name
|
176
|
+
logger = log
|
177
|
+
|
178
|
+
begin
|
179
|
+
thumbnails = client.thumbnail(source_image.data) do
|
180
|
+
rendered_specs.each_pair do |name, spec|
|
181
|
+
begin
|
182
|
+
thumbnail(*spec)
|
183
|
+
rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
|
184
|
+
logger.warn 'got thumbnailer error while passing specs', error
|
185
|
+
raise ThumbnailingError.new(source_image_name, name, error)
|
186
|
+
end
|
187
|
+
end
|
188
|
+
end
|
189
|
+
rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
|
190
|
+
logger.warn 'got thumbnailer error while sending input data', error
|
191
|
+
raise ThumbnailingError.new(source_image_name, nil, error)
|
192
|
+
end
|
193
|
+
|
194
|
+
input_mime_type = thumbnails.input_mime_type
|
195
|
+
|
196
|
+
# check each thumbnail for errors
|
197
|
+
thumbnails = Hash[rendered_specs.keys.zip(thumbnails)]
|
198
|
+
thumbnails.each do |name, thumbnail|
|
199
|
+
if thumbnail.kind_of? HTTPThumbnailerClient::HTTPThumbnailerClientError
|
200
|
+
error = thumbnail
|
201
|
+
log.warn 'got single thumbnail error', error
|
202
|
+
raise ThumbnailingError.new(@source_image_name, name, error)
|
203
|
+
end
|
204
|
+
end
|
205
|
+
|
206
|
+
# borrow from memory limit - note that we might have already used too much memory
|
207
|
+
thumbnails.each do |name, thumbnail|
|
208
|
+
request_state.memory_limit.borrow thumbnail.data.bytesize
|
209
|
+
end
|
210
|
+
else
|
211
|
+
name, rendered_spec = *rendered_specs.first
|
212
|
+
log.info "thumbnailing '#{@source_image_name}' to '#{name}' with spec: #{rendered_spec}"
|
213
|
+
|
214
|
+
begin
|
215
|
+
thumbnail = client.thumbnail(source_image.data, *rendered_spec)
|
216
|
+
request_state.memory_limit.borrow thumbnail.data.bytesize
|
217
|
+
input_mime_type = thumbnail.input_mime_type
|
218
|
+
thumbnails[name] = thumbnail
|
219
|
+
rescue HTTPThumbnailerClient::HTTPThumbnailerClientError => error
|
220
|
+
log.warn 'got thumbnailer error', error
|
221
|
+
raise ThumbnailingError.new(@source_image_name, name, error)
|
222
|
+
end
|
223
|
+
end
|
224
|
+
|
225
|
+
# copy input source path and url
|
226
|
+
thumbnails.each do |name, thumbnail|
|
227
|
+
thumbnail.extend ImageMetaData
|
228
|
+
thumbnail.source_path = source_image.source_path
|
229
|
+
thumbnail.source_url = source_image.source_url
|
230
|
+
|
231
|
+
Thumbnail.stats.incr_total_thumbnail_thumbnails
|
232
|
+
Thumbnail.stats.incr_total_thumbnail_thumbnails_bytes thumbnail.data.bytesize
|
233
|
+
end
|
234
|
+
|
235
|
+
# update input image mime type from httpthumbnailer provided information
|
236
|
+
source_image.mime_type = input_mime_type unless source_image.mime_type
|
237
|
+
|
238
|
+
request_state.images.merge! thumbnails
|
239
|
+
end
|
240
|
+
end
|
241
|
+
Handler::register_node_parser Thumbnail
|
242
|
+
StatsReporter << Thumbnail.stats
|
243
|
+
end
|
244
|
+
|
@@ -1,55 +1,152 @@
|
|
1
|
-
require '
|
1
|
+
require 'sdl4r'
|
2
2
|
require 'pathname'
|
3
|
+
require 'ostruct'
|
4
|
+
require 'unicorn-cuba-base'
|
3
5
|
|
4
|
-
|
5
|
-
|
6
|
-
|
7
|
-
|
6
|
+
module Configuration
|
7
|
+
# parsing errors
|
8
|
+
class SyntaxError < ArgumentError
|
9
|
+
def initialize(node, message)
|
10
|
+
super "syntax error while parsing '#{node}': #{message}"
|
8
11
|
end
|
9
12
|
end
|
10
13
|
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
+
class NoAttributeError < SyntaxError
|
15
|
+
def initialize(node, attribute)
|
16
|
+
super node, "expected '#{attribute}' attribute to be set"
|
14
17
|
end
|
18
|
+
end
|
15
19
|
|
16
|
-
|
20
|
+
class NoValueError < SyntaxError
|
21
|
+
def initialize(node, value)
|
22
|
+
super node, "expected #{value}"
|
23
|
+
end
|
24
|
+
end
|
17
25
|
|
18
|
-
|
26
|
+
class BadAttributeValueError < SyntaxError
|
27
|
+
def initialize(node, attribute, value, valid)
|
28
|
+
super node, "expected '#{attribute}' attribute value to be #{valid.map(&:inspect).join(' or ')}; got: #{value.inspect}"
|
29
|
+
end
|
19
30
|
end
|
20
31
|
|
21
|
-
|
22
|
-
|
23
|
-
|
24
|
-
eval(file.read, nil, file.to_s)
|
32
|
+
class UnexpectedValueError < SyntaxError
|
33
|
+
def initialize(node, values)
|
34
|
+
super node, "unexpected values: #{values.map(&:inspect).join(', ')}"
|
25
35
|
end
|
26
36
|
end
|
27
37
|
|
28
|
-
|
29
|
-
|
38
|
+
class UnexpectedAttributesError < SyntaxError
|
39
|
+
def initialize(node, attributes)
|
40
|
+
super node, "unexpected attributes: #{attributes.keys.map{|a| "'#{a}'"}.join(', ')}"
|
41
|
+
end
|
30
42
|
end
|
31
43
|
|
32
|
-
|
33
|
-
|
34
|
-
|
44
|
+
class StatementCollisionError < SyntaxError
|
45
|
+
def initialize(node, type)
|
46
|
+
super node, "only one #{type} type statement can be specified within context"
|
47
|
+
end
|
35
48
|
end
|
36
49
|
|
37
|
-
|
38
|
-
|
50
|
+
# runtime errors
|
51
|
+
ConfigurationError = Class.new ArgumentError
|
52
|
+
|
53
|
+
module SDL4RTagExtensions
|
54
|
+
def required_attributes(*list)
|
55
|
+
list.each do |attribute|
|
56
|
+
attribute(attribute) or raise NoAttributeError.new(self, attribute)
|
57
|
+
end
|
58
|
+
true
|
59
|
+
end
|
60
|
+
|
61
|
+
def grab_attributes_with_remaining(*list)
|
62
|
+
attributes = self.attributes.dup
|
63
|
+
values = list.map do |attribute|
|
64
|
+
attributes.delete(attribute)
|
65
|
+
end
|
66
|
+
values + [attributes]
|
67
|
+
end
|
68
|
+
|
69
|
+
def grab_attributes(*list)
|
70
|
+
*values, remaining = *grab_attributes_with_remaining(*list)
|
71
|
+
remaining.empty? or raise UnexpectedAttributesError.new(self, remaining)
|
72
|
+
values
|
73
|
+
end
|
74
|
+
|
75
|
+
def valid_attribute_values(attribute, *valid)
|
76
|
+
value = self.attribute(attribute)
|
77
|
+
valid.include? value or raise BadAttributeValueError.new(self, attribute, value, valid)
|
78
|
+
end
|
79
|
+
|
80
|
+
def grab_values(*list)
|
81
|
+
values = self.values.dup
|
82
|
+
out = []
|
83
|
+
list.each do |name|
|
84
|
+
val = values.shift or raise NoValueError.new(self, name)
|
85
|
+
out << val
|
86
|
+
end
|
87
|
+
values.empty? or raise UnexpectedValueError.new(self, values)
|
88
|
+
out
|
89
|
+
end
|
39
90
|
end
|
40
91
|
|
41
|
-
|
42
|
-
|
92
|
+
class Scope
|
93
|
+
include ClassLogging
|
94
|
+
|
95
|
+
def self.node_parsers
|
96
|
+
@node_parsers ||= []
|
97
|
+
end
|
98
|
+
|
99
|
+
def self.register_node_parser(parser)
|
100
|
+
parser.logger = logger_for(parser) if parser.respond_to? :logger=
|
101
|
+
node_parsers << parser
|
102
|
+
end
|
103
|
+
|
104
|
+
def initialize(configuration)
|
105
|
+
@configuration = configuration
|
106
|
+
end
|
107
|
+
|
108
|
+
def parse(node)
|
109
|
+
self.class.node_parsers.each do |parser|
|
110
|
+
parser.pre(@configuration) if parser.respond_to? :pre
|
111
|
+
end
|
112
|
+
|
113
|
+
node.children.each do |node|
|
114
|
+
parser = self.class.node_parsers.find do |parser|
|
115
|
+
parser.match node
|
116
|
+
end
|
117
|
+
if parser
|
118
|
+
parser.parse(@configuration, node)
|
119
|
+
else
|
120
|
+
log.warn "unexpected statement: #{node.name}"
|
121
|
+
end
|
122
|
+
end
|
123
|
+
|
124
|
+
self.class.node_parsers.each do |parser|
|
125
|
+
parser.post(@configuration) if parser.respond_to? :post
|
126
|
+
end
|
127
|
+
@configuration
|
128
|
+
end
|
43
129
|
end
|
44
130
|
|
45
|
-
|
46
|
-
Struct.new(:thumbnail_classes, :s3_key_id, :s3_key_secret, :s3_bucket, :thumbnailer_url).new(@thumbnail_classes, @s3_key_id, @s3_key_secret, @s3_bucket, @thumbnailer_url)
|
131
|
+
class Global < Scope
|
47
132
|
end
|
48
133
|
|
49
|
-
def
|
50
|
-
|
51
|
-
|
52
|
-
|
134
|
+
def self.from_file(config_file, defaults = {})
|
135
|
+
read Pathname.new(config_file), defaults
|
136
|
+
end
|
137
|
+
|
138
|
+
def self.read(config, defaults = {})
|
139
|
+
parse SDL4R::read(config), defaults
|
53
140
|
end
|
141
|
+
|
142
|
+
def self.parse(root, defaults = {})
|
143
|
+
configuration = OpenStruct.new
|
144
|
+
configuration.defaults = defaults
|
145
|
+
Global.new(configuration).parse(root)
|
146
|
+
end
|
147
|
+
end
|
148
|
+
|
149
|
+
class SDL4R::Tag
|
150
|
+
include Configuration::SDL4RTagExtensions
|
54
151
|
end
|
55
152
|
|