httpimagestore 1.0.0 → 1.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +43 -2
- data/VERSION +1 -1
- data/bin/httpimagestore +13 -9
- data/features/s3-store-and-thumbnail.feature +24 -0
- data/httpimagestore.gemspec +2 -2
- data/lib/httpimagestore/configuration/handler.rb +15 -3
- data/spec/configuration_file_spec.rb +2 -2
- data/spec/configuration_handler_spec.rb +1 -1
- data/spec/configuration_output_spec.rb +1 -1
- data/spec/configuration_s3_spec.rb +4 -4
- data/spec/configuration_thumbnailer_spec.rb +44 -32
- metadata +3 -3
data/README.md
CHANGED
@@ -89,7 +89,7 @@ Variables:
|
|
89
89
|
* `extension` - file extension determined from `path`
|
90
90
|
* `mimeextension` - image extension based on mime type; mime type will be updated based on information from [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) for input image and output thumbnails - content determined
|
91
91
|
* `imagename` - name of the image that is being stored or sourced
|
92
|
-
* URL matches - other variables can be matched from URL pattern - see API configuration
|
92
|
+
* URL matches and query string parameters - other variables can be matched from URL pattern and query string parameters - see API configuration
|
93
93
|
|
94
94
|
Example:
|
95
95
|
|
@@ -119,6 +119,7 @@ Statement should start with one of the following HTTP verbs in lowercase: `get`,
|
|
119
119
|
* `:symbol/regexp/` - in this format symbol will be matched only if `/` surrounded [regular expression](http://rubular.com) matches the URI section
|
120
120
|
|
121
121
|
Note that any remaining URI are is stored in `path` variable.
|
122
|
+
Also any query string parameters are available as variables. Additionally `query_string_options` is build from query string parameters and can be used to specify options to [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer).
|
122
123
|
|
123
124
|
Example:
|
124
125
|
|
@@ -540,6 +541,14 @@ get "v1" "thumbnail" ":path" ":operation" ":width" ":height" ":options?" {
|
|
540
541
|
|
541
542
|
output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
|
542
543
|
}
|
544
|
+
|
545
|
+
get "v2" "thumbnail" ":operation" ":width" ":height" {
|
546
|
+
source_s3 "original" bucket="mybucket_v1" path="path"
|
547
|
+
|
548
|
+
thumbnail "original" "thumbnail" operation="#{operation}" width="#{width}" height="#{height}" options="#{query_string_options}" quality=84 format="png"
|
549
|
+
|
550
|
+
output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
|
551
|
+
}
|
543
552
|
```
|
544
553
|
|
545
554
|
Compatibility API works by storing input image and selected (via URI) classes of thumbnails generated during image upload. Once the image is uploaded thumbnails can be served directly from S3. There are two endpoints defined for that API to handle URIs that contain optional image storage name that results in usage of different storage key.
|
@@ -617,6 +626,31 @@ $ curl 10.1.1.24:3000/v1/thumbnail/4006450256177f4a.jpg/fit/100/1000 -v -s -o /t
|
|
617
626
|
|
618
627
|
$ identify /tmp/test.jpeg
|
619
628
|
/tmp/test.jpeg JPEG 100x141 100x141+0+0 8-bit sRGB 4.68KB 0.000u 0:00.000
|
629
|
+
|
630
|
+
# Also form with query string passed options can be used to retrieve thumbnails
|
631
|
+
$ curl 10.1.1.24:3000/v2/thumbnail/pad/100/100/4006450256177f4a.jpg?background-color=green -v -s -o /tmp/test.jpg
|
632
|
+
* About to connect() to 10.1.1.24 port 3000 (#0)
|
633
|
+
* Trying 10.1.1.24... connected
|
634
|
+
> GET /v2/thumbnail/pad/100/100/4006450256177f4a.jpg?background-color=green HTTP/1.1
|
635
|
+
> User-Agent: curl/7.22.0 (x86_64-apple-darwin10.8.0) libcurl/7.22.0 OpenSSL/1.0.1c zlib/1.2.7 libidn/1.25
|
636
|
+
> Host: 10.1.1.24:3000
|
637
|
+
> Accept: */*
|
638
|
+
>
|
639
|
+
< HTTP/1.1 200 OK
|
640
|
+
< Server: nginx/1.2.9
|
641
|
+
< Date: Wed, 24 Jul 2013 11:38:39 GMT
|
642
|
+
< Content-Type: image/jpeg
|
643
|
+
< Content-Length: 3310
|
644
|
+
< Connection: keep-alive
|
645
|
+
< Status: 200 OK
|
646
|
+
< Cache-Control: public, max-age=31557600, s-maxage=0
|
647
|
+
<
|
648
|
+
{ [data not shown]
|
649
|
+
* Connection #0 to host 10.1.1.24 left intact
|
650
|
+
* Closing connection #0
|
651
|
+
|
652
|
+
$ identify /tmp/test.jpg
|
653
|
+
/tmp/test.jpg JPEG 100x100 100x100+0+0 8-bit sRGB 3.31KB 0.000u 0:00.000
|
620
654
|
```
|
621
655
|
|
622
656
|
## Usage
|
@@ -680,7 +714,8 @@ httpthumbnailer --pid-file /var/run/httpthumbnailer/pidfile --log-file /var/log/
|
|
680
714
|
```
|
681
715
|
|
682
716
|
To start [nginx](http://nginx.org) we need to configure it to run as reverse HTTP proxy for our UNIX socket based `httpimagestore` backend.
|
683
|
-
Also we set it up so that it does request and response buffering.
|
717
|
+
Also we set it up so that it does request and response buffering and on disk caching of GET requests.
|
718
|
+
You may want to disable caching if your GET URL resource is not immutable.
|
684
719
|
Here is the example `/etc/nginx/nginx.conf` file:
|
685
720
|
|
686
721
|
```nginx
|
@@ -716,6 +751,9 @@ http {
|
|
716
751
|
server unix:/var/run/httpimagestore.sock fail_timeout=0;
|
717
752
|
}
|
718
753
|
|
754
|
+
# cache GET requests up to 256MiB in RAM and 130GiB on disk for up to 30 days of no access
|
755
|
+
proxy_cache_path /var/cache/nginx/httpimagestore levels=2:2 keys_zone=httpimagestore:256m max_size=130g inactive=30d;
|
756
|
+
|
719
757
|
server {
|
720
758
|
listen *:3000;
|
721
759
|
server_name localhost;
|
@@ -737,6 +775,9 @@ http {
|
|
737
775
|
proxy_read_timeout 120s;
|
738
776
|
proxy_connect_timeout 10s;
|
739
777
|
|
778
|
+
proxy_cache httpimagestore;
|
779
|
+
proxy_cache_key "$request_uri";
|
780
|
+
|
740
781
|
proxy_pass http://httpimagestore;
|
741
782
|
}
|
742
783
|
}
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.1.0
|
data/bin/httpimagestore
CHANGED
@@ -68,17 +68,21 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
|
|
68
68
|
|
69
69
|
env['app.configuration'].handlers.each do |handler|
|
70
70
|
on eval(handler.http_method), *handler.uri_matchers.map{|m| instance_eval(&m.matcher)} do |*args|
|
71
|
-
|
72
|
-
|
73
|
-
{
|
74
|
-
|
75
|
-
}.merge(
|
76
|
-
Hash[handler.uri_matchers.select{|m| m.name}.map{|m| m.name}.zip(args)]
|
77
|
-
).each do |name, value|
|
78
|
-
locals[name] = URI.decode(value).force_encoding('UTF-8')
|
71
|
+
# map and decode matched URI componetns
|
72
|
+
matches = {}
|
73
|
+
Hash[handler.uri_matchers.select{|m| m.name}.map{|m| m.name}.zip(args)].each do |name, value|
|
74
|
+
matches[name] = URI.decode(value).force_encoding('UTF-8')
|
79
75
|
end
|
80
76
|
|
81
|
-
|
77
|
+
# decode remaining URI components
|
78
|
+
path = (env["PATH_INFO"][1..-1] || '').split('/').map do |part|
|
79
|
+
URI.decode(part).force_encoding('UTF-8')
|
80
|
+
end.join('/')
|
81
|
+
|
82
|
+
# query string already doceded by Rack
|
83
|
+
query_string = req.GET
|
84
|
+
|
85
|
+
state = Configuration::RequestState.new(req.body.read, matches, path, query_string, memory_limit)
|
82
86
|
|
83
87
|
handler.image_sources.each do |image_source|
|
84
88
|
image_source.realize(state) unless image_source.respond_to? :excluded? and image_source.excluded?(state)
|
@@ -28,6 +28,13 @@ Feature: Store limited original image in S3 and thumbnail based on request
|
|
28
28
|
output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
|
29
29
|
}
|
30
30
|
|
31
|
+
get "thumbnail" "v2" ":operation" ":width" ":height" {
|
32
|
+
source_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="path"
|
33
|
+
|
34
|
+
thumbnail "original" "thumbnail" operation="#{operation}" width="#{width}" height="#{height}" options="#{query_string_options}" quality=84 format="png"
|
35
|
+
|
36
|
+
output_image "thumbnail" cache-control="public, max-age=31557600, s-maxage=0"
|
37
|
+
}
|
31
38
|
"""
|
32
39
|
Given httpthumbnailer server is running at http://localhost:3100/
|
33
40
|
|
@@ -63,3 +70,20 @@ Feature: Store limited original image in S3 and thumbnail based on request
|
|
63
70
|
Then response body will contain PNG image of size 50x50
|
64
71
|
And that image pixel at 2x2 should be of color green
|
65
72
|
|
73
|
+
@s3-store-and-thumbnail @v2
|
74
|
+
Scenario: Getting thumbnail to spec based on uploaded S3 image - v2
|
75
|
+
Given test.jpg file content is stored in S3 under 4006450256177f4a.jpg
|
76
|
+
When I do GET request http://localhost:3000/thumbnail/v2/pad/50/50/4006450256177f4a.jpg
|
77
|
+
Then response status will be 200
|
78
|
+
And response content type will be image/png
|
79
|
+
Then response body will contain PNG image of size 50x50
|
80
|
+
|
81
|
+
@s3-store-and-thumbnail @v2
|
82
|
+
Scenario: Getting thumbnail to spec based on uploaded S3 image - with options passed
|
83
|
+
Given test.jpg file content is stored in S3 under 4006450256177f4a.jpg
|
84
|
+
When I do GET request http://localhost:3000/thumbnail/v2/pad/50/50/4006450256177f4a.jpg?background-color=green
|
85
|
+
Then response status will be 200
|
86
|
+
And response content type will be image/png
|
87
|
+
Then response body will contain PNG image of size 50x50
|
88
|
+
And that image pixel at 2x2 should be of color green
|
89
|
+
|
data/httpimagestore.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "httpimagestore"
|
8
|
-
s.version = "1.
|
8
|
+
s.version = "1.1.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Jakub Pastuszek"]
|
12
|
-
s.date = "2013-07-
|
12
|
+
s.date = "2013-07-24"
|
13
13
|
s.description = "Thumbnails images using httpthumbnailer and stored data on HTTP server (S3)"
|
14
14
|
s.email = "jpastuszek@gmail.com"
|
15
15
|
s.executables = ["httpimagestore"]
|
@@ -14,6 +14,8 @@ module Configuration
|
|
14
14
|
end
|
15
15
|
|
16
16
|
class RequestState
|
17
|
+
include ClassLogging
|
18
|
+
|
17
19
|
class Images < Hash
|
18
20
|
def initialize(memory_limit)
|
19
21
|
@memory_limit = memory_limit
|
@@ -31,10 +33,18 @@ module Configuration
|
|
31
33
|
fetch(name){|image_name| raise ImageNotLoadedError.new(image_name)}
|
32
34
|
end
|
33
35
|
end
|
34
|
-
|
35
|
-
def initialize(body = '',
|
36
|
+
|
37
|
+
def initialize(body = '', matches = {}, path = '', query_string = {}, memory_limit = MemoryLimit.new)
|
38
|
+
@locals = {}
|
39
|
+
@locals.merge! query_string
|
40
|
+
@locals[:path] = path
|
41
|
+
@locals.merge! matches
|
42
|
+
@locals[:query_string_options] = query_string.sort.map{|kv| kv.join(':')}.join(',')
|
43
|
+
log.debug "processing request with body length: #{body.bytesize} bytes and locals: #{@locals} "
|
44
|
+
|
45
|
+
@locals[:body] = body
|
46
|
+
|
36
47
|
@images = Images.new(memory_limit)
|
37
|
-
@locals = {body: body}.merge(locals)
|
38
48
|
@memory_limit = memory_limit
|
39
49
|
@output_callback = nil
|
40
50
|
end
|
@@ -234,6 +244,8 @@ module Configuration
|
|
234
244
|
log.warn 'no handlers configured' if configuration.handlers.empty?
|
235
245
|
end
|
236
246
|
end
|
247
|
+
RequestState.logger = Global.logger_for(RequestState)
|
248
|
+
|
237
249
|
Global.register_node_parser Handler
|
238
250
|
end
|
239
251
|
|
@@ -146,7 +146,7 @@ describe Configuration do
|
|
146
146
|
|
147
147
|
describe 'memory limit' do
|
148
148
|
let :state do
|
149
|
-
Configuration::RequestState.new('abc', {}, MemoryLimit.new(1))
|
149
|
+
Configuration::RequestState.new('abc', {}, '', {}, MemoryLimit.new(1))
|
150
150
|
end
|
151
151
|
|
152
152
|
it 'should rais MemoryLimit::MemoryLimitedExceededError error if limit exceeded runing file sourcing' do
|
@@ -214,7 +214,7 @@ describe Configuration do
|
|
214
214
|
end
|
215
215
|
|
216
216
|
let :state do
|
217
|
-
Configuration::RequestState.new('abc', list: 'input1,input3')
|
217
|
+
Configuration::RequestState.new('abc', {list: 'input1,input3'})
|
218
218
|
end
|
219
219
|
|
220
220
|
it 'should mark stores to ib included when image name match if-image-name-on list' do
|
@@ -47,7 +47,7 @@ describe Configuration do
|
|
47
47
|
|
48
48
|
it 'should free memory limit if overwritting image' do
|
49
49
|
limit = MemoryLimit.new(2)
|
50
|
-
request_state = Configuration::RequestState.new('abc', {}, limit)
|
50
|
+
request_state = Configuration::RequestState.new('abc', {}, '', {}, limit)
|
51
51
|
|
52
52
|
limit.borrow 1
|
53
53
|
request_state.images['test'] = Configuration::Image.new('x')
|
@@ -155,7 +155,7 @@ describe Configuration do
|
|
155
155
|
|
156
156
|
describe 'conditional inclusion support' do
|
157
157
|
let :state do
|
158
|
-
Configuration::RequestState.new('abc', list: 'input,image2')
|
158
|
+
Configuration::RequestState.new('abc', {list: 'input,image2'})
|
159
159
|
end
|
160
160
|
|
161
161
|
subject do
|
@@ -68,7 +68,7 @@ else
|
|
68
68
|
|
69
69
|
describe Configuration::S3Source do
|
70
70
|
let :state do
|
71
|
-
Configuration::RequestState.new('abc', test_image: 'test.jpg')
|
71
|
+
Configuration::RequestState.new('abc', {test_image: 'test.jpg'})
|
72
72
|
end
|
73
73
|
|
74
74
|
subject do
|
@@ -259,7 +259,7 @@ else
|
|
259
259
|
|
260
260
|
describe 'memory limit' do
|
261
261
|
let :state do
|
262
|
-
Configuration::RequestState.new('abc', {test_image: 'test.jpg'}, MemoryLimit.new(10))
|
262
|
+
Configuration::RequestState.new('abc', {test_image: 'test.jpg'}, '', {}, MemoryLimit.new(10))
|
263
263
|
end
|
264
264
|
|
265
265
|
it 'should raise MemoryLimit::MemoryLimitedExceededError when sourcing bigger image than limit' do
|
@@ -272,7 +272,7 @@ else
|
|
272
272
|
|
273
273
|
describe Configuration::S3Store do
|
274
274
|
let :state do
|
275
|
-
Configuration::RequestState.new(@test_data, test_image: 'test_out.jpg')
|
275
|
+
Configuration::RequestState.new(@test_data, {test_image: 'test_out.jpg'})
|
276
276
|
end
|
277
277
|
|
278
278
|
subject do
|
@@ -435,7 +435,7 @@ else
|
|
435
435
|
|
436
436
|
describe 'conditional inclusion support' do
|
437
437
|
let :state do
|
438
|
-
Configuration::RequestState.new(@test_data, test_image: 'test_out.jpg', list: 'input,input2')
|
438
|
+
Configuration::RequestState.new(@test_data, {test_image: 'test_out.jpg', list: 'input,input2'})
|
439
439
|
end
|
440
440
|
|
441
441
|
subject do
|
@@ -136,11 +136,12 @@ describe Configuration do
|
|
136
136
|
let :state do
|
137
137
|
Configuration::RequestState.new(
|
138
138
|
(support_dir + 'compute.jpg').read,
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
139
|
+
{
|
140
|
+
operation: 'pad',
|
141
|
+
width: '10',
|
142
|
+
height: '10',
|
143
|
+
options: 'background-color:green'
|
144
|
+
}
|
144
145
|
)
|
145
146
|
end
|
146
147
|
|
@@ -187,11 +188,14 @@ describe Configuration do
|
|
187
188
|
let :state do
|
188
189
|
Configuration::RequestState.new(
|
189
190
|
(support_dir + 'compute.jpg').read,
|
190
|
-
{
|
191
|
-
|
192
|
-
|
193
|
-
|
194
|
-
|
191
|
+
{
|
192
|
+
operation: 'pad',
|
193
|
+
width: '10',
|
194
|
+
height: '10',
|
195
|
+
options: 'background-color:green'
|
196
|
+
},
|
197
|
+
'',
|
198
|
+
{},
|
195
199
|
MemoryLimit.new(10)
|
196
200
|
)
|
197
201
|
end
|
@@ -207,11 +211,12 @@ describe Configuration do
|
|
207
211
|
it 'should raise Thumbnail::ThumbnailingError on realization of bad thumbnail sepc' do
|
208
212
|
state = Configuration::RequestState.new(
|
209
213
|
(support_dir + 'compute.jpg').read,
|
210
|
-
|
211
|
-
|
212
|
-
|
213
|
-
|
214
|
-
|
214
|
+
{
|
215
|
+
operation: 'pad',
|
216
|
+
width: '0',
|
217
|
+
height: '10',
|
218
|
+
options: 'background-color:green'
|
219
|
+
}
|
215
220
|
)
|
216
221
|
|
217
222
|
expect {
|
@@ -293,11 +298,14 @@ describe Configuration do
|
|
293
298
|
let :state do
|
294
299
|
Configuration::RequestState.new(
|
295
300
|
(support_dir + 'compute.jpg').read,
|
296
|
-
{
|
297
|
-
|
298
|
-
|
299
|
-
|
300
|
-
|
301
|
+
{
|
302
|
+
operation: 'pad',
|
303
|
+
width: '10',
|
304
|
+
height: '10',
|
305
|
+
options: 'background-color:green'
|
306
|
+
},
|
307
|
+
'',
|
308
|
+
{},
|
301
309
|
MemoryLimit.new(10)
|
302
310
|
)
|
303
311
|
end
|
@@ -325,12 +333,13 @@ describe Configuration do
|
|
325
333
|
let :state do
|
326
334
|
Configuration::RequestState.new(
|
327
335
|
(support_dir + 'compute.jpg').read,
|
328
|
-
|
329
|
-
|
330
|
-
|
331
|
-
|
332
|
-
|
333
|
-
|
336
|
+
{
|
337
|
+
operation: 'pad',
|
338
|
+
width: '10',
|
339
|
+
height: '10',
|
340
|
+
options: 'background-color:green',
|
341
|
+
list: 'small,padded'
|
342
|
+
}
|
334
343
|
)
|
335
344
|
end
|
336
345
|
|
@@ -346,11 +355,12 @@ describe Configuration do
|
|
346
355
|
it 'should raise Thumbnail::ThumbnailingError on realization of bad thumbnail sepc' do
|
347
356
|
state = Configuration::RequestState.new(
|
348
357
|
(support_dir + 'compute.jpg').read,
|
349
|
-
|
350
|
-
|
351
|
-
|
352
|
-
|
353
|
-
|
358
|
+
{
|
359
|
+
operation: 'pad',
|
360
|
+
width: '0',
|
361
|
+
height: '10',
|
362
|
+
options: 'background-color:green'
|
363
|
+
}
|
354
364
|
)
|
355
365
|
|
356
366
|
subject.handlers[0].image_sources[0].realize(state)
|
@@ -378,7 +388,9 @@ describe Configuration do
|
|
378
388
|
let :state do
|
379
389
|
Configuration::RequestState.new(
|
380
390
|
(support_dir + 'compute.jpg').read,
|
381
|
-
|
391
|
+
{
|
392
|
+
list: 'thumbnail1,input4,thumbnail5,input6'
|
393
|
+
}
|
382
394
|
)
|
383
395
|
end
|
384
396
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: httpimagestore
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2013-07-
|
12
|
+
date: 2013-07-24 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: unicorn-cuba-base
|
@@ -291,7 +291,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
291
291
|
version: '0'
|
292
292
|
segments:
|
293
293
|
- 0
|
294
|
-
hash:
|
294
|
+
hash: 1029095012750983706
|
295
295
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
296
296
|
none: false
|
297
297
|
requirements:
|