httpimagestore 1.6.0 → 1.7.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +4 -3
- data/Gemfile.lock +12 -16
- data/README.md +73 -0
- data/VERSION +1 -1
- data/bin/httpimagestore +7 -7
- data/features/data-uri.feature +55 -0
- data/features/error-reporting.feature +21 -3
- data/features/source-failover.feature +71 -0
- data/features/step_definitions/httpimagestore_steps.rb +27 -12
- data/features/storage.feature +26 -25
- data/features/support/tiny.png +0 -0
- data/features/xid-forwarding.feature +49 -0
- data/httpimagestore.gemspec +19 -23
- data/lib/httpimagestore/configuration/file.rb +6 -0
- data/lib/httpimagestore/configuration/handler.rb +4 -1
- data/lib/httpimagestore/configuration/identify.rb +2 -2
- data/lib/httpimagestore/configuration/output.rb +20 -1
- data/lib/httpimagestore/configuration/s3.rb +15 -9
- data/lib/httpimagestore/configuration/source_failover.rb +51 -0
- data/lib/httpimagestore/configuration/thumbnailer.rb +7 -7
- data/lib/httpimagestore/error_reporter.rb +9 -1
- data/lib/httpimagestore/ruby_string_template.rb +12 -7
- data/load_test/load_test.jmx +50 -77
- data/load_test/thumbnail_specs_v2.csv +10 -0
- data/spec/configuration_file_spec.rb +27 -2
- data/spec/configuration_identify_spec.rb +25 -2
- data/spec/configuration_s3_spec.rb +29 -3
- data/spec/configuration_source_failover_spec.rb +101 -0
- data/spec/configuration_thumbnailer_spec.rb +63 -8
- data/spec/ruby_string_template_spec.rb +4 -0
- data/spec/support/full.cfg +167 -33
- metadata +19 -23
- data/.idea/.name +0 -1
- data/.idea/.rakeTasks +0 -7
- data/.idea/codeStyleSettings.xml +0 -13
- data/.idea/dictionaries/wcc.xml +0 -8
- data/.idea/encodings.xml +0 -5
- data/.idea/httpimagestore.iml +0 -69
- data/.idea/jenkinsSettings.xml +0 -9
- data/.idea/misc.xml +0 -5
- data/.idea/modules.xml +0 -9
- data/.idea/scopes/scope_settings.xml +0 -5
- data/.idea/vcs.xml +0 -7
data/Gemfile
CHANGED
@@ -1,9 +1,9 @@
|
|
1
1
|
source "http://rubygems.org"
|
2
2
|
ruby "1.9.3"
|
3
3
|
|
4
|
-
gem "unicorn-cuba-base", "~> 1.
|
4
|
+
gem "unicorn-cuba-base", "~> 1.2.0"
|
5
5
|
#gem "unicorn-cuba-base", path: "../unicorn-cuba-base"
|
6
|
-
gem "httpthumbnailer-client", "~> 1.
|
6
|
+
gem "httpthumbnailer-client", "~> 1.2.0"
|
7
7
|
#gem "httpthumbnailer-client", path: "../httpthumbnailer-client"
|
8
8
|
gem "aws-sdk", "~> 1.10"
|
9
9
|
gem "mime-types", "~> 1.17"
|
@@ -20,6 +20,7 @@ group :development do
|
|
20
20
|
gem "rdoc", "~> 3.9"
|
21
21
|
gem "daemon", "~> 1"
|
22
22
|
gem "prawn", "= 0.8.4"
|
23
|
-
gem "httpthumbnailer", path: '../httpthumbnailer'
|
23
|
+
#gem "httpthumbnailer", path: '../httpthumbnailer'
|
24
|
+
gem "httpthumbnailer", "~> 1.2.0"
|
24
25
|
end
|
25
26
|
|
data/Gemfile.lock
CHANGED
@@ -1,10 +1,3 @@
|
|
1
|
-
PATH
|
2
|
-
remote: ../httpthumbnailer
|
3
|
-
specs:
|
4
|
-
httpthumbnailer (1.1.1)
|
5
|
-
rmagick (~> 2)
|
6
|
-
unicorn-cuba-base (~> 1.1.1)
|
7
|
-
|
8
1
|
GEM
|
9
2
|
remote: http://rubygems.org/
|
10
3
|
specs:
|
@@ -15,7 +8,7 @@ GEM
|
|
15
8
|
uuidtools (~> 2.1)
|
16
9
|
builder (3.2.2)
|
17
10
|
cli (1.3.1)
|
18
|
-
cuba (3.1.
|
11
|
+
cuba (3.1.1)
|
19
12
|
rack
|
20
13
|
cucumber (1.3.8)
|
21
14
|
builder (>= 2.1.2)
|
@@ -42,7 +35,10 @@ GEM
|
|
42
35
|
highline (1.6.20)
|
43
36
|
httpauth (0.2.0)
|
44
37
|
httpclient (2.3.4.1)
|
45
|
-
httpthumbnailer
|
38
|
+
httpthumbnailer (1.2.0)
|
39
|
+
rmagick (~> 2)
|
40
|
+
unicorn-cuba-base
|
41
|
+
httpthumbnailer-client (1.2.0)
|
46
42
|
cli (~> 1.3)
|
47
43
|
httpclient (>= 2.3)
|
48
44
|
multipart-parser (~> 0.1.1)
|
@@ -58,7 +54,7 @@ GEM
|
|
58
54
|
json (1.8.1)
|
59
55
|
jwt (0.1.8)
|
60
56
|
multi_json (>= 1.5)
|
61
|
-
kgio (2.
|
57
|
+
kgio (2.9.2)
|
62
58
|
mime-types (1.25)
|
63
59
|
msgpack (0.5.5)
|
64
60
|
multi_json (1.8.2)
|
@@ -82,7 +78,7 @@ GEM
|
|
82
78
|
prawn-layout (0.8.4)
|
83
79
|
prawn-security (0.8.4)
|
84
80
|
rack (1.5.2)
|
85
|
-
raindrops (0.
|
81
|
+
raindrops (0.13.0)
|
86
82
|
rake (10.1.0)
|
87
83
|
rdoc (3.12.2)
|
88
84
|
json (~> 1.4)
|
@@ -97,11 +93,11 @@ GEM
|
|
97
93
|
rspec-mocks (2.14.3)
|
98
94
|
ruby-ip (0.9.1)
|
99
95
|
sdl4r (0.9.11)
|
100
|
-
unicorn (4.
|
96
|
+
unicorn (4.8.2)
|
101
97
|
kgio (~> 2.6)
|
102
98
|
rack
|
103
99
|
raindrops (~> 0.7)
|
104
|
-
unicorn-cuba-base (1.
|
100
|
+
unicorn-cuba-base (1.2.0)
|
105
101
|
cli (~> 1.3)
|
106
102
|
cuba (~> 3.0)
|
107
103
|
facter (~> 1.6.11)
|
@@ -118,8 +114,8 @@ DEPENDENCIES
|
|
118
114
|
cucumber
|
119
115
|
daemon (~> 1)
|
120
116
|
httpclient (>= 2.3)
|
121
|
-
httpthumbnailer
|
122
|
-
httpthumbnailer-client (~> 1.
|
117
|
+
httpthumbnailer (~> 1.2.0)
|
118
|
+
httpthumbnailer-client (~> 1.2.0)
|
123
119
|
jeweler (~> 1.8.4)
|
124
120
|
mime-types (~> 1.17)
|
125
121
|
msgpack (~> 0.5)
|
@@ -127,4 +123,4 @@ DEPENDENCIES
|
|
127
123
|
rdoc (~> 3.9)
|
128
124
|
rspec (~> 2.13)
|
129
125
|
sdl4r (~> 0.9)
|
130
|
-
unicorn-cuba-base (~> 1.
|
126
|
+
unicorn-cuba-base (~> 1.2.0)
|
data/README.md
CHANGED
@@ -17,6 +17,14 @@ It is using [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) as
|
|
17
17
|
|
18
18
|
## Changelog
|
19
19
|
|
20
|
+
### 1.7.0
|
21
|
+
* `output_data_uri_image` support
|
22
|
+
* `source_failover` support
|
23
|
+
* fixed possible data leak with nested template processing
|
24
|
+
* reporting `400 Bad Request` for non UTF-8 characters encoded in URLs
|
25
|
+
* syslog logging
|
26
|
+
* transaction ID tracking
|
27
|
+
|
20
28
|
### 1.6.0
|
21
29
|
* `output_store_path` and `output_store_url` `path` argument support
|
22
30
|
* encoding resulting file storage URL
|
@@ -261,6 +269,33 @@ get "small" {
|
|
261
269
|
|
262
270
|
Requesting `/small` URI will result with image fetched from S3 bucket `mybucket` and key `myimage.jpg` and named `original`.
|
263
271
|
|
272
|
+
|
273
|
+
#### source_failover
|
274
|
+
|
275
|
+
This statement can be used to wrap around other sources.
|
276
|
+
HTTP Image Store will try wrapped sources one by one until one will succeed.
|
277
|
+
If all sources fail status code for error from the first source will be returned.
|
278
|
+
|
279
|
+
This source has no options.
|
280
|
+
|
281
|
+
Example:
|
282
|
+
|
283
|
+
```sdl
|
284
|
+
s3 key="AIAITCKMELYWQZPJP7HQ" secret="V37lCu0F48Tv9s7QVqIT/sLf/wwqhNSB4B0Em7Ei" ssl=false
|
285
|
+
path "myimage" "myimage.jpg"
|
286
|
+
|
287
|
+
get "small" {
|
288
|
+
source_failover {
|
289
|
+
source_s3 "original" bucket="mybucket" path="myimage"
|
290
|
+
source_s3 "original" bucket="mybucket-backup" path="myimage"
|
291
|
+
}
|
292
|
+
}
|
293
|
+
```
|
294
|
+
|
295
|
+
Requesting `/small` URI will result with image fetched from S3 bucket `mybucket`.
|
296
|
+
If `mybucket` does not contain key `myimage.jpg` than `mybucket-backup` will be tried.
|
297
|
+
If `mybucket-backup` does not contain the key than `404` status code will be returned.
|
298
|
+
|
264
299
|
### API endpoint processing operations
|
265
300
|
|
266
301
|
#### thumbnail
|
@@ -440,6 +475,38 @@ put "test" {
|
|
440
475
|
|
441
476
|
The output will contain the posted image with `Content-Type` header set to `application/octet-stream` and `Cache-Control` to `public, max-age=999, s-maxage=666`.
|
442
477
|
|
478
|
+
#### output_data_uri_image
|
479
|
+
|
480
|
+
This statement will produce `200 OK` response containing base64 encoded image data within data URI in format:
|
481
|
+
```
|
482
|
+
data:<image mime type>;base64,<base64 encoded image data>
|
483
|
+
```
|
484
|
+
No tailing new line/line feed is added to the output.
|
485
|
+
|
486
|
+
This may be used to include images directly in HTML pages or CSS.
|
487
|
+
|
488
|
+
The `Content-Type` of the response will be `text/uri-list`.
|
489
|
+
Image mime type will be provided within data URI and must be known before output is constructed or `500 Internal Server Error` will be served.
|
490
|
+
|
491
|
+
Arguments:
|
492
|
+
|
493
|
+
1. image name - image to be sent in response body
|
494
|
+
|
495
|
+
Options:
|
496
|
+
|
497
|
+
* `cache-control` - value of response `Cache-Control` header can be specified with this option
|
498
|
+
|
499
|
+
Example:
|
500
|
+
|
501
|
+
```sdl
|
502
|
+
put "test" {
|
503
|
+
identify "input"
|
504
|
+
output_data_uri_image "input" cache-control="public, max-age=31557600, s-maxage=0"
|
505
|
+
}
|
506
|
+
```
|
507
|
+
|
508
|
+
The output will contain base64 encoded posted image within data URI with `Content-Type` header set to `text/uri-list` and `Cache-Control` to `public, max-age=999, s-maxage=666`.
|
509
|
+
|
443
510
|
#### output_store_path
|
444
511
|
|
445
512
|
This statement will output actual storage path on the file system (without root) or S3 key under witch the image was stored.
|
@@ -901,6 +968,12 @@ If running as root you can use `--user` option to specify user with whose privil
|
|
901
968
|
|
902
969
|
Additionally `httpimagestore` will log requests in [common NCSA format](http://en.wikipedia.org/wiki/Common_Log_Format) to `httpimagestore_access.log` file. Use `--access-log-file` option to change location of access log.
|
903
970
|
|
971
|
+
Syslog logging can be enabled with `--syslog-facility` option followed by name of syslog facility to use. When enabled log files are not created and both application logs and access logs are sent to syslog.
|
972
|
+
Access logs will gain meta information that will include `type="http-access"` that can be used to filter access log entries out from application log entries.
|
973
|
+
|
974
|
+
With `--xid-header` option name of HTTP request header can be specified. Value of this header will be logged in meta information tag `xid` along side all request related log entries.
|
975
|
+
Named header will be passed down with requests to HTTP Thumbnailer for grater traceability.
|
976
|
+
|
904
977
|
### Running with nginx
|
905
978
|
|
906
979
|
[nginx](http://nginx.org) if configured properly will buffer incoming requests before sending them to the backend and server response before sending them to client.
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.7.0
|
data/bin/httpimagestore
CHANGED
@@ -14,13 +14,13 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
|
|
14
14
|
end
|
15
15
|
|
16
16
|
settings do |settings|
|
17
|
-
|
17
|
+
Controller.settings[:config_file] = settings.config
|
18
18
|
end
|
19
19
|
|
20
20
|
main do |settings|
|
21
21
|
require 'httpimagestore/error_reporter'
|
22
22
|
|
23
|
-
class HTTPImageStore <
|
23
|
+
class HTTPImageStore < Controller
|
24
24
|
extend Stats
|
25
25
|
def_stats(
|
26
26
|
:workers,
|
@@ -85,18 +85,18 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
|
|
85
85
|
names.zip(args)
|
86
86
|
.each do |name, value|
|
87
87
|
fail "name should be a symbol" unless name.is_a? Symbol
|
88
|
-
matches[name] = URI.
|
88
|
+
matches[name] = URI.utf_decode(value)
|
89
89
|
end
|
90
90
|
|
91
91
|
# decode remaining URI components
|
92
92
|
path = (env["PATH_INFO"][1..-1] || '').split('/').map do |part|
|
93
|
-
URI.
|
93
|
+
URI.utf_decode(part)
|
94
94
|
end.join('/')
|
95
95
|
|
96
96
|
# query string already decoded by Rack
|
97
97
|
query_string = req.GET
|
98
98
|
|
99
|
-
state = Configuration::RequestState.new(req.body.read, matches, path, query_string, memory_limit)
|
99
|
+
state = Configuration::RequestState.new(req.body.read, matches, path, query_string, memory_limit, env['xid'] || {})
|
100
100
|
|
101
101
|
handler.sources.each do |source|
|
102
102
|
source.realize(state) unless source.respond_to? :excluded? and source.excluded?(state)
|
@@ -133,8 +133,8 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
|
|
133
133
|
|
134
134
|
require 'httpimagestore/configuration'
|
135
135
|
|
136
|
-
# connect Scope tree with
|
137
|
-
Configuration::Scope.logger =
|
136
|
+
# connect Scope tree with Controller logger
|
137
|
+
Configuration::Scope.logger = Controller.logger_for(Configuration::Scope)
|
138
138
|
|
139
139
|
# load builin supported set
|
140
140
|
require 'httpimagestore/configuration/path'
|
@@ -0,0 +1,55 @@
|
|
1
|
+
Feature: Data URI style image output
|
2
|
+
HTTP Image Store can provide images base64 encoded in data URI format so they can be included directly in HTML pages.
|
3
|
+
|
4
|
+
Background:
|
5
|
+
Given httpthumbnailer server is running at http://localhost:3100/health_check
|
6
|
+
Given httpimagestore server is running at http://localhost:3000/ with the following configuration
|
7
|
+
"""
|
8
|
+
post "data-uri" "no-identify" {
|
9
|
+
output_data_uri_image "input"
|
10
|
+
}
|
11
|
+
|
12
|
+
post "data-uri" "cache-control" {
|
13
|
+
identify "input"
|
14
|
+
output_data_uri_image "input" cache-control="public, max-age=31557600, s-maxage=0"
|
15
|
+
}
|
16
|
+
|
17
|
+
post "data-uri" {
|
18
|
+
identify "input"
|
19
|
+
output_data_uri_image "input"
|
20
|
+
}
|
21
|
+
"""
|
22
|
+
|
23
|
+
@data-uri
|
24
|
+
Scenario: Getting image encoded in data URI scheme with base64 encoding
|
25
|
+
Given tiny.png file content as request body
|
26
|
+
When I do POST request http://localhost:3000/data-uri
|
27
|
+
Then response status will be 200
|
28
|
+
And response content type will be text/uri-list
|
29
|
+
And response body will be
|
30
|
+
"""
|
31
|
+
data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAgUlEQVQI10XNsQ2CUBhF4XOTfwJeog0U2kLJNlZOxQ4swBLGwkhhY6MNRkic4Fq8ENqTLznqhuvlOY2v5f6egaYs6iq1x30YAENdJskgEBAYUFMlYSMw2CYk2sMOkTFgI6Fp/mEpt7Ua69Sf2TRgLESMt8dW5bwE4vtZvOK8xkj8Ac8BL8dFQipPAAAAAElFTkSuQmCC
|
32
|
+
"""
|
33
|
+
|
34
|
+
@data-uri
|
35
|
+
Scenario: Getting image encoded in data URI scheme with base64 encoding - cache-control should be supported
|
36
|
+
Given tiny.png file content as request body
|
37
|
+
When I do POST request http://localhost:3000/data-uri/cache-control
|
38
|
+
Then response status will be 200
|
39
|
+
And response content type will be text/uri-list
|
40
|
+
And response Cache-Control will be public, max-age=31557600, s-maxage=0
|
41
|
+
And response body will be
|
42
|
+
"""
|
43
|
+
data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAgUlEQVQI10XNsQ2CUBhF4XOTfwJeog0U2kLJNlZOxQ4swBLGwkhhY6MNRkic4Fq8ENqTLznqhuvlOY2v5f6egaYs6iq1x30YAENdJskgEBAYUFMlYSMw2CYk2sMOkTFgI6Fp/mEpt7Ua69Sf2TRgLESMt8dW5bwE4vtZvOK8xkj8Ac8BL8dFQipPAAAAAElFTkSuQmCC
|
44
|
+
"""
|
45
|
+
|
46
|
+
@data-uri
|
47
|
+
Scenario: Error 500 served when data URI output is used on unidentified image
|
48
|
+
Given tiny.png file content as request body
|
49
|
+
When I do POST request http://localhost:3000/data-uri/no-identify
|
50
|
+
Then response status will be 500
|
51
|
+
And response content type will be text/plain
|
52
|
+
And response body will be CRLF ended lines
|
53
|
+
"""
|
54
|
+
image 'input' needs to be identified first to be used in data URI output
|
55
|
+
"""
|
@@ -1,6 +1,5 @@
|
|
1
|
-
Feature:
|
2
|
-
|
3
|
-
This configuration should be mostly compatible with pre v1.0 release.
|
1
|
+
Feature: Error handling
|
2
|
+
API should provide different status codes with text/plain description body for different error modes.
|
4
3
|
|
5
4
|
Background:
|
6
5
|
Given S3 settings in AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_TEST_BUCKET environment variables
|
@@ -9,9 +8,11 @@ Feature: Image list based thumbnailing and S3 storage
|
|
9
8
|
"""
|
10
9
|
s3 key="@AWS_ACCESS_KEY_ID@" secret="@AWS_SECRET_ACCESS_KEY@" ssl=false
|
11
10
|
|
11
|
+
path "uri_part" "/thumbnails/#{input_digest}/#{name}.#{image_mime_extension}"
|
12
12
|
path "structured-name" "#{dirname}/#{input_digest}/#{basename}-#{image_name}.#{image_mime_extension}"
|
13
13
|
path "missing" "blah"
|
14
14
|
path "zero" "zero"
|
15
|
+
path "hash" "#{input_digest}"
|
15
16
|
|
16
17
|
put "multipart" ":name_list" {
|
17
18
|
thumbnail "input" {
|
@@ -33,6 +34,12 @@ Feature: Image list based thumbnailing and S3 storage
|
|
33
34
|
thumbnail "input" "bad_opts" operation="crop" width=128 height=128 options="foo=bar" if-image-name-on="#{name_list}"
|
34
35
|
}
|
35
36
|
|
37
|
+
post "filename" "/(?<name>.+?)(\\.[^\\.]+)?$/" {
|
38
|
+
identify "input"
|
39
|
+
store_s3 "input" bucket="@AWS_S3_TEST_BUCKET@" path="hash" cache-root="/tmp"
|
40
|
+
output_store_path "input" path="uri_part"
|
41
|
+
}
|
42
|
+
|
36
43
|
get "s3" {
|
37
44
|
source_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="missing"
|
38
45
|
}
|
@@ -199,6 +206,17 @@ Feature: Image list based thumbnailing and S3 storage
|
|
199
206
|
thumbnailing of 'input' into 'superlarge' failed: image too large: cache resources exhausted
|
200
207
|
"""
|
201
208
|
|
209
|
+
@error-reporting
|
210
|
+
Scenario: Bad URI encoding
|
211
|
+
Given test.jpg file content as request body
|
212
|
+
When I do POST request http://localhost:3000/filename/hello%e9world.jpg
|
213
|
+
Then response status will be 400
|
214
|
+
And response content type will be text/plain
|
215
|
+
And response body will be CRLF ended lines like
|
216
|
+
"""
|
217
|
+
invalid UTF-8 encoding in URI: "hello\\xE9world"
|
218
|
+
"""
|
219
|
+
|
202
220
|
@error-reporting
|
203
221
|
Scenario: Zero body length
|
204
222
|
Given test.empty file content as request body
|
@@ -0,0 +1,71 @@
|
|
1
|
+
Feature: More than one source can be tried in canse of source problem
|
2
|
+
Sources can be grouped under source_failover group.
|
3
|
+
HTTP Image Store will try each source until working one is found.
|
4
|
+
|
5
|
+
Background:
|
6
|
+
Given S3 settings in AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_TEST_BUCKET environment variables
|
7
|
+
Given httpimagestore server is running at http://localhost:3000/ with the following configuration
|
8
|
+
"""
|
9
|
+
s3 key="@AWS_ACCESS_KEY_ID@" secret="@AWS_SECRET_ACCESS_KEY@" ssl=false
|
10
|
+
|
11
|
+
path "test-file" "test.file"
|
12
|
+
path "bogus" "bogus"
|
13
|
+
|
14
|
+
put "input" {
|
15
|
+
store_s3 "input" bucket="@AWS_S3_TEST_BUCKET@" path="test-file"
|
16
|
+
}
|
17
|
+
|
18
|
+
get "s3_fail" {
|
19
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="bogus"
|
20
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="test-file"
|
21
|
+
output_image "image"
|
22
|
+
}
|
23
|
+
|
24
|
+
get "s3_failover" {
|
25
|
+
source_failover {
|
26
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="bogus"
|
27
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="test-file"
|
28
|
+
}
|
29
|
+
output_image "image"
|
30
|
+
}
|
31
|
+
|
32
|
+
get "s3_all_fail" {
|
33
|
+
source_failover {
|
34
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="bogus"
|
35
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="bogus"
|
36
|
+
}
|
37
|
+
output_image "image"
|
38
|
+
}
|
39
|
+
"""
|
40
|
+
|
41
|
+
@source-failover
|
42
|
+
Scenario: Sourcing S3 object without failover
|
43
|
+
Given there is no test.file file in S3 bucket
|
44
|
+
Given test.jpg file content as request body
|
45
|
+
When I do PUT request http://localhost:3000/input
|
46
|
+
Then response status will be 200
|
47
|
+
When I do GET request http://localhost:3000/s3_fail
|
48
|
+
Then response status will be 404
|
49
|
+
|
50
|
+
@source-failover
|
51
|
+
Scenario: Sourcing S3 object with failover
|
52
|
+
Given there is no test.file file in S3 bucket
|
53
|
+
Given test.jpg file content as request body
|
54
|
+
When I do PUT request http://localhost:3000/input
|
55
|
+
Then response status will be 200
|
56
|
+
When I do GET request http://localhost:3000/s3_failover
|
57
|
+
Then response status will be 200
|
58
|
+
|
59
|
+
@source-failover
|
60
|
+
Scenario: Sourcing S3 object with all sources failing
|
61
|
+
Given there is no test.file file in S3 bucket
|
62
|
+
Given test.jpg file content as request body
|
63
|
+
When I do PUT request http://localhost:3000/input
|
64
|
+
Then response status will be 200
|
65
|
+
When I do GET request http://localhost:3000/s3_all_fail
|
66
|
+
Then response status will be 404
|
67
|
+
And response content type will be text/plain
|
68
|
+
And response body will be CRLF ended lines
|
69
|
+
"""
|
70
|
+
all sources failed: S3Source[image_name: 'image' bucket: 'httpimagestoretest' prefix: '' path_spec: 'bogus'](Configuration::S3NoSuchKeyError: S3 bucket 'httpimagestoretest' does not contain key 'bogus'), S3Source[image_name: 'image' bucket: 'httpimagestoretest' prefix: '' path_spec: 'bogus'](Configuration::S3NoSuchKeyError: S3 bucket 'httpimagestoretest' does not contain key 'bogus')
|
71
|
+
"""
|
@@ -4,7 +4,7 @@ end
|
|
4
4
|
|
5
5
|
Given /httpimagestore server is running at (.*) with the following configuration/ do |url, config|
|
6
6
|
$temp_dir = Pathname.new(Dir.mktmpdir) unless $temp_dir
|
7
|
-
|
7
|
+
|
8
8
|
cfile = $temp_dir + Digest.hexencode(Digest::MD5.digest(config))
|
9
9
|
cfile.open('w') do |io|
|
10
10
|
io.write(config.replace_s3_variables)
|
@@ -13,7 +13,7 @@ Given /httpimagestore server is running at (.*) with the following configuration
|
|
13
13
|
begin
|
14
14
|
log = support_dir + 'server.log'
|
15
15
|
start_server(
|
16
|
-
"bundle exec #{script('httpimagestore')} -f -d -l #{log} -w 1 #{(@httpimagestore_args ||= []).join(' ')} #{cfile.to_s}",
|
16
|
+
"bundle exec #{script('httpimagestore')} -f -d -x XID -l #{log} -w 1 #{(@httpimagestore_args ||= []).join(' ')} #{cfile.to_s}",
|
17
17
|
'/tmp/httpimagestore.pid',
|
18
18
|
log,
|
19
19
|
url
|
@@ -26,7 +26,7 @@ end
|
|
26
26
|
Given /httpthumbnailer server is running at (.*)/ do |url|
|
27
27
|
log = support_dir + 'thumbniler.log'
|
28
28
|
start_server(
|
29
|
-
"httpthumbnailer -f -d -l #{log} -w 1",
|
29
|
+
"httpthumbnailer -f -d -x XID -l #{log} -a #{log}.access -w 1",
|
30
30
|
'/tmp/httpthumbnailer.pid',
|
31
31
|
log,
|
32
32
|
url
|
@@ -34,7 +34,7 @@ Given /httpthumbnailer server is running at (.*)/ do |url|
|
|
34
34
|
end
|
35
35
|
|
36
36
|
Given /httpthumbnailer server is not running/ do
|
37
|
-
stop_server('/tmp/httpthumbnailer.pid')
|
37
|
+
stop_server('/tmp/httpthumbnailer.pid')
|
38
38
|
end
|
39
39
|
|
40
40
|
Given /httpimagestore log is empty/ do
|
@@ -56,7 +56,7 @@ Given /there is no file (.*)/ do |file|
|
|
56
56
|
end
|
57
57
|
|
58
58
|
Given /S3 settings in AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_TEST_BUCKET environment variables/ do
|
59
|
-
|
59
|
+
|
60
60
|
unless ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY'] and ENV['AWS_S3_TEST_BUCKET']
|
61
61
|
fail "AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY or AWS_S3_TEST_BUCKET environment variables not set"
|
62
62
|
end
|
@@ -83,7 +83,7 @@ end
|
|
83
83
|
|
84
84
|
When /I do (.*) request (.*)/ do |method, uri|
|
85
85
|
@request_body = nil if method == 'GET'
|
86
|
-
@response = HTTPClient.new.request(method,
|
86
|
+
@response = HTTPClient.new.request(method, uri.replace_s3_variables, nil, @request_body, (@request_headers or {}))
|
87
87
|
end
|
88
88
|
|
89
89
|
Then /response status will be (.*)/ do |status|
|
@@ -94,14 +94,22 @@ Then /response content type will be (.*)/ do |content_type|
|
|
94
94
|
@response.header['Content-Type'].first.should == content_type
|
95
95
|
end
|
96
96
|
|
97
|
-
Then /response
|
97
|
+
Then /response Cache-Control will be (.*)/ do |content_type|
|
98
|
+
@response.header['Cache-Control'].first.should == content_type
|
99
|
+
end
|
100
|
+
|
101
|
+
Then /response body will be CRLF ended lines like/ do |body|
|
98
102
|
@response.body.should match(body.replace_s3_variables)
|
99
103
|
@response.body.each_line do |line|
|
100
104
|
line[-2,2].should == "\r\n"
|
101
105
|
end
|
102
106
|
end
|
103
107
|
|
104
|
-
Then /response body will be
|
108
|
+
Then /response body will be$/ do |body|
|
109
|
+
@response.body.should == body.replace_s3_variables
|
110
|
+
end
|
111
|
+
|
112
|
+
Then /response body will be CRLF ended lines$/ do |body|
|
105
113
|
@response.body.should == body.replace_s3_variables.gsub("\n", "\r\n") + "\r\n"
|
106
114
|
end
|
107
115
|
|
@@ -119,7 +127,7 @@ end
|
|
119
127
|
|
120
128
|
Then /(http.*) will contain (.*) image of size (.*)x(.*)/ do |url, format, width, height|
|
121
129
|
data = get(url.replace_s3_variables)
|
122
|
-
|
130
|
+
|
123
131
|
@image.destroy! if @image
|
124
132
|
@image = Magick::Image.from_blob(data).first
|
125
133
|
|
@@ -130,7 +138,7 @@ end
|
|
130
138
|
|
131
139
|
Then /S3 object (.*) will contain (.*) image of size (.*)x(.*)/ do |key, format, width, height|
|
132
140
|
data = @bucket.objects[key].read
|
133
|
-
|
141
|
+
|
134
142
|
@image.destroy! if @image
|
135
143
|
@image = Magick::Image.from_blob(data).first
|
136
144
|
|
@@ -146,7 +154,7 @@ end
|
|
146
154
|
Then /response body will contain (.*) image of size (.*)x(.*)/ do |format, width, height|
|
147
155
|
data = @response.body
|
148
156
|
Pathname.new('/tmp/out.jpg').open('w'){|io| io.write data}
|
149
|
-
|
157
|
+
|
150
158
|
@image.destroy! if @image
|
151
159
|
@image = Magick::Image.from_blob(data).first
|
152
160
|
|
@@ -165,7 +173,7 @@ end
|
|
165
173
|
|
166
174
|
Then /file (.*) will contain (.*) image of size (.*)x(.*)/ do |file, format, width, height|
|
167
175
|
data = Pathname.new(file).read
|
168
|
-
|
176
|
+
|
169
177
|
@image.destroy! if @image
|
170
178
|
@image = Magick::Image.from_blob(data).first
|
171
179
|
|
@@ -174,3 +182,10 @@ Then /file (.*) will contain (.*) image of size (.*)x(.*)/ do |file, format, wid
|
|
174
182
|
@image.rows.should == height.to_i
|
175
183
|
end
|
176
184
|
|
185
|
+
Then /httpimagestore log will contain (.*)/ do |entry|
|
186
|
+
(support_dir + 'server.log').read.should include entry
|
187
|
+
end
|
188
|
+
|
189
|
+
Then /httpthumbnailer log will contain (.*)/ do |entry|
|
190
|
+
(support_dir + 'thumbniler.log').read.should include entry
|
191
|
+
end
|
data/features/storage.feature
CHANGED
@@ -110,31 +110,32 @@ Feature: Storing images under different names
|
|
110
110
|
"""
|
111
111
|
Then file /tmp/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e will contain PNG image of size 50x50
|
112
112
|
|
113
|
-
|
114
|
-
|
115
|
-
|
116
|
-
|
117
|
-
|
118
|
-
|
119
|
-
|
120
|
-
And response
|
121
|
-
|
122
|
-
|
123
|
-
|
124
|
-
|
125
|
-
|
126
|
-
|
127
|
-
|
128
|
-
|
129
|
-
Given
|
130
|
-
|
131
|
-
|
132
|
-
|
133
|
-
And response
|
134
|
-
|
135
|
-
|
136
|
-
|
137
|
-
|
113
|
+
# Following tests depend on libpng/ImageMagick version
|
114
|
+
#@storage @image_digest
|
115
|
+
#Scenario: Posting picture to file system under input data digest
|
116
|
+
#Given there is no file /tmp/b0fe25319ba5909a
|
117
|
+
#Given test.png file content as request body
|
118
|
+
#When I do POST request http://localhost:3000/images/image_digest
|
119
|
+
#Then response status will be 200
|
120
|
+
#And response content type will be text/plain
|
121
|
+
#And response body will be CRLF ended lines
|
122
|
+
#"""
|
123
|
+
#091000e2c0aee836
|
124
|
+
#"""
|
125
|
+
#Then file /tmp/091000e2c0aee836 will contain PNG image of size 50x50
|
126
|
+
|
127
|
+
#@storage @image_sha256
|
128
|
+
#Scenario: Posting picture to file system under input data digest
|
129
|
+
#Given there is no file /tmp/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e
|
130
|
+
#Given test.png file content as request body
|
131
|
+
#When I do POST request http://localhost:3000/images/image_sha256
|
132
|
+
#Then response status will be 200
|
133
|
+
#And response content type will be text/plain
|
134
|
+
#And response body will be CRLF ended lines
|
135
|
+
#"""
|
136
|
+
#091000e2c0aee836fff432c1151faba86d46690c900c0f6355247a353defa37f
|
137
|
+
#"""
|
138
|
+
#Then file /tmp/091000e2c0aee836fff432c1151faba86d46690c900c0f6355247a353defa37f will contain PNG image of size 50x50
|
138
139
|
|
139
140
|
@storage @uuid
|
140
141
|
Scenario: Posting picture to file system under input data digest
|
Binary file
|
@@ -0,0 +1,49 @@
|
|
1
|
+
Feature: Forwarding of transaction ID header to thumbnailer
|
2
|
+
HTTP Image Store should forward transaction ID header found in the request to HTTP Thumbnailer
|
3
|
+
|
4
|
+
Background:
|
5
|
+
Given httpthumbnailer server is running at http://localhost:3100/health_check
|
6
|
+
Given httpimagestore server is running at http://localhost:3000/ with the following configuration
|
7
|
+
"""
|
8
|
+
post "identify" {
|
9
|
+
identify "input"
|
10
|
+
}
|
11
|
+
|
12
|
+
post "single" {
|
13
|
+
thumbnail "input" "thumbnail" operation="pad" width="8" height="8"
|
14
|
+
}
|
15
|
+
|
16
|
+
post "multi" {
|
17
|
+
thumbnail "input" {
|
18
|
+
"small" operation="crop" width=128 height=128 format="jpeg"
|
19
|
+
"tiny_png" operation="crop" width=32 height=32 format="png"
|
20
|
+
}
|
21
|
+
}
|
22
|
+
"""
|
23
|
+
|
24
|
+
@xid-forwarding
|
25
|
+
Scenario: Should forward XID with identify requests
|
26
|
+
Given tiny.png file content as request body
|
27
|
+
And XID header set to 123
|
28
|
+
When I do POST request http://localhost:3000/identify
|
29
|
+
Then response status will be 200
|
30
|
+
And httpimagestore log will contain xid="123"
|
31
|
+
And httpthumbnailer log will contain xid="123"
|
32
|
+
|
33
|
+
@xid-forwarding
|
34
|
+
Scenario: Should forward XID with single thumbnail requests
|
35
|
+
Given tiny.png file content as request body
|
36
|
+
And XID header set to 123
|
37
|
+
When I do POST request http://localhost:3000/single
|
38
|
+
Then response status will be 200
|
39
|
+
And httpimagestore log will contain xid="123"
|
40
|
+
And httpthumbnailer log will contain xid="123"
|
41
|
+
|
42
|
+
@xid-forwarding
|
43
|
+
Scenario: Should forward XID with multi thumbnail requests
|
44
|
+
Given tiny.png file content as request body
|
45
|
+
And XID header set to 123
|
46
|
+
When I do POST request http://localhost:3000/multi
|
47
|
+
Then response status will be 200
|
48
|
+
And httpimagestore log will contain xid="123"
|
49
|
+
And httpthumbnailer log will contain xid="123"
|