httpimagestore 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +4 -3
- data/Gemfile.lock +12 -16
- data/README.md +73 -0
- data/VERSION +1 -1
- data/bin/httpimagestore +7 -7
- data/features/data-uri.feature +55 -0
- data/features/error-reporting.feature +21 -3
- data/features/source-failover.feature +71 -0
- data/features/step_definitions/httpimagestore_steps.rb +27 -12
- data/features/storage.feature +26 -25
- data/features/support/tiny.png +0 -0
- data/features/xid-forwarding.feature +49 -0
- data/httpimagestore.gemspec +19 -23
- data/lib/httpimagestore/configuration/file.rb +6 -0
- data/lib/httpimagestore/configuration/handler.rb +4 -1
- data/lib/httpimagestore/configuration/identify.rb +2 -2
- data/lib/httpimagestore/configuration/output.rb +20 -1
- data/lib/httpimagestore/configuration/s3.rb +15 -9
- data/lib/httpimagestore/configuration/source_failover.rb +51 -0
- data/lib/httpimagestore/configuration/thumbnailer.rb +7 -7
- data/lib/httpimagestore/error_reporter.rb +9 -1
- data/lib/httpimagestore/ruby_string_template.rb +12 -7
- data/load_test/load_test.jmx +50 -77
- data/load_test/thumbnail_specs_v2.csv +10 -0
- data/spec/configuration_file_spec.rb +27 -2
- data/spec/configuration_identify_spec.rb +25 -2
- data/spec/configuration_s3_spec.rb +29 -3
- data/spec/configuration_source_failover_spec.rb +101 -0
- data/spec/configuration_thumbnailer_spec.rb +63 -8
- data/spec/ruby_string_template_spec.rb +4 -0
- data/spec/support/full.cfg +167 -33
- metadata +19 -23
- data/.idea/.name +0 -1
- data/.idea/.rakeTasks +0 -7
- data/.idea/codeStyleSettings.xml +0 -13
- data/.idea/dictionaries/wcc.xml +0 -8
- data/.idea/encodings.xml +0 -5
- data/.idea/httpimagestore.iml +0 -69
- data/.idea/jenkinsSettings.xml +0 -9
- data/.idea/misc.xml +0 -5
- data/.idea/modules.xml +0 -9
- data/.idea/scopes/scope_settings.xml +0 -5
- data/.idea/vcs.xml +0 -7
data/Gemfile
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
source "http://rubygems.org"
|
|
2
2
|
ruby "1.9.3"
|
|
3
3
|
|
|
4
|
-
gem "unicorn-cuba-base", "~> 1.
|
|
4
|
+
gem "unicorn-cuba-base", "~> 1.2.0"
|
|
5
5
|
#gem "unicorn-cuba-base", path: "../unicorn-cuba-base"
|
|
6
|
-
gem "httpthumbnailer-client", "~> 1.
|
|
6
|
+
gem "httpthumbnailer-client", "~> 1.2.0"
|
|
7
7
|
#gem "httpthumbnailer-client", path: "../httpthumbnailer-client"
|
|
8
8
|
gem "aws-sdk", "~> 1.10"
|
|
9
9
|
gem "mime-types", "~> 1.17"
|
|
@@ -20,6 +20,7 @@ group :development do
|
|
|
20
20
|
gem "rdoc", "~> 3.9"
|
|
21
21
|
gem "daemon", "~> 1"
|
|
22
22
|
gem "prawn", "= 0.8.4"
|
|
23
|
-
gem "httpthumbnailer", path: '../httpthumbnailer'
|
|
23
|
+
#gem "httpthumbnailer", path: '../httpthumbnailer'
|
|
24
|
+
gem "httpthumbnailer", "~> 1.2.0"
|
|
24
25
|
end
|
|
25
26
|
|
data/Gemfile.lock
CHANGED
|
@@ -1,10 +1,3 @@
|
|
|
1
|
-
PATH
|
|
2
|
-
remote: ../httpthumbnailer
|
|
3
|
-
specs:
|
|
4
|
-
httpthumbnailer (1.1.1)
|
|
5
|
-
rmagick (~> 2)
|
|
6
|
-
unicorn-cuba-base (~> 1.1.1)
|
|
7
|
-
|
|
8
1
|
GEM
|
|
9
2
|
remote: http://rubygems.org/
|
|
10
3
|
specs:
|
|
@@ -15,7 +8,7 @@ GEM
|
|
|
15
8
|
uuidtools (~> 2.1)
|
|
16
9
|
builder (3.2.2)
|
|
17
10
|
cli (1.3.1)
|
|
18
|
-
cuba (3.1.
|
|
11
|
+
cuba (3.1.1)
|
|
19
12
|
rack
|
|
20
13
|
cucumber (1.3.8)
|
|
21
14
|
builder (>= 2.1.2)
|
|
@@ -42,7 +35,10 @@ GEM
|
|
|
42
35
|
highline (1.6.20)
|
|
43
36
|
httpauth (0.2.0)
|
|
44
37
|
httpclient (2.3.4.1)
|
|
45
|
-
httpthumbnailer
|
|
38
|
+
httpthumbnailer (1.2.0)
|
|
39
|
+
rmagick (~> 2)
|
|
40
|
+
unicorn-cuba-base
|
|
41
|
+
httpthumbnailer-client (1.2.0)
|
|
46
42
|
cli (~> 1.3)
|
|
47
43
|
httpclient (>= 2.3)
|
|
48
44
|
multipart-parser (~> 0.1.1)
|
|
@@ -58,7 +54,7 @@ GEM
|
|
|
58
54
|
json (1.8.1)
|
|
59
55
|
jwt (0.1.8)
|
|
60
56
|
multi_json (>= 1.5)
|
|
61
|
-
kgio (2.
|
|
57
|
+
kgio (2.9.2)
|
|
62
58
|
mime-types (1.25)
|
|
63
59
|
msgpack (0.5.5)
|
|
64
60
|
multi_json (1.8.2)
|
|
@@ -82,7 +78,7 @@ GEM
|
|
|
82
78
|
prawn-layout (0.8.4)
|
|
83
79
|
prawn-security (0.8.4)
|
|
84
80
|
rack (1.5.2)
|
|
85
|
-
raindrops (0.
|
|
81
|
+
raindrops (0.13.0)
|
|
86
82
|
rake (10.1.0)
|
|
87
83
|
rdoc (3.12.2)
|
|
88
84
|
json (~> 1.4)
|
|
@@ -97,11 +93,11 @@ GEM
|
|
|
97
93
|
rspec-mocks (2.14.3)
|
|
98
94
|
ruby-ip (0.9.1)
|
|
99
95
|
sdl4r (0.9.11)
|
|
100
|
-
unicorn (4.
|
|
96
|
+
unicorn (4.8.2)
|
|
101
97
|
kgio (~> 2.6)
|
|
102
98
|
rack
|
|
103
99
|
raindrops (~> 0.7)
|
|
104
|
-
unicorn-cuba-base (1.
|
|
100
|
+
unicorn-cuba-base (1.2.0)
|
|
105
101
|
cli (~> 1.3)
|
|
106
102
|
cuba (~> 3.0)
|
|
107
103
|
facter (~> 1.6.11)
|
|
@@ -118,8 +114,8 @@ DEPENDENCIES
|
|
|
118
114
|
cucumber
|
|
119
115
|
daemon (~> 1)
|
|
120
116
|
httpclient (>= 2.3)
|
|
121
|
-
httpthumbnailer
|
|
122
|
-
httpthumbnailer-client (~> 1.
|
|
117
|
+
httpthumbnailer (~> 1.2.0)
|
|
118
|
+
httpthumbnailer-client (~> 1.2.0)
|
|
123
119
|
jeweler (~> 1.8.4)
|
|
124
120
|
mime-types (~> 1.17)
|
|
125
121
|
msgpack (~> 0.5)
|
|
@@ -127,4 +123,4 @@ DEPENDENCIES
|
|
|
127
123
|
rdoc (~> 3.9)
|
|
128
124
|
rspec (~> 2.13)
|
|
129
125
|
sdl4r (~> 0.9)
|
|
130
|
-
unicorn-cuba-base (~> 1.
|
|
126
|
+
unicorn-cuba-base (~> 1.2.0)
|
data/README.md
CHANGED
|
@@ -17,6 +17,14 @@ It is using [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) as
|
|
|
17
17
|
|
|
18
18
|
## Changelog
|
|
19
19
|
|
|
20
|
+
### 1.7.0
|
|
21
|
+
* `output_data_uri_image` support
|
|
22
|
+
* `source_failover` support
|
|
23
|
+
* fixed possible data leak with nested template processing
|
|
24
|
+
* reporting `400 Bad Request` for non UTF-8 characters encoded in URLs
|
|
25
|
+
* syslog logging
|
|
26
|
+
* transaction ID tracking
|
|
27
|
+
|
|
20
28
|
### 1.6.0
|
|
21
29
|
* `output_store_path` and `output_store_url` `path` argument support
|
|
22
30
|
* encoding resulting file storage URL
|
|
@@ -261,6 +269,33 @@ get "small" {
|
|
|
261
269
|
|
|
262
270
|
Requesting `/small` URI will result with image fetched from S3 bucket `mybucket` and key `myimage.jpg` and named `original`.
|
|
263
271
|
|
|
272
|
+
|
|
273
|
+
#### source_failover
|
|
274
|
+
|
|
275
|
+
This statement can be used to wrap around other sources.
|
|
276
|
+
HTTP Image Store will try wrapped sources one by one until one will succeed.
|
|
277
|
+
If all sources fail status code for error from the first source will be returned.
|
|
278
|
+
|
|
279
|
+
This source has no options.
|
|
280
|
+
|
|
281
|
+
Example:
|
|
282
|
+
|
|
283
|
+
```sdl
|
|
284
|
+
s3 key="AIAITCKMELYWQZPJP7HQ" secret="V37lCu0F48Tv9s7QVqIT/sLf/wwqhNSB4B0Em7Ei" ssl=false
|
|
285
|
+
path "myimage" "myimage.jpg"
|
|
286
|
+
|
|
287
|
+
get "small" {
|
|
288
|
+
source_failover {
|
|
289
|
+
source_s3 "original" bucket="mybucket" path="myimage"
|
|
290
|
+
source_s3 "original" bucket="mybucket-backup" path="myimage"
|
|
291
|
+
}
|
|
292
|
+
}
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
Requesting `/small` URI will result with image fetched from S3 bucket `mybucket`.
|
|
296
|
+
If `mybucket` does not contain key `myimage.jpg` than `mybucket-backup` will be tried.
|
|
297
|
+
If `mybucket-backup` does not contain the key than `404` status code will be returned.
|
|
298
|
+
|
|
264
299
|
### API endpoint processing operations
|
|
265
300
|
|
|
266
301
|
#### thumbnail
|
|
@@ -440,6 +475,38 @@ put "test" {
|
|
|
440
475
|
|
|
441
476
|
The output will contain the posted image with `Content-Type` header set to `application/octet-stream` and `Cache-Control` to `public, max-age=999, s-maxage=666`.
|
|
442
477
|
|
|
478
|
+
#### output_data_uri_image
|
|
479
|
+
|
|
480
|
+
This statement will produce `200 OK` response containing base64 encoded image data within data URI in format:
|
|
481
|
+
```
|
|
482
|
+
data:<image mime type>;base64,<base64 encoded image data>
|
|
483
|
+
```
|
|
484
|
+
No tailing new line/line feed is added to the output.
|
|
485
|
+
|
|
486
|
+
This may be used to include images directly in HTML pages or CSS.
|
|
487
|
+
|
|
488
|
+
The `Content-Type` of the response will be `text/uri-list`.
|
|
489
|
+
Image mime type will be provided within data URI and must be known before output is constructed or `500 Internal Server Error` will be served.
|
|
490
|
+
|
|
491
|
+
Arguments:
|
|
492
|
+
|
|
493
|
+
1. image name - image to be sent in response body
|
|
494
|
+
|
|
495
|
+
Options:
|
|
496
|
+
|
|
497
|
+
* `cache-control` - value of response `Cache-Control` header can be specified with this option
|
|
498
|
+
|
|
499
|
+
Example:
|
|
500
|
+
|
|
501
|
+
```sdl
|
|
502
|
+
put "test" {
|
|
503
|
+
identify "input"
|
|
504
|
+
output_data_uri_image "input" cache-control="public, max-age=31557600, s-maxage=0"
|
|
505
|
+
}
|
|
506
|
+
```
|
|
507
|
+
|
|
508
|
+
The output will contain base64 encoded posted image within data URI with `Content-Type` header set to `text/uri-list` and `Cache-Control` to `public, max-age=999, s-maxage=666`.
|
|
509
|
+
|
|
443
510
|
#### output_store_path
|
|
444
511
|
|
|
445
512
|
This statement will output actual storage path on the file system (without root) or S3 key under witch the image was stored.
|
|
@@ -901,6 +968,12 @@ If running as root you can use `--user` option to specify user with whose privil
|
|
|
901
968
|
|
|
902
969
|
Additionally `httpimagestore` will log requests in [common NCSA format](http://en.wikipedia.org/wiki/Common_Log_Format) to `httpimagestore_access.log` file. Use `--access-log-file` option to change location of access log.
|
|
903
970
|
|
|
971
|
+
Syslog logging can be enabled with `--syslog-facility` option followed by name of syslog facility to use. When enabled log files are not created and both application logs and access logs are sent to syslog.
|
|
972
|
+
Access logs will gain meta information that will include `type="http-access"` that can be used to filter access log entries out from application log entries.
|
|
973
|
+
|
|
974
|
+
With `--xid-header` option name of HTTP request header can be specified. Value of this header will be logged in meta information tag `xid` along side all request related log entries.
|
|
975
|
+
Named header will be passed down with requests to HTTP Thumbnailer for grater traceability.
|
|
976
|
+
|
|
904
977
|
### Running with nginx
|
|
905
978
|
|
|
906
979
|
[nginx](http://nginx.org) if configured properly will buffer incoming requests before sending them to the backend and server response before sending them to client.
|
data/VERSION
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
1.
|
|
1
|
+
1.7.0
|
data/bin/httpimagestore
CHANGED
|
@@ -14,13 +14,13 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
|
|
|
14
14
|
end
|
|
15
15
|
|
|
16
16
|
settings do |settings|
|
|
17
|
-
|
|
17
|
+
Controller.settings[:config_file] = settings.config
|
|
18
18
|
end
|
|
19
19
|
|
|
20
20
|
main do |settings|
|
|
21
21
|
require 'httpimagestore/error_reporter'
|
|
22
22
|
|
|
23
|
-
class HTTPImageStore <
|
|
23
|
+
class HTTPImageStore < Controller
|
|
24
24
|
extend Stats
|
|
25
25
|
def_stats(
|
|
26
26
|
:workers,
|
|
@@ -85,18 +85,18 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
|
|
|
85
85
|
names.zip(args)
|
|
86
86
|
.each do |name, value|
|
|
87
87
|
fail "name should be a symbol" unless name.is_a? Symbol
|
|
88
|
-
matches[name] = URI.
|
|
88
|
+
matches[name] = URI.utf_decode(value)
|
|
89
89
|
end
|
|
90
90
|
|
|
91
91
|
# decode remaining URI components
|
|
92
92
|
path = (env["PATH_INFO"][1..-1] || '').split('/').map do |part|
|
|
93
|
-
URI.
|
|
93
|
+
URI.utf_decode(part)
|
|
94
94
|
end.join('/')
|
|
95
95
|
|
|
96
96
|
# query string already decoded by Rack
|
|
97
97
|
query_string = req.GET
|
|
98
98
|
|
|
99
|
-
state = Configuration::RequestState.new(req.body.read, matches, path, query_string, memory_limit)
|
|
99
|
+
state = Configuration::RequestState.new(req.body.read, matches, path, query_string, memory_limit, env['xid'] || {})
|
|
100
100
|
|
|
101
101
|
handler.sources.each do |source|
|
|
102
102
|
source.realize(state) unless source.respond_to? :excluded? and source.excluded?(state)
|
|
@@ -133,8 +133,8 @@ Application.new('httpimagestore', port: 3000, processor_count_factor: 2) do
|
|
|
133
133
|
|
|
134
134
|
require 'httpimagestore/configuration'
|
|
135
135
|
|
|
136
|
-
# connect Scope tree with
|
|
137
|
-
Configuration::Scope.logger =
|
|
136
|
+
# connect Scope tree with Controller logger
|
|
137
|
+
Configuration::Scope.logger = Controller.logger_for(Configuration::Scope)
|
|
138
138
|
|
|
139
139
|
# load builin supported set
|
|
140
140
|
require 'httpimagestore/configuration/path'
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
Feature: Data URI style image output
|
|
2
|
+
HTTP Image Store can provide images base64 encoded in data URI format so they can be included directly in HTML pages.
|
|
3
|
+
|
|
4
|
+
Background:
|
|
5
|
+
Given httpthumbnailer server is running at http://localhost:3100/health_check
|
|
6
|
+
Given httpimagestore server is running at http://localhost:3000/ with the following configuration
|
|
7
|
+
"""
|
|
8
|
+
post "data-uri" "no-identify" {
|
|
9
|
+
output_data_uri_image "input"
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
post "data-uri" "cache-control" {
|
|
13
|
+
identify "input"
|
|
14
|
+
output_data_uri_image "input" cache-control="public, max-age=31557600, s-maxage=0"
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
post "data-uri" {
|
|
18
|
+
identify "input"
|
|
19
|
+
output_data_uri_image "input"
|
|
20
|
+
}
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
@data-uri
|
|
24
|
+
Scenario: Getting image encoded in data URI scheme with base64 encoding
|
|
25
|
+
Given tiny.png file content as request body
|
|
26
|
+
When I do POST request http://localhost:3000/data-uri
|
|
27
|
+
Then response status will be 200
|
|
28
|
+
And response content type will be text/uri-list
|
|
29
|
+
And response body will be
|
|
30
|
+
"""
|
|
31
|
+
data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAgUlEQVQI10XNsQ2CUBhF4XOTfwJeog0U2kLJNlZOxQ4swBLGwkhhY6MNRkic4Fq8ENqTLznqhuvlOY2v5f6egaYs6iq1x30YAENdJskgEBAYUFMlYSMw2CYk2sMOkTFgI6Fp/mEpt7Ua69Sf2TRgLESMt8dW5bwE4vtZvOK8xkj8Ac8BL8dFQipPAAAAAElFTkSuQmCC
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
@data-uri
|
|
35
|
+
Scenario: Getting image encoded in data URI scheme with base64 encoding - cache-control should be supported
|
|
36
|
+
Given tiny.png file content as request body
|
|
37
|
+
When I do POST request http://localhost:3000/data-uri/cache-control
|
|
38
|
+
Then response status will be 200
|
|
39
|
+
And response content type will be text/uri-list
|
|
40
|
+
And response Cache-Control will be public, max-age=31557600, s-maxage=0
|
|
41
|
+
And response body will be
|
|
42
|
+
"""
|
|
43
|
+
data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAgAAAAICAIAAABLbSncAAAAgUlEQVQI10XNsQ2CUBhF4XOTfwJeog0U2kLJNlZOxQ4swBLGwkhhY6MNRkic4Fq8ENqTLznqhuvlOY2v5f6egaYs6iq1x30YAENdJskgEBAYUFMlYSMw2CYk2sMOkTFgI6Fp/mEpt7Ua69Sf2TRgLESMt8dW5bwE4vtZvOK8xkj8Ac8BL8dFQipPAAAAAElFTkSuQmCC
|
|
44
|
+
"""
|
|
45
|
+
|
|
46
|
+
@data-uri
|
|
47
|
+
Scenario: Error 500 served when data URI output is used on unidentified image
|
|
48
|
+
Given tiny.png file content as request body
|
|
49
|
+
When I do POST request http://localhost:3000/data-uri/no-identify
|
|
50
|
+
Then response status will be 500
|
|
51
|
+
And response content type will be text/plain
|
|
52
|
+
And response body will be CRLF ended lines
|
|
53
|
+
"""
|
|
54
|
+
image 'input' needs to be identified first to be used in data URI output
|
|
55
|
+
"""
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
Feature:
|
|
2
|
-
|
|
3
|
-
This configuration should be mostly compatible with pre v1.0 release.
|
|
1
|
+
Feature: Error handling
|
|
2
|
+
API should provide different status codes with text/plain description body for different error modes.
|
|
4
3
|
|
|
5
4
|
Background:
|
|
6
5
|
Given S3 settings in AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_TEST_BUCKET environment variables
|
|
@@ -9,9 +8,11 @@ Feature: Image list based thumbnailing and S3 storage
|
|
|
9
8
|
"""
|
|
10
9
|
s3 key="@AWS_ACCESS_KEY_ID@" secret="@AWS_SECRET_ACCESS_KEY@" ssl=false
|
|
11
10
|
|
|
11
|
+
path "uri_part" "/thumbnails/#{input_digest}/#{name}.#{image_mime_extension}"
|
|
12
12
|
path "structured-name" "#{dirname}/#{input_digest}/#{basename}-#{image_name}.#{image_mime_extension}"
|
|
13
13
|
path "missing" "blah"
|
|
14
14
|
path "zero" "zero"
|
|
15
|
+
path "hash" "#{input_digest}"
|
|
15
16
|
|
|
16
17
|
put "multipart" ":name_list" {
|
|
17
18
|
thumbnail "input" {
|
|
@@ -33,6 +34,12 @@ Feature: Image list based thumbnailing and S3 storage
|
|
|
33
34
|
thumbnail "input" "bad_opts" operation="crop" width=128 height=128 options="foo=bar" if-image-name-on="#{name_list}"
|
|
34
35
|
}
|
|
35
36
|
|
|
37
|
+
post "filename" "/(?<name>.+?)(\\.[^\\.]+)?$/" {
|
|
38
|
+
identify "input"
|
|
39
|
+
store_s3 "input" bucket="@AWS_S3_TEST_BUCKET@" path="hash" cache-root="/tmp"
|
|
40
|
+
output_store_path "input" path="uri_part"
|
|
41
|
+
}
|
|
42
|
+
|
|
36
43
|
get "s3" {
|
|
37
44
|
source_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="missing"
|
|
38
45
|
}
|
|
@@ -199,6 +206,17 @@ Feature: Image list based thumbnailing and S3 storage
|
|
|
199
206
|
thumbnailing of 'input' into 'superlarge' failed: image too large: cache resources exhausted
|
|
200
207
|
"""
|
|
201
208
|
|
|
209
|
+
@error-reporting
|
|
210
|
+
Scenario: Bad URI encoding
|
|
211
|
+
Given test.jpg file content as request body
|
|
212
|
+
When I do POST request http://localhost:3000/filename/hello%e9world.jpg
|
|
213
|
+
Then response status will be 400
|
|
214
|
+
And response content type will be text/plain
|
|
215
|
+
And response body will be CRLF ended lines like
|
|
216
|
+
"""
|
|
217
|
+
invalid UTF-8 encoding in URI: "hello\\xE9world"
|
|
218
|
+
"""
|
|
219
|
+
|
|
202
220
|
@error-reporting
|
|
203
221
|
Scenario: Zero body length
|
|
204
222
|
Given test.empty file content as request body
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
Feature: More than one source can be tried in canse of source problem
|
|
2
|
+
Sources can be grouped under source_failover group.
|
|
3
|
+
HTTP Image Store will try each source until working one is found.
|
|
4
|
+
|
|
5
|
+
Background:
|
|
6
|
+
Given S3 settings in AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_TEST_BUCKET environment variables
|
|
7
|
+
Given httpimagestore server is running at http://localhost:3000/ with the following configuration
|
|
8
|
+
"""
|
|
9
|
+
s3 key="@AWS_ACCESS_KEY_ID@" secret="@AWS_SECRET_ACCESS_KEY@" ssl=false
|
|
10
|
+
|
|
11
|
+
path "test-file" "test.file"
|
|
12
|
+
path "bogus" "bogus"
|
|
13
|
+
|
|
14
|
+
put "input" {
|
|
15
|
+
store_s3 "input" bucket="@AWS_S3_TEST_BUCKET@" path="test-file"
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
get "s3_fail" {
|
|
19
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="bogus"
|
|
20
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="test-file"
|
|
21
|
+
output_image "image"
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
get "s3_failover" {
|
|
25
|
+
source_failover {
|
|
26
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="bogus"
|
|
27
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="test-file"
|
|
28
|
+
}
|
|
29
|
+
output_image "image"
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
get "s3_all_fail" {
|
|
33
|
+
source_failover {
|
|
34
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="bogus"
|
|
35
|
+
source_s3 "image" bucket="@AWS_S3_TEST_BUCKET@" path="bogus"
|
|
36
|
+
}
|
|
37
|
+
output_image "image"
|
|
38
|
+
}
|
|
39
|
+
"""
|
|
40
|
+
|
|
41
|
+
@source-failover
|
|
42
|
+
Scenario: Sourcing S3 object without failover
|
|
43
|
+
Given there is no test.file file in S3 bucket
|
|
44
|
+
Given test.jpg file content as request body
|
|
45
|
+
When I do PUT request http://localhost:3000/input
|
|
46
|
+
Then response status will be 200
|
|
47
|
+
When I do GET request http://localhost:3000/s3_fail
|
|
48
|
+
Then response status will be 404
|
|
49
|
+
|
|
50
|
+
@source-failover
|
|
51
|
+
Scenario: Sourcing S3 object with failover
|
|
52
|
+
Given there is no test.file file in S3 bucket
|
|
53
|
+
Given test.jpg file content as request body
|
|
54
|
+
When I do PUT request http://localhost:3000/input
|
|
55
|
+
Then response status will be 200
|
|
56
|
+
When I do GET request http://localhost:3000/s3_failover
|
|
57
|
+
Then response status will be 200
|
|
58
|
+
|
|
59
|
+
@source-failover
|
|
60
|
+
Scenario: Sourcing S3 object with all sources failing
|
|
61
|
+
Given there is no test.file file in S3 bucket
|
|
62
|
+
Given test.jpg file content as request body
|
|
63
|
+
When I do PUT request http://localhost:3000/input
|
|
64
|
+
Then response status will be 200
|
|
65
|
+
When I do GET request http://localhost:3000/s3_all_fail
|
|
66
|
+
Then response status will be 404
|
|
67
|
+
And response content type will be text/plain
|
|
68
|
+
And response body will be CRLF ended lines
|
|
69
|
+
"""
|
|
70
|
+
all sources failed: S3Source[image_name: 'image' bucket: 'httpimagestoretest' prefix: '' path_spec: 'bogus'](Configuration::S3NoSuchKeyError: S3 bucket 'httpimagestoretest' does not contain key 'bogus'), S3Source[image_name: 'image' bucket: 'httpimagestoretest' prefix: '' path_spec: 'bogus'](Configuration::S3NoSuchKeyError: S3 bucket 'httpimagestoretest' does not contain key 'bogus')
|
|
71
|
+
"""
|
|
@@ -4,7 +4,7 @@ end
|
|
|
4
4
|
|
|
5
5
|
Given /httpimagestore server is running at (.*) with the following configuration/ do |url, config|
|
|
6
6
|
$temp_dir = Pathname.new(Dir.mktmpdir) unless $temp_dir
|
|
7
|
-
|
|
7
|
+
|
|
8
8
|
cfile = $temp_dir + Digest.hexencode(Digest::MD5.digest(config))
|
|
9
9
|
cfile.open('w') do |io|
|
|
10
10
|
io.write(config.replace_s3_variables)
|
|
@@ -13,7 +13,7 @@ Given /httpimagestore server is running at (.*) with the following configuration
|
|
|
13
13
|
begin
|
|
14
14
|
log = support_dir + 'server.log'
|
|
15
15
|
start_server(
|
|
16
|
-
"bundle exec #{script('httpimagestore')} -f -d -l #{log} -w 1 #{(@httpimagestore_args ||= []).join(' ')} #{cfile.to_s}",
|
|
16
|
+
"bundle exec #{script('httpimagestore')} -f -d -x XID -l #{log} -w 1 #{(@httpimagestore_args ||= []).join(' ')} #{cfile.to_s}",
|
|
17
17
|
'/tmp/httpimagestore.pid',
|
|
18
18
|
log,
|
|
19
19
|
url
|
|
@@ -26,7 +26,7 @@ end
|
|
|
26
26
|
Given /httpthumbnailer server is running at (.*)/ do |url|
|
|
27
27
|
log = support_dir + 'thumbniler.log'
|
|
28
28
|
start_server(
|
|
29
|
-
"httpthumbnailer -f -d -l #{log} -w 1",
|
|
29
|
+
"httpthumbnailer -f -d -x XID -l #{log} -a #{log}.access -w 1",
|
|
30
30
|
'/tmp/httpthumbnailer.pid',
|
|
31
31
|
log,
|
|
32
32
|
url
|
|
@@ -34,7 +34,7 @@ Given /httpthumbnailer server is running at (.*)/ do |url|
|
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
Given /httpthumbnailer server is not running/ do
|
|
37
|
-
stop_server('/tmp/httpthumbnailer.pid')
|
|
37
|
+
stop_server('/tmp/httpthumbnailer.pid')
|
|
38
38
|
end
|
|
39
39
|
|
|
40
40
|
Given /httpimagestore log is empty/ do
|
|
@@ -56,7 +56,7 @@ Given /there is no file (.*)/ do |file|
|
|
|
56
56
|
end
|
|
57
57
|
|
|
58
58
|
Given /S3 settings in AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY and AWS_S3_TEST_BUCKET environment variables/ do
|
|
59
|
-
|
|
59
|
+
|
|
60
60
|
unless ENV['AWS_ACCESS_KEY_ID'] and ENV['AWS_SECRET_ACCESS_KEY'] and ENV['AWS_S3_TEST_BUCKET']
|
|
61
61
|
fail "AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY or AWS_S3_TEST_BUCKET environment variables not set"
|
|
62
62
|
end
|
|
@@ -83,7 +83,7 @@ end
|
|
|
83
83
|
|
|
84
84
|
When /I do (.*) request (.*)/ do |method, uri|
|
|
85
85
|
@request_body = nil if method == 'GET'
|
|
86
|
-
@response = HTTPClient.new.request(method,
|
|
86
|
+
@response = HTTPClient.new.request(method, uri.replace_s3_variables, nil, @request_body, (@request_headers or {}))
|
|
87
87
|
end
|
|
88
88
|
|
|
89
89
|
Then /response status will be (.*)/ do |status|
|
|
@@ -94,14 +94,22 @@ Then /response content type will be (.*)/ do |content_type|
|
|
|
94
94
|
@response.header['Content-Type'].first.should == content_type
|
|
95
95
|
end
|
|
96
96
|
|
|
97
|
-
Then /response
|
|
97
|
+
Then /response Cache-Control will be (.*)/ do |content_type|
|
|
98
|
+
@response.header['Cache-Control'].first.should == content_type
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
Then /response body will be CRLF ended lines like/ do |body|
|
|
98
102
|
@response.body.should match(body.replace_s3_variables)
|
|
99
103
|
@response.body.each_line do |line|
|
|
100
104
|
line[-2,2].should == "\r\n"
|
|
101
105
|
end
|
|
102
106
|
end
|
|
103
107
|
|
|
104
|
-
Then /response body will be
|
|
108
|
+
Then /response body will be$/ do |body|
|
|
109
|
+
@response.body.should == body.replace_s3_variables
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
Then /response body will be CRLF ended lines$/ do |body|
|
|
105
113
|
@response.body.should == body.replace_s3_variables.gsub("\n", "\r\n") + "\r\n"
|
|
106
114
|
end
|
|
107
115
|
|
|
@@ -119,7 +127,7 @@ end
|
|
|
119
127
|
|
|
120
128
|
Then /(http.*) will contain (.*) image of size (.*)x(.*)/ do |url, format, width, height|
|
|
121
129
|
data = get(url.replace_s3_variables)
|
|
122
|
-
|
|
130
|
+
|
|
123
131
|
@image.destroy! if @image
|
|
124
132
|
@image = Magick::Image.from_blob(data).first
|
|
125
133
|
|
|
@@ -130,7 +138,7 @@ end
|
|
|
130
138
|
|
|
131
139
|
Then /S3 object (.*) will contain (.*) image of size (.*)x(.*)/ do |key, format, width, height|
|
|
132
140
|
data = @bucket.objects[key].read
|
|
133
|
-
|
|
141
|
+
|
|
134
142
|
@image.destroy! if @image
|
|
135
143
|
@image = Magick::Image.from_blob(data).first
|
|
136
144
|
|
|
@@ -146,7 +154,7 @@ end
|
|
|
146
154
|
Then /response body will contain (.*) image of size (.*)x(.*)/ do |format, width, height|
|
|
147
155
|
data = @response.body
|
|
148
156
|
Pathname.new('/tmp/out.jpg').open('w'){|io| io.write data}
|
|
149
|
-
|
|
157
|
+
|
|
150
158
|
@image.destroy! if @image
|
|
151
159
|
@image = Magick::Image.from_blob(data).first
|
|
152
160
|
|
|
@@ -165,7 +173,7 @@ end
|
|
|
165
173
|
|
|
166
174
|
Then /file (.*) will contain (.*) image of size (.*)x(.*)/ do |file, format, width, height|
|
|
167
175
|
data = Pathname.new(file).read
|
|
168
|
-
|
|
176
|
+
|
|
169
177
|
@image.destroy! if @image
|
|
170
178
|
@image = Magick::Image.from_blob(data).first
|
|
171
179
|
|
|
@@ -174,3 +182,10 @@ Then /file (.*) will contain (.*) image of size (.*)x(.*)/ do |file, format, wid
|
|
|
174
182
|
@image.rows.should == height.to_i
|
|
175
183
|
end
|
|
176
184
|
|
|
185
|
+
Then /httpimagestore log will contain (.*)/ do |entry|
|
|
186
|
+
(support_dir + 'server.log').read.should include entry
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
Then /httpthumbnailer log will contain (.*)/ do |entry|
|
|
190
|
+
(support_dir + 'thumbniler.log').read.should include entry
|
|
191
|
+
end
|
data/features/storage.feature
CHANGED
|
@@ -110,31 +110,32 @@ Feature: Storing images under different names
|
|
|
110
110
|
"""
|
|
111
111
|
Then file /tmp/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e will contain PNG image of size 50x50
|
|
112
112
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
And response
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
Given
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
And response
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
113
|
+
# Following tests depend on libpng/ImageMagick version
|
|
114
|
+
#@storage @image_digest
|
|
115
|
+
#Scenario: Posting picture to file system under input data digest
|
|
116
|
+
#Given there is no file /tmp/b0fe25319ba5909a
|
|
117
|
+
#Given test.png file content as request body
|
|
118
|
+
#When I do POST request http://localhost:3000/images/image_digest
|
|
119
|
+
#Then response status will be 200
|
|
120
|
+
#And response content type will be text/plain
|
|
121
|
+
#And response body will be CRLF ended lines
|
|
122
|
+
#"""
|
|
123
|
+
#091000e2c0aee836
|
|
124
|
+
#"""
|
|
125
|
+
#Then file /tmp/091000e2c0aee836 will contain PNG image of size 50x50
|
|
126
|
+
|
|
127
|
+
#@storage @image_sha256
|
|
128
|
+
#Scenario: Posting picture to file system under input data digest
|
|
129
|
+
#Given there is no file /tmp/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e
|
|
130
|
+
#Given test.png file content as request body
|
|
131
|
+
#When I do POST request http://localhost:3000/images/image_sha256
|
|
132
|
+
#Then response status will be 200
|
|
133
|
+
#And response content type will be text/plain
|
|
134
|
+
#And response body will be CRLF ended lines
|
|
135
|
+
#"""
|
|
136
|
+
#091000e2c0aee836fff432c1151faba86d46690c900c0f6355247a353defa37f
|
|
137
|
+
#"""
|
|
138
|
+
#Then file /tmp/091000e2c0aee836fff432c1151faba86d46690c900c0f6355247a353defa37f will contain PNG image of size 50x50
|
|
138
139
|
|
|
139
140
|
@storage @uuid
|
|
140
141
|
Scenario: Posting picture to file system under input data digest
|
|
Binary file
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
Feature: Forwarding of transaction ID header to thumbnailer
|
|
2
|
+
HTTP Image Store should forward transaction ID header found in the request to HTTP Thumbnailer
|
|
3
|
+
|
|
4
|
+
Background:
|
|
5
|
+
Given httpthumbnailer server is running at http://localhost:3100/health_check
|
|
6
|
+
Given httpimagestore server is running at http://localhost:3000/ with the following configuration
|
|
7
|
+
"""
|
|
8
|
+
post "identify" {
|
|
9
|
+
identify "input"
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
post "single" {
|
|
13
|
+
thumbnail "input" "thumbnail" operation="pad" width="8" height="8"
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
post "multi" {
|
|
17
|
+
thumbnail "input" {
|
|
18
|
+
"small" operation="crop" width=128 height=128 format="jpeg"
|
|
19
|
+
"tiny_png" operation="crop" width=32 height=32 format="png"
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
@xid-forwarding
|
|
25
|
+
Scenario: Should forward XID with identify requests
|
|
26
|
+
Given tiny.png file content as request body
|
|
27
|
+
And XID header set to 123
|
|
28
|
+
When I do POST request http://localhost:3000/identify
|
|
29
|
+
Then response status will be 200
|
|
30
|
+
And httpimagestore log will contain xid="123"
|
|
31
|
+
And httpthumbnailer log will contain xid="123"
|
|
32
|
+
|
|
33
|
+
@xid-forwarding
|
|
34
|
+
Scenario: Should forward XID with single thumbnail requests
|
|
35
|
+
Given tiny.png file content as request body
|
|
36
|
+
And XID header set to 123
|
|
37
|
+
When I do POST request http://localhost:3000/single
|
|
38
|
+
Then response status will be 200
|
|
39
|
+
And httpimagestore log will contain xid="123"
|
|
40
|
+
And httpthumbnailer log will contain xid="123"
|
|
41
|
+
|
|
42
|
+
@xid-forwarding
|
|
43
|
+
Scenario: Should forward XID with multi thumbnail requests
|
|
44
|
+
Given tiny.png file content as request body
|
|
45
|
+
And XID header set to 123
|
|
46
|
+
When I do POST request http://localhost:3000/multi
|
|
47
|
+
Then response status will be 200
|
|
48
|
+
And httpimagestore log will contain xid="123"
|
|
49
|
+
And httpthumbnailer log will contain xid="123"
|