httpimagestore 1.7.0 → 1.8.0
Sign up to get free protection for your applications and to get access to all the features.
- data/Gemfile +2 -2
- data/Gemfile.lock +7 -7
- data/README.md +59 -4
- data/VERSION +1 -1
- data/features/compatibility.feature +3 -3
- data/features/encoding.feature +35 -0
- data/features/error-reporting.feature +11 -0
- data/features/rewrite.feature +122 -0
- data/features/s3-store-and-thumbnail.feature +4 -4
- data/features/step_definitions/httpimagestore_steps.rb +18 -6
- data/features/support/env.rb +11 -9
- data/httpimagestore.gemspec +12 -9
- data/lib/httpimagestore/configuration/file.rb +2 -2
- data/lib/httpimagestore/configuration/handler.rb +7 -3
- data/lib/httpimagestore/configuration/output.rb +55 -17
- data/lib/httpimagestore/configuration/s3.rb +12 -4
- data/spec/configuration_file_spec.rb +3 -4
- data/spec/configuration_handler_spec.rb +1 -2
- data/spec/configuration_identify_spec.rb +1 -2
- data/spec/configuration_output_spec.rb +289 -10
- data/spec/configuration_path_spec.rb +2 -3
- data/spec/configuration_s3_spec.rb +39 -38
- data/spec/configuration_source_failover_spec.rb +1 -1
- data/spec/configuration_spec.rb +1 -2
- data/spec/configuration_thumbnailer_spec.rb +1 -2
- data/spec/spec_helper.rb +15 -7
- data/spec/support/utf_string.txt +1 -0
- metadata +11 -8
data/Gemfile
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
source "http://rubygems.org"
|
2
2
|
ruby "1.9.3"
|
3
3
|
|
4
|
-
gem "unicorn-cuba-base", "~> 1.2.
|
4
|
+
gem "unicorn-cuba-base", "~> 1.2.2"
|
5
5
|
#gem "unicorn-cuba-base", path: "../unicorn-cuba-base"
|
6
6
|
gem "httpthumbnailer-client", "~> 1.2.0"
|
7
7
|
#gem "httpthumbnailer-client", path: "../httpthumbnailer-client"
|
@@ -13,7 +13,7 @@ gem "msgpack", "~> 0.5"
|
|
13
13
|
# Add dependencies to develop your gem here.
|
14
14
|
# Include everything needed to run rake, tests, features, etc.
|
15
15
|
group :development do
|
16
|
-
gem "
|
16
|
+
gem "faraday", ">= 0.8"
|
17
17
|
gem "rspec", "~> 2.13"
|
18
18
|
gem "cucumber", ">= 0"
|
19
19
|
gem "jeweler", "~> 1.8.4"
|
data/Gemfile.lock
CHANGED
@@ -8,7 +8,7 @@ GEM
|
|
8
8
|
uuidtools (~> 2.1)
|
9
9
|
builder (3.2.2)
|
10
10
|
cli (1.3.1)
|
11
|
-
cuba (3.
|
11
|
+
cuba (3.3.0)
|
12
12
|
rack
|
13
13
|
cucumber (1.3.8)
|
14
14
|
builder (>= 2.1.2)
|
@@ -34,7 +34,7 @@ GEM
|
|
34
34
|
hashie (2.0.5)
|
35
35
|
highline (1.6.20)
|
36
36
|
httpauth (0.2.0)
|
37
|
-
httpclient (2.
|
37
|
+
httpclient (2.4.0)
|
38
38
|
httpthumbnailer (1.2.0)
|
39
39
|
rmagick (~> 2)
|
40
40
|
unicorn-cuba-base
|
@@ -91,13 +91,13 @@ GEM
|
|
91
91
|
rspec-expectations (2.14.2)
|
92
92
|
diff-lcs (>= 1.1.3, < 2.0)
|
93
93
|
rspec-mocks (2.14.3)
|
94
|
-
ruby-ip (0.9.
|
94
|
+
ruby-ip (0.9.3)
|
95
95
|
sdl4r (0.9.11)
|
96
|
-
unicorn (4.8.
|
96
|
+
unicorn (4.8.3)
|
97
97
|
kgio (~> 2.6)
|
98
98
|
rack
|
99
99
|
raindrops (~> 0.7)
|
100
|
-
unicorn-cuba-base (1.2.
|
100
|
+
unicorn-cuba-base (1.2.2)
|
101
101
|
cli (~> 1.3)
|
102
102
|
cuba (~> 3.0)
|
103
103
|
facter (~> 1.6.11)
|
@@ -113,7 +113,7 @@ DEPENDENCIES
|
|
113
113
|
aws-sdk (~> 1.10)
|
114
114
|
cucumber
|
115
115
|
daemon (~> 1)
|
116
|
-
|
116
|
+
faraday (>= 0.8)
|
117
117
|
httpthumbnailer (~> 1.2.0)
|
118
118
|
httpthumbnailer-client (~> 1.2.0)
|
119
119
|
jeweler (~> 1.8.4)
|
@@ -123,4 +123,4 @@ DEPENDENCIES
|
|
123
123
|
rdoc (~> 3.9)
|
124
124
|
rspec (~> 2.13)
|
125
125
|
sdl4r (~> 0.9)
|
126
|
-
unicorn-cuba-base (~> 1.2.
|
126
|
+
unicorn-cuba-base (~> 1.2.2)
|
data/README.md
CHANGED
@@ -17,6 +17,13 @@ It is using [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) as
|
|
17
17
|
|
18
18
|
## Changelog
|
19
19
|
|
20
|
+
### 1.8.0
|
21
|
+
* `output_store_url` support additional arguments: `scheme`, `port` and `host`
|
22
|
+
* `output_store_uri` support added
|
23
|
+
* fixed `output_store_url` URL formatting for `file_store`
|
24
|
+
* fixed parsing of POST body when query string matches are used
|
25
|
+
* decoding UTF-8 characters encoded with JavaScript encode() (%uXXXX) to support legacy/broken clients
|
26
|
+
|
20
27
|
### 1.7.0
|
21
28
|
* `output_data_uri_image` support
|
22
29
|
* `source_failover` support
|
@@ -562,7 +569,7 @@ Putting image data to `/multi` URI will result with image `original` sourced fro
|
|
562
569
|
|
563
570
|
#### output_store_url
|
564
571
|
|
565
|
-
This is similar statement to `output_store_file` but it will output `file
|
572
|
+
This is similar statement to `output_store_file` but it will output `file:` URL for file stored images and valid S3 access URL for S3 stored images.
|
566
573
|
|
567
574
|
For S3 stored image if `ssl` is set to `true` on the S3 client statement (`s3 ssl="true"`) the URL will start with `https://`. If `public` is set to `true` when storing image in S3 (`store_s3 public="true"`) then the URL will not contain query string options, otherwise authentication tokens and expiration token (set to expire in 20 years) will be include in the query string.
|
568
575
|
|
@@ -575,7 +582,10 @@ Arguments:
|
|
575
582
|
|
576
583
|
Options:
|
577
584
|
|
578
|
-
* `
|
585
|
+
* `scheme` - rewrite URL scheme with provided one; variables can be used; `#{path}` variable and all derivative will be replaced with given image storage path; `#{url}` variable will contain full original URL
|
586
|
+
* `host` - add/rewrite provided host to the URL; variables can be used; `#{path}` variable and all derivative will be replaced with given image storage path; `#{url}` variable will contain full original URL
|
587
|
+
* `port` - add provided port to the URL; variables can be used; `#{path}` variable and all derivative will be replaced with given image storage path; `#{url}` variable will contain full original URL
|
588
|
+
* `path` - name of predefined path that will be used to generate output URL path part; `#{path}` variable and all derivative will be replaced with given image storage path; if not specified the original URL will be provided; `#{url}` variable will contain full original URL
|
579
589
|
|
580
590
|
Example:
|
581
591
|
|
@@ -611,9 +621,54 @@ http://mybucket.s3.amazonaws.com/4006450256177f4a/small.jpg
|
|
611
621
|
http://mybucket.s3.amazonaws.com/4006450256177f4a/tiny_png.png
|
612
622
|
```
|
613
623
|
|
614
|
-
####
|
624
|
+
#### output_store_uri
|
625
|
+
|
626
|
+
This is similar statement to `output_store_url` but it will output only path component of the URL for stored images.
|
627
|
+
|
628
|
+
The `Content-Type` header of this response is `text/uri-list`.
|
629
|
+
Each output URL is `\r\n` ended.
|
630
|
+
|
631
|
+
Arguments:
|
632
|
+
|
633
|
+
1. image names - names of images
|
634
|
+
|
635
|
+
Options:
|
615
636
|
|
616
|
-
|
637
|
+
* `path` - name of predefined path that will be used to generate output URL path part; `#{path}` variable and all derivative will be replaced with given image storage path; if not specified the original URL will be provided; `#{url}` variable will contain full original URL
|
638
|
+
|
639
|
+
Example:
|
640
|
+
|
641
|
+
```sdl
|
642
|
+
s3 key="AIAITCKMELYWQZPJP7HQ" secret="V37lCu0F48Tv9s7QVqIT/sLf/wwqhNSB4B0Em7Ei" ssl=false
|
643
|
+
|
644
|
+
path "hash" "#{input_digest}.#{image_mime_extension}"
|
645
|
+
path "hash-name" "#{input_digest}/#{image_name}.#{image_mime_extension}"
|
646
|
+
|
647
|
+
put "thumbnail" {
|
648
|
+
thumbnail "input" {
|
649
|
+
"small" operation="crop" width=128 height=128 format="jpeg"
|
650
|
+
"tiny_png" operation="crop" width=32 height=32 format="png"
|
651
|
+
}
|
652
|
+
|
653
|
+
store_s3 "input" bucket="mybucket" path="hash" public=true
|
654
|
+
store_s3 "small" bucket="mybucket" path="hash-name" public=true
|
655
|
+
store_s3 "tiny_png" bucket="mybucket" path="hash-name" public=true
|
656
|
+
|
657
|
+
output_store_uri {
|
658
|
+
"input"
|
659
|
+
"small"
|
660
|
+
"tiny_png"
|
661
|
+
}
|
662
|
+
}
|
663
|
+
```
|
664
|
+
|
665
|
+
Putting image data will result in storage of that image and two thumbnails generated from it. The output will contain `\r\n` ended lines like:
|
666
|
+
|
667
|
+
```
|
668
|
+
/4006450256177f4a.jpg
|
669
|
+
/4006450256177f4a/small.jpg
|
670
|
+
/4006450256177f4a/tiny_png.png
|
671
|
+
```
|
617
672
|
|
618
673
|
### API endpoint meta options
|
619
674
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
1.
|
1
|
+
1.8.0
|
@@ -130,7 +130,7 @@ Feature: Image list based thumbnailing and S3 storage
|
|
130
130
|
Given there is no test/图像/4006450256177f4a/测试.jpg file in S3 bucket
|
131
131
|
And there is no test/图像/4006450256177f4a/测试-small.jpg file in S3 bucket
|
132
132
|
Given test.jpg file content as request body
|
133
|
-
When I do PUT request http://localhost:3000/thumbnail/small/test/图像/测试
|
133
|
+
When I do PUT request with encoded URL http://localhost:3000/thumbnail/small/test/图像/测试
|
134
134
|
Then response status will be 200
|
135
135
|
And response content type will be text/uri-list
|
136
136
|
And response body will be CRLF ended lines
|
@@ -138,8 +138,8 @@ Feature: Image list based thumbnailing and S3 storage
|
|
138
138
|
http://@AWS_S3_TEST_BUCKET@.s3.amazonaws.com/test/%E5%9B%BE%E5%83%8F/4006450256177f4a/%E6%B5%8B%E8%AF%95.jpg
|
139
139
|
http://@AWS_S3_TEST_BUCKET@.s3.amazonaws.com/test/%E5%9B%BE%E5%83%8F/4006450256177f4a/%E6%B5%8B%E8%AF%95-small.jpg
|
140
140
|
"""
|
141
|
-
And http://@AWS_S3_TEST_BUCKET@.s3.amazonaws.com/test/图像/4006450256177f4a/测试.jpg will contain JPEG image of size 509x719
|
142
|
-
And http://@AWS_S3_TEST_BUCKET@.s3.amazonaws.com/test/图像/4006450256177f4a/测试-small.jpg will contain JPEG image of size 128x128
|
141
|
+
And Encoded URL http://@AWS_S3_TEST_BUCKET@.s3.amazonaws.com/test/图像/4006450256177f4a/测试.jpg will contain JPEG image of size 509x719
|
142
|
+
And Encoded URL http://@AWS_S3_TEST_BUCKET@.s3.amazonaws.com/test/图像/4006450256177f4a/测试-small.jpg will contain JPEG image of size 128x128
|
143
143
|
|
144
144
|
@compatibility @forward @test
|
145
145
|
Scenario: Getting thumbanils requested with compatibility API from flexi bucket
|
@@ -0,0 +1,35 @@
|
|
1
|
+
Feature: Encoded UTF-8 URI support
|
2
|
+
HTTP Image Store should be able to decode UTF-8 characters form URI using URI decode and also support JavaScript encode() format.
|
3
|
+
|
4
|
+
Background:
|
5
|
+
Given httpthumbnailer server is running at http://localhost:3100/health_check
|
6
|
+
Given httpimagestore server is running at http://localhost:3000/health_check with the following configuration
|
7
|
+
"""
|
8
|
+
path "path" "#{path}"
|
9
|
+
|
10
|
+
post "encoding" "encoded" {
|
11
|
+
store_file "input" root="/tmp" path="path"
|
12
|
+
output_store_uri "input" path="path"
|
13
|
+
}
|
14
|
+
|
15
|
+
post "encoding" "decoded" {
|
16
|
+
store_file "input" root="/tmp" path="path"
|
17
|
+
output_store_path "input" path="path"
|
18
|
+
}
|
19
|
+
"""
|
20
|
+
|
21
|
+
Scenario: JavaScript encode() + URL encoded variable decoding and URL encoding
|
22
|
+
Given test.png file content as request body
|
23
|
+
When I do POST request http://localhost:3000/encoding/encoded/triple%20kro%25u0301l.png
|
24
|
+
And response body will be CRLF ended lines
|
25
|
+
"""
|
26
|
+
/triple%20kro%CC%81l.png
|
27
|
+
"""
|
28
|
+
|
29
|
+
Scenario: URL encoded variable decoding
|
30
|
+
Given test.png file content as request body
|
31
|
+
When I do POST request http://localhost:3000/encoding/decoded/triple%20kr%C3%B3l.png
|
32
|
+
And response body will be CRLF ended lines
|
33
|
+
"""
|
34
|
+
triple król.png
|
35
|
+
"""
|
@@ -217,6 +217,17 @@ Feature: Error handling
|
|
217
217
|
invalid UTF-8 encoding in URI: "hello\\xE9world"
|
218
218
|
"""
|
219
219
|
|
220
|
+
@error-reporting
|
221
|
+
Scenario: Bad URI encoding + JavaScript encode
|
222
|
+
Given test.jpg file content as request body
|
223
|
+
When I do POST request http://localhost:3000/filename/triple%20kro%25udfcfl.png
|
224
|
+
Then response status will be 400
|
225
|
+
And response content type will be text/plain
|
226
|
+
And response body will be CRLF ended lines like
|
227
|
+
"""
|
228
|
+
invalid UTF-8 encoding in URI: "triple kro\\xED\\xBF\\x8Fl"
|
229
|
+
"""
|
230
|
+
|
220
231
|
@error-reporting
|
221
232
|
Scenario: Zero body length
|
222
233
|
Given test.empty file content as request body
|
@@ -0,0 +1,122 @@
|
|
1
|
+
Feature: Rewrite of the output path and URL
|
2
|
+
Output path or URL can be rewritten with variables to customise further the API output.
|
3
|
+
|
4
|
+
Background:
|
5
|
+
Given httpthumbnailer server is running at http://localhost:3100/health_check
|
6
|
+
Given httpimagestore server is running at http://localhost:3000/health_check with the following configuration
|
7
|
+
"""
|
8
|
+
path "input_digest" "#{input_digest}"
|
9
|
+
path "rewritten" "hello/#{prefix}/#{input_sha256}.jpg"
|
10
|
+
path "rewritten-absolute" "/hello/#{prefix}/#{input_sha256}.jpg"
|
11
|
+
path "demo" "image/#{input_sha256}.jpg"
|
12
|
+
|
13
|
+
post "rewrite" "path" "&:prefix" {
|
14
|
+
store_file "input" root="/tmp" path="input_digest"
|
15
|
+
output_store_path "input" path="rewritten"
|
16
|
+
}
|
17
|
+
|
18
|
+
post "rewrite" "uri" "path" "&:prefix" {
|
19
|
+
store_file "input" root="/tmp" path="input_digest"
|
20
|
+
output_store_uri "input" path="rewritten"
|
21
|
+
}
|
22
|
+
|
23
|
+
post "rewrite" "uri" "path-absolute" "&:prefix" {
|
24
|
+
store_file "input" root="/tmp" path="input_digest"
|
25
|
+
output_store_uri "input" path="rewritten-absolute"
|
26
|
+
}
|
27
|
+
|
28
|
+
post "rewrite" "url" "path" "&:prefix" {
|
29
|
+
store_file "input" root="/tmp" path="input_digest"
|
30
|
+
output_store_url "input" path="rewritten"
|
31
|
+
}
|
32
|
+
|
33
|
+
post "rewrite" "url" "path-absolute" "&:prefix" {
|
34
|
+
store_file "input" root="/tmp" path="input_digest"
|
35
|
+
output_store_url "input" path="rewritten-absolute"
|
36
|
+
}
|
37
|
+
|
38
|
+
post "rewrite" "url" "scheme" "&:proto" {
|
39
|
+
store_file "input" root="/tmp" path="input_digest"
|
40
|
+
output_store_url "input" scheme="#{proto}"
|
41
|
+
}
|
42
|
+
|
43
|
+
post "rewrite" "url" "host" "&:target" {
|
44
|
+
store_file "input" root="/tmp" path="input_digest"
|
45
|
+
output_store_url "input" host="www.#{target}"
|
46
|
+
}
|
47
|
+
|
48
|
+
post "rewrite" "url" "port" "&:number" {
|
49
|
+
store_file "input" root="/tmp" path="input_digest"
|
50
|
+
output_store_url "input" port="#{number}"
|
51
|
+
}
|
52
|
+
|
53
|
+
post "rewrite" "demo" {
|
54
|
+
store_file "input" root="/tmp" path="input_digest"
|
55
|
+
output_store_url "input" scheme="ftp" host="example.com" port="21" path="demo"
|
56
|
+
}
|
57
|
+
"""
|
58
|
+
|
59
|
+
Scenario: Store path rewriting
|
60
|
+
Given test.png file content as request body
|
61
|
+
When I do POST request http://localhost:3000/rewrite/path?prefix=world
|
62
|
+
And response body will be CRLF ended lines
|
63
|
+
"""
|
64
|
+
hello/world/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e.jpg
|
65
|
+
"""
|
66
|
+
|
67
|
+
Scenario: Store URI path rewriting
|
68
|
+
Given test.png file content as request body
|
69
|
+
When I do POST request http://localhost:3000/rewrite/uri/path?prefix=world
|
70
|
+
And response body will be CRLF ended lines
|
71
|
+
"""
|
72
|
+
/hello/world/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e.jpg
|
73
|
+
"""
|
74
|
+
|
75
|
+
Scenario: Store URI path rewriting - absolute path
|
76
|
+
Given test.png file content as request body
|
77
|
+
When I do POST request http://localhost:3000/rewrite/uri/path-absolute?prefix=world
|
78
|
+
And response body will be CRLF ended lines
|
79
|
+
"""
|
80
|
+
/hello/world/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e.jpg
|
81
|
+
"""
|
82
|
+
|
83
|
+
Scenario: Store URL path rewriting
|
84
|
+
Given test.png file content as request body
|
85
|
+
When I do POST request http://localhost:3000/rewrite/url/path?prefix=world
|
86
|
+
And response body will be CRLF ended lines
|
87
|
+
"""
|
88
|
+
file:/hello/world/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e.jpg
|
89
|
+
"""
|
90
|
+
|
91
|
+
Scenario: Store URL path rewriting - absolute path
|
92
|
+
Given test.png file content as request body
|
93
|
+
When I do POST request http://localhost:3000/rewrite/url/path-absolute?prefix=world
|
94
|
+
And response body will be CRLF ended lines
|
95
|
+
"""
|
96
|
+
file:/hello/world/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e.jpg
|
97
|
+
"""
|
98
|
+
|
99
|
+
Scenario: Store URL scheme rewriting
|
100
|
+
Given test.png file content as request body
|
101
|
+
When I do POST request http://localhost:3000/rewrite/url/host?target=example.com
|
102
|
+
And response body will be CRLF ended lines
|
103
|
+
"""
|
104
|
+
file://www.example.com/b0fe25319ba5909a
|
105
|
+
"""
|
106
|
+
|
107
|
+
Scenario: Store URL port rewriting
|
108
|
+
Given test.png file content as request body
|
109
|
+
When I do POST request http://localhost:3000/rewrite/url/port?number=41
|
110
|
+
And response body will be CRLF ended lines
|
111
|
+
"""
|
112
|
+
file::41/b0fe25319ba5909a
|
113
|
+
"""
|
114
|
+
|
115
|
+
Scenario: Store URL rewriting demo
|
116
|
+
Given test.png file content as request body
|
117
|
+
When I do POST request http://localhost:3000/rewrite/demo
|
118
|
+
And response body will be CRLF ended lines
|
119
|
+
"""
|
120
|
+
ftp://example.com:21/image/b0fe25319ba5909aa97fded546847a96d7fdf26e18715b0cfccfcbee52dce57e.jpg
|
121
|
+
"""
|
122
|
+
|
@@ -16,7 +16,7 @@ Feature: Store limited original image in S3 and thumbnail based on request
|
|
16
16
|
put "original" {
|
17
17
|
thumbnail "input" "original" operation="limit" width=100 height=100 format="jpeg" quality=95
|
18
18
|
store_s3 "original" bucket="@AWS_S3_TEST_BUCKET@" path="original-hash" cache-root="/tmp"
|
19
|
-
|
19
|
+
output_store_uri "original"
|
20
20
|
}
|
21
21
|
|
22
22
|
get "thumbnail" "v1" ":path" ":operation" ":width" ":height" ":options?" {
|
@@ -38,10 +38,10 @@ Feature: Store limited original image in S3 and thumbnail based on request
|
|
38
38
|
Given test.jpg file content as request body
|
39
39
|
When I do PUT request http://localhost:3000/original
|
40
40
|
Then response status will be 200
|
41
|
-
And response content type will be text/
|
41
|
+
And response content type will be text/uri-list
|
42
42
|
And response body will be CRLF ended lines
|
43
43
|
"""
|
44
|
-
4006450256177f4a.jpg
|
44
|
+
/4006450256177f4a.jpg
|
45
45
|
"""
|
46
46
|
Then S3 object 4006450256177f4a.jpg will contain JPEG image of size 71x100
|
47
47
|
When I do GET request http://@AWS_S3_TEST_BUCKET@.s3.amazonaws.com/4006450256177f4a.jpg
|
@@ -79,4 +79,4 @@ Feature: Store limited original image in S3 and thumbnail based on request
|
|
79
79
|
Then response status will be 200
|
80
80
|
And response content type will be image/png
|
81
81
|
Then response body will contain PNG image of size 50x50
|
82
|
-
And that image pixel at 2x2 should be of color green
|
82
|
+
And that image pixel at 2x2 should be of color green
|
@@ -81,9 +81,14 @@ Then /S3 bucket will not contain (.*)/ do |key|
|
|
81
81
|
@bucket.objects[key].exists?.should_not be_true
|
82
82
|
end
|
83
83
|
|
84
|
-
When /I do (.*) request (
|
84
|
+
When /I do (.*) request ([^ ]+)$/ do |method, uri|
|
85
85
|
@request_body = nil if method == 'GET'
|
86
|
-
|
86
|
+
|
87
|
+
@response = request(method, uri, @request_body, (@request_headers or {}))
|
88
|
+
end
|
89
|
+
|
90
|
+
When /I do (.*) request with encoded URL (.*)/ do |method, uri|
|
91
|
+
step "I do #{method} request #{URI.encode(uri)}"
|
87
92
|
end
|
88
93
|
|
89
94
|
Then /response status will be (.*)/ do |status|
|
@@ -91,11 +96,11 @@ Then /response status will be (.*)/ do |status|
|
|
91
96
|
end
|
92
97
|
|
93
98
|
Then /response content type will be (.*)/ do |content_type|
|
94
|
-
@response.
|
99
|
+
@response.headers['Content-Type'].should == content_type
|
95
100
|
end
|
96
101
|
|
97
102
|
Then /response Cache-Control will be (.*)/ do |content_type|
|
98
|
-
@response.
|
103
|
+
@response.headers['Cache-Control'].should == content_type
|
99
104
|
end
|
100
105
|
|
101
106
|
Then /response body will be CRLF ended lines like/ do |body|
|
@@ -110,7 +115,9 @@ Then /response body will be$/ do |body|
|
|
110
115
|
end
|
111
116
|
|
112
117
|
Then /response body will be CRLF ended lines$/ do |body|
|
113
|
-
@response.body.
|
118
|
+
res = @response.body.dup
|
119
|
+
res.force_encoding('UTF-8')
|
120
|
+
res.should == body.replace_s3_variables.gsub("\n", "\r\n") + "\r\n"
|
114
121
|
end
|
115
122
|
|
116
123
|
Then /(http.*) content type will be (.*)/ do |url, content_type|
|
@@ -125,7 +132,7 @@ Then /(http.*) ([^ ]+) header will not be set/ do |url, header|
|
|
125
132
|
get_headers(url.replace_s3_variables)[header].should be_nil
|
126
133
|
end
|
127
134
|
|
128
|
-
Then
|
135
|
+
Then /^(http[^ ]+) will contain (.*) image of size (.*)x(.*)/ do |url, format, width, height|
|
129
136
|
data = get(url.replace_s3_variables)
|
130
137
|
|
131
138
|
@image.destroy! if @image
|
@@ -136,6 +143,11 @@ Then /(http.*) will contain (.*) image of size (.*)x(.*)/ do |url, format, width
|
|
136
143
|
@image.rows.should == height.to_i
|
137
144
|
end
|
138
145
|
|
146
|
+
|
147
|
+
Then /Encoded URL (http.*) will contain (.*) image of size (.*)x(.*)/ do |url, format, width, height|
|
148
|
+
step "#{URI.encode(url)} will contain #{format} image of size #{width}x#{height}"
|
149
|
+
end
|
150
|
+
|
139
151
|
Then /S3 object (.*) will contain (.*) image of size (.*)x(.*)/ do |key, format, width, height|
|
140
152
|
data = @bucket.objects[key].read
|
141
153
|
|
data/features/support/env.rb
CHANGED
@@ -12,7 +12,7 @@ require 'rspec/expectations'
|
|
12
12
|
|
13
13
|
require 'daemon'
|
14
14
|
require 'timeout'
|
15
|
-
require '
|
15
|
+
require 'faraday'
|
16
16
|
require "open3"
|
17
17
|
require "thread"
|
18
18
|
require 'tempfile'
|
@@ -48,24 +48,26 @@ def script(file)
|
|
48
48
|
end
|
49
49
|
|
50
50
|
def http_client
|
51
|
-
|
52
|
-
|
53
|
-
|
51
|
+
@faraday ||= Faraday.new
|
52
|
+
end
|
53
|
+
|
54
|
+
def request(method, uri, body, headers)
|
55
|
+
http_client.run_request(method.downcase.to_sym, uri.replace_s3_variables, body, headers || {})
|
54
56
|
end
|
55
57
|
|
56
58
|
def get(url)
|
57
|
-
http_client.
|
59
|
+
http_client.get(url).body
|
58
60
|
end
|
59
61
|
|
60
62
|
def get_headers(url)
|
61
|
-
http_client.get(
|
63
|
+
http_client.get(url).headers
|
62
64
|
end
|
63
65
|
|
64
66
|
@@running_cmd = {}
|
65
67
|
def start_server(cmd, pid_file, log_file, test_url)
|
66
68
|
if @@running_cmd[pid_file]
|
67
69
|
return if @@running_cmd[pid_file] == cmd
|
68
|
-
stop_server(pid_file)
|
70
|
+
stop_server(pid_file)
|
69
71
|
end
|
70
72
|
|
71
73
|
fork do
|
@@ -85,7 +87,7 @@ def start_server(cmd, pid_file, log_file, test_url)
|
|
85
87
|
Timeout.timeout(10) do
|
86
88
|
begin
|
87
89
|
get test_url
|
88
|
-
rescue
|
90
|
+
rescue Faraday::Error::ConnectionFailed
|
89
91
|
sleep 0.1
|
90
92
|
retry
|
91
93
|
end
|
@@ -96,7 +98,7 @@ def stop_server(pid_file)
|
|
96
98
|
pid_file = Pathname.new(pid_file)
|
97
99
|
return unless pid_file.exist?
|
98
100
|
|
99
|
-
STDERR.puts http_client.
|
101
|
+
#STDERR.puts http_client.get("http://localhost:3000/stats").body if pid_file.to_s.include? 'httpimagestore'
|
100
102
|
pid = pid_file.read.strip.to_i
|
101
103
|
|
102
104
|
Timeout.timeout(20) do
|
data/httpimagestore.gemspec
CHANGED
@@ -5,11 +5,11 @@
|
|
5
5
|
|
6
6
|
Gem::Specification.new do |s|
|
7
7
|
s.name = "httpimagestore"
|
8
|
-
s.version = "1.
|
8
|
+
s.version = "1.8.0"
|
9
9
|
|
10
10
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
11
11
|
s.authors = ["Jakub Pastuszek"]
|
12
|
-
s.date = "2014-
|
12
|
+
s.date = "2014-08-22"
|
13
13
|
s.description = "Thumbnails images using httpthumbnailer and stored data on HTTP server (S3)"
|
14
14
|
s.email = "jpastuszek@gmail.com"
|
15
15
|
s.executables = ["httpimagestore"]
|
@@ -30,10 +30,12 @@ Gem::Specification.new do |s|
|
|
30
30
|
"features/cache-control.feature",
|
31
31
|
"features/compatibility.feature",
|
32
32
|
"features/data-uri.feature",
|
33
|
+
"features/encoding.feature",
|
33
34
|
"features/error-reporting.feature",
|
34
35
|
"features/flexi.feature",
|
35
36
|
"features/health-check.feature",
|
36
37
|
"features/request-matching.feature",
|
38
|
+
"features/rewrite.feature",
|
37
39
|
"features/s3-store-and-thumbnail.feature",
|
38
40
|
"features/source-failover.feature",
|
39
41
|
"features/step_definitions/httpimagestore_steps.rb",
|
@@ -77,7 +79,8 @@ Gem::Specification.new do |s|
|
|
77
79
|
"spec/spec_helper.rb",
|
78
80
|
"spec/support/compute.jpg",
|
79
81
|
"spec/support/cuba_response_env.rb",
|
80
|
-
"spec/support/full.cfg"
|
82
|
+
"spec/support/full.cfg",
|
83
|
+
"spec/support/utf_string.txt"
|
81
84
|
]
|
82
85
|
s.homepage = "http://github.com/jpastuszek/httpimagestore"
|
83
86
|
s.licenses = ["MIT"]
|
@@ -89,13 +92,13 @@ Gem::Specification.new do |s|
|
|
89
92
|
s.specification_version = 3
|
90
93
|
|
91
94
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
92
|
-
s.add_runtime_dependency(%q<unicorn-cuba-base>, ["~> 1.2.
|
95
|
+
s.add_runtime_dependency(%q<unicorn-cuba-base>, ["~> 1.2.2"])
|
93
96
|
s.add_runtime_dependency(%q<httpthumbnailer-client>, ["~> 1.2.0"])
|
94
97
|
s.add_runtime_dependency(%q<aws-sdk>, ["~> 1.10"])
|
95
98
|
s.add_runtime_dependency(%q<mime-types>, ["~> 1.17"])
|
96
99
|
s.add_runtime_dependency(%q<sdl4r>, ["~> 0.9"])
|
97
100
|
s.add_runtime_dependency(%q<msgpack>, ["~> 0.5"])
|
98
|
-
s.add_development_dependency(%q<
|
101
|
+
s.add_development_dependency(%q<faraday>, [">= 0.8"])
|
99
102
|
s.add_development_dependency(%q<rspec>, ["~> 2.13"])
|
100
103
|
s.add_development_dependency(%q<cucumber>, [">= 0"])
|
101
104
|
s.add_development_dependency(%q<jeweler>, ["~> 1.8.4"])
|
@@ -104,13 +107,13 @@ Gem::Specification.new do |s|
|
|
104
107
|
s.add_development_dependency(%q<prawn>, ["= 0.8.4"])
|
105
108
|
s.add_development_dependency(%q<httpthumbnailer>, ["~> 1.2.0"])
|
106
109
|
else
|
107
|
-
s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.2.
|
110
|
+
s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.2.2"])
|
108
111
|
s.add_dependency(%q<httpthumbnailer-client>, ["~> 1.2.0"])
|
109
112
|
s.add_dependency(%q<aws-sdk>, ["~> 1.10"])
|
110
113
|
s.add_dependency(%q<mime-types>, ["~> 1.17"])
|
111
114
|
s.add_dependency(%q<sdl4r>, ["~> 0.9"])
|
112
115
|
s.add_dependency(%q<msgpack>, ["~> 0.5"])
|
113
|
-
s.add_dependency(%q<
|
116
|
+
s.add_dependency(%q<faraday>, [">= 0.8"])
|
114
117
|
s.add_dependency(%q<rspec>, ["~> 2.13"])
|
115
118
|
s.add_dependency(%q<cucumber>, [">= 0"])
|
116
119
|
s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
|
@@ -120,13 +123,13 @@ Gem::Specification.new do |s|
|
|
120
123
|
s.add_dependency(%q<httpthumbnailer>, ["~> 1.2.0"])
|
121
124
|
end
|
122
125
|
else
|
123
|
-
s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.2.
|
126
|
+
s.add_dependency(%q<unicorn-cuba-base>, ["~> 1.2.2"])
|
124
127
|
s.add_dependency(%q<httpthumbnailer-client>, ["~> 1.2.0"])
|
125
128
|
s.add_dependency(%q<aws-sdk>, ["~> 1.10"])
|
126
129
|
s.add_dependency(%q<mime-types>, ["~> 1.17"])
|
127
130
|
s.add_dependency(%q<sdl4r>, ["~> 0.9"])
|
128
131
|
s.add_dependency(%q<msgpack>, ["~> 0.5"])
|
129
|
-
s.add_dependency(%q<
|
132
|
+
s.add_dependency(%q<faraday>, [">= 0.8"])
|
130
133
|
s.add_dependency(%q<rspec>, ["~> 2.13"])
|
131
134
|
s.add_dependency(%q<cucumber>, [">= 0"])
|
132
135
|
s.add_dependency(%q<jeweler>, ["~> 1.8.4"])
|
@@ -86,7 +86,7 @@ module Configuration
|
|
86
86
|
FileSourceStoreBase.stats.incr_total_file_source_bytes(data.bytesize)
|
87
87
|
|
88
88
|
image = Image.new(data)
|
89
|
-
image.source_url = "
|
89
|
+
image.source_url = URI::Generic.new('file', nil, nil, nil, nil, "/#{URI.encode(rendered_path.to_s)}", nil, nil, nil, URI::DEFAULT_PARSER, true)
|
90
90
|
image
|
91
91
|
rescue Errno::ENOENT
|
92
92
|
raise NoSuchFileError.new(image_name, rendered_path)
|
@@ -112,7 +112,7 @@ module Configuration
|
|
112
112
|
get_named_image_for_storage(request_state) do |image_name, image, rendered_path|
|
113
113
|
storage_path = storage_path(rendered_path)
|
114
114
|
|
115
|
-
image.store_url = "
|
115
|
+
image.store_url = URI::Generic.new('file', nil, nil, nil, nil, "/#{URI.encode(rendered_path.to_s)}", nil, nil, nil, URI::DEFAULT_PARSER, true)
|
116
116
|
|
117
117
|
log.info "storing '#{image_name}' in file '#{storage_path}' (#{image.data.length} bytes)"
|
118
118
|
storage_path.open('wb'){|io| io.write image.data}
|
@@ -98,6 +98,10 @@ module Configuration
|
|
98
98
|
@output_callback or fail 'no output callback'
|
99
99
|
end
|
100
100
|
|
101
|
+
def render_template(template)
|
102
|
+
RubyStringTemplate.new(template).render(self)
|
103
|
+
end
|
104
|
+
|
101
105
|
def fetch_base_variable(name, base_name)
|
102
106
|
fetch(base_name, nil) or generate_meta_variable(base_name) or raise NoVariableToGenerateMetaVariableError.new(base_name, name)
|
103
107
|
end
|
@@ -346,18 +350,18 @@ module Configuration
|
|
346
350
|
name = $1.to_sym
|
347
351
|
value = $2
|
348
352
|
Matcher.new([name], 'QueryKeyValue', "#{name}=#{value}") do
|
349
|
-
->{req[name] && req[name] == value && captures.push(req[name])}
|
353
|
+
->{req.GET[name.to_s] && req.GET[name.to_s] == value && captures.push(req.GET[name.to_s])}
|
350
354
|
end
|
351
355
|
when /^\&:(.+)\?(.*)$/# &:foo?bar
|
352
356
|
name = $1.to_sym
|
353
357
|
default = $2
|
354
358
|
Matcher.new([name], 'QueryKeyDefault', "#{name}=<key>|#{default}") do
|
355
|
-
->{captures.push(req[name] || default)}
|
359
|
+
->{captures.push(req.GET[name.to_s] || default)}
|
356
360
|
end
|
357
361
|
when /^\&:(.+)$/# &:foo
|
358
362
|
name = $1.to_sym
|
359
363
|
Matcher.new([name], 'QueryKey', "#{name}=<key>") do
|
360
|
-
->{req[name] && captures.push(req[name])}
|
364
|
+
->{req.GET[name.to_s] && captures.push(req.GET[name.to_s])}
|
361
365
|
end
|
362
366
|
# Literal URI segment matcher
|
363
367
|
else # foobar
|