httpimagestore 0.5.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +10 -12
- data/Gemfile.lock +57 -55
- data/README.md +829 -0
- data/VERSION +1 -1
- data/bin/httpimagestore +114 -180
- data/features/cache-control.feature +26 -90
- data/features/compatibility.feature +129 -0
- data/features/error-reporting.feature +207 -0
- data/features/health-check.feature +30 -0
- data/features/s3-store-and-thumbnail.feature +65 -0
- data/features/step_definitions/httpimagestore_steps.rb +66 -26
- data/features/support/env.rb +32 -5
- data/features/support/test.empty +0 -0
- data/httpimagestore.gemspec +60 -47
- data/lib/httpimagestore/aws_sdk_regions_hack.rb +23 -0
- data/lib/httpimagestore/configuration/file.rb +120 -0
- data/lib/httpimagestore/configuration/handler.rb +239 -0
- data/lib/httpimagestore/configuration/output.rb +119 -0
- data/lib/httpimagestore/configuration/path.rb +77 -0
- data/lib/httpimagestore/configuration/s3.rb +194 -0
- data/lib/httpimagestore/configuration/thumbnailer.rb +244 -0
- data/lib/httpimagestore/configuration.rb +126 -29
- data/lib/httpimagestore/error_reporter.rb +36 -0
- data/lib/httpimagestore/ruby_string_template.rb +26 -0
- data/load_test/load_test.1k.23a022f6e.m1.small-comp.csv +3 -0
- data/load_test/load_test.1k.ec9bde794.m1.small.csv +4 -0
- data/load_test/load_test.jmx +344 -0
- data/load_test/thumbnail_specs.csv +11 -0
- data/spec/configuration_file_spec.rb +309 -0
- data/spec/configuration_handler_spec.rb +124 -0
- data/spec/configuration_output_spec.rb +338 -0
- data/spec/configuration_path_spec.rb +92 -0
- data/spec/configuration_s3_spec.rb +571 -0
- data/spec/configuration_spec.rb +80 -105
- data/spec/configuration_thumbnailer_spec.rb +417 -0
- data/spec/ruby_string_template_spec.rb +43 -0
- data/spec/spec_helper.rb +61 -0
- data/spec/support/compute.jpg +0 -0
- data/spec/support/cuba_response_env.rb +40 -0
- data/spec/support/full.cfg +49 -0
- metadata +138 -84
- data/README.rdoc +0 -23
- data/features/httpimagestore.feature +0 -167
- data/lib/httpimagestore/image_path.rb +0 -54
- data/lib/httpimagestore/s3_service.rb +0 -37
- data/lib/httpimagestore/thumbnail_class.rb +0 -13
- data/spec/image_path_spec.rb +0 -72
- data/spec/test.cfg +0 -8
@@ -1,167 +0,0 @@
|
|
1
|
-
Feature: Storing of original image and specified classes of its thumbnails on S3
|
2
|
-
In order to store original image and its thumbnails in preconfigured S3 bucket
|
3
|
-
A user must PUT the image data to URI representing its path within the bucket
|
4
|
-
The response will be paths to files stored in S3
|
5
|
-
|
6
|
-
Background:
|
7
|
-
Given issthumbtest S3 bucket with key AKIAJMUYVYOSACNXLPTQ and secret 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
|
8
|
-
Given httpimagestore log is empty
|
9
|
-
Given httpimagestore server is running at http://localhost:3000/ with the following configuration
|
10
|
-
"""
|
11
|
-
s3_key 'AKIAJMUYVYOSACNXLPTQ', 'MAeGhvW+clN7kzK3NboASf3/kZ6a81PRtvwMZj4Y'
|
12
|
-
s3_bucket 'issthumbtest'
|
13
|
-
|
14
|
-
thumbnail_class 'small', 'crop', 128, 128
|
15
|
-
thumbnail_class 'tiny', 'crop', 32, 32
|
16
|
-
thumbnail_class 'tiny_png', 'crop', 32, 32, 'PNG'
|
17
|
-
thumbnail_class 'bad', 'crop', 0, 0
|
18
|
-
thumbnail_class 'superlarge', 'crop', 16000, 16000
|
19
|
-
thumbnail_class 'large_png', 'crop', 7000, 7000, 'PNG'
|
20
|
-
"""
|
21
|
-
Given httpthumbnailer log is empty
|
22
|
-
Given httpthumbnailer server is running at http://localhost:3100/
|
23
|
-
Given Content-Type header set to image/autodetect
|
24
|
-
|
25
|
-
Scenario: Putting original and its thumbnails to S3 bucket
|
26
|
-
Given there is no 4006450256177f4a.jpg file in S3 bucket
|
27
|
-
And there is no 4006450256177f4a/small.jpg file in S3 bucket
|
28
|
-
And there is no 4006450256177f4a/tiny_png.png file in S3 bucket
|
29
|
-
Given test.jpg file content as request body
|
30
|
-
When I do PUT request http://localhost:3000/thumbnail/small,tiny_png
|
31
|
-
Then response status will be 200
|
32
|
-
And response content type will be text/uri-list
|
33
|
-
And response body will be CRLF ended lines
|
34
|
-
"""
|
35
|
-
http://issthumbtest.s3.amazonaws.com/4006450256177f4a.jpg
|
36
|
-
http://issthumbtest.s3.amazonaws.com/4006450256177f4a/small.jpg
|
37
|
-
http://issthumbtest.s3.amazonaws.com/4006450256177f4a/tiny_png.png
|
38
|
-
"""
|
39
|
-
Then http://issthumbtest.s3.amazonaws.com/4006450256177f4a.jpg will contain JPEG image of size 509x719
|
40
|
-
And http://issthumbtest.s3.amazonaws.com/4006450256177f4a.jpg content type will be image/jpeg
|
41
|
-
Then http://issthumbtest.s3.amazonaws.com/4006450256177f4a/small.jpg will contain JPEG image of size 128x128
|
42
|
-
And http://issthumbtest.s3.amazonaws.com/4006450256177f4a/small.jpg content type will be image/jpeg
|
43
|
-
Then http://issthumbtest.s3.amazonaws.com/4006450256177f4a/tiny_png.png will contain PNG image of size 32x32
|
44
|
-
And http://issthumbtest.s3.amazonaws.com/4006450256177f4a/tiny_png.png content type will be image/png
|
45
|
-
|
46
|
-
Scenario: Putting original and its thumbnails to S3 bucket under custom path
|
47
|
-
Given there is no test/image/4006450256177f4a/test.jpg file in S3 bucket
|
48
|
-
And there is no test/image/4006450256177f4a/test-small.jpg file in S3 bucket
|
49
|
-
And there is no test/image/4006450256177f4a/test-tiny_png.png file in S3 bucket
|
50
|
-
Given test.jpg file content as request body
|
51
|
-
When I do PUT request http://localhost:3000/thumbnail/small,tiny_png/test/image/test
|
52
|
-
Then response status will be 200
|
53
|
-
And response content type will be text/uri-list
|
54
|
-
And response body will be CRLF ended lines
|
55
|
-
"""
|
56
|
-
http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test.jpg
|
57
|
-
http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test-small.jpg
|
58
|
-
http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test-tiny_png.png
|
59
|
-
"""
|
60
|
-
Then http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test.jpg will contain JPEG image of size 509x719
|
61
|
-
And http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test.jpg content type will be image/jpeg
|
62
|
-
Then http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test-small.jpg will contain JPEG image of size 128x128
|
63
|
-
And http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test-small.jpg content type will be image/jpeg
|
64
|
-
Then http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test-tiny_png.png will contain PNG image of size 32x32
|
65
|
-
And http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test-tiny_png.png content type will be image/png
|
66
|
-
|
67
|
-
Scenario: Custom path name can contain file name extension that may be used as failback to content based detection
|
68
|
-
Given there is no test/image/4006450256177f4a/test.jpg file in S3 bucket
|
69
|
-
And there is no test/image/4006450256177f4a/test-tiny_png.jpg file in S3 bucket
|
70
|
-
Given test.jpg file content as request body
|
71
|
-
When I do PUT request http://localhost:3000/thumbnail/tiny_png/test/image/test.jpg
|
72
|
-
Then response status will be 200
|
73
|
-
And response content type will be text/uri-list
|
74
|
-
And response body will be CRLF ended lines
|
75
|
-
"""
|
76
|
-
http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test.jpg
|
77
|
-
http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test-tiny_png.png
|
78
|
-
"""
|
79
|
-
And http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test.jpg content type will be image/jpeg
|
80
|
-
And http://issthumbtest.s3.amazonaws.com/test/image/4006450256177f4a/test-tiny_png.png content type will be image/png
|
81
|
-
|
82
|
-
Scenario: Custom path name encoding when UTF-8 characters can be used
|
83
|
-
Given there is no test/图像/4006450256177f4a/测试.jpg file in S3 bucket
|
84
|
-
And there is no test/图像/4006450256177f4a/测试-small.jpg file in S3 bucket
|
85
|
-
Given test.jpg file content as request body
|
86
|
-
When I do PUT request http://localhost:3000/thumbnail/small/test/图像/测试
|
87
|
-
Then response status will be 200
|
88
|
-
And response content type will be text/uri-list
|
89
|
-
And response body will be CRLF ended lines
|
90
|
-
"""
|
91
|
-
http://issthumbtest.s3.amazonaws.com/test/%E5%9B%BE%E5%83%8F/4006450256177f4a/%E6%B5%8B%E8%AF%95.jpg
|
92
|
-
http://issthumbtest.s3.amazonaws.com/test/%E5%9B%BE%E5%83%8F/4006450256177f4a/%E6%B5%8B%E8%AF%95-small.jpg
|
93
|
-
"""
|
94
|
-
And http://issthumbtest.s3.amazonaws.com/test/图像/4006450256177f4a/测试.jpg will contain JPEG image of size 509x719
|
95
|
-
And http://issthumbtest.s3.amazonaws.com/test/图像/4006450256177f4a/测试-small.jpg will contain JPEG image of size 128x128
|
96
|
-
|
97
|
-
Scenario: Reporting of missing resource
|
98
|
-
When I do GET request http://localhost:3000/blah
|
99
|
-
Then response status will be 404
|
100
|
-
And response content type will be text/plain
|
101
|
-
And response body will be CRLF ended lines
|
102
|
-
"""
|
103
|
-
Resource '/blah' not found
|
104
|
-
"""
|
105
|
-
|
106
|
-
Scenario: Reporting of unsupported media type
|
107
|
-
Given there is no test/image/4006450256177f4a/test.jpg file in S3 bucket
|
108
|
-
And there is no test/image/4006450256177f4a/test-small.jpg file in S3 bucket
|
109
|
-
And there is no test/image/4006450256177f4a/test-tiny.jpg file in S3 bucket
|
110
|
-
Given test.txt file content as request body
|
111
|
-
When I do PUT request http://localhost:3000/thumbnail/small,tiny/test/image/test.jpg
|
112
|
-
Then response status will be 415
|
113
|
-
And response content type will be text/plain
|
114
|
-
And response body will be CRLF ended lines like
|
115
|
-
"""
|
116
|
-
Error: HTTPThumbnailerClient::UnsupportedMediaTypeError:
|
117
|
-
"""
|
118
|
-
And S3 bucket will not contain test/image/4006450256177f4a/test.jpg
|
119
|
-
And S3 bucket will not contain test/image/4006450256177f4a/test-small.jpg
|
120
|
-
And S3 bucket will not contain test/image/4006450256177f4a/test-tiny.jpg
|
121
|
-
|
122
|
-
Scenario: Reporting and handling of thumbnailing errors
|
123
|
-
Given there is no test/image/4006450256177f4a/test.jpg file in S3 bucket
|
124
|
-
And there is no test/image/4006450256177f4a/test-small.jpg file in S3 bucket
|
125
|
-
And there is no test/image/4006450256177f4a/test-tiny.jpg file in S3 bucket
|
126
|
-
Given test.jpg file content as request body
|
127
|
-
When I do PUT request http://localhost:3000/thumbnail/small,tiny,bad/test/image/test.jpg
|
128
|
-
Then response status will be 500
|
129
|
-
And response content type will be text/plain
|
130
|
-
And response body will be CRLF ended lines like
|
131
|
-
"""
|
132
|
-
Error: ThumbnailingError: Thumbnailing for class 'bad' failed: Error: ArgumentError:
|
133
|
-
"""
|
134
|
-
And S3 bucket will not contain test/image/4006450256177f4a/test.jpg
|
135
|
-
And S3 bucket will not contain test/image/4006450256177f4a/test-small.jpg
|
136
|
-
And S3 bucket will not contain test/image/4006450256177f4a/test-tiny.jpg
|
137
|
-
|
138
|
-
Scenario: Reporting of missing class error
|
139
|
-
Given test.jpg file content as request body
|
140
|
-
When I do PUT request http://localhost:3000/thumbnail/small,bogous,bad/test/image/test.jpg
|
141
|
-
Then response status will be 404
|
142
|
-
And response content type will be text/plain
|
143
|
-
And response body will be CRLF ended lines like
|
144
|
-
"""
|
145
|
-
Error: Configuration::ThumbnailClassDoesNotExistError: Class 'bogous' does not exist
|
146
|
-
"""
|
147
|
-
|
148
|
-
Scenario: Too large image - uploaded image too big to fit in memory limit
|
149
|
-
Given test-large.jpg file content as request body
|
150
|
-
When I do PUT request http://localhost:3000/thumbnail/large_png/test/image/test.jpg
|
151
|
-
Then response status will be 413
|
152
|
-
And response content type will be text/plain
|
153
|
-
And response body will be CRLF ended lines like
|
154
|
-
"""
|
155
|
-
Error: HTTPThumbnailerClient::ImageTooLargeError:
|
156
|
-
"""
|
157
|
-
|
158
|
-
Scenario: Too large image - memory exhausted when thmbnailing
|
159
|
-
Given test.jpg file content as request body
|
160
|
-
When I do PUT request http://localhost:3000/thumbnail/superlarge/test/image/test.jpg
|
161
|
-
Then response status will be 413
|
162
|
-
And response content type will be text/plain
|
163
|
-
And response body will be CRLF ended lines like
|
164
|
-
"""
|
165
|
-
Error: ThumbnailingError: Thumbnailing for class 'superlarge' failed: Error: Thumbnailer::ImageTooLargeError:
|
166
|
-
"""
|
167
|
-
|
@@ -1,54 +0,0 @@
|
|
1
|
-
require 'mime/types'
|
2
|
-
require 'pathname'
|
3
|
-
|
4
|
-
class ImagePath
|
5
|
-
class CouldNotDetermineFileExtensionError < ArgumentError
|
6
|
-
def initialize(mime_type)
|
7
|
-
super "could not determine file extension for mime type: #{mime_type}"
|
8
|
-
end
|
9
|
-
end
|
10
|
-
|
11
|
-
def initialize(id)
|
12
|
-
@id = id.to_s
|
13
|
-
end
|
14
|
-
|
15
|
-
private
|
16
|
-
|
17
|
-
def mime_extension(mime_type)
|
18
|
-
mime = MIME::Types[mime_type].first or raise CouldNotDetermineFileExtensionError, mime_type
|
19
|
-
'.' + (mime.extensions.select{|e| e.length == 3}.first or mime.extensions.first)
|
20
|
-
end
|
21
|
-
|
22
|
-
class Auto < ImagePath
|
23
|
-
def original_image(mime_type)
|
24
|
-
"#{@id}#{mime_extension(mime_type)}"
|
25
|
-
end
|
26
|
-
|
27
|
-
def thumbnail_image(mime_type, thumbnail_class)
|
28
|
-
"#{@id}/#{thumbnail_class}#{mime_extension(mime_type)}"
|
29
|
-
end
|
30
|
-
end
|
31
|
-
|
32
|
-
class Custom < ImagePath
|
33
|
-
def initialize(id, path)
|
34
|
-
super(id)
|
35
|
-
@path = Pathname.new(path)
|
36
|
-
end
|
37
|
-
|
38
|
-
def original_image(mime_type)
|
39
|
-
extension = begin
|
40
|
-
mime_extension(mime_type)
|
41
|
-
rescue CouldNotDetermineFileExtensionError
|
42
|
-
raise if @path.extname.empty?
|
43
|
-
@path.extname
|
44
|
-
end
|
45
|
-
|
46
|
-
(@path.dirname + @id + "#{@path.basename(@path.extname)}#{extension}").to_s
|
47
|
-
end
|
48
|
-
|
49
|
-
def thumbnail_image(mime_type, thumbnail_class)
|
50
|
-
(@path.dirname + @id + "#{@path.basename(@path.extname)}-#{thumbnail_class}#{mime_extension(mime_type)}").to_s
|
51
|
-
end
|
52
|
-
end
|
53
|
-
end
|
54
|
-
|
@@ -1,37 +0,0 @@
|
|
1
|
-
require 's3'
|
2
|
-
require 'retry-this'
|
3
|
-
|
4
|
-
class S3Service
|
5
|
-
def initialize(key_id, key_secret, bucket, options = {})
|
6
|
-
@options = options
|
7
|
-
@logger = (options[:logger] or Logger.new('/dev/null'))
|
8
|
-
|
9
|
-
@s3 = S3::Service.new(:access_key_id => key_id, :secret_access_key => key_secret)
|
10
|
-
|
11
|
-
@logger.info "Getting bucket: #{bucket}"
|
12
|
-
@bucket = @s3.buckets.find(bucket) or fail "no buckte '#{bucket}' found"
|
13
|
-
end
|
14
|
-
|
15
|
-
def put_image(image_path, content_type, data)
|
16
|
-
@logger.info "Putting image in bucket '#{@bucket.name}': #{image_path}"
|
17
|
-
|
18
|
-
file = @bucket.objects.build(image_path)
|
19
|
-
cache_control = @options[:cache_control].join(', ') if @options.include?(:cache_control) and not @options[:cache_control].empty?
|
20
|
-
file.cache_control = cache_control unless cache_control.empty?
|
21
|
-
|
22
|
-
file.content_type = content_type
|
23
|
-
file.content = data
|
24
|
-
|
25
|
-
RetryThis.retry_this(
|
26
|
-
:times => (@options[:upload_retry_times] or 1),
|
27
|
-
:sleep => (@options[:upload_retry_delay] or 0.0),
|
28
|
-
:error_types => [Errno::ECONNRESET, Timeout::Error, S3::Error::RequestTimeout]
|
29
|
-
) do |attempt|
|
30
|
-
@logger.warn "Retrying S3 save operation" if attempt > 1
|
31
|
-
file.save
|
32
|
-
end
|
33
|
-
|
34
|
-
"http://#{@bucket.name}.s3.amazonaws.com/#{image_path}"
|
35
|
-
end
|
36
|
-
end
|
37
|
-
|
@@ -1,13 +0,0 @@
|
|
1
|
-
class ThumbnailClass
|
2
|
-
def initialize(name, method, width, height, format = 'JPEG', options = {})
|
3
|
-
@name = name
|
4
|
-
@method = method
|
5
|
-
@width = width
|
6
|
-
@height = height
|
7
|
-
@format = format
|
8
|
-
@options = options
|
9
|
-
end
|
10
|
-
|
11
|
-
attr_reader :name, :method, :width, :height, :format, :options
|
12
|
-
end
|
13
|
-
|
data/spec/image_path_spec.rb
DELETED
@@ -1,72 +0,0 @@
|
|
1
|
-
require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
|
2
|
-
require 'httpimagestore/image_path'
|
3
|
-
require 'pathname'
|
4
|
-
|
5
|
-
shared_examples "extension handling" do |image_path|
|
6
|
-
describe "#original_image" do
|
7
|
-
it "determines extension based on mime type" do
|
8
|
-
Pathname.new(image_path.original_image("image/jpeg")).extname.should == ".jpg"
|
9
|
-
Pathname.new(image_path.original_image("image/tiff")).extname.should == ".tif"
|
10
|
-
Pathname.new(image_path.original_image("image/png")).extname.should == ".png"
|
11
|
-
end
|
12
|
-
|
13
|
-
it "should fail if provided extension from mime type could not be determined" do
|
14
|
-
lambda {
|
15
|
-
image_path.original_image("image/xyz")
|
16
|
-
}.should raise_error ImagePath::CouldNotDetermineFileExtensionError, "could not determine file extension for mime type: image/xyz"
|
17
|
-
end
|
18
|
-
end
|
19
|
-
|
20
|
-
describe "#thumbnail_image" do
|
21
|
-
it "determines extension based on mime type" do
|
22
|
-
Pathname.new(image_path.thumbnail_image("image/jpeg", "small")).extname.should == ".jpg"
|
23
|
-
Pathname.new(image_path.thumbnail_image("image/tiff", "small")).extname.should == ".tif"
|
24
|
-
Pathname.new(image_path.thumbnail_image("image/png", "small")).extname.should == ".png"
|
25
|
-
end
|
26
|
-
|
27
|
-
it "should fail if provided extension from mime type could not be determined" do
|
28
|
-
lambda {
|
29
|
-
image_path.thumbnail_image("image/xyz", "small")
|
30
|
-
}.should raise_error ImagePath::CouldNotDetermineFileExtensionError, "could not determine file extension for mime type: image/xyz"
|
31
|
-
end
|
32
|
-
end
|
33
|
-
end
|
34
|
-
|
35
|
-
describe ImagePath do
|
36
|
-
describe ImagePath::Auto do
|
37
|
-
describe "#original_image" do
|
38
|
-
it "returns path in format <id>.<ext>" do
|
39
|
-
ImagePath::Auto.new(123).original_image("image/jpeg").should == "123.jpg"
|
40
|
-
end
|
41
|
-
end
|
42
|
-
|
43
|
-
describe "#thumbnail_image" do
|
44
|
-
it "returns path in format <id>/<class>.<ext>" do
|
45
|
-
ImagePath::Auto.new(123).thumbnail_image("image/jpeg", "small").should == "123/small.jpg"
|
46
|
-
end
|
47
|
-
end
|
48
|
-
|
49
|
-
include_examples "extension handling", ImagePath::Auto.new(123)
|
50
|
-
end
|
51
|
-
|
52
|
-
describe ImagePath::Custom do
|
53
|
-
describe "#original_image" do
|
54
|
-
it "returns path in format abc/<id>/xyz.<ext>" do
|
55
|
-
ImagePath::Custom.new(123, "test/file/path.jpg").original_image("image/jpeg").should == "test/file/123/path.jpg"
|
56
|
-
end
|
57
|
-
|
58
|
-
it "should fail back to provided extension if extension from mime type could not be determined" do
|
59
|
-
Pathname.new(ImagePath::Custom.new(123, "test/file/path.abc").original_image("image/xyz")).extname.should == ".abc"
|
60
|
-
end
|
61
|
-
end
|
62
|
-
|
63
|
-
describe "#thumbnail_image" do
|
64
|
-
it "returns path in format abc/<id>/xyz-<class>.<ext>" do
|
65
|
-
ImagePath::Custom.new(123, "test/file/path.jpg").thumbnail_image("image/jpeg", "small").should == "test/file/123/path-small.jpg"
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
include_examples "extension handling", ImagePath::Custom.new(123, "test/file/path")
|
70
|
-
end
|
71
|
-
end
|
72
|
-
|
data/spec/test.cfg
DELETED