httpthumbnailer 0.3.1 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/Gemfile +10 -11
- data/Gemfile.lock +75 -51
- data/README.md +184 -54
- data/VERSION +1 -1
- data/bin/httpthumbnailer +76 -133
- data/features/httpthumbnailer.feature +11 -260
- data/features/step_definitions/httpthumbnailer_steps.rb +89 -44
- data/features/support/env.rb +22 -9
- data/features/support/test-large.jpg +0 -0
- data/features/thumbnail.feature +241 -0
- data/features/thumbnails.feature +142 -0
- data/httpthumbnailer.gemspec +39 -40
- data/lib/httpthumbnailer/error_reporter.rb +38 -0
- data/lib/httpthumbnailer/plugin/thumbnailer.rb +396 -0
- data/lib/httpthumbnailer/thumbnail_specs.rb +50 -47
- data/lib/httpthumbnailer/thumbnailer.rb +52 -229
- data/load_test/load_test-cd9679c.csv +10 -0
- data/load_test/load_test-v0.3.1.csv +10 -0
- data/load_test/load_test.jmx +49 -34
- data/load_test/soak_test-ac0c6bcbe5e-broken-libjpeg-tatoos.csv +11 -0
- data/load_test/soak_test-cd9679c.csv +10 -0
- data/load_test/soak_test-f98334a-tatoos.csv +11 -0
- data/load_test/soak_test.jmx +697 -0
- data/spec/image_processing_spec.rb +148 -0
- data/spec/spec_helper.rb +4 -1
- data/spec/thumbnail_specs_spec.rb +18 -5
- metadata +101 -71
- data/lib/httpthumbnailer/multipart_response.rb +0 -45
- data/spec/multipart_response_spec.rb +0 -95
- data/spec/thumbnailer_spec.rb +0 -33
data/Gemfile
CHANGED
@@ -1,21 +1,20 @@
|
|
1
1
|
source "http://rubygems.org"
|
2
|
+
ruby "1.9.3"
|
2
3
|
|
3
|
-
gem "
|
4
|
-
gem "mongrel", ">= 1.2.0.pre2"
|
4
|
+
gem "unicorn-cuba-base", "~> 1.0"
|
5
5
|
gem "rmagick", "~> 2"
|
6
|
-
gem "haml", "~> 3"
|
7
|
-
gem "ruby-ip", "~> 0.9"
|
8
|
-
gem "cli", "~> 1.1.0"
|
9
6
|
|
10
7
|
# Add dependencies to develop your gem here.
|
11
8
|
# Include everything needed to run rake, tests, features, etc.
|
12
9
|
group :development do
|
13
|
-
gem "rspec", "~> 2.
|
10
|
+
gem "rspec", "~> 2.13"
|
11
|
+
gem "rspec-mocks", "~> 2.13"
|
14
12
|
gem "cucumber", ">= 0"
|
15
|
-
gem "
|
16
|
-
gem "jeweler", "~> 1.
|
17
|
-
gem "
|
18
|
-
gem "daemon", "~> 1"
|
19
|
-
gem "httpclient", "~> 2.2"
|
13
|
+
gem "capybara", "~> 1.1"
|
14
|
+
gem "jeweler", "~> 1.8.4"
|
15
|
+
gem "httpclient", "~> 2.3"
|
20
16
|
gem "rdoc", "~> 3.9"
|
17
|
+
gem "multipart-parser", "~> 0.1.1"
|
18
|
+
gem "daemon", "~> 1.1"
|
21
19
|
end
|
20
|
+
|
data/Gemfile.lock
CHANGED
@@ -1,69 +1,93 @@
|
|
1
1
|
GEM
|
2
2
|
remote: http://rubygems.org/
|
3
3
|
specs:
|
4
|
-
builder (3.
|
5
|
-
|
6
|
-
|
4
|
+
builder (3.2.0)
|
5
|
+
capybara (1.1.4)
|
6
|
+
mime-types (>= 1.16)
|
7
|
+
nokogiri (>= 1.3.3)
|
8
|
+
rack (>= 1.0.0)
|
9
|
+
rack-test (>= 0.5.4)
|
10
|
+
selenium-webdriver (~> 2.0)
|
11
|
+
xpath (~> 0.1.4)
|
12
|
+
childprocess (0.3.9)
|
13
|
+
ffi (~> 1.0, >= 1.0.11)
|
14
|
+
cli (1.1.1)
|
15
|
+
cuba (3.1.0)
|
16
|
+
rack
|
17
|
+
cucumber (1.3.1)
|
7
18
|
builder (>= 2.1.2)
|
8
|
-
diff-lcs (>= 1.1.
|
9
|
-
gherkin (~> 2.
|
10
|
-
|
11
|
-
term-ansicolor (>= 1.0.6)
|
19
|
+
diff-lcs (>= 1.1.3)
|
20
|
+
gherkin (~> 2.12.0)
|
21
|
+
multi_json (~> 1.3)
|
12
22
|
daemon (1.1.0)
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
gherkin (2.
|
17
|
-
|
23
|
+
diff-lcs (1.2.4)
|
24
|
+
facter (1.6.18)
|
25
|
+
ffi (1.8.1)
|
26
|
+
gherkin (2.12.0)
|
27
|
+
multi_json (~> 1.3)
|
18
28
|
git (1.2.5)
|
19
|
-
|
20
|
-
|
21
|
-
jeweler (1.6.4)
|
29
|
+
httpclient (2.3.3)
|
30
|
+
jeweler (1.8.4)
|
22
31
|
bundler (~> 1.0)
|
23
32
|
git (>= 1.2.5)
|
24
33
|
rake
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
34
|
+
rdoc
|
35
|
+
json (1.7.7)
|
36
|
+
kgio (2.8.0)
|
37
|
+
mime-types (1.23)
|
38
|
+
multi_json (1.7.3)
|
39
|
+
multipart-parser (0.1.1)
|
40
|
+
nokogiri (1.5.9)
|
41
|
+
rack (1.5.2)
|
42
|
+
rack-test (0.6.2)
|
43
|
+
rack (>= 1.0)
|
44
|
+
raindrops (0.11.0)
|
45
|
+
rake (10.0.4)
|
46
|
+
rdoc (3.12.2)
|
47
|
+
json (~> 1.4)
|
48
|
+
rmagick (2.13.2)
|
49
|
+
rspec (2.13.0)
|
50
|
+
rspec-core (~> 2.13.0)
|
51
|
+
rspec-expectations (~> 2.13.0)
|
52
|
+
rspec-mocks (~> 2.13.0)
|
53
|
+
rspec-core (2.13.1)
|
54
|
+
rspec-expectations (2.13.0)
|
55
|
+
diff-lcs (>= 1.1.3, < 2.0)
|
56
|
+
rspec-mocks (2.13.1)
|
57
|
+
ruby-ip (0.9.1)
|
58
|
+
rubyzip (0.9.9)
|
59
|
+
selenium-webdriver (2.32.1)
|
60
|
+
childprocess (>= 0.2.5)
|
61
|
+
multi_json (~> 1.0)
|
62
|
+
rubyzip
|
63
|
+
websocket (~> 1.0.4)
|
64
|
+
unicorn (4.6.3)
|
65
|
+
kgio (~> 2.6)
|
31
66
|
rack
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
rspec-mocks (2.3.0)
|
44
|
-
ruby-ip (0.9.0)
|
45
|
-
sinatra (1.3.1)
|
46
|
-
rack (~> 1.3, >= 1.3.4)
|
47
|
-
rack-protection (~> 1.1, >= 1.1.2)
|
48
|
-
tilt (~> 1.3, >= 1.3.3)
|
49
|
-
term-ansicolor (1.0.7)
|
50
|
-
tilt (1.3.3)
|
67
|
+
raindrops (~> 0.7)
|
68
|
+
unicorn-cuba-base (1.0.0)
|
69
|
+
cli (~> 1.1.0)
|
70
|
+
cuba (~> 3.0)
|
71
|
+
facter (~> 1.6.11)
|
72
|
+
raindrops (~> 0.11)
|
73
|
+
ruby-ip (~> 0.9)
|
74
|
+
unicorn (>= 4.6.2)
|
75
|
+
websocket (1.0.7)
|
76
|
+
xpath (0.1.4)
|
77
|
+
nokogiri (~> 1.3)
|
51
78
|
|
52
79
|
PLATFORMS
|
53
80
|
ruby
|
54
81
|
|
55
82
|
DEPENDENCIES
|
56
|
-
|
57
|
-
cli (~> 1.1.0)
|
83
|
+
capybara (~> 1.1)
|
58
84
|
cucumber
|
59
|
-
daemon (~> 1)
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
mongrel (>= 1.2.0.pre2)
|
64
|
-
rcov
|
85
|
+
daemon (~> 1.1)
|
86
|
+
httpclient (~> 2.3)
|
87
|
+
jeweler (~> 1.8.4)
|
88
|
+
multipart-parser (~> 0.1.1)
|
65
89
|
rdoc (~> 3.9)
|
66
90
|
rmagick (~> 2)
|
67
|
-
rspec (~> 2.
|
68
|
-
|
69
|
-
|
91
|
+
rspec (~> 2.13)
|
92
|
+
rspec-mocks (~> 2.13)
|
93
|
+
unicorn-cuba-base (~> 1.0)
|
data/README.md
CHANGED
@@ -1,99 +1,229 @@
|
|
1
|
-
#
|
1
|
+
# HTTP Thumbnailer
|
2
2
|
|
3
3
|
HTTP API server for image thumbnailing and format conversion.
|
4
4
|
|
5
|
-
It is using
|
5
|
+
It is using [ImageMagick](http://www.imagemagick.org) or [GraphicsMagick](http://www.graphicsmagick.org) via [RMagick](http://rmagick.rubyforge.org) gem as the image processing library.
|
6
|
+
|
7
|
+
## Features
|
8
|
+
|
9
|
+
* thumbnailing images with different aspect ratio keeping methods
|
10
|
+
* support of many input and output formats
|
11
|
+
* efficient API for generating multiple thumbnails from single input image with just one request
|
12
|
+
* many image scaling and loading performance optimizations
|
13
|
+
* efficient memory usage
|
14
|
+
* memory limits and disk memory offloading support
|
15
|
+
* based on [Unicorn HTTP server](http://unicorn.bogomips.org) with UNIX socket communication support
|
6
16
|
|
7
17
|
## Installing
|
8
18
|
|
9
|
-
You will need the following system packages installed: `imagemagick`, `
|
10
|
-
|
19
|
+
You will need the following system packages installed: `imagemagick`, `pkg-config` and `make`.
|
20
|
+
For PNG support install `libpng`. You may want to consult [ImageMagick](http://www.imagemagick.org) installation guide for more information on supported formats and required libraries.
|
11
21
|
|
12
22
|
For Arch Linux you can use this commands:
|
13
23
|
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
24
|
+
```bash
|
25
|
+
pacman -S imagemagick
|
26
|
+
pacman -S libpng
|
27
|
+
pacman -S pkg-config
|
28
|
+
pacman -S make
|
29
|
+
```
|
18
30
|
|
19
31
|
Then you can install the gem as usual:
|
20
32
|
|
21
|
-
|
33
|
+
```bash
|
34
|
+
gem install httpthumbnailer
|
35
|
+
```
|
36
|
+
|
37
|
+
Optionally install Ruby client library and tool:
|
38
|
+
|
39
|
+
```bash
|
40
|
+
gem install httpthumbnailer-client
|
41
|
+
```
|
22
42
|
|
23
43
|
## Usage
|
24
44
|
|
25
|
-
|
45
|
+
### Getting started
|
46
|
+
|
47
|
+
```bash
|
48
|
+
# install httpthumbnailer (see above)
|
49
|
+
# install httpthumbnailer-client
|
50
|
+
gem install httpthumbnailer-client
|
51
|
+
|
52
|
+
# start thumbnailing server in foreground (to stop hit Ctlr-C)
|
53
|
+
httpthumbnailer --foreground --verbose
|
54
|
+
|
55
|
+
# in another console thumbnail to standard output
|
56
|
+
cat ~/Pictures/compute.jpg | httpthumbnailer-client -t crop,100,200,png > thumbnail.png
|
57
|
+
|
58
|
+
# generate multiple thumbnails
|
59
|
+
cat ~/Pictures/compute.jpg | httpthumbnailer-client -t crop,100,200,jpeg,quality:100 -t pad,200,200,png thumbnail1.jpg thumbnail2.png
|
60
|
+
```
|
61
|
+
|
62
|
+
In this example we use [httpthumbnailer-client](http://github.com/jpastuszek/httpthumbnailer-client) gem CLI tool that will use HTTP API of the server to generate thumbnails.
|
63
|
+
|
64
|
+
### Running the server
|
26
65
|
|
27
|
-
|
66
|
+
HTTP Thumbnailer uses worker based server model thanks to [Unicorn HTTP server](http://unicorn.bogomips.org) gem.
|
28
67
|
|
29
|
-
|
30
|
-
By default
|
68
|
+
To start the thumbnailer use `httpthumbnailer` command.
|
69
|
+
By default it will start in background and will spawn CPU core number + 1 number of worker processes.
|
70
|
+
It will be listening on **localhost** port **3100**.
|
31
71
|
|
32
|
-
To start
|
72
|
+
To start in foreground with verbose output use `httpthumbnailer --verbose --foreground`.
|
73
|
+
To see available switches and options use `httpthumbnailer --help`.
|
33
74
|
|
34
|
-
|
75
|
+
When running in background the master server process will store it's PID in `httpthumbnailer.pid` file. You can change pid file location with `--pid-file` option.
|
76
|
+
If running as root you can use `--user` option to specify user with whose privileges the worker processes will be running.
|
35
77
|
|
36
|
-
|
78
|
+
### Logging
|
37
79
|
|
38
|
-
|
80
|
+
`httpthumbnailer` logs to `httpthumbnailer.log` file in current directory by default. You can change log file location with `--log-file` option and verbosity with `--verbose` or `--debug` switch.
|
39
81
|
|
40
|
-
|
41
|
-
Since it is single threaded (some ImageMagick operations may be multithreaded) it will be able to max out only single CPU core.
|
42
|
-
Therefore it is recommended to run as many instances as there are CPU cores available.
|
82
|
+
Additionally `httpthumbnailer` will log requests in [common NCSA format](http://en.wikipedia.org/wiki/Common_Log_Format) to `httpthumbnailer_access.log` file. Use `--access-log-file` option to change location of access log.
|
43
83
|
|
44
|
-
|
84
|
+
### Supported operations
|
45
85
|
|
46
|
-
|
86
|
+
As operation type you can select one of the following options:
|
87
|
+
* fit - fit image within given dimensions keeping aspect ratio
|
88
|
+
* crop - cut image to fit within given dimensions keeping aspect ratio
|
89
|
+
* pad - fit scale image and pad image with background colour to given dimensions keeping aspect ratio
|
90
|
+
* limit - fit scale image to given dimensions if it is larger than that dimensions
|
47
91
|
|
48
|
-
|
92
|
+
### Supported formats
|
49
93
|
|
50
|
-
|
94
|
+
List of supported formats can be displayed with `httpthumbnailer --formats`.
|
95
|
+
Optionally format `input` can be used to use the same thumbnail format as input image.
|
51
96
|
|
52
|
-
|
97
|
+
### Thumbnail width and height
|
53
98
|
|
54
|
-
|
99
|
+
Width and height values are interpreted depending on operation.
|
100
|
+
`input` string can be used for width and/or height to use input image width or height.
|
55
101
|
|
56
|
-
|
57
|
-
server.reject-expect-100-with-417 = "disable"
|
58
|
-
fastcgi.debug = 1
|
59
|
-
fastcgi.server = ( "/" =>
|
60
|
-
((
|
61
|
-
"socket" => "/var/run/lighttpd/httpthumbniler.sock",
|
62
|
-
"bin-path" => "/usr/bin/httpthumbnailer -s fastcgi --no-bind --no-logging",
|
63
|
-
"max-procs" => 2,
|
64
|
-
"check-local" => "disable",
|
65
|
-
"fix-root-scriptname" => "enable",
|
66
|
-
))
|
67
|
-
)
|
68
|
-
}
|
102
|
+
### Thumbnail options
|
69
103
|
|
70
|
-
|
71
|
-
|
104
|
+
Following options can be used with thumbnail specification:
|
105
|
+
* quality - set output image quality; this is format specific: for JPEG 0 is maximum compression and 100 is maximum quality, for PNG first digit is zlib compression level and second one is filter level
|
106
|
+
* background-color - color in HTML notation or textual description ('red', 'green' etc.) used for background when processing transparent images or padding; by default white background is used
|
72
107
|
|
73
108
|
### API
|
74
109
|
|
75
|
-
|
110
|
+
#### Single thumbnail API
|
111
|
+
|
112
|
+
To generate single thumbnail send input image with **PUT** request to URI in format:
|
113
|
+
|
114
|
+
/thumbnail/<operation type>,<width>,<height>,<format>[,<option key>:<option value>]*
|
115
|
+
|
116
|
+
Server will respond with thumbnail data with correct **Content-Type** header value.
|
117
|
+
|
118
|
+
For example the URI may look like this:
|
119
|
+
|
120
|
+
/thumbnail/pad,100,100,png,background-color:green
|
121
|
+
|
122
|
+
For detailed information about the API see [cucumber features](http://github.com/jpastuszek/httpthumbnailer/blob/master/features/thumbnail.feature).
|
123
|
+
|
124
|
+
#### Multipart API
|
125
|
+
|
126
|
+
To generate multiple thumbnails of single image send that image with **PUT** request to URI in format:
|
127
|
+
|
128
|
+
/thumbnails/<operation type>,<width>,<height>,<format>[,<option key>:<option value>]*[/<operation type>,<width>,<height>,<format>[,<option key>:<option value>]*]*
|
76
129
|
|
77
|
-
|
78
|
-
2. the server will respond with **multi-part content** with parts containing data of your thumbnails in order with proper **Content-type** headers set
|
130
|
+
Server will respond with **multi-part content** with each part containing **Content-Type** header and thumbnail data corresponding to format defined in the URI.
|
79
131
|
|
80
132
|
For example the URI may look like this:
|
81
133
|
|
82
|
-
/
|
134
|
+
/thumbnails/crop,16,16,png/crop,4,8,jpg/pad,16,32,jpeg
|
135
|
+
|
136
|
+
HTTP Thumbnailer will generate 3 thumbnails:
|
137
|
+
1. 16x16 cropped PNG
|
138
|
+
2. 4x8 cropped JPEG
|
139
|
+
3. 16x32 colour padded JPEG
|
140
|
+
|
141
|
+
For detailed information about the API see [cucumber features](http://github.com/jpastuszek/httpthumbnailer/blob/master/features/thumbnails.feature).
|
142
|
+
|
143
|
+
### Ruby API client
|
144
|
+
|
145
|
+
To make it easier to use this server [httpthumbnailer-client](http://github.com/jpastuszek/httpthumbnailer-client) gem provides useful class.
|
146
|
+
|
147
|
+
### Memory limits
|
148
|
+
|
149
|
+
Each worker uses [ImageMagick](http://www.imagemagick.org) memory usage limit feature.
|
150
|
+
By default it will use up to 128MiB of RAM and up to 1GiB of disk backed virtual memory.
|
151
|
+
To change this defaults use `--limit-memory` option for RAM limit and `--limit-disk` to control file backed memory mapping limit in MiB.
|
152
|
+
|
153
|
+
## Status codes
|
154
|
+
|
155
|
+
HTTP Thumbnailer will respond with different status codes on different situations.
|
156
|
+
If all goes well 200 OK will be returned otherwise:
|
157
|
+
|
158
|
+
### 400
|
159
|
+
|
160
|
+
* requested thumbnail method is not supported
|
161
|
+
* at least one image dimension is zero in thumbnail spec
|
162
|
+
* missing option key or value in thumbnail spec
|
163
|
+
* missing argument in in thumbnail spec
|
164
|
+
* bad argument value
|
165
|
+
|
166
|
+
### 413
|
167
|
+
|
168
|
+
* request body is too long
|
169
|
+
* input image is too big to fit in memory
|
170
|
+
* memory or pixel cache limit has been exceeded
|
171
|
+
|
172
|
+
### 415
|
173
|
+
|
174
|
+
* unsupported media type - see **Supported formats** section
|
175
|
+
|
176
|
+
### 500
|
177
|
+
|
178
|
+
* unexpected error has occurred - see the log file
|
179
|
+
|
180
|
+
### Multipart API
|
181
|
+
|
182
|
+
In multipart API when error relates to single thumbnail `Content-Type: plain/text` header will be used for that part.
|
183
|
+
In addition `Status` header will be set for failing part with number corresponding to above status codes.
|
184
|
+
The body will contain description of the error.
|
185
|
+
|
186
|
+
## Statistics API
|
187
|
+
|
188
|
+
HTTP Thumbnailer comes with statistics API that shows various runtime collected statistics.
|
189
|
+
It is set up under `/stats` URI. You can also request single stat with `/stats/<stat name>` request.
|
83
190
|
|
84
|
-
|
191
|
+
Example:
|
85
192
|
|
86
|
-
|
87
|
-
|
88
|
-
|
193
|
+
```bash
|
194
|
+
$ curl 127.0.0.1:3100/stats
|
195
|
+
total_requests: 119
|
196
|
+
total_errors: 1
|
197
|
+
calling: 1
|
198
|
+
writing: 0
|
199
|
+
total_images_loaded: 115
|
200
|
+
total_images_prescaled: 30
|
201
|
+
total_thumbnails_created: 147
|
202
|
+
images_loaded: 0
|
203
|
+
max_images_loaded: 3
|
204
|
+
max_images_loaded_worker: 3
|
205
|
+
total_images_created: 312
|
206
|
+
total_images_destroyed: 312
|
207
|
+
total_images_created_from_blob: 115
|
208
|
+
total_images_created_initialize: 53
|
209
|
+
total_images_created_resize: 101
|
210
|
+
total_images_created_crop: 13
|
211
|
+
total_images_created_sample: 30
|
212
|
+
total_write_multipart: 16
|
213
|
+
total_write: 101
|
214
|
+
total_write_part: 48
|
215
|
+
total_write_error: 1
|
216
|
+
total_write_error_part: 0
|
89
217
|
|
90
|
-
|
218
|
+
$ curl 127.0.0.1:3100/stats/total_write_multipart
|
219
|
+
16
|
220
|
+
```
|
91
221
|
|
92
|
-
|
222
|
+
## See also
|
93
223
|
|
94
|
-
|
224
|
+
[HTTP Image Store](https://github.com/jpastuszek/httpimagestore) service is configurable image storage and processing HTTP API server that uses this service as thumbnailing backend.
|
95
225
|
|
96
|
-
## Contributing to
|
226
|
+
## Contributing to HTTP Thumbnailer
|
97
227
|
|
98
228
|
* Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
|
99
229
|
* Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
|
@@ -105,6 +235,6 @@ To make it easy to use this server there is [httpthumbnailer-client](http://gith
|
|
105
235
|
|
106
236
|
## Copyright
|
107
237
|
|
108
|
-
Copyright (c)
|
238
|
+
Copyright (c) 2013 Jakub Pastuszek. See LICENSE.txt for
|
109
239
|
further details.
|
110
240
|
|
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
1.0.0
|