image_vise 0.0.25 → 0.0.26
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/DEVELOPMENT.md +98 -0
- data/Gemfile +2 -1
- data/README.md +9 -75
- data/SECURITY.md +32 -0
- data/examples/custom_image_operator.rb +27 -0
- data/image_vise.gemspec +12 -6
- data/lib/image_vise/image_request.rb +10 -5
- data/lib/image_vise/render_engine.rb +115 -41
- data/lib/image_vise.rb +2 -2
- data/spec/image_vise/image_request_spec.rb +14 -4
- data/spec/image_vise/render_engine_spec.rb +19 -0
- metadata +33 -16
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: cb3f1e2fdb272ce247b39af0ac465622ae4aab47
|
4
|
+
data.tar.gz: af4c923a506fa1b8e960f06cf6ac6686a0cda9d0
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 436cf6a7fcbafd57c1d531530cf42e08f8aa746fa85aa99dfd3d672e1c6ec9a49c6775387cc7f0fbd183e9abb01c75b698e7a0664b4cd65d7115294bdf334bb5
|
7
|
+
data.tar.gz: b72ead80e081805e1705ea8f22a69dccca06a70967a5ba731c27938aa3a20469f482c23f62789bce1f9a90d71666d570e304dd2f35e22aee85ba0fea42872a72
|
data/DEVELOPMENT.md
ADDED
@@ -0,0 +1,98 @@
|
|
1
|
+
# NOTE: Until 1.0.0 ImageVise API is somewhat in flux
|
2
|
+
|
3
|
+
## Defining image operators
|
4
|
+
|
5
|
+
To add an image operator, define a class roughly like so, and add it to the list of operators.
|
6
|
+
|
7
|
+
class MyBlur
|
8
|
+
# The constructor must accept keyword arguments if your operator accepts them
|
9
|
+
def initialize(radius:, sigma:)
|
10
|
+
@radius = radius.to_f
|
11
|
+
@sigma = sigma.to_f
|
12
|
+
end
|
13
|
+
|
14
|
+
# Needed for the operator to be serialized to parameters
|
15
|
+
def to_h
|
16
|
+
{radius: @radius, sigma: @sigma}
|
17
|
+
end
|
18
|
+
|
19
|
+
def apply!(magick_image) # The argument is a Magick::Image
|
20
|
+
# Apply the method to the function argument
|
21
|
+
blurred_image = magick_image.blur_image(@radius, @sigma)
|
22
|
+
|
23
|
+
# Some methods (without !) return a new Magick::Image.
|
24
|
+
# That image has to be composited back into the original.
|
25
|
+
magick_image.composite!(blurred_image, Magick::CenterGravity, Magick::CopyCompositeOp)
|
26
|
+
ensure
|
27
|
+
# Make sure to explicitly destroy() all the intermediate images
|
28
|
+
ImageVise.destroy(blurred_image)
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# This will also make `ImageVise::Pipeline` respond to `blur(radius:.., sigma:..)`
|
33
|
+
ImageVise.add_operator 'my_blur', MyBlur
|
34
|
+
|
35
|
+
## Defining fetchers
|
36
|
+
|
37
|
+
Fetchers are based on the scheme of the image URL. Default fetchers are already defined for `http`, `https`
|
38
|
+
and `file` schemes. If you need to grab data from a database, for instance, you can define a fetcher and register
|
39
|
+
it:
|
40
|
+
|
41
|
+
module DatabaseFetcher
|
42
|
+
def self.fetch_uri_to_tempfile(uri_object)
|
43
|
+
tf = Tempfile.new 'data-fetch'
|
44
|
+
|
45
|
+
object_id = uri_object.path[/\d+/]
|
46
|
+
data_model = ProfilePictures.find(object_id)
|
47
|
+
tf << data_model.body
|
48
|
+
|
49
|
+
tf.rewind; tf
|
50
|
+
rescue Exception => e
|
51
|
+
ImageVise.close_and_unlink(tf) # do not litter Tempfiles
|
52
|
+
raise e
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
56
|
+
ImageVise.register_fetcher 'profilepictures', self
|
57
|
+
|
58
|
+
Once done, you can use URLs like `profilepictures:/5674`
|
59
|
+
|
60
|
+
## Overriding the render engine
|
61
|
+
|
62
|
+
By default, `ImageVise.call` delegates to `ImageVise::RenderEngine.new.call`. You can mount your own subclass
|
63
|
+
instead, and it will handle everything the same way:
|
64
|
+
|
65
|
+
class MyThumbnailer < ImageVise::RenderEngine
|
66
|
+
...
|
67
|
+
end
|
68
|
+
|
69
|
+
map '/thumbs' do
|
70
|
+
run MyThumbnailer.new
|
71
|
+
end
|
72
|
+
|
73
|
+
Note that since the API is in flux the methods you can override in `RenderEngine` can change.
|
74
|
+
So far none of them are private.
|
75
|
+
|
76
|
+
## Performance and memory
|
77
|
+
|
78
|
+
ImageVise uses ImageMagick and RMagick. It _does not_ shell out to `convert` or `mogrify`, because shelling out
|
79
|
+
is _expensive_ in terms of wall clock. It _does_ do it's best to deallocate (`#destroy!`) the image it works on,
|
80
|
+
but it is not 100% bullet proof.
|
81
|
+
|
82
|
+
Additionally, in contrast to `convert` and `mogrify` ImageVise supports _stackable_ operations, and these operations
|
83
|
+
might be repeated with different parameters. Unfortunately, `convert` is _not_ Shake, so we cannot pass a DAG of
|
84
|
+
image operators to it and just expect it to work. If we want to do processing of multiple steps that `convert` is
|
85
|
+
unable to execute in one call, we have to do
|
86
|
+
|
87
|
+
[fork+exec read + convert + write] -> [fork+exec read + convert + write] + ...
|
88
|
+
|
89
|
+
for each operator we want to apply in a consistent fashion. We cannot stuff all the operators into one `convert`
|
90
|
+
command because the order the operators get applied within `convert` is not clear, whereas we need a reproducible
|
91
|
+
deterministic order of operations (as set in the pipeline). A much better solution is - load the image into memory
|
92
|
+
**once**, do all the transformations, save. Additionally, if you use things like OpenCL with ImageMagick, the overhead
|
93
|
+
of loading the library and compiling the compute kernels will outweigh _any_ performance gains you might get when
|
94
|
+
actually using them. If you are using a library it is a one-time cost, with very fast processing afterwards.
|
95
|
+
|
96
|
+
Also note that image operators are not per definition Imagemagick-specific - it's completely possible to not only use
|
97
|
+
a different library for processing them, but even to use a different image processing server complying to the
|
98
|
+
same protocol (a signed JSON-encodded waybill of HTTP(S) source-URL + pipeline instructions).
|
data/Gemfile
CHANGED
@@ -1,6 +1,5 @@
|
|
1
1
|
source 'https://rubygems.org'
|
2
2
|
|
3
|
-
gem 'bundler'
|
4
3
|
gem 'patron', '~> 0.6'
|
5
4
|
gem 'rmagick', '~> 2.15', :require => 'rmagick'
|
6
5
|
gem 'exceptional_fork', '~> 1.2'
|
@@ -8,6 +7,8 @@ gem 'ks'
|
|
8
7
|
gem 'magic_bytes'
|
9
8
|
|
10
9
|
group :development do
|
10
|
+
gem 'bundler'
|
11
|
+
gem 'yard'
|
11
12
|
gem 'simplecov'
|
12
13
|
gem 'rack-cache'
|
13
14
|
gem 'strenv'
|
data/README.md
CHANGED
@@ -129,40 +129,6 @@ image = Magick::Image.read(my_image_path)[0]
|
|
129
129
|
pipe.apply!(image)
|
130
130
|
```
|
131
131
|
|
132
|
-
## Performance and memory
|
133
|
-
|
134
|
-
ImageVise uses ImageMagick and RMagick. It _does not_ shell out to `convert` or `mogrify`, because shelling out
|
135
|
-
is _expensive_ in terms of wall clock. It _does_ do it's best to deallocate (`#destroy!`) the image it works on,
|
136
|
-
but it is not 100% bullet proof.
|
137
|
-
|
138
|
-
Additionally, in contrast to `convert` and `mogrify` ImageVise supports _stackable_ operations, and these operations
|
139
|
-
might be repeated with different parameters. Unfortunately, `convert` is _not_ Shake, so we cannot pass a DAG of
|
140
|
-
image operators to it and just expect it to work. If we want to do processing of multiple steps that `convert` is
|
141
|
-
unable to execute in one call, we have to do
|
142
|
-
|
143
|
-
[fork+exec read + convert + write] -> [fork+exec read + convert + write] + ...
|
144
|
-
|
145
|
-
for each operator we want to apply in a consistent fashion. We cannot stuff all the operators into one `convert`
|
146
|
-
command because the order the operators get applied within `convert` is not clear, whereas we need a reproducible
|
147
|
-
deterministic order of operations (as set in the pipeline). A much better solution is - load the image into memory
|
148
|
-
**once**, do all the transformations, save. Additionally, if you use things like OpenCL with ImageMagick, the overhead
|
149
|
-
of loading the library and compiling the compute kernels will outweigh _any_ performance gains you might get when
|
150
|
-
actually using them. If you are using a library it is a one-time cost, with very fast processing afterwards.
|
151
|
-
|
152
|
-
Also note that image operators are not per definition Imagemagick-specific - it's completely possible to not only use
|
153
|
-
a different library for processing them, but even to use a different image processing server complying to the
|
154
|
-
same protocol (a signed JSON-encodded waybill of HTTP(S) source-URL + pipeline instructions).
|
155
|
-
|
156
|
-
## Using forked child processes for RMagick tasks
|
157
|
-
|
158
|
-
You can optionally set the `IMAGE_VISE_ENABLE_FORK` environment variable to `yes` to enable forking. When this
|
159
|
-
variable is set, ImageVise will fork a child process and perform the image processing task within that process,
|
160
|
-
killing it afterwards and deallocating all the memory. This can be extremely efficient for dealing with potential
|
161
|
-
memory bloat issues in ImageMagick/RMagick. However, loading images into RMagick may hang in a forked child. This
|
162
|
-
will lead to the child being timeout-terminated, and no image is going to be rendered. This issue is known and
|
163
|
-
also platform-dependent (it does not happen on OSX but does happen on Docker within Ubuntu Trusty for instance).
|
164
|
-
|
165
|
-
So, this feature _does_ exist but your mileage may vary with regards to it's use.
|
166
132
|
|
167
133
|
## Caching
|
168
134
|
|
@@ -207,50 +173,18 @@ Note that these are _glob_ patterns. The image path will be checked against them
|
|
207
173
|
By default, the Rack app within ImageVise swallows all exceptions and returns the error message
|
208
174
|
within a machine-readable JSON payload. If that doesn't work for you, or you want to add error
|
209
175
|
handling using some error tracking provider, either subclass `ImageVise::RenderEngine` or prepend
|
210
|
-
a module into it that will intercept the errors.
|
211
|
-
property of picking up `rack.exception` from the Rack request env. Using the hooks in the render engine,
|
212
|
-
you can add Sentry support by using the following module:
|
213
|
-
|
214
|
-
```ruby
|
215
|
-
module ImageViseSentrySupport
|
216
|
-
ImageVise::RenderEngine.prepend self
|
217
|
-
|
218
|
-
def setup_error_handling(rack_env)
|
219
|
-
@env = rack_env
|
220
|
-
end
|
221
|
-
|
222
|
-
def handle_request_error(err)
|
223
|
-
@env['rack.exception'] = err
|
224
|
-
end
|
225
|
-
|
226
|
-
def handle_generic_error(err)
|
227
|
-
@env['rack.exception'] = err
|
228
|
-
end
|
229
|
-
end
|
230
|
-
```
|
231
|
-
|
232
|
-
For [Appsignal](https://appsignal.com) you can use the following module instead:
|
176
|
+
a module into it that will intercept the errors. See error handling in `examples/` for more.
|
233
177
|
|
234
|
-
|
235
|
-
module ImageViseAppsignal
|
236
|
-
ImageVise::RenderEngine.prepend self
|
237
|
-
|
238
|
-
def setup_error_handling(rack_env)
|
239
|
-
txn = Appsignal::Transaction.current
|
240
|
-
txn.set_action('%s#%s' % [self.class, 'call'])
|
241
|
-
end
|
242
|
-
|
243
|
-
def handle_request_error(err)
|
244
|
-
Appsignal.add_exception(err)
|
245
|
-
end
|
178
|
+
## Using forked child processes for RMagick tasks
|
246
179
|
|
247
|
-
|
248
|
-
|
249
|
-
|
250
|
-
|
251
|
-
|
180
|
+
You can optionally set the `IMAGE_VISE_ENABLE_FORK` environment variable to `yes` to enable forking. When this
|
181
|
+
variable is set, ImageVise will fork a child process and perform the image processing task within that process,
|
182
|
+
killing it afterwards and deallocating all the memory. This can be extremely efficient for dealing with potential
|
183
|
+
memory bloat issues in ImageMagick/RMagick. However, loading images into RMagick may hang in a forked child. This
|
184
|
+
will lead to the child being timeout-terminated, and no image is going to be rendered. This issue is known and
|
185
|
+
also platform-dependent (it does not happen on OSX but does happen on Docker within Ubuntu Trusty for instance).
|
252
186
|
|
253
|
-
|
187
|
+
So, this feature _does_ exist but your mileage may vary with regards to it's use.
|
254
188
|
|
255
189
|
## State
|
256
190
|
|
data/SECURITY.md
ADDED
@@ -0,0 +1,32 @@
|
|
1
|
+
# Security implications for ImageVise
|
2
|
+
|
3
|
+
This lists out the implementation details of security-sensitive parts of ImageVise.
|
4
|
+
|
5
|
+
## Protection of the URLs
|
6
|
+
|
7
|
+
URLs are passed as Base64-encoded JSON. The JSON payload is signed using HMAC with SHA256. This should be
|
8
|
+
sufficient to prevent too much probing. If this is a critical issue you need to put throttling in front of the application.
|
9
|
+
For checking HMAC values `Rack::Utils.secure_compare` constant-time comparison is used.
|
10
|
+
|
11
|
+
## Protection for remote URLs from HTTP(s) origins
|
12
|
+
|
13
|
+
Only URLs referring to permitted hosts are going to be permitted for fetching. If there are no host added,
|
14
|
+
any remote URL is going to cause an exception. No special verification for whether the upstream must be HTTP
|
15
|
+
or HTTPS is performed at this time.
|
16
|
+
|
17
|
+
## Protection for "file:/" URLs
|
18
|
+
|
19
|
+
The file URLs are going to be decoded, and the path component will be matched against permitted _glob patterns._
|
20
|
+
The matching takes links (hard and soft) into account, and uses Ruby's `File.fnmatch?` under the hood. The path
|
21
|
+
is always expanded first using `File.expand_path`. The data is not read into ImageMagick from the original location,
|
22
|
+
but gets copied into a tempfile first.
|
23
|
+
|
24
|
+
## ImageMagick memory constraints
|
25
|
+
|
26
|
+
ImageVise does not set RMagick limits by itself. You should
|
27
|
+
[set them according to the RMagick documentation.](https://rmagick.github.io/magick.html#limit_resource)
|
28
|
+
|
29
|
+
## Processing time constraints
|
30
|
+
|
31
|
+
If you are using forking, there will be a timeout used for how long the forked child process may run,
|
32
|
+
which is the default timeout used in ExceptionalFork.
|
@@ -0,0 +1,27 @@
|
|
1
|
+
class MyBlur
|
2
|
+
# The constructor must accept keyword arguments if your operator accepts them
|
3
|
+
def initialize(radius:, sigma:)
|
4
|
+
@radius = radius.to_f
|
5
|
+
@sigma = sigma.to_f
|
6
|
+
end
|
7
|
+
|
8
|
+
# Needed for the operator to be serialized to parameters
|
9
|
+
def to_h
|
10
|
+
{radius: @radius, sigma: @sigma}
|
11
|
+
end
|
12
|
+
|
13
|
+
def apply!(magick_image) # The argument is a Magick::Image
|
14
|
+
# Apply the method to the function argument
|
15
|
+
blurred_image = magick_image.blur_image(@radius, @sigma)
|
16
|
+
|
17
|
+
# Some methods (without !) return a new Magick::Image.
|
18
|
+
# That image has to be composited back into the original.
|
19
|
+
magick_image.composite!(blurred_image, Magick::CenterGravity, Magick::CopyCompositeOp)
|
20
|
+
ensure
|
21
|
+
# Make sure to explicitly destroy() all the intermediate images
|
22
|
+
ImageVise.destroy(blurred_image)
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
# This will also make `ImageVise::Pipeline` respond to `blur(radius:.., sigma:..)`
|
27
|
+
ImageVise.add_operator 'my_blur', MyBlur
|
data/image_vise.gemspec
CHANGED
@@ -2,16 +2,16 @@
|
|
2
2
|
# DO NOT EDIT THIS FILE DIRECTLY
|
3
3
|
# Instead, edit Jeweler::Tasks in Rakefile, and run 'rake gemspec'
|
4
4
|
# -*- encoding: utf-8 -*-
|
5
|
-
# stub: image_vise 0.0.
|
5
|
+
# stub: image_vise 0.0.26 ruby lib
|
6
6
|
|
7
7
|
Gem::Specification.new do |s|
|
8
8
|
s.name = "image_vise"
|
9
|
-
s.version = "0.0.
|
9
|
+
s.version = "0.0.26"
|
10
10
|
|
11
11
|
s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
|
12
12
|
s.require_paths = ["lib"]
|
13
13
|
s.authors = ["Julik Tarkhanov"]
|
14
|
-
s.date = "2016-10-
|
14
|
+
s.date = "2016-10-20"
|
15
15
|
s.description = "Image processing via URLs"
|
16
16
|
s.email = "me@julik.nl"
|
17
17
|
s.extra_rdoc_files = [
|
@@ -19,11 +19,14 @@ Gem::Specification.new do |s|
|
|
19
19
|
"README.md"
|
20
20
|
]
|
21
21
|
s.files = [
|
22
|
+
"DEVELOPMENT.md",
|
22
23
|
"Gemfile",
|
23
24
|
"LICENSE.txt",
|
24
25
|
"README.md",
|
25
26
|
"Rakefile",
|
27
|
+
"SECURITY.md",
|
26
28
|
"examples/config.ru",
|
29
|
+
"examples/custom_image_operator.rb",
|
27
30
|
"examples/error_handline_appsignal.rb",
|
28
31
|
"examples/error_handling_sentry.rb",
|
29
32
|
"image_vise.gemspec",
|
@@ -72,12 +75,13 @@ Gem::Specification.new do |s|
|
|
72
75
|
s.specification_version = 4
|
73
76
|
|
74
77
|
if Gem::Version.new(Gem::VERSION) >= Gem::Version.new('1.2.0') then
|
75
|
-
s.add_runtime_dependency(%q<bundler>, [">= 0"])
|
76
78
|
s.add_runtime_dependency(%q<patron>, ["~> 0.6"])
|
77
79
|
s.add_runtime_dependency(%q<rmagick>, ["~> 2.15"])
|
78
80
|
s.add_runtime_dependency(%q<exceptional_fork>, ["~> 1.2"])
|
79
81
|
s.add_runtime_dependency(%q<ks>, [">= 0"])
|
80
82
|
s.add_runtime_dependency(%q<magic_bytes>, [">= 0"])
|
83
|
+
s.add_development_dependency(%q<bundler>, [">= 0"])
|
84
|
+
s.add_development_dependency(%q<yard>, [">= 0"])
|
81
85
|
s.add_development_dependency(%q<simplecov>, [">= 0"])
|
82
86
|
s.add_development_dependency(%q<rack-cache>, [">= 0"])
|
83
87
|
s.add_development_dependency(%q<strenv>, [">= 0"])
|
@@ -89,12 +93,13 @@ Gem::Specification.new do |s|
|
|
89
93
|
s.add_development_dependency(%q<rake>, ["~> 10"])
|
90
94
|
s.add_development_dependency(%q<jeweler>, [">= 0"])
|
91
95
|
else
|
92
|
-
s.add_dependency(%q<bundler>, [">= 0"])
|
93
96
|
s.add_dependency(%q<patron>, ["~> 0.6"])
|
94
97
|
s.add_dependency(%q<rmagick>, ["~> 2.15"])
|
95
98
|
s.add_dependency(%q<exceptional_fork>, ["~> 1.2"])
|
96
99
|
s.add_dependency(%q<ks>, [">= 0"])
|
97
100
|
s.add_dependency(%q<magic_bytes>, [">= 0"])
|
101
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
102
|
+
s.add_dependency(%q<yard>, [">= 0"])
|
98
103
|
s.add_dependency(%q<simplecov>, [">= 0"])
|
99
104
|
s.add_dependency(%q<rack-cache>, [">= 0"])
|
100
105
|
s.add_dependency(%q<strenv>, [">= 0"])
|
@@ -107,12 +112,13 @@ Gem::Specification.new do |s|
|
|
107
112
|
s.add_dependency(%q<jeweler>, [">= 0"])
|
108
113
|
end
|
109
114
|
else
|
110
|
-
s.add_dependency(%q<bundler>, [">= 0"])
|
111
115
|
s.add_dependency(%q<patron>, ["~> 0.6"])
|
112
116
|
s.add_dependency(%q<rmagick>, ["~> 2.15"])
|
113
117
|
s.add_dependency(%q<exceptional_fork>, ["~> 1.2"])
|
114
118
|
s.add_dependency(%q<ks>, [">= 0"])
|
115
119
|
s.add_dependency(%q<magic_bytes>, [">= 0"])
|
120
|
+
s.add_dependency(%q<bundler>, [">= 0"])
|
121
|
+
s.add_dependency(%q<yard>, [">= 0"])
|
116
122
|
s.add_dependency(%q<simplecov>, [">= 0"])
|
117
123
|
s.add_dependency(%q<rack-cache>, [">= 0"])
|
118
124
|
s.add_dependency(%q<strenv>, [">= 0"])
|
@@ -9,14 +9,18 @@ class ImageVise::ImageRequest < Ks.strict(:src_url, :pipeline)
|
|
9
9
|
def self.to_request(qs_params:, secrets:)
|
10
10
|
base64_encoded_params = qs_params.fetch(:q) rescue qs_params.fetch('q')
|
11
11
|
given_signature = qs_params.fetch(:sig) rescue qs_params.fetch('sig')
|
12
|
-
|
13
|
-
# Decode Base64 first - this gives us a stable serialized form of the request parameters
|
12
|
+
|
13
|
+
# Decode Base64 first - this gives us a stable serialized form of the request parameters.
|
14
|
+
# The encoded parameters might _not_ include ==-padding at the end.
|
14
15
|
decoded_json = Base64.decode64(base64_encoded_params)
|
15
16
|
|
16
|
-
# Check the signature before decoding JSON (since we will be creating symbols
|
17
|
-
|
17
|
+
# Check the signature before decoding JSON (since we will be creating symbols)
|
18
|
+
unless valid_signature?(decoded_json, given_signature, secrets)
|
19
|
+
raise SignatureError, "Invalid or missing signature"
|
20
|
+
end
|
18
21
|
|
19
22
|
# Decode the JSON
|
23
|
+
# (only AFTER the signature has been validated, so we can use symbol keys)
|
20
24
|
params = JSON.parse(decoded_json, symbolize_names: true)
|
21
25
|
|
22
26
|
# Pick up the URL and validate it
|
@@ -30,7 +34,8 @@ class ImageVise::ImageRequest < Ks.strict(:src_url, :pipeline)
|
|
30
34
|
|
31
35
|
def to_query_string_params(signed_with_secret)
|
32
36
|
payload = JSON.dump(to_h)
|
33
|
-
|
37
|
+
base64_enc = Base64.strict_encode64(payload).gsub(/\=+$/, '')
|
38
|
+
{q: base64_enc, sig: OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, signed_with_secret, payload)}
|
34
39
|
end
|
35
40
|
|
36
41
|
def to_h
|
@@ -43,17 +43,24 @@ class ImageVise::RenderEngine
|
|
43
43
|
throw :__bail, response
|
44
44
|
end
|
45
45
|
|
46
|
-
# The main entry point
|
47
|
-
#
|
48
|
-
#
|
46
|
+
# The main entry point for the Rack app. Wraps a call to {#handle_request} in a `catch{}` block
|
47
|
+
# so that any method can abort the request by calling {#bail}
|
48
|
+
#
|
49
|
+
# @param env[Hash] the Rack env
|
50
|
+
# @return [Array] the Rack response
|
49
51
|
def call(env)
|
50
52
|
catch(:__bail) { handle_request(env) }
|
51
53
|
end
|
52
54
|
|
55
|
+
# Hadles the Rack request. If one of the steps calls {#bail} the `:__bail` symbol will be
|
56
|
+
# thrown and the execution will abort. Any errors will cause either an error response in
|
57
|
+
# JSON format or an Exception will be raised (depending on the return value of `raise_exceptions?`)
|
58
|
+
#
|
59
|
+
# @param env[Hash] the Rack env
|
60
|
+
# @return [Array] the Rack response
|
53
61
|
def handle_request(env)
|
54
62
|
setup_error_handling(env)
|
55
|
-
|
56
|
-
|
63
|
+
|
57
64
|
# Assume that if _any_ ETag is given the image is being requested anew as a refetch,
|
58
65
|
# and the client already has it. Just respond with a 304.
|
59
66
|
return [304, DEFAULT_HEADERS.dup, []] if env['HTTP_IF_NONE_MATCH']
|
@@ -61,9 +68,31 @@ class ImageVise::RenderEngine
|
|
61
68
|
req = Rack::Request.new(env)
|
62
69
|
bail(405, 'Only GET supported') unless req.get?
|
63
70
|
|
64
|
-
# Parse and reinstate the URL and pipeline
|
65
71
|
image_request = ImageVise::ImageRequest.to_request(qs_params: req.params, secrets: ImageVise.secret_keys)
|
72
|
+
render_destination_file, render_file_type, etag = process_image_request(image_request)
|
73
|
+
image_rack_response(render_destination_file, render_file_type, etag)
|
74
|
+
rescue *permanent_failures => e
|
75
|
+
handle_request_error(e)
|
76
|
+
http_status_code = e.respond_to?(:http_status) ? e.http_status : 422
|
77
|
+
raise_exception_or_error_response(e, http_status_code)
|
78
|
+
rescue Exception => e
|
79
|
+
if http_status_code = (e.respond_to?(:http_status) && e.http_status)
|
80
|
+
handle_request_error(e)
|
81
|
+
raise_exception_or_error_response(e, http_status_code)
|
82
|
+
else
|
83
|
+
handle_generic_error(e)
|
84
|
+
raise_exception_or_error_response(e, 500)
|
85
|
+
end
|
86
|
+
end
|
66
87
|
|
88
|
+
# Processes the ImageRequest object created from the request parameters,
|
89
|
+
# and returns a triplet of the File object containing the rendered image,
|
90
|
+
# the MagicBytes::FileType object of the render, and the cache ETag value
|
91
|
+
# representing the processing pipeline
|
92
|
+
#
|
93
|
+
# @param image_request[ImageVise::ImageRequest] the request for the image
|
94
|
+
# @return [Array<File, MagicBytes::FileType, String]
|
95
|
+
def process_image_request(image_request)
|
67
96
|
# Recover the source image URL and the pipeline instructions (all the image ops)
|
68
97
|
source_image_uri, pipeline = image_request.src_url, image_request.pipeline
|
69
98
|
raise 'Image pipeline has no operators' if pipeline.empty?
|
@@ -81,21 +110,39 @@ class ImageVise::RenderEngine
|
|
81
110
|
unless source_file_type_permitted?(source_file_type)
|
82
111
|
raise UnsupportedInputFormat.new("Unsupported/unknown input file format .%s" % source_file_type.ext)
|
83
112
|
end
|
84
|
-
|
113
|
+
|
114
|
+
render_destination_file = Tempfile.new('imagevise-render').tap{|f| f.binmode }
|
115
|
+
|
85
116
|
# Perform the processing
|
86
117
|
if enable_forking?
|
87
118
|
require 'exceptional_fork'
|
88
|
-
ExceptionalFork.fork_and_wait
|
119
|
+
ExceptionalFork.fork_and_wait do
|
120
|
+
apply_pipeline(source_file.path, pipeline, source_file_type, render_destination_file.path)
|
121
|
+
end
|
89
122
|
else
|
90
123
|
apply_pipeline(source_file.path, pipeline, source_file_type, render_destination_file.path)
|
91
124
|
end
|
92
|
-
|
125
|
+
|
93
126
|
# Catch this one early
|
127
|
+
render_destination_file.rewind
|
94
128
|
raise EmptyRender, "The rendered image was empty" if render_destination_file.size.zero?
|
95
129
|
|
96
|
-
render_destination_file.rewind
|
97
130
|
render_file_type = detect_file_type(render_destination_file)
|
98
|
-
|
131
|
+
[render_destination_file, render_file_type, etag]
|
132
|
+
ensure
|
133
|
+
ImageVise.close_and_unlink(source_file)
|
134
|
+
end
|
135
|
+
|
136
|
+
# Returns a Rack response triplet. Accepts the return value of
|
137
|
+
# `process_image_request` unsplatted, and returns a triplet that
|
138
|
+
# can be returned as a Rack response. The Rack response will contain
|
139
|
+
# an iterable body object that is designed to automatically delete
|
140
|
+
# the Tempfile it wraps on close.
|
141
|
+
#
|
142
|
+
# @param render_destination_file[File] the File handle to the rendered image
|
143
|
+
# @param render_file_type[MagicBytes::FileType] the rendered file type
|
144
|
+
# @param etag[String] the ETag for the response
|
145
|
+
def image_rack_response(render_destination_file, render_file_type, etag)
|
99
146
|
response_headers = DEFAULT_HEADERS.merge({
|
100
147
|
'Content-Type' => render_file_type.mime,
|
101
148
|
'Content-Length' => '%d' % render_destination_file.size,
|
@@ -106,22 +153,13 @@ class ImageVise::RenderEngine
|
|
106
153
|
# Wrap the body Tempfile with a self-closing response.
|
107
154
|
# Once the response is read in full, the tempfile is going to be closed and unlinked.
|
108
155
|
[200, response_headers, ImageVise::FileResponse.new(render_destination_file)]
|
109
|
-
rescue *permanent_failures => e
|
110
|
-
handle_request_error(e)
|
111
|
-
http_status_code = e.respond_to?(:http_status) ? e.http_status : 422
|
112
|
-
raise_exception_or_error_response(e, http_status_code)
|
113
|
-
rescue Exception => e
|
114
|
-
if http_status_code = (e.respond_to?(:http_status) && e.http_status)
|
115
|
-
handle_request_error(e)
|
116
|
-
raise_exception_or_error_response(e, http_status_code)
|
117
|
-
else
|
118
|
-
handle_generic_error(e)
|
119
|
-
raise_exception_or_error_response(e, 500)
|
120
|
-
end
|
121
|
-
ensure
|
122
|
-
ImageVise.close_and_unlink(source_file)
|
123
156
|
end
|
124
|
-
|
157
|
+
|
158
|
+
# Depending on `raise_exceptions?` will either raise the passed Exception,
|
159
|
+
# or force the application to return the error in the Rack response.
|
160
|
+
#
|
161
|
+
# @param exception[Exception] the error that has to be captured
|
162
|
+
# @param status_code[Fixnum] the HTTP status code
|
125
163
|
def raise_exception_or_error_response(exception, status_code)
|
126
164
|
if raise_exceptions?
|
127
165
|
raise exception
|
@@ -130,25 +168,37 @@ class ImageVise::RenderEngine
|
|
130
168
|
end
|
131
169
|
end
|
132
170
|
|
133
|
-
|
134
|
-
|
135
|
-
|
136
|
-
|
171
|
+
# Detects the file type of the given File and returns
|
172
|
+
# a MagicBytes::FileType object that contains the extension and
|
173
|
+
# the MIME type.
|
174
|
+
#
|
175
|
+
# @param tempfile[File] the file to perform detection on
|
176
|
+
# @return [MagicBytes::FileType] the detected file type
|
137
177
|
def detect_file_type(tempfile)
|
138
178
|
tempfile.rewind
|
139
|
-
MagicBytes.read_and_detect(tempfile)
|
179
|
+
MagicBytes.read_and_detect(tempfile).tap { tempfile.rewind }
|
140
180
|
end
|
141
181
|
|
142
|
-
|
143
|
-
|
182
|
+
# Tells whether the given file type may be loaded into the image processor.
|
183
|
+
#
|
184
|
+
# @param magic_bytes_file_info[MagicBytes::FileType] the filetype
|
185
|
+
# @return [Boolean]
|
186
|
+
def source_file_type_permitted?(magic_bytes_file_info)
|
187
|
+
PERMITTED_SOURCE_FILE_EXTENSIONS.include?(magic_bytes_file_info.ext)
|
144
188
|
end
|
145
189
|
|
146
|
-
|
147
|
-
|
190
|
+
# Tells whether the given file type may be returned
|
191
|
+
# as the result of the render
|
192
|
+
#
|
193
|
+
# @param magic_bytes_file_info[MagicBytes::FileType] the filetype
|
194
|
+
# @return [Boolean]
|
195
|
+
def output_file_type_permitted?(magic_bytes_file_info)
|
196
|
+
PERMITTED_OUTPUT_FILE_EXTENSIONS.include?(magic_bytes_file_info.ext)
|
148
197
|
end
|
149
198
|
|
150
199
|
# Lists exceptions that should lead to the request being flagged
|
151
|
-
# as invalid (
|
200
|
+
# as invalid (4xx as opposed to 5xx for a generic server error).
|
201
|
+
# Decent clients should _not_ retry those requests.
|
152
202
|
def permanent_failures
|
153
203
|
[
|
154
204
|
Magick::ImageMagickError,
|
@@ -158,32 +208,56 @@ class ImageVise::RenderEngine
|
|
158
208
|
end
|
159
209
|
|
160
210
|
# Is meant to be overridden by subclasses,
|
161
|
-
# will be called at the start of each
|
211
|
+
# will be called at the start of each request to set up the error handling
|
212
|
+
# library (Appsignal, Honeybadger, Sentry...)
|
213
|
+
#
|
214
|
+
# @param rack_env[Hash] the Rack env
|
215
|
+
# @return [void]
|
162
216
|
def setup_error_handling(rack_env)
|
163
217
|
end
|
164
218
|
|
165
219
|
# Is meant to be overridden by subclasses,
|
166
220
|
# will be called when a request fails due to a malformed query string,
|
167
|
-
# unrecognized signature or other client-induced problems
|
168
|
-
|
221
|
+
# unrecognized signature or other client-induced problems. The method
|
222
|
+
# should _not_ re-raise the exception.
|
223
|
+
#
|
224
|
+
# @param exception[Exception] the exception to be handled
|
225
|
+
# @return [void]
|
226
|
+
def handle_request_error(exception)
|
169
227
|
end
|
170
228
|
|
171
229
|
# Is meant to be overridden by subclasses,
|
172
230
|
# will be called when a request fails due to an error on the server
|
173
|
-
# (like an unexpected error in an image operator)
|
174
|
-
|
231
|
+
# (like an unexpected error in an image operator). The method
|
232
|
+
# should _not_ re-raise the exception.
|
233
|
+
#
|
234
|
+
# @param exception[Exception] the exception to be handled
|
235
|
+
# @return [void]
|
236
|
+
def handle_generic_error(exception)
|
175
237
|
end
|
176
238
|
|
177
239
|
# Tells whether the engine must raise the exceptions further up the Rack stack,
|
178
240
|
# or they should be suppressed and a JSON response must be returned.
|
241
|
+
#
|
242
|
+
# @return [Boolean]
|
179
243
|
def raise_exceptions?
|
180
244
|
false
|
181
245
|
end
|
182
246
|
|
247
|
+
# Tells whether image processing in a forked subproces should be turned on
|
248
|
+
#
|
249
|
+
# @return [Boolean]
|
183
250
|
def enable_forking?
|
184
251
|
ENV['IMAGE_VISE_ENABLE_FORK'] == 'yes'
|
185
252
|
end
|
186
253
|
|
254
|
+
# Applies the given {ImageVise::Pipeline} to the image, and writes the render to
|
255
|
+
# the given path.
|
256
|
+
#
|
257
|
+
# @param source_file_path[String] the path to the file containing the source image
|
258
|
+
# @param pipeline[#apply!(Magick::Image)] the processing pipeline
|
259
|
+
# @param render_to_path[String] the path to write the rendered image to
|
260
|
+
# @return [void]
|
187
261
|
def apply_pipeline(source_file_path, pipeline, source_file_type, render_to_path)
|
188
262
|
render_file_type = source_file_type
|
189
263
|
magick_image = Magick::Image.read(source_file_path)[0]
|
data/lib/image_vise.rb
CHANGED
@@ -8,7 +8,7 @@ require 'base64'
|
|
8
8
|
require 'rack'
|
9
9
|
|
10
10
|
class ImageVise
|
11
|
-
VERSION = '0.0.
|
11
|
+
VERSION = '0.0.26'
|
12
12
|
S_MUTEX = Mutex.new
|
13
13
|
private_constant :S_MUTEX
|
14
14
|
|
@@ -75,7 +75,7 @@ class ImageVise
|
|
75
75
|
#
|
76
76
|
# The query string elements can be then passed on to RenderEngine for validation and execution.
|
77
77
|
#
|
78
|
-
# @
|
78
|
+
# @yield {ImageVise::Pipeline}
|
79
79
|
# @return [Hash]
|
80
80
|
def image_params(src_url:, secret:)
|
81
81
|
p = Pipeline.new
|
@@ -4,11 +4,10 @@ describe ImageVise::ImageRequest do
|
|
4
4
|
it 'accepts a set of params and secrets, and returns a Pipeline' do
|
5
5
|
img_params = {src_url: 'http://bucket.s3.aws.com/image.jpg', pipeline: [[:crop, {width: 10, height: 10, gravity: 's'}]]}
|
6
6
|
img_params_json = JSON.dump(img_params)
|
7
|
+
|
8
|
+
q = Base64.encode64(img_params_json)
|
7
9
|
signature = OpenSSL::HMAC.hexdigest(OpenSSL::Digest::SHA256.new, 'this is a secret', img_params_json)
|
8
|
-
params = {
|
9
|
-
q: Base64.encode64(img_params_json),
|
10
|
-
sig: signature
|
11
|
-
}
|
10
|
+
params = {q: q, sig: signature}
|
12
11
|
|
13
12
|
image_request = described_class.to_request(qs_params: params, secrets: ['this is a secret'])
|
14
13
|
request_qs_params = image_request.to_query_string_params('this is a secret')
|
@@ -29,6 +28,17 @@ describe ImageVise::ImageRequest do
|
|
29
28
|
expect(image_request.src_url).to be_kind_of(URI)
|
30
29
|
end
|
31
30
|
|
31
|
+
|
32
|
+
it 'never apppends "="-padding to the Base64-encoded "q"' do
|
33
|
+
parametrized = double(to_params: {foo: 'bar'})
|
34
|
+
(1..12).each do |num_chars_in_url|
|
35
|
+
uri = URI('http://ex.com/%s' % ('i' * num_chars_in_url))
|
36
|
+
image_request = described_class.new(src_url: uri, pipeline: parametrized)
|
37
|
+
q = image_request.to_query_string_params('password').fetch(:q)
|
38
|
+
expect(q).not_to include('=')
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
32
42
|
describe 'fails with an invalid pipeline' do
|
33
43
|
it 'when the pipe param is missing'
|
34
44
|
it 'when the pipe param is empty'
|
@@ -158,6 +158,25 @@ describe ImageVise::RenderEngine do
|
|
158
158
|
expect(parsed_image.columns).to eq(10)
|
159
159
|
end
|
160
160
|
|
161
|
+
it 'calls all of the internal methods during execution' do
|
162
|
+
uri = Addressable::URI.parse(public_url)
|
163
|
+
ImageVise.add_allowed_host!(uri.host)
|
164
|
+
ImageVise.add_secret_key!('l33tness')
|
165
|
+
|
166
|
+
p = ImageVise::Pipeline.new.geom(geometry_string: '512x335').fit_crop(width: 10, height: 10, gravity: 'c')
|
167
|
+
image_request = ImageVise::ImageRequest.new(src_url: uri.to_s, pipeline: p)
|
168
|
+
params = image_request.to_query_string_params('l33tness')
|
169
|
+
|
170
|
+
expect(app).to receive(:process_image_request).and_call_original
|
171
|
+
expect(app).to receive(:image_rack_response).and_call_original
|
172
|
+
expect(app).to receive(:source_file_type_permitted?).and_call_original
|
173
|
+
expect(app).to receive(:output_file_type_permitted?).and_call_original
|
174
|
+
expect(app).to receive(:enable_forking?).and_call_original
|
175
|
+
|
176
|
+
get '/', params
|
177
|
+
expect(last_response.status).to eq(200)
|
178
|
+
end
|
179
|
+
|
161
180
|
it 'picks the image from the filesystem if that is permitted' do
|
162
181
|
uri = 'file://' + test_image_path
|
163
182
|
ImageVise.allow_filesystem_source!(File.dirname(test_image_path) + '/*.*')
|
metadata
CHANGED
@@ -1,29 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: image_vise
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.26
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Julik Tarkhanov
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2016-10-
|
11
|
+
date: 2016-10-20 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
|
-
- !ruby/object:Gem::Dependency
|
14
|
-
name: bundler
|
15
|
-
requirement: !ruby/object:Gem::Requirement
|
16
|
-
requirements:
|
17
|
-
- - ">="
|
18
|
-
- !ruby/object:Gem::Version
|
19
|
-
version: '0'
|
20
|
-
type: :runtime
|
21
|
-
prerelease: false
|
22
|
-
version_requirements: !ruby/object:Gem::Requirement
|
23
|
-
requirements:
|
24
|
-
- - ">="
|
25
|
-
- !ruby/object:Gem::Version
|
26
|
-
version: '0'
|
27
13
|
- !ruby/object:Gem::Dependency
|
28
14
|
name: patron
|
29
15
|
requirement: !ruby/object:Gem::Requirement
|
@@ -94,6 +80,34 @@ dependencies:
|
|
94
80
|
- - ">="
|
95
81
|
- !ruby/object:Gem::Version
|
96
82
|
version: '0'
|
83
|
+
- !ruby/object:Gem::Dependency
|
84
|
+
name: bundler
|
85
|
+
requirement: !ruby/object:Gem::Requirement
|
86
|
+
requirements:
|
87
|
+
- - ">="
|
88
|
+
- !ruby/object:Gem::Version
|
89
|
+
version: '0'
|
90
|
+
type: :development
|
91
|
+
prerelease: false
|
92
|
+
version_requirements: !ruby/object:Gem::Requirement
|
93
|
+
requirements:
|
94
|
+
- - ">="
|
95
|
+
- !ruby/object:Gem::Version
|
96
|
+
version: '0'
|
97
|
+
- !ruby/object:Gem::Dependency
|
98
|
+
name: yard
|
99
|
+
requirement: !ruby/object:Gem::Requirement
|
100
|
+
requirements:
|
101
|
+
- - ">="
|
102
|
+
- !ruby/object:Gem::Version
|
103
|
+
version: '0'
|
104
|
+
type: :development
|
105
|
+
prerelease: false
|
106
|
+
version_requirements: !ruby/object:Gem::Requirement
|
107
|
+
requirements:
|
108
|
+
- - ">="
|
109
|
+
- !ruby/object:Gem::Version
|
110
|
+
version: '0'
|
97
111
|
- !ruby/object:Gem::Dependency
|
98
112
|
name: simplecov
|
99
113
|
requirement: !ruby/object:Gem::Requirement
|
@@ -248,11 +262,14 @@ extra_rdoc_files:
|
|
248
262
|
- LICENSE.txt
|
249
263
|
- README.md
|
250
264
|
files:
|
265
|
+
- DEVELOPMENT.md
|
251
266
|
- Gemfile
|
252
267
|
- LICENSE.txt
|
253
268
|
- README.md
|
254
269
|
- Rakefile
|
270
|
+
- SECURITY.md
|
255
271
|
- examples/config.ru
|
272
|
+
- examples/custom_image_operator.rb
|
256
273
|
- examples/error_handline_appsignal.rb
|
257
274
|
- examples/error_handling_sentry.rb
|
258
275
|
- image_vise.gemspec
|