httpimagestore 1.8.1 → 1.9.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZWExYjg5NWQ2ZjU4MDVlN2RhYmRiYWUxNDFiYTVhMmNmMzYwNGNiMw==
+  data.tar.gz: !binary |-
+    NDc4YmY4NjE1ZWNiNDgzNDlkZTE4MjA2NjEzYjcxYTE4ZmU1ODU3Nw==
+SHA512:
+  metadata.gz: !binary |-
+    MzY4MjVlNTY4YjI0ZmM1M2I0OTg1NDM2ODI3YmQxMTczYzEyNWJhMjZlMDE0
+    ZmVmNjc1YTMwY2JjZjY0Njk2MWVhMGU1OTY0MDA0ZjU1NGRkMDYyZGQ5ZjRj
+    NTM0MDRlM2YwMDAxZjg5NGVmMmU0YmRjMTA0NWQ5YWI0M2FlYTc=
+  data.tar.gz: !binary |-
+    ZGE1ZGY1NWZiYzE0OWY3NmU1NzM3NTUxNTYyYmY0MDJjMTAxMGRlNGZjODdk
+    ZTBjY2Y5OTZiNzg3ODc5NTEwZDExZTRhMGE1ZjgyNmJlZmY3YmQ0NzViYjM0
+    NjQ3NzgyOTZmZTQxZWIzOWU4M2RiYTlmYTk2ZjUzOGQ2YjI4M2I=

data/Gemfile

@@ -1,12 +1,12 @@
 source 'http://rubygems.org'
 ruby '1.9.3'
 
-gem 'unicorn-cuba-base', '~> 1.2.2'
+gem 'unicorn-cuba-base', '~> 1.6'
 #gem 'unicorn-cuba-base', path: '../unicorn-cuba-base'
-gem 'httpthumbnailer-client', '~> 1.2.0'
+gem 'httpthumbnailer-client', '~> 1.3'
 #gem 'httpthumbnailer-client', path: '../httpthumbnailer-client'
 gem 'aws-sdk', '~> 1.10'
-gem 'mime-types', '~> 1.17'
+gem 'mime-types', '~> 2.6', '< 2.99'
 gem 'sdl4r', '~> 0.9'
 gem 'msgpack', '~> 0.5'
 gem 'addressable', '~> 2.3'
@@ -14,14 +14,14 @@ gem 'addressable', '~> 2.3'
 # Add dependencies to develop your gem here.
 # Include everything needed to run rake, tests, features, etc.
 group :development do
-  gem 'faraday', '>= 0.8'
+  gem 'faraday', '~> 0.8'
   gem 'rspec', '~> 2.13'
-  gem 'cucumber', '>= 0'
-  gem 'jeweler', '~> 1.8.4'
+  gem 'cucumber', '~> 1.3'
+  gem "jeweler", "~> 1.8", ">= 1.8.8"
   gem 'rdoc', '~> 3.9'
   gem 'daemon', '~> 1'
   gem 'prawn', '= 0.8.4'
   #gem 'httpthumbnailer', path: '../httpthumbnailer'
-  gem 'httpthumbnailer', '~> 1.2.0'
+  gem 'httpthumbnailer', '~> 1.3'
 end
 

data/Gemfile.lock

@@ -8,7 +8,7 @@ GEM
       uuidtools (~> 2.1)
     builder (3.2.2)
     cli (1.3.1)
-    cuba (3.3.0)
+    cuba (3.4.0)
       rack
     cucumber (1.3.8)
       builder (>= 2.1.2)
@@ -18,7 +18,7 @@ GEM
       multi_test (>= 0.0.2)
     daemon (1.1.0)
     diff-lcs (1.2.4)
-    facter (1.6.18)
+    facter (1.7.6)
     faraday (0.8.8)
       multipart-post (~> 1.2.0)
     gherkin (2.12.1)
@@ -34,11 +34,11 @@ GEM
     hashie (2.0.5)
     highline (1.6.20)
     httpauth (0.2.0)
-    httpclient (2.4.0)
-    httpthumbnailer (1.2.0)
+    httpclient (2.6.0.1)
+    httpthumbnailer (1.3.0)
       rmagick (~> 2)
-      unicorn-cuba-base
-    httpthumbnailer-client (1.2.0)
+      unicorn-cuba-base (~> 1.6.0)
+    httpthumbnailer-client (1.3.0)
       cli (~> 1.3)
       httpclient (>= 2.3)
       multipart-parser (~> 0.1.1)
@@ -54,8 +54,8 @@ GEM
     json (1.8.1)
     jwt (0.1.8)
       multi_json (>= 1.5)
-    kgio (2.9.2)
-    mime-types (1.25)
+    kgio (2.9.3)
+    mime-types (2.6.1)
     msgpack (0.5.5)
     multi_json (1.8.2)
     multi_test (0.0.2)
@@ -77,7 +77,7 @@ GEM
     prawn-core (0.8.4)
     prawn-layout (0.8.4)
     prawn-security (0.8.4)
-    rack (1.5.2)
+    rack (1.6.1)
     raindrops (0.13.0)
     rake (10.1.0)
     rdoc (3.12.2)
@@ -93,17 +93,17 @@ GEM
     rspec-mocks (2.14.3)
     ruby-ip (0.9.3)
     sdl4r (0.9.11)
-    unicorn (4.8.3)
+    unicorn (4.6.3)
       kgio (~> 2.6)
       rack
       raindrops (~> 0.7)
-    unicorn-cuba-base (1.2.2)
+    unicorn-cuba-base (1.6.0)
       cli (~> 1.3)
       cuba (~> 3.0)
-      facter (~> 1.6.11)
+      facter (~> 1.6, >= 1.6.18)
       raindrops (~> 0.11)
       ruby-ip (~> 0.9)
-      unicorn (>= 4.6.2)
+      unicorn (~> 4.6.2)
     uuidtools (2.1.4)
 
 PLATFORMS
@@ -112,16 +112,16 @@ PLATFORMS
 DEPENDENCIES
   addressable (~> 2.3)
   aws-sdk (~> 1.10)
-  cucumber
+  cucumber (~> 1.3)
   daemon (~> 1)
-  faraday (>= 0.8)
-  httpthumbnailer (~> 1.2.0)
-  httpthumbnailer-client (~> 1.2.0)
-  jeweler (~> 1.8.4)
-  mime-types (~> 1.17)
+  faraday (~> 0.8)
+  httpthumbnailer (~> 1.3)
+  httpthumbnailer-client (~> 1.3)
+  jeweler (~> 1.8, >= 1.8.8)
+  mime-types (~> 2.6, < 2.99)
   msgpack (~> 0.5)
   prawn (= 0.8.4)
   rdoc (~> 3.9)
   rspec (~> 2.13)
   sdl4r (~> 0.9)
-  unicorn-cuba-base (~> 1.2.2)
+  unicorn-cuba-base (~> 1.6)

data/README.md

@@ -1,22 +1,28 @@
 # HTTP Image Store
 
-HTTP API server for image thumbnailing and storage.
+Configurable S3 or file system image storage and processing HTTP API server.
 It is using [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) as image processing backend.
 
 ## Features
 
 * fully configurable image processing and storage pipeline with custom API configuration capabilities
-* thumbnailing of sourced or input image into one or more thumbnails
+* thumbnailing and editing (rotate, crop, blur, etc.) of input or sourced image into one or more thumbnails
 * sourcing and storage of images on file system
 * sourcing and storage of images on [Amazon S3](http://aws.amazon.com/s3/)
 * image output with Cache-Control header
 * S3 public or private and http:// or https:// URL list output for stored images
 * S3 read-through and write-through object cache
-* storage under custom paths including image hash, content determined extension or used URL path
+* storage under custom paths including image hash, content determined extension or used URI path
+* data URI encoding of image body (for HTML inlining)
+* HMAC based URI authentication
 * based on [Unicorn HTTP server](http://unicorn.bogomips.org) with UNIX socket communication support
 
 ## Changelog
 
+### 1.9.0
+* added support for thumbnail edits
+* `validate_uri_hmac` and `validate_header_hmac` support
+
 ### 1.8.0
 * `output_store_url` support additional arguments: `scheme`, `port` and `host`
 * `output_store_uri` support added
@@ -82,25 +88,24 @@ Configuration consists of:
 * S3 client configuration (optional)
 * storage path definitions
 * API endpoint definitions
+  * request validation
   * image sourcing operations
-  * image storage operation
-  * output operations
+  * image storage operations
+  * output setup
 
 ### Top level configuration elements
 
 #### thumbnailer
 
-Configures [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) client. It will be used to perform image processing operations.
+Configures [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) client. It will be used to perform image processing operations with `thumbnail` and `identify` statements.
 
 Options:
-* `url` - URL of [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) service
-
-If omitted [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) service located at `http://localhost:3100` will be used.
+* `url` - URL of [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) service; default: `http://localhost:3100`
 
 Example:
 
 ```sdl
-thumbnailer url="http://2.2.2.2:1000"
+thumbnailer url="http://10.2.2.2:3100"
 ```
 
 #### s3
@@ -121,7 +126,7 @@ s3 key="AIAITCKMELYWQZPJP7HQ" secret="V37lCu0F48Tv9s7QVqIT/sLf/wwqhNSB4B0Em7Ei"
 
 #### path
 
-This directive is used to define storage and retrieval paths that will be used when storing and sourcing file on file system or S3 service.
+This directive is used to define storage and retrieval paths that will be used when storing and sourcing file on file system or S3 service. They may also be used for generating output paths or URIs.
 You can declare one path per statement or use `{}` brackets syntax to define more than one with single statement - they are semantically equal.
 
 Arguments:
@@ -169,12 +174,12 @@ path {
 }
 ```
 
-#### API endpoint
+#### HTTP API endpoint
 
 This group of statements allows to configure single API endpoint.
-Each endpoint can have one or more operation defined that will be performed on request matching that endpoint.
+Each endpoint can have one or more operation defined that will be performed on HTTP request matching that endpoint.
 
-Endpoints will be evaluated in order of definition until one is matched.
+Endpoints will be evaluated in order of definition until one is matched or **404 Not Found** is returned.
 
 Statement should start with one of the following HTTP verbs in lowercase: `get`, `post`, `put`, `delete`, followed by list of URI segment matchers:
 
@@ -215,7 +220,72 @@ In this example first endpoint will be matched for **GET** request with URI like
 Second endpoint will be matched only for **PUT** requests with URIs beginning with `/thumbnail/v1/small` and `/thumbnail/v1/large`.
 Third endpoint will match any **POST** request.
 
-### API endpoint image sourcing operations
+### API endpoint validator operations
+
+First any defined validators are executed against request to see if we should accept the request or reject it.
+
+#### validate_uri_hmac
+
+This statement sets up HMAC based URI authentication.
+If HMAC provided with the request does not have the expected value the server will return **403 Forbidden** response.
+
+Arguments:
+
+1. query string key for HMAC value - query string parameter key that contains hex encoded HMAC value to be validated against
+
+Options:
+
+* `secret` - shared secret used to generate HMAC
+* `digest` - digest used for HMAC; see OpenSSL digests for valid values; default: sha1 (for HMAC-SHA1 scheme)
+* `exclude` - comma separated list of query string parameter keys that will be excluded form HMAC computation
+* `remove` - comma separated list of query string parameter keys that will be removed from request for further processing
+
+`exclude` is useful when your URI contains extra query string parameters not related to this API that would otherwise brake HMAC computation.
+`remove` can be used to remove all extra (apart form the HMAC parameter itself) HMAC related query string parameters like date or nonce so they don't interact with further statements.
+Adding additional `nonce` or `date` query string parameter can help with HMAC security by 'randomizing' it's value for same URIs - this won't play well with caching though.
+
+Example:
+
+```sdl
+get "small" {
+	validate_uri_hmac "hmac" secret="key"
+}
+```
+
+In this example request URI `/small?hmac=a49182838f1b0b306614cea1d0fbcdab980717f7` will be valid.
+The actual HMAC is computed from `/small` URI and secret `key` using HMAC-SHA1 scheme.
+Note that `hmac` query string parameter will be removed from the request and won't be accessible by further statements.
+
+#### validate_header_hmac
+
+This statement is identical in function as `validate_uri_hmac` but takes value for URI from given request header instead.
+This is useful when HTTP Image Store is behind proxy that rewrites URIs. This would normally brake HMAC computation but if it can store the original URI in a header this can be used instead for HMAC computation.
+
+Arguments:
+
+1. header name - name of request header that will contain original URI value that will be used for HMAC computation instead of the actual request URI
+2. query string key for HMAC value - query string parameter key of the actual request URI that contains hex encoded HMAC value to be validated against
+
+Options:
+
+* `secret` - shared secret used to generate HMAC
+* `digest` - digest used for HMAC; see OpenSSL digests for valid values; default: sha1 (for HMAC-SHA1 scheme)
+* `exclude` - comma separated list of query string parameter keys that will be excluded form HMAC computation
+* `remove` - comma separated list of query string parameter keys that will be removed from request for further processing
+
+Example:
+
+```sdl
+get "small" {
+	validate_header_hmac "X-ORIGINAL-URI" "hmac" secret="key"
+}
+```
+
+In this example request URI `/my-rewritten-uri?hmac=a49182838f1b0b306614cea1d0fbcdab980717f7` will be valid if request comes with `X-ORIGINAL-URI` header of value `/small`.
+Note that the actual request URI needs only to contain valid `hamc` query string parameter while the value of header needs only the URI necessary for validation - it is OK if it contains the `hmac` query string parameter since it will be removed before computation.
+`remove` will operate on request as normal. `exclude` will operate on the value of the header for the computation to be valid.
+
+## API endpoint image sourcing operations
 
 Sourcing will load images into memory for further processing.
 Each image is stored under predefined or user defined name.
@@ -260,7 +330,7 @@ Options:
 
 * `bucket` - name of bucket to source image from
 * `path` - name of predefined path that will be used to generate key to object to source
-* `prefix` - prefix object key with given prefix value; this does not affect format of output URL; prefix will not be included in source path output; default: ``
+* `prefix` - prefix object key with given prefix value; this does not affect format of output URL; prefix will not be included in source path output; default: empty
 * `cache-root` - path to directory where S3 objects and meta-data will be cached when sourced from S3; read-through mode; if required information was found in cache object no S3 requests will be made; same directory can be used with different buckets since cache key consists of S3 bucket name and object key
 
 Example:
@@ -323,6 +393,7 @@ Options:
 * `height` - requested thumbnail height in pixels
 * `format` - format in which the resulting image will be encoded; all backend supported formats can be specified like `jpeg` or `png`; default: `jpeg`
 * `options` - list of options in format `key:value[,key:value]*` to be passed to thumbnailer; this can be a list of any backend supported options like `background-color` or `quality`
+* `edits` - list of edits in format supported by [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) (`[!<edit name>[,<edit arg>]*[,<edit option>:<edit option value>]*]*`)
 * backend supported options - options can also be defined as statement options
 
 Note that you can use `#{variable}` expansion within all of this options.
@@ -350,6 +421,40 @@ put ":operation" ":width" ":height" ":options" {
 }
 ```
 
+##### edit
+
+`thumbnail` statement supports sub statement `edit` that will apply additional edits to the thumbnail image.
+This edits are applied before `edits` option specified edits.
+
+Arguments:
+
+1. edit name - name of [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer) supported edit
+2. argument 1 - first argument
+3. argument 2 - second argument
+4. argument n - etc. depending on how many argument the edit needs
+
+Options
+
+* any options supported by named edit
+
+Example:
+
+```sdl
+post "example" "&:rotate?0" "&:crop_x?0.0" "&:crop_y?0.0" "&:crop_w?1.0" "&:crop_h?1.0" "&:edits?" {
+	thumbnail "input" "thumbnail" operation="pad" width=100 height=100 format="jpeg" edits="#{edits}" {
+		edit "rotate" "#{rotate}"
+		edit "crop" "#{crop_x}" "#{crop_y}" "#{crop_w}" "#{crop_h}"
+	}
+	output_image "thumbnail"
+}
+```
+
+In this example each thumbnail will be first rotated by `rotate` query string parameter value and cropped by query string provided values `crop_x`, `crop_y`, `crop_w`, `crop_h`.
+If query string parameter `edits` is provided it will be used to do additional edits after rotation and cropping is applied.
+
+Example URI can look like this: `/example?rotate=90&crop_x=0.1&crop_y=0.1&crop_w=0.8&crop_h=0.8&edits=pixelate,0.2,0.1,0.6,0.6,size:0.03!rectangle,0.1,0.8,0.8,0.1,color:blue`.
+Posted image will be rotated by 90 degree. Than the result will be cropped by 10% from each sied. Next middle region of the image will be pixelated and then blue rectangle will be drawn at the bottom of it. Finally thumbnail of dimensions 100 by 100 will be generated and provided as JPEG encoded image in response body.
+
 #### identify
 
 This statement allows for image mime type, width and height identification based on image content with use of [HTTP Thumbnailer](https://github.com/jpastuszek/httpthumbnailer).
@@ -415,7 +520,7 @@ Options:
 * `bucket` - name of bucket to store image in
 * `path` - name of predefined path that will be used to generate key to store object under
 * `public` - if set to `true` the image will be readable by everybody; this affects fromat of output URL; default: `false`
-* `prefix` - prefix storeage key with given prefix value; this does not affect fromat of output URL; prefix will not be included in storage path output; default: ``
+* `prefix` - prefix storeage key with given prefix value; this does not affect fromat of output URL; prefix will not be included in storage path output; default: empty
 * `cache-root` - path to directory where stored S3 objects and meta-data will be cached for future sourcing with `source_s3`; write-through mode; note that same directory needs to be configured with corresponding `source_s3` statement
 
 Example:
@@ -485,9 +590,11 @@ The output will contain the posted image with `Content-Type` header set to `appl
 #### output_data_uri_image
 
 This statement will produce `200 OK` response containing base64 encoded image data within data URI in format:
+
 ```
 data:<image mime type>;base64,<base64 encoded image data>
 ```
+
 No tailing new line/line feed is added to the output.
 
 This may be used to include images directly in HTML pages or CSS.
@@ -684,6 +791,16 @@ The list is in format `image name[,image name]*`.
 
 This option is useful when building API that works on predefined set of image operations and allows to select witch set of operations to perform with list included in the URL.
 
+#### if-variable-matches
+
+It can be used on most operation statements and sub statements (`edit`).
+
+The argument can contain name of variable to or be in format `<name of variable>:<expected value>`.
+In first from the statement will be executed if variable of given name exists and is non empty.
+In second form the statement will be executed if variable value equals to expected value.
+
+This option is useful for example to selectively execute statements based on query string parameters.
+
 ## Configuration examples
 
 ### Flexible API example
@@ -1023,12 +1140,14 @@ If running as root you can use `--user` option to specify user with whose privil
 
 Additionally `httpimagestore` will log requests in [common NCSA format](http://en.wikipedia.org/wiki/Common_Log_Format) to `httpimagestore_access.log` file. Use `--access-log-file` option to change location of access log.
 
-Syslog logging can be enabled with `--syslog-facility` option followed by name of syslog facility to use. When enabled log files are not created and both application logs and access logs are sent to syslog. 
-Access logs will gain meta information that will include `type="http-access"` that can be used to filter access log entries out from application log entries.
+Syslog logging can be enabled with `--syslog-facility` option followed by name of syslog facility to use. When enabled log files are not created and both application logs and access logs are sent to syslog.
+Access logs will gain meta information that will include `"type":"http-access"` that can be used to filter access log entries out from application log entries.
 
 With `--xid-header` option name of HTTP request header can be specified. Value of this header will be logged in meta information tag `xid` along side all request related log entries.
 Named header will be passed down with requests to HTTP Thumbnailer for grater traceability.
 
+Using `--perf-stats` switch will enable logging of performance statistics. This log entries will be logged with `"type":"perf-stats"`.
+
 ### Running with nginx
 
 [nginx](http://nginx.org) if configured properly will buffer incoming requests before sending them to the backend and server response before sending them to client.
@@ -1129,6 +1248,10 @@ If all goes well `200 OK` will be returned otherwise:
 * bad thumbnail specification
 * empty body when image data expected
 
+### 403
+
+* request not authentic
+
 ### 404
 
 * no API endpoint found for given URL
@@ -1159,31 +1282,37 @@ It is set up under `/stats` URI. You can also request single stat with `/stats/<
 Example:
 
 ```bash
-$ curl 10.1.1.24:3000/stats
-total_requests: 2
-total_errors: 0
+$ curl 127.0.0.1:3000/stats
+workers: 10
+total_requests: 27032164
+total_errors: 3785333
 calling: 1
 writing: 0
 total_write_multipart: 0
-total_write: 1
+total_write: 23246830
 total_write_part: 0
-total_write_error: 0
+total_write_error: 3785333
 total_write_error_part: 0
-total_thumbnail_requests: 1
-total_thumbnail_requests_bytes: 43308
-total_thumbnail_thumbnails: 3
-total_thumbnail_thumbnails_bytes: 102914
+total_thumbnail_requests: 16415809
+total_thumbnail_requests_bytes: 1624475247201
+total_thumbnail_thumbnails: 16414688
+total_thumbnail_thumbnails_bytes: 61806698184
+total_identify_requests: 9
+total_identify_requests_bytes: 73948
 total_file_store: 0
 total_file_store_bytes: 0
 total_file_source: 0
 total_file_source_bytes: 0
-total_s3_store: 4
-total_s3_store_bytes: 146222
-total_s3_source: 1
-total_s3_source_bytes: 51581
-
-$ curl 10.1.1.24:3000/stats/total_s3_source
-1
+total_s3_store: 115337
+total_s3_store_bytes: 11420368878
+total_s3_source: 17150098
+total_s3_source_bytes: 1694190265856
+total_s3_cache_hits: 17149566
+total_s3_cache_misses: 1416670
+total_s3_cache_errors: 0
+
+$ curl 127.0.0.1:3000/stats/total_thumbnail_thumbnails
+16414688
 ```
 
 ## Contributing to HTTP Image Store
@@ -1198,6 +1327,5 @@ $ curl 10.1.1.24:3000/stats/total_s3_source
 
 ## Copyright
 
-Copyright (c) 2013 Jakub Pastuszek. See LICENSE.txt for
-further details.
+Copyright (c) 2013 - 2015 Jakub Pastuszek. See LICENSE.txt for further details.