ungulate 0.1.4 → 0.2.0
Sign up to get free protection for your applications and to get access to all the features.
- data/VERSION +1 -1
- data/bin/ungulate_server.rb +22 -7
- data/features/image_resize.feature +2 -12
- data/features/notification_url.feature +14 -0
- data/features/resize_then_composite.feature +2 -1
- data/features/step_definitions/bucket_steps.rb +4 -0
- data/features/step_definitions/command_steps.rb +10 -1
- data/features/step_definitions/notification_steps.rb +23 -0
- data/features/step_definitions/queue_steps.rb +7 -21
- data/features/step_definitions/version_steps.rb +2 -2
- data/features/{support.rb → support/config.rb} +4 -0
- data/features/support/helpers.rb +21 -0
- data/features/support/put_server.rb +22 -0
- data/lib/ungulate.rb +64 -0
- data/lib/ungulate/blob_processor.rb +34 -0
- data/lib/ungulate/curl_http.rb +25 -0
- data/lib/ungulate/file_upload.rb +22 -16
- data/lib/ungulate/job.rb +23 -143
- data/lib/ungulate/rmagick_version_creator.rb +77 -0
- data/lib/ungulate/s3_storage.rb +41 -0
- data/lib/ungulate/server.rb +27 -7
- data/lib/ungulate/sqs_message_queue.rb +38 -0
- data/spec/fixtures/chuckle.jpg +0 -0
- data/spec/fixtures/chuckle.png +0 -0
- data/spec/fixtures/chuckle_converted.png +0 -0
- data/spec/fixtures/chuckle_converted_badly.png +0 -0
- data/spec/fixtures/chuckle_thumbnail.png +0 -0
- data/spec/fixtures/watermark.png +0 -0
- data/spec/lib/ungulate/blob_processor_spec.rb +68 -0
- data/spec/lib/ungulate/curl_http_spec.rb +20 -0
- data/spec/{ungulate → lib/ungulate}/file_upload_spec.rb +10 -27
- data/spec/lib/ungulate/job_spec.rb +120 -0
- data/spec/lib/ungulate/rmagick_version_creator_spec.rb +97 -0
- data/spec/lib/ungulate/s3_storage_spec.rb +74 -0
- data/spec/lib/ungulate/server_spec.rb +44 -0
- data/spec/lib/ungulate/sqs_message_queue_spec.rb +28 -0
- data/spec/{ungulate → lib/ungulate}/view_helpers_spec.rb +1 -1
- data/spec/lib/ungulate_spec.rb +24 -0
- data/spec/spec_helper.rb +73 -0
- metadata +67 -25
- data/spec/ungulate/job_spec.rb +0 -386
- data/spec/ungulate/server_spec.rb +0 -42
data/VERSION
CHANGED
@@ -1 +1 @@
|
|
1
|
-
0.
|
1
|
+
0.2.0
|
data/bin/ungulate_server.rb
CHANGED
@@ -1,18 +1,33 @@
|
|
1
1
|
#!/usr/bin/env ruby
|
2
2
|
|
3
|
-
require 'ungulate
|
3
|
+
require File.expand_path('../lib/ungulate', File.dirname(__FILE__))
|
4
|
+
require 'optparse'
|
4
5
|
|
5
|
-
|
6
|
-
|
7
|
-
|
6
|
+
logger = Logger.new $stderr
|
7
|
+
|
8
|
+
options = {
|
9
|
+
:config => File.expand_path('../config/ungulate', File.dirname(__FILE__))
|
10
|
+
}
|
11
|
+
|
12
|
+
cmdline_options = OptionParser.new do |opts|
|
13
|
+
opts.banner = "Usage: ungulate_server.rb [options]"
|
14
|
+
|
15
|
+
opts.on '-c', '--config CONFIG' do |config|
|
16
|
+
options[:config] = config
|
17
|
+
end
|
8
18
|
end
|
9
19
|
|
10
|
-
|
20
|
+
cmdline_options.parse!
|
21
|
+
|
22
|
+
require options[:config]
|
11
23
|
|
12
24
|
loop do
|
13
25
|
begin
|
14
|
-
processed_something = Ungulate::Server.run
|
15
|
-
sleep(
|
26
|
+
processed_something = Ungulate::Server.run
|
27
|
+
sleep(Ungulate.configuration.server_sleep || 2) unless processed_something
|
28
|
+
rescue Ungulate::MissingConfiguration => e
|
29
|
+
logger.error e.message
|
30
|
+
exit 1
|
16
31
|
rescue StandardError => e
|
17
32
|
logger.error e.message
|
18
33
|
end
|
@@ -5,19 +5,9 @@ Feature: Image resize
|
|
5
5
|
|
6
6
|
Background:
|
7
7
|
Given an empty queue
|
8
|
+
And an empty bucket
|
8
9
|
|
9
|
-
Scenario: Run queue
|
10
|
-
Given a request to resize "image.jpg" to sizes:
|
11
|
-
| label | width | height |
|
12
|
-
| large | 200 | 100 |
|
13
|
-
| small | 100 | 50 |
|
14
|
-
When I run Ungulate
|
15
|
-
Then there should be the following public versions:
|
16
|
-
| key |
|
17
|
-
| image_large.jpg |
|
18
|
-
| image_small.jpg |
|
19
|
-
|
20
|
-
Scenario: Run queue on image key with path separator
|
10
|
+
Scenario: Run queue that has one image job
|
21
11
|
Given a request to resize "some/path/to/image.jpg" to sizes:
|
22
12
|
| label | width | height |
|
23
13
|
| large | 200 | 100 |
|
@@ -0,0 +1,14 @@
|
|
1
|
+
Feature: Notification URL
|
2
|
+
As a developer
|
3
|
+
I want Ungulate to PUT to a URL when it's finished a job
|
4
|
+
So that I don't have to poll for the completed resource
|
5
|
+
|
6
|
+
Background:
|
7
|
+
Given an empty queue
|
8
|
+
And an empty bucket
|
9
|
+
|
10
|
+
Scenario: Run queue that has one image job
|
11
|
+
Given a request that has a notification URL
|
12
|
+
When I run Ungulate
|
13
|
+
Then the notification URL should receive a PUT
|
14
|
+
|
@@ -1,10 +1,11 @@
|
|
1
|
-
Feature: Image resize
|
1
|
+
Feature: Image resize then composite
|
2
2
|
As a site owner
|
3
3
|
I want Ungulate to resize then composite images
|
4
4
|
So that I don't have to watermark images by hand
|
5
5
|
|
6
6
|
Scenario: Run queue on image key with no path separator
|
7
7
|
Given an empty queue
|
8
|
+
And an empty bucket
|
8
9
|
And a request to resize "image.jpg" and then composite with "https://dmxno528jhfy0.cloudfront.net/superhug-watermark.png"
|
9
10
|
When I run Ungulate
|
10
11
|
Then there should be a public watermarked version
|
@@ -1,9 +1,18 @@
|
|
1
1
|
require 'ostruct'
|
2
|
+
require 'ungulate'
|
2
3
|
|
3
4
|
When /^I run Ungulate$/ do
|
4
5
|
@errors = OpenStruct.new :write => ''
|
5
6
|
$stderr = @errors
|
6
|
-
|
7
|
+
|
8
|
+
Ungulate.configure do |config|
|
9
|
+
config.queue_name = QUEUE_NAME
|
10
|
+
config.queue_server = sqs_server
|
11
|
+
end
|
12
|
+
|
13
|
+
10.times do
|
14
|
+
break if Ungulate::Server.run
|
15
|
+
end
|
7
16
|
end
|
8
17
|
|
9
18
|
Then /^there should be no errors$/ do
|
@@ -0,0 +1,23 @@
|
|
1
|
+
Given /^a request that has a notification URL$/ do
|
2
|
+
key = 'bobbyjpeg'
|
3
|
+
|
4
|
+
bucket.put key, File.open('features/camels.jpg').read
|
5
|
+
|
6
|
+
message = {
|
7
|
+
:bucket => BUCKET_NAME,
|
8
|
+
:key => key,
|
9
|
+
:notification_url => 'http://localhost:9999/bob',
|
10
|
+
:versions => {
|
11
|
+
:medium => [
|
12
|
+
[:resize_to_fill, 100, 100],
|
13
|
+
]
|
14
|
+
}
|
15
|
+
}.to_yaml
|
16
|
+
|
17
|
+
send_message(message)
|
18
|
+
end
|
19
|
+
|
20
|
+
Then /^the notification URL should receive a PUT$/ do
|
21
|
+
File.read(TEST_FILE.path).should == 'http://localhost:9999/bob'
|
22
|
+
end
|
23
|
+
|
@@ -1,19 +1,10 @@
|
|
1
1
|
Given /^an empty queue$/ do
|
2
|
-
|
3
|
-
ENV['AMAZON_SECRET_ACCESS_KEY'])
|
4
|
-
@queue_name = 'ungulate-test-queue'
|
5
|
-
@q = sqs.queue @queue_name
|
2
|
+
@q = sqs.queue QUEUE_NAME
|
6
3
|
@q.clear
|
7
4
|
end
|
8
5
|
|
9
6
|
Given /^a request to resize "([^\"]*)" to sizes:$/ do |key, table|
|
10
|
-
|
11
|
-
|
12
|
-
@s3 = RightAws::S3.new(ENV['AMAZON_ACCESS_KEY_ID'],
|
13
|
-
ENV['AMAZON_SECRET_ACCESS_KEY'])
|
14
|
-
@bucket = @s3.bucket @bucket_name
|
15
|
-
@bucket.put key, File.open('features/camels.jpg').read
|
16
|
-
|
7
|
+
bucket.put key, File.open('features/camels.jpg').read
|
17
8
|
|
18
9
|
versions = table.rows.inject({}) do |hash, row|
|
19
10
|
label, width, height = row
|
@@ -22,24 +13,19 @@ Given /^a request to resize "([^\"]*)" to sizes:$/ do |key, table|
|
|
22
13
|
end
|
23
14
|
|
24
15
|
message = {
|
25
|
-
:bucket =>
|
16
|
+
:bucket => BUCKET_NAME,
|
26
17
|
:key => key,
|
27
18
|
:versions => versions
|
28
19
|
}.to_yaml
|
29
20
|
|
30
|
-
|
21
|
+
send_message(message)
|
31
22
|
end
|
32
23
|
|
33
24
|
Given /^a request to resize "([^"]*)" and then composite with "([^"]*)"$/ do |key, composite_url|
|
34
|
-
|
35
|
-
|
36
|
-
@s3 = RightAws::S3.new(ENV['AMAZON_ACCESS_KEY_ID'],
|
37
|
-
ENV['AMAZON_SECRET_ACCESS_KEY'])
|
38
|
-
@bucket = @s3.bucket @bucket_name
|
39
|
-
@bucket.put key, File.open('features/camels.jpg').read
|
25
|
+
bucket.put key, File.open('features/camels.jpg').read
|
40
26
|
|
41
27
|
message = {
|
42
|
-
:bucket =>
|
28
|
+
:bucket => BUCKET_NAME,
|
43
29
|
:key => key,
|
44
30
|
:versions => {
|
45
31
|
:watermarked => [
|
@@ -49,5 +35,5 @@ Given /^a request to resize "([^"]*)" and then composite with "([^"]*)"$/ do |ke
|
|
49
35
|
}
|
50
36
|
}.to_yaml
|
51
37
|
|
52
|
-
|
38
|
+
send_message(message)
|
53
39
|
end
|
@@ -1,5 +1,5 @@
|
|
1
1
|
Then /^there should be the following public versions:$/ do |table|
|
2
|
-
Net::HTTP.start("#{
|
2
|
+
Net::HTTP.start("#{BUCKET_NAME}.s3.amazonaws.com", 80) do |http|
|
3
3
|
table.rows.flatten.each do |key|
|
4
4
|
response = http.get("/#{key}")
|
5
5
|
response.should be_a(Net::HTTPSuccess)
|
@@ -8,7 +8,7 @@ Then /^there should be the following public versions:$/ do |table|
|
|
8
8
|
end
|
9
9
|
|
10
10
|
Then /^there should be a public watermarked version$/ do
|
11
|
-
Net::HTTP.start("#{
|
11
|
+
Net::HTTP.start("#{BUCKET_NAME}.s3.amazonaws.com", 80) do |http|
|
12
12
|
response = http.get("/image_watermarked.jpg")
|
13
13
|
response.should be_a(Net::HTTPSuccess)
|
14
14
|
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
require 'right_aws'
|
2
|
+
|
3
|
+
def bucket
|
4
|
+
s3 = RightAws::S3.new(ENV['AMAZON_ACCESS_KEY_ID'],
|
5
|
+
ENV['AMAZON_SECRET_ACCESS_KEY'])
|
6
|
+
s3.bucket BUCKET_NAME
|
7
|
+
end
|
8
|
+
|
9
|
+
def sqs_server
|
10
|
+
'sqs.eu-west-1.amazonaws.com'
|
11
|
+
end
|
12
|
+
|
13
|
+
def sqs
|
14
|
+
sqs = RightAws::SqsGen2.new(ENV['AMAZON_ACCESS_KEY_ID'],
|
15
|
+
ENV['AMAZON_SECRET_ACCESS_KEY'],
|
16
|
+
:server => sqs_server)
|
17
|
+
end
|
18
|
+
|
19
|
+
def send_message(message)
|
20
|
+
@q.send_message message
|
21
|
+
end
|
@@ -0,0 +1,22 @@
|
|
1
|
+
require 'tempfile'
|
2
|
+
|
3
|
+
TEST_FILE = Tempfile.new('put request')
|
4
|
+
|
5
|
+
pid = fork do
|
6
|
+
require 'webrick'
|
7
|
+
|
8
|
+
class PutServlet < WEBrick::HTTPServlet::AbstractServlet
|
9
|
+
def do_PUT(req, resp)
|
10
|
+
TEST_FILE << req.request_uri
|
11
|
+
TEST_FILE.close
|
12
|
+
raise WEBrick::HTTPStatus::OK
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
server = WEBrick::HTTPServer.new(:Port => 9999)
|
17
|
+
server.mount('/bob', PutServlet)
|
18
|
+
server.start
|
19
|
+
end
|
20
|
+
|
21
|
+
at_exit { Process.kill('KILL', pid) }
|
22
|
+
|
data/lib/ungulate.rb
CHANGED
@@ -1,7 +1,71 @@
|
|
1
1
|
require 'rubygems'
|
2
|
+
require 'bundler/setup'
|
3
|
+
|
4
|
+
require 'hashie'
|
2
5
|
require 'ungulate/server'
|
6
|
+
require 'ungulate/job'
|
7
|
+
require 'ungulate/sqs_message_queue'
|
8
|
+
require 'ungulate/blob_processor'
|
9
|
+
require 'ungulate/rmagick_version_creator'
|
10
|
+
require 'ungulate/s3_storage'
|
11
|
+
require 'ungulate/curl_http'
|
3
12
|
|
4
13
|
module Ungulate
|
14
|
+
class Configuration < Hashie::Mash; end
|
15
|
+
class MissingConfiguration < StandardError; end
|
16
|
+
|
17
|
+
def self.configuration
|
18
|
+
@config ||=
|
19
|
+
begin
|
20
|
+
config = Configuration.new
|
21
|
+
|
22
|
+
amazon_credentials = lambda {
|
23
|
+
{
|
24
|
+
:access_key_id => config.access_key_id || ENV['AMAZON_ACCESS_KEY_ID'],
|
25
|
+
:secret_access_key => config.secret_access_key || ENV['AMAZON_SECRET_ACCESS_KEY']
|
26
|
+
}
|
27
|
+
}
|
28
|
+
|
29
|
+
config.queue = lambda {
|
30
|
+
SqsMessageQueue.new(
|
31
|
+
config.queue_name,
|
32
|
+
amazon_credentials.call.merge(:server => config.queue_server)
|
33
|
+
)
|
34
|
+
}
|
35
|
+
|
36
|
+
config.http = lambda {
|
37
|
+
CurlHttp.new
|
38
|
+
}
|
39
|
+
|
40
|
+
config.version_creator = lambda {
|
41
|
+
RmagickVersionCreator.new(:http => config.http.call)
|
42
|
+
}
|
43
|
+
|
44
|
+
config.storage = lambda {
|
45
|
+
S3Storage.new(amazon_credentials.call)
|
46
|
+
}
|
47
|
+
|
48
|
+
config.blob_processor = lambda {
|
49
|
+
BlobProcessor.new(
|
50
|
+
:version_creator => config.version_creator.call
|
51
|
+
)
|
52
|
+
}
|
53
|
+
|
54
|
+
config.job_processor = lambda {
|
55
|
+
Job.new(
|
56
|
+
:blob_processor => config.blob_processor.call,
|
57
|
+
:storage => config.storage.call,
|
58
|
+
:http => config.http.call
|
59
|
+
)
|
60
|
+
}
|
61
|
+
|
62
|
+
config
|
63
|
+
end
|
64
|
+
end
|
65
|
+
|
66
|
+
def self.configure(&block)
|
67
|
+
yield configuration
|
68
|
+
end
|
5
69
|
end
|
6
70
|
|
7
71
|
if defined? ActionView::Base
|
@@ -0,0 +1,34 @@
|
|
1
|
+
module Ungulate
|
2
|
+
class BlobProcessor
|
3
|
+
def initialize(options)
|
4
|
+
@creator = options[:version_creator]
|
5
|
+
end
|
6
|
+
|
7
|
+
def process(options)
|
8
|
+
versions = options.delete(:versions)
|
9
|
+
blob = options.delete(:blob)
|
10
|
+
original_key = options.delete(:original_key)
|
11
|
+
bucket = options.delete(:bucket)
|
12
|
+
|
13
|
+
versions.each_pair do |name, instructions|
|
14
|
+
stored_data = @creator.create(blob, instructions)
|
15
|
+
|
16
|
+
bucket.store(
|
17
|
+
new_key(original_key, name),
|
18
|
+
stored_data[:blob],
|
19
|
+
:version => name,
|
20
|
+
:content_type => stored_data[:content_type]
|
21
|
+
)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
|
25
|
+
protected
|
26
|
+
|
27
|
+
def new_key(original, version)
|
28
|
+
dirname = File.dirname(original)
|
29
|
+
extname = File.extname(original)
|
30
|
+
basename = File.basename(original, extname)
|
31
|
+
"#{dirname}/#{basename}_#{version}#{extname}".sub(/^\.\//, '')
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
@@ -0,0 +1,25 @@
|
|
1
|
+
require 'curb'
|
2
|
+
|
3
|
+
module Ungulate
|
4
|
+
class CurlHttp
|
5
|
+
def initialize(options = {})
|
6
|
+
@easy = Curl::Easy
|
7
|
+
@logger = options[:logger] || ::Logger.new($stdout)
|
8
|
+
end
|
9
|
+
|
10
|
+
def get_body(url)
|
11
|
+
@logger.info "GET via HTTP: #{url}"
|
12
|
+
@easy.http_get(url).body_str
|
13
|
+
end
|
14
|
+
|
15
|
+
def put(url)
|
16
|
+
@logger.info "PUT #{url}"
|
17
|
+
@response = @easy.http_put(url, '')
|
18
|
+
self
|
19
|
+
end
|
20
|
+
|
21
|
+
def code
|
22
|
+
@response.response_code
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
data/lib/ungulate/file_upload.rb
CHANGED
@@ -1,23 +1,21 @@
|
|
1
1
|
require 'active_support/core_ext/class/attribute_accessors'
|
2
|
+
require 'active_support/json'
|
3
|
+
|
2
4
|
class Ungulate::FileUpload
|
3
|
-
attr_accessor
|
4
|
-
:bucket_url,
|
5
|
-
:key
|
6
|
-
)
|
5
|
+
attr_accessor :bucket_url, :key
|
7
6
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
)
|
7
|
+
class << self
|
8
|
+
def config
|
9
|
+
Ungulate.configuration
|
10
|
+
end
|
13
11
|
|
14
|
-
|
15
|
-
|
16
|
-
|
12
|
+
def queue
|
13
|
+
@queue ||= config.queue.call
|
14
|
+
end
|
17
15
|
|
18
|
-
|
19
|
-
|
20
|
-
|
16
|
+
def enqueue(job_description)
|
17
|
+
queue.push(job_description.to_yaml)
|
18
|
+
end
|
21
19
|
end
|
22
20
|
|
23
21
|
def initialize(options = {})
|
@@ -29,6 +27,14 @@ class Ungulate::FileUpload
|
|
29
27
|
end
|
30
28
|
end
|
31
29
|
|
30
|
+
def config
|
31
|
+
self.class.config
|
32
|
+
end
|
33
|
+
|
34
|
+
def access_key_id
|
35
|
+
config.access_key_id
|
36
|
+
end
|
37
|
+
|
32
38
|
def acl
|
33
39
|
condition 'acl'
|
34
40
|
end
|
@@ -62,7 +68,7 @@ class Ungulate::FileUpload
|
|
62
68
|
def signature
|
63
69
|
Base64.encode64(
|
64
70
|
OpenSSL::HMAC.digest(OpenSSL::Digest::Digest.new('sha1'),
|
65
|
-
secret_access_key,
|
71
|
+
config.secret_access_key,
|
66
72
|
policy)
|
67
73
|
).gsub("\n", '')
|
68
74
|
end
|