squeese 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/README.md +90 -0
- data/Rakefile +19 -0
- data/VERSION +1 -0
- data/bin/squeese +16 -0
- data/examples/enqueue.rb +5 -0
- data/examples/jobs.rb +16 -0
- data/lib/squeese.rb +99 -0
- metadata +90 -0
data/README.md
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
SQueeSe - a job queueing DSL for SQS
|
2
|
+
==========================================
|
3
|
+
|
4
|
+
SQS is a queueing system from Amazon. SQueeSe is a friendly wrapper around it in the style of Stalker or Minion.
|
5
|
+
|
6
|
+
Queueing jobs
|
7
|
+
-------------
|
8
|
+
|
9
|
+
From anywhere in your app:
|
10
|
+
|
11
|
+
require 'squeese'
|
12
|
+
|
13
|
+
Squeese.enqueue('email.send', :to => 'joe@example.com')
|
14
|
+
Squeese.enqueue('post.cleanup.all')
|
15
|
+
Squeese.enqueue('post.cleanup', :id => post.id)
|
16
|
+
|
17
|
+
Working jobs
|
18
|
+
------------
|
19
|
+
|
20
|
+
In a standalone file, typically jobs.rb or worker.rb:
|
21
|
+
|
22
|
+
require 'squeese'
|
23
|
+
include Squeese
|
24
|
+
|
25
|
+
job 'email.send' do |args|
|
26
|
+
Pony.send(:to => args['to'], :subject => "Hello there")
|
27
|
+
end
|
28
|
+
|
29
|
+
job 'post.cleanup.all' do |args|
|
30
|
+
Post.all.each do |post|
|
31
|
+
enqueue('post.cleanup', :id => post.all)
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
job 'post.cleanup' do |args|
|
36
|
+
Post.find(args['id']).cleanup
|
37
|
+
end
|
38
|
+
|
39
|
+
Running
|
40
|
+
-------
|
41
|
+
|
42
|
+
First, make sure you have your AWS secret keys configured.
|
43
|
+
|
44
|
+
$ export AWS_SECRET_ACCESS_KEY=[...]
|
45
|
+
$ export AWS_ACCESS_KEY_ID=[...]
|
46
|
+
|
47
|
+
Now get your squeese on:
|
48
|
+
|
49
|
+
$ sudo gem install squeese
|
50
|
+
|
51
|
+
Now squeese tight with a worker:
|
52
|
+
|
53
|
+
$ squeese jobs.rb
|
54
|
+
[Sat Apr 17 14:13:40 -0700 2010] Working 3 jobs :: [ email.send post.cleanup.all post.cleanup ]
|
55
|
+
|
56
|
+
Squeese will log to stdout as it starts working each job.
|
57
|
+
|
58
|
+
Filter to a list of jobs you wish to run with an argument:
|
59
|
+
|
60
|
+
$ squeese jobs.rb post.cleanup.all,post.cleanup
|
61
|
+
[Sat Apr 17 14:13:40 -0700 2010] Working 2 jobs :: [ post.cleanup.all post.cleanup ]
|
62
|
+
|
63
|
+
In a production environment you may run one or more high-priority workers (limited to short/urgent jobs) and any number of regular workers (working all jobs). For example, two workers working just the email.send job, and four running all jobs:
|
64
|
+
|
65
|
+
$ for i in 1 2; do squeese jobs.rb email.send > log/urgent-worker.log 2>&1; end
|
66
|
+
$ for i in 1 2 3 4; do squeese jobs.rb > log/worker.log 2>&1; end
|
67
|
+
|
68
|
+
NOTE:
|
69
|
+
Filtering squeese jobs by worker is not yet supported!
|
70
|
+
|
71
|
+
Tidbits
|
72
|
+
-------
|
73
|
+
|
74
|
+
* Jobs are serialized as JSON, so you should stick to strings, integers, arrays, and hashes as arguments to jobs. e.g. don't pass full Ruby objects - use something like an ActiveRecord/MongoMapper/CouchRest id instead.
|
75
|
+
* Because there are no class definitions associated with jobs, you can queue jobs from anywhere without needing to include your full app's environment.
|
76
|
+
* The default queue name used by squeese is "squeese", but you can select a different queue with ENV['SQUEESE_QUEUE'].
|
77
|
+
* The squeese binary is just for convenience, you can also run a worker with a straight Ruby command:
|
78
|
+
$ ruby -r jobs -e Squeese.work
|
79
|
+
|
80
|
+
Meta
|
81
|
+
----
|
82
|
+
|
83
|
+
Created by Adam Wiggins
|
84
|
+
|
85
|
+
Heavily inspired by [Minion](http://github.com/orionz/minion) and [Stalker](http://github.com/adamwiggins/stalker) by Orion Henry and Adam Wiggins, respectively.
|
86
|
+
|
87
|
+
Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
|
88
|
+
|
89
|
+
http://github.com/pvh/squeese
|
90
|
+
|
data/Rakefile
ADDED
@@ -0,0 +1,19 @@
|
|
1
|
+
require 'jeweler'
|
2
|
+
|
3
|
+
Jeweler::Tasks.new do |s|
|
4
|
+
s.name = "squeese"
|
5
|
+
s.summary = "A job queueing and background workers system using SQS."
|
6
|
+
s.description = "A job queueing and background workers system using SQS. Inspired by the Stalker gem."
|
7
|
+
s.author = "Peter van Hardenberg"
|
8
|
+
s.email = "pvh@heroku.com"
|
9
|
+
s.homepage = "http://github.com/pvh/squeese"
|
10
|
+
s.executables = [ "squeese" ]
|
11
|
+
s.rubyforge_project = "squeese"
|
12
|
+
|
13
|
+
s.add_dependency 'right_aws'
|
14
|
+
s.add_dependency 'json_pure'
|
15
|
+
|
16
|
+
s.files = FileList["[A-Z]*", "{bin,lib}/**/*"]
|
17
|
+
end
|
18
|
+
|
19
|
+
Jeweler::GemcutterTasks.new
|
data/VERSION
ADDED
@@ -0,0 +1 @@
|
|
1
|
+
0.1.0
|
data/bin/squeese
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require File.dirname(__FILE__) + '/../lib/squeese'
|
4
|
+
|
5
|
+
usage = "squeese <jobs.rb>"
|
6
|
+
file = ARGV.shift or abort usage
|
7
|
+
jobs = ARGV.shift.split(',') rescue nil
|
8
|
+
|
9
|
+
require file
|
10
|
+
|
11
|
+
trap('INT') do
|
12
|
+
puts "\rExiting"
|
13
|
+
exit
|
14
|
+
end
|
15
|
+
|
16
|
+
Squeese.work
|
data/examples/enqueue.rb
ADDED
data/examples/jobs.rb
ADDED
@@ -0,0 +1,16 @@
|
|
1
|
+
$LOAD_PATH.unshift '../lib'
|
2
|
+
require 'squeese'
|
3
|
+
|
4
|
+
include Squeese
|
5
|
+
|
6
|
+
job 'send.email' do |args|
|
7
|
+
log "Sending email to #{args['email']}"
|
8
|
+
end
|
9
|
+
|
10
|
+
job 'transform.image' do |args|
|
11
|
+
log "Image transform"
|
12
|
+
end
|
13
|
+
|
14
|
+
job 'cleanup.strays' do |args|
|
15
|
+
log "Cleaning up"
|
16
|
+
end
|
data/lib/squeese.rb
ADDED
@@ -0,0 +1,99 @@
|
|
1
|
+
require 'right_aws'
|
2
|
+
require 'json'
|
3
|
+
require 'uri'
|
4
|
+
|
5
|
+
module Squeese
|
6
|
+
extend self
|
7
|
+
|
8
|
+
def purge
|
9
|
+
queue.delete
|
10
|
+
end
|
11
|
+
|
12
|
+
def enqueue(job, args={})
|
13
|
+
queue.send_message [ job, args ].to_json
|
14
|
+
end
|
15
|
+
|
16
|
+
def job(j, &block)
|
17
|
+
@@handlers ||= {}
|
18
|
+
@@handlers[j] = block
|
19
|
+
end
|
20
|
+
|
21
|
+
class NoJobsDefined < RuntimeError; end
|
22
|
+
class NoSuchJob < RuntimeError; end
|
23
|
+
|
24
|
+
def work
|
25
|
+
raise NoJobsDefined unless defined?(@@handlers)
|
26
|
+
|
27
|
+
# this makes more sense when we get support for working a subset
|
28
|
+
# of the available jobs.
|
29
|
+
jobs = all_jobs
|
30
|
+
|
31
|
+
jobs.each do |job|
|
32
|
+
raise(NoSuchJob, job) unless @@handlers[job]
|
33
|
+
end
|
34
|
+
|
35
|
+
log "Working #{jobs.size} jobs :: [ #{jobs.join(' ')} ]"
|
36
|
+
|
37
|
+
loop do
|
38
|
+
work_one_job
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def work_one_job
|
43
|
+
msg = queue.receive
|
44
|
+
|
45
|
+
# don't be CPU greedy on a quiet queue
|
46
|
+
unless msg
|
47
|
+
sleep 2
|
48
|
+
return
|
49
|
+
end
|
50
|
+
|
51
|
+
name, args = JSON.parse msg.body
|
52
|
+
log_job(name, args)
|
53
|
+
handler = @@handlers[name]
|
54
|
+
raise(NoSuchJob, name) unless handler
|
55
|
+
handler.call(args)
|
56
|
+
msg.delete
|
57
|
+
rescue => e
|
58
|
+
log exception_message(e)
|
59
|
+
end
|
60
|
+
|
61
|
+
def log_job(name, args)
|
62
|
+
args_flat = args.inject("") do |accum, (key,value)|
|
63
|
+
accum += "#{key}=#{value} "
|
64
|
+
end
|
65
|
+
|
66
|
+
log sprintf("%-15s :: #{args_flat}", name)
|
67
|
+
end
|
68
|
+
|
69
|
+
def log(msg)
|
70
|
+
puts "[#{Time.now}] #{msg}"
|
71
|
+
end
|
72
|
+
|
73
|
+
def sqs
|
74
|
+
@sqs ||= RightAws::SqsGen2.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], :logger => Logger.new(nil))
|
75
|
+
end
|
76
|
+
|
77
|
+
def queue_name
|
78
|
+
ENV["SQS_QUEUE"] || "squeese"
|
79
|
+
end
|
80
|
+
|
81
|
+
def queue
|
82
|
+
sqs.queue(queue_name, true)
|
83
|
+
end
|
84
|
+
|
85
|
+
def exception_message(e)
|
86
|
+
msg = [ "Exception #{e.class} -> #{e.message}" ]
|
87
|
+
|
88
|
+
base = File.expand_path(Dir.pwd) + '/'
|
89
|
+
e.backtrace.each do |t|
|
90
|
+
msg << " #{File.expand_path(t).gsub(/#{base}/, '')}"
|
91
|
+
end
|
92
|
+
|
93
|
+
msg.join("\n")
|
94
|
+
end
|
95
|
+
|
96
|
+
def all_jobs
|
97
|
+
@@handlers.keys
|
98
|
+
end
|
99
|
+
end
|
metadata
ADDED
@@ -0,0 +1,90 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: squeese
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
prerelease: false
|
5
|
+
segments:
|
6
|
+
- 0
|
7
|
+
- 1
|
8
|
+
- 0
|
9
|
+
version: 0.1.0
|
10
|
+
platform: ruby
|
11
|
+
authors:
|
12
|
+
- Peter van Hardenberg
|
13
|
+
autorequire:
|
14
|
+
bindir: bin
|
15
|
+
cert_chain: []
|
16
|
+
|
17
|
+
date: 2010-05-18 00:00:00 -07:00
|
18
|
+
default_executable: squeese
|
19
|
+
dependencies:
|
20
|
+
- !ruby/object:Gem::Dependency
|
21
|
+
name: right_aws
|
22
|
+
prerelease: false
|
23
|
+
requirement: &id001 !ruby/object:Gem::Requirement
|
24
|
+
requirements:
|
25
|
+
- - ">="
|
26
|
+
- !ruby/object:Gem::Version
|
27
|
+
segments:
|
28
|
+
- 0
|
29
|
+
version: "0"
|
30
|
+
type: :runtime
|
31
|
+
version_requirements: *id001
|
32
|
+
- !ruby/object:Gem::Dependency
|
33
|
+
name: json_pure
|
34
|
+
prerelease: false
|
35
|
+
requirement: &id002 !ruby/object:Gem::Requirement
|
36
|
+
requirements:
|
37
|
+
- - ">="
|
38
|
+
- !ruby/object:Gem::Version
|
39
|
+
segments:
|
40
|
+
- 0
|
41
|
+
version: "0"
|
42
|
+
type: :runtime
|
43
|
+
version_requirements: *id002
|
44
|
+
description: A job queueing and background workers system using SQS. Inspired by the Stalker gem.
|
45
|
+
email: pvh@heroku.com
|
46
|
+
executables:
|
47
|
+
- squeese
|
48
|
+
extensions: []
|
49
|
+
|
50
|
+
extra_rdoc_files:
|
51
|
+
- README.md
|
52
|
+
files:
|
53
|
+
- README.md
|
54
|
+
- Rakefile
|
55
|
+
- VERSION
|
56
|
+
- bin/squeese
|
57
|
+
- lib/squeese.rb
|
58
|
+
has_rdoc: true
|
59
|
+
homepage: http://github.com/pvh/squeese
|
60
|
+
licenses: []
|
61
|
+
|
62
|
+
post_install_message:
|
63
|
+
rdoc_options:
|
64
|
+
- --charset=UTF-8
|
65
|
+
require_paths:
|
66
|
+
- lib
|
67
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
68
|
+
requirements:
|
69
|
+
- - ">="
|
70
|
+
- !ruby/object:Gem::Version
|
71
|
+
segments:
|
72
|
+
- 0
|
73
|
+
version: "0"
|
74
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
75
|
+
requirements:
|
76
|
+
- - ">="
|
77
|
+
- !ruby/object:Gem::Version
|
78
|
+
segments:
|
79
|
+
- 0
|
80
|
+
version: "0"
|
81
|
+
requirements: []
|
82
|
+
|
83
|
+
rubyforge_project: squeese
|
84
|
+
rubygems_version: 1.3.6
|
85
|
+
signing_key:
|
86
|
+
specification_version: 3
|
87
|
+
summary: A job queueing and background workers system using SQS.
|
88
|
+
test_files:
|
89
|
+
- examples/enqueue.rb
|
90
|
+
- examples/jobs.rb
|