squeese 0.1.0

data/README.md

@@ -0,0 +1,90 @@
+SQueeSe - a job queueing DSL for SQS
+==========================================
+
+SQS is a queueing system from Amazon. SQueeSe is a friendly wrapper around it in the style of Stalker or Minion.
+
+Queueing jobs
+-------------
+
+From anywhere in your app:
+
+    require 'squeese'
+
+    Squeese.enqueue('email.send', :to => 'joe@example.com')
+    Squeese.enqueue('post.cleanup.all')
+    Squeese.enqueue('post.cleanup', :id => post.id)
+
+Working jobs
+------------
+
+In a standalone file, typically jobs.rb or worker.rb:
+
+    require 'squeese'
+    include Squeese
+
+    job 'email.send' do |args|
+      Pony.send(:to => args['to'], :subject => "Hello there")
+    end
+
+    job 'post.cleanup.all' do |args|
+      Post.all.each do |post|
+        enqueue('post.cleanup', :id => post.all)
+      end
+    end
+
+    job 'post.cleanup' do |args|
+      Post.find(args['id']).cleanup
+    end
+
+Running
+-------
+
+First, make sure you have your AWS secret keys configured.
+
+    $ export AWS_SECRET_ACCESS_KEY=[...]
+    $ export AWS_ACCESS_KEY_ID=[...]
+
+Now get your squeese on:
+
+    $ sudo gem install squeese
+
+Now squeese tight with a worker:
+
+    $ squeese jobs.rb
+    [Sat Apr 17 14:13:40 -0700 2010] Working 3 jobs  :: [ email.send post.cleanup.all post.cleanup ]
+
+Squeese will log to stdout as it starts working each job.
+
+Filter to a list of jobs you wish to run with an argument:
+
+    $ squeese jobs.rb post.cleanup.all,post.cleanup
+    [Sat Apr 17 14:13:40 -0700 2010] Working 2 jobs  :: [ post.cleanup.all post.cleanup ]
+
+In a production environment you may run one or more high-priority workers (limited to short/urgent jobs) and any number of regular workers (working all jobs).  For example, two workers working just the email.send job, and four running all jobs:
+
+    $ for i in 1 2; do squeese jobs.rb email.send > log/urgent-worker.log 2>&1; end
+    $ for i in 1 2 3 4; do squeese jobs.rb > log/worker.log 2>&1; end
+
+NOTE:
+Filtering squeese jobs by worker is not yet supported!
+
+Tidbits
+-------
+
+* Jobs are serialized as JSON, so you should stick to strings, integers, arrays, and hashes as arguments to jobs.  e.g. don't pass full Ruby objects - use something like an ActiveRecord/MongoMapper/CouchRest id instead.
+* Because there are no class definitions associated with jobs, you can queue jobs from anywhere without needing to include your full app's environment.
+* The default queue name used by squeese is "squeese", but you can select a different queue with ENV['SQUEESE_QUEUE'].
+* The squeese binary is just for convenience, you can also run a worker with a straight Ruby command:
+    $ ruby -r jobs -e Squeese.work
+
+Meta
+----
+
+Created by Adam Wiggins
+
+Heavily inspired by [Minion](http://github.com/orionz/minion) and [Stalker](http://github.com/adamwiggins/stalker) by Orion Henry and Adam Wiggins, respectively.
+
+Released under the MIT License: http://www.opensource.org/licenses/mit-license.php
+
+http://github.com/pvh/squeese
+

data/Rakefile

@@ -0,0 +1,19 @@
+require 'jeweler'
+
+Jeweler::Tasks.new do |s|
+	s.name = "squeese"
+	s.summary = "A job queueing and background workers system using SQS."
+	s.description = "A job queueing and background workers system using SQS.  Inspired by the Stalker gem."
+	s.author = "Peter van Hardenberg"
+	s.email = "pvh@heroku.com"
+	s.homepage = "http://github.com/pvh/squeese"
+	s.executables = [ "squeese" ]
+	s.rubyforge_project = "squeese"
+
+	s.add_dependency 'right_aws'
+	s.add_dependency 'json_pure'
+
+	s.files = FileList["[A-Z]*", "{bin,lib}/**/*"]
+end
+
+Jeweler::GemcutterTasks.new

data/VERSION

@@ -0,0 +1 @@
+0.1.0

data/bin/squeese

@@ -0,0 +1,16 @@
+#!/usr/bin/env ruby
+
+require File.dirname(__FILE__) + '/../lib/squeese'
+
+usage = "squeese <jobs.rb>"
+file = ARGV.shift or abort usage
+jobs = ARGV.shift.split(',') rescue nil
+
+require file
+
+trap('INT') do
+	puts "\rExiting"
+	exit
+end
+
+Squeese.work

data/examples/enqueue.rb

@@ -0,0 +1,5 @@
+$LOAD_PATH.unshift '../lib'
+require 'squeese'
+
+Squeese.enqueue('send.email', :email => 'hello@example.com')
+Squeese.enqueue('cleanup.strays')

data/examples/jobs.rb

@@ -0,0 +1,16 @@
+$LOAD_PATH.unshift '../lib'
+require 'squeese'
+
+include Squeese
+
+job 'send.email' do |args|
+	log "Sending email to #{args['email']}"
+end
+
+job 'transform.image' do |args|
+	log "Image transform"
+end
+
+job 'cleanup.strays' do |args|
+	log "Cleaning up"
+end

data/lib/squeese.rb

@@ -0,0 +1,99 @@
+require 'right_aws'
+require 'json'
+require 'uri'
+
+module Squeese
+	extend self
+
+	def purge
+		queue.delete
+	end
+
+	def enqueue(job, args={})
+		queue.send_message [ job, args ].to_json
+	end
+
+	def job(j, &block)
+		@@handlers ||= {}
+		@@handlers[j] = block
+	end
+
+	class NoJobsDefined < RuntimeError; end
+	class NoSuchJob < RuntimeError; end
+
+	def work
+		raise NoJobsDefined unless defined?(@@handlers)
+
+		# this makes more sense when we get support for working a subset
+		# of the available jobs.
+		jobs = all_jobs
+
+		jobs.each do |job|
+			raise(NoSuchJob, job) unless @@handlers[job]
+		end
+
+		log "Working #{jobs.size} jobs  :: [ #{jobs.join(' ')} ]"
+
+		loop do
+			work_one_job
+		end
+	end
+
+	def work_one_job
+		msg = queue.receive
+
+		# don't be CPU greedy on a quiet queue
+		unless msg
+			sleep 2
+			return
+		end
+
+		name, args = JSON.parse msg.body
+		log_job(name, args)
+		handler = @@handlers[name]
+		raise(NoSuchJob, name) unless handler
+		handler.call(args)
+		msg.delete
+	rescue => e
+		log exception_message(e)
+	end
+
+	def log_job(name, args)
+		args_flat = args.inject("") do |accum, (key,value)|
+			accum += "#{key}=#{value} "
+		end
+
+		log sprintf("%-15s :: #{args_flat}", name)
+	end
+
+	def log(msg)
+		puts "[#{Time.now}] #{msg}"
+	end
+
+	def sqs
+		@sqs ||= RightAws::SqsGen2.new(ENV['AWS_ACCESS_KEY_ID'], ENV['AWS_SECRET_ACCESS_KEY'], :logger => Logger.new(nil))
+	end
+
+	def queue_name
+		ENV["SQS_QUEUE"] || "squeese"
+	end
+
+	def queue
+		sqs.queue(queue_name, true)
+	end
+
+	def exception_message(e)
+		msg = [ "Exception #{e.class} -> #{e.message}" ]
+
+		base = File.expand_path(Dir.pwd) + '/'
+		e.backtrace.each do |t|
+			msg << "   #{File.expand_path(t).gsub(/#{base}/, '')}"
+		end
+
+		msg.join("\n")
+	end
+
+	def all_jobs
+		@@handlers.keys
+	end
+end

metadata

@@ -0,0 +1,90 @@
+--- !ruby/object:Gem::Specification 
+name: squeese
+version: !ruby/object:Gem::Version 
+  prerelease: false
+  segments: 
+  - 0
+  - 1
+  - 0
+  version: 0.1.0
+platform: ruby
+authors: 
+- Peter van Hardenberg
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2010-05-18 00:00:00 -07:00
+default_executable: squeese
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: right_aws
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: json_pure
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        segments: 
+        - 0
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+description: A job queueing and background workers system using SQS.  Inspired by the Stalker gem.
+email: pvh@heroku.com
+executables: 
+- squeese
+extensions: []
+
+extra_rdoc_files: 
+- README.md
+files: 
+- README.md
+- Rakefile
+- VERSION
+- bin/squeese
+- lib/squeese.rb
+has_rdoc: true
+homepage: http://github.com/pvh/squeese
+licenses: []
+
+post_install_message: 
+rdoc_options: 
+- --charset=UTF-8
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: squeese
+rubygems_version: 1.3.6
+signing_key: 
+specification_version: 3
+summary: A job queueing and background workers system using SQS.
+test_files: 
+- examples/enqueue.rb
+- examples/jobs.rb