RubyGems - opengotham_resque - Versions diffs - 1.8.2 - Mend

opengotham_resque 1.8.2

Files changed (70) hide show

data/.gitignore +2 -0
data/.kick +26 -0
data/HISTORY.md +142 -0
data/LICENSE +20 -0
data/README.markdown +794 -0
data/Rakefile +112 -0
data/bin/resque +57 -0
data/bin/resque-web +23 -0
data/config.ru +14 -0
data/deps.rip +7 -0
data/docs/HOOKS.md +121 -0
data/docs/PLUGINS.md +93 -0
data/examples/async_helper.rb +31 -0
data/examples/demo/README.markdown +71 -0
data/examples/demo/Rakefile +8 -0
data/examples/demo/app.rb +38 -0
data/examples/demo/config.ru +19 -0
data/examples/demo/job.rb +22 -0
data/examples/god/resque.god +53 -0
data/examples/god/stale.god +26 -0
data/examples/instance.rb +11 -0
data/examples/monit/resque.monit +6 -0
data/examples/simple.rb +30 -0
data/init.rb +1 -0
data/lib/resque.rb +287 -0
data/lib/resque/errors.rb +10 -0
data/lib/resque/failure.rb +66 -0
data/lib/resque/failure/base.rb +61 -0
data/lib/resque/failure/hoptoad.rb +132 -0
data/lib/resque/failure/multiple.rb +48 -0
data/lib/resque/failure/redis.rb +40 -0
data/lib/resque/helpers.rb +63 -0
data/lib/resque/job.rb +207 -0
data/lib/resque/plugin.rb +46 -0
data/lib/resque/server.rb +201 -0
data/lib/resque/server/public/idle.png +0 -0
data/lib/resque/server/public/jquery-1.3.2.min.js +19 -0
data/lib/resque/server/public/jquery.relatize_date.js +95 -0
data/lib/resque/server/public/poll.png +0 -0
data/lib/resque/server/public/ranger.js +67 -0
data/lib/resque/server/public/reset.css +48 -0
data/lib/resque/server/public/style.css +81 -0
data/lib/resque/server/public/working.png +0 -0
data/lib/resque/server/test_helper.rb +19 -0
data/lib/resque/server/views/error.erb +1 -0
data/lib/resque/server/views/failed.erb +53 -0
data/lib/resque/server/views/key_sets.erb +20 -0
data/lib/resque/server/views/key_string.erb +11 -0
data/lib/resque/server/views/layout.erb +44 -0
data/lib/resque/server/views/next_more.erb +10 -0
data/lib/resque/server/views/overview.erb +4 -0
data/lib/resque/server/views/queues.erb +49 -0
data/lib/resque/server/views/stats.erb +62 -0
data/lib/resque/server/views/workers.erb +78 -0
data/lib/resque/server/views/working.erb +69 -0
data/lib/resque/stat.rb +53 -0
data/lib/resque/tasks.rb +39 -0
data/lib/resque/version.rb +3 -0
data/lib/resque/worker.rb +478 -0
data/tasks/redis.rake +159 -0
data/tasks/resque.rake +2 -0
data/test/job_hooks_test.rb +302 -0
data/test/job_plugins_test.rb +209 -0
data/test/plugin_test.rb +116 -0
data/test/redis-test.conf +132 -0
data/test/resque-web_test.rb +54 -0
data/test/resque_test.rb +225 -0
data/test/test_helper.rb +111 -0
data/test/worker_test.rb +302 -0
metadata +199 -0

data/examples/demo/README.markdown ADDED

@@ -0,0 +1,71 @@
+Resque Demo
+-----------
+This is a dirt simple Resque setup for you to play with.
+### Starting the Demo App
+Here's how to run the Sinatra app:
+    $ git clone git://github.com/defunkt/resque.git
+    $ cd resque/examples/demo
+    $ rackup config.ru
+    $ open http://localhost:9292/
+Click 'Create New Job' a few times. You should see the number of
+pending jobs rising.
+### Starting the Demo Worker
+Now in another shell terminal start the worker:
+    $ cd resque/examples/demo
+    $ VERBOSE=true QUEUE=default rake resque:work
+You should see the following output:
+    *** Starting worker hostname:90185:default
+    *** got: (Job{default} | Demo::Job | [{}])
+    Processed a job!
+    *** done: (Job{default} | Demo::Job | [{}])
+You can also use `VVERBOSE` (very verbose) if you want to see more:
+    $ VERBOSE=true QUEUE=default rake resque:work
+    *** Starting worker hostname:90399:default
+    ** [05:55:09 2009-09-16] 90399: Registered signals
+    ** [05:55:09 2009-09-16] 90399: Checking default
+    ** [05:55:09 2009-09-16] 90399: Found job on default
+    ** [05:55:09 2009-09-16] 90399: got: (Job{default} | Demo::Job | [{}])
+    ** [05:55:09 2009-09-16] 90399: resque: Forked 90401 at 1253141709
+    ** [05:55:09 2009-09-16] 90401: resque: Processing default since 1253141709
+    Processed a job!
+    ** [05:55:10 2009-09-16] 90401: done: (Job{default} | Demo::Job | [{}])
+Notice that our workers `require 'job'` in our `Rakefile`. This
+ensures they have our app loaded and can access the job classes.
+### Starting the Resque frontend
+Great, now let's check out the Resque frontend. Either click on 'View
+Resque' in your web browser or run:
+    $ open http://localhost:9292/resque/
+You should see the Resque web frontend. 404 page? Don't forget the
+trailing slash!
+### config.ru
+The `config.ru` shows you how to mount multiple Rack apps. Resque
+should work fine on a subpath - feel free to load it up in your
+Passenger app and protect it with some basic auth.
+### That's it!
+Click around, add some more queues, add more jobs, do whatever, have fun.

data/examples/demo/Rakefile ADDED

@@ -0,0 +1,8 @@
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'resque/tasks'
+require 'job'
+desc "Start the demo using `rackup`"
+task :start do
+  exec "rackup config.ru"
+end

data/examples/demo/app.rb ADDED

@@ -0,0 +1,38 @@
+require 'sinatra/base'
+require 'resque'
+require 'job'
+module Demo
+  class App < Sinatra::Base
+    get '/' do
+      info = Resque.info
+      out = "<html><head><title>Resque Demo</title></head><body>"
+      out << "<p>"
+      out << "There are #{info[:pending]} pending and "
+      out << "#{info[:processed]} processed jobs across #{info[:queues]} queues."
+      out << "</p>"
+      out << '<form method="POST">'
+      out << '<input type="submit" value="Create New Job"/>'
+      out << '&nbsp;&nbsp;<a href="/resque/">View Resque</a>'
+      out << '</form>'
+       out << "<form action='/failing' method='POST''>"
+       out << '<input type="submit" value="Create Failing New Job"/>'
+       out << '&nbsp;&nbsp;<a href="/resque/">View Resque</a>'
+       out << '</form>'
+      out << "</body></html>"
+      out
+    end
+    post '/' do
+      Resque.enqueue(Job, params)
+      redirect "/"
+    end
+    post '/failing' do
+      Resque.enqueue(FailingJob, params)
+      redirect "/"
+    end
+  end
+end

data/examples/demo/config.ru ADDED

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+require 'logger'
+$LOAD_PATH.unshift File.dirname(__FILE__) + '/../../lib'
+require 'app'
+require 'resque/server'
+use Rack::ShowExceptions
+# Set the AUTH env variable to your basic auth password to protect Resque.
+AUTH_PASSWORD = ENV['AUTH']
+if AUTH_PASSWORD
+  Resque::Server.use Rack::Auth::Basic do |username, password|
+    password == AUTH_PASSWORD
+  end
+end
+run Rack::URLMap.new \
+  "/"       => Demo::App.new,
+  "/resque" => Resque::Server.new

data/examples/demo/job.rb ADDED

@@ -0,0 +1,22 @@
+require 'resque'
+module Demo
+  module Job
+    @queue = :default
+    def self.perform(params)
+      sleep 1
+      puts "Processed a job!"
+    end
+  end
+  module FailingJob
+    @queue = :failing
+    def self.perform(params)
+      sleep 1
+      raise 'not processable!'
+      puts "Processed a job!"
+    end
+  end
+end

data/examples/god/resque.god ADDED

@@ -0,0 +1,53 @@
+rails_env   = ENV['RAILS_ENV']  || "production"
+rails_root  = ENV['RAILS_ROOT'] || "/data/github/current"
+num_workers = rails_env == 'production' ? 5 : 2
+num_workers.times do |num|
+  God.watch do |w|
+    w.name     = "resque-#{num}"
+    w.group    = 'resque'
+    w.interval = 30.seconds
+    w.env      = {"QUEUE"=>"critical,high,low", "RAILS_ENV"=>rails_env}
+    w.start    = "/usr/bin/rake -f #{rails_root}/Rakefile environment resque:work"
+    w.uid = 'git'
+    w.gid = 'git'
+    # retart if memory gets too high
+    w.transition(:up, :restart) do |on|
+      on.condition(:memory_usage) do |c|
+        c.above = 350.megabytes
+        c.times = 2
+      end
+    end
+    # determine the state on startup
+    w.transition(:init, { true => :up, false => :start }) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+      end
+    end
+    # determine when process has finished starting
+    w.transition([:start, :restart], :up) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+        c.interval = 5.seconds
+      end
+      # failsafe
+      on.condition(:tries) do |c|
+        c.times = 5
+        c.transition = :start
+        c.interval = 5.seconds
+      end
+    end
+    # start if process is not running
+    w.transition(:up, :start) do |on|
+      on.condition(:process_running) do |c|
+        c.running = false
+      end
+    end
+  end
+end

data/examples/god/stale.god ADDED

@@ -0,0 +1,26 @@
+# This will ride alongside god and kill any rogue stale worker
+# processes. Their sacrifice is for the greater good.
+WORKER_TIMEOUT = 60 * 10 # 10 minutes
+Thread.new do
+  loop do
+    begin
+      `ps -e -o pid,command | grep [r]esque`.split("\n").each do |line|
+        parts   = line.split(' ')
+        next if parts[-2] != "at"
+        started = parts[-1].to_i
+        elapsed = Time.now - Time.at(started)
+        if elapsed >= WORKER_TIMEOUT
+          ::Process.kill('USR1', parts[0].to_i)
+        end
+      end
+    rescue
+      # don't die because of stupid exceptions
+      nil
+    end
+    sleep 30
+  end
+end

data/examples/instance.rb ADDED

@@ -0,0 +1,11 @@
+# DelayedJob wants you to create instances. No problem.
+class Archive < Struct.new(:repo_id, :branch)
+  def self.perform(*args)
+    new(*args).perform
+  end
+  def perform
+    # do work!
+  end
+end

data/examples/monit/resque.monit ADDED

@@ -0,0 +1,6 @@
+check process resque_worker_QUEUE
+  with pidfile /data/APP_NAME/current/tmp/pids/resque_worker_QUEUE.pid
+  start program = "/bin/sh -c 'cd /data/APP_NAME/current; RAILS_ENV=production QUEUE=queue_name VERBOSE=1 nohup rake resque:work& &> log/resque_worker_QUEUE.log && echo $! > tmp/pids/resque_worker_QUEUE.pid'" as uid deploy and gid deploy
+  stop program = "/bin/sh -c 'cd /data/APP_NAME/current && kill -s QUIT `cat tmp/pids/resque_worker_QUEUE.pid` && rm -f tmp/pids/resque_worker_QUEUE.pid; exit 0;'"
+  if totalmem is greater than 300 MB for 10 cycles then restart  # eating up memory?
+  group resque_workers

data/examples/simple.rb ADDED

@@ -0,0 +1,30 @@
+# This is a simple Resque job.
+class Archive
+  @queue = :file_serve
+  def self.perform(repo_id, branch = 'master')
+    repo = Repository.find(repo_id)
+    repo.create_archive(branch)
+  end
+end
+# This is in our app code
+class Repository < Model
+  # ... stuff ...
+  def async_create_archive(branch)
+    Resque.enqueue(Archive, self.id, branch)
+  end
+  # ... more stuff ...
+end
+# Calling this code:
+repo = Repository.find(22)
+repo.async_create_archive('homebrew')
+# Will return immediately and create a Resque job which is later
+# processed.
+# Essentially, this code is run by the worker when processing:
+Archive.perform(22, 'homebrew')

data/init.rb ADDED

	@@ -0,0 +1 @@
1	+ require 'resque'

data/lib/resque.rb ADDED

@@ -0,0 +1,287 @@
+require 'redis/namespace'
+begin
+  require 'yajl'
+rescue LoadError
+  require 'json'
+end
+require 'resque/version'
+require 'resque/errors'
+require 'resque/failure'
+require 'resque/failure/base'
+require 'resque/helpers'
+require 'resque/stat'
+require 'resque/job'
+require 'resque/worker'
+require 'resque/plugin'
+module Resque
+  include Helpers
+  extend self
+  # Accepts:
+  #   1. A 'hostname:port' string
+  #   2. A 'hostname:port:db' string (to select the Redis db)
+  #   3. An instance of `Redis`, `Redis::Client`, `Redis::DistRedis`,
+  #      or `Redis::Namespace`.
+  def redis=(server)
+    case server
+    when String
+      host, port, db = server.split(':')
+      redis = Redis.new(:host => host, :port => port,
+        :thread_safe => true, :db => db)
+      @redis = Redis::Namespace.new(:resque, :redis => redis)
+    when Redis, Redis::Client, Redis::DistRedis
+      @redis = Redis::Namespace.new(:resque, :redis => server)
+    when Redis::Namespace
+      @redis = server
+    else
+      raise "I don't know what to do with #{server.inspect}"
+    end
+  end
+  # Returns the current Redis connection. If none has been created, will
+  # create a new one.
+  def redis
+    return @redis if @redis
+    self.redis = 'localhost:6379'
+    self.redis
+  end
+  # The `before_first_fork` hook will be run in the **parent** process
+  # only once, before forking to run the first job. Be careful- any
+  # changes you make will be permanent for the lifespan of the
+  # worker.
+  #
+  # Call with a block to set the hook.
+  # Call with no arguments to return the hook.
+  def before_first_fork(&block)
+    block ? (@before_first_fork = block) : @before_first_fork
+  end
+  # Set a proc that will be called in the parent process before the
+  # worker forks for the first time.
+  def before_first_fork=(before_first_fork)
+    @before_first_fork = before_first_fork
+  end
+  # The `before_fork` hook will be run in the **parent** process
+  # before every job, so be careful- any changes you make will be
+  # permanent for the lifespan of the worker.
+  #
+  # Call with a block to set the hook.
+  # Call with no arguments to return the hook.
+  def before_fork(&block)
+    block ? (@before_fork = block) : @before_fork
+  end
+  # Set the before_fork proc.
+  def before_fork=(before_fork)
+    @before_fork = before_fork
+  end
+  # The `after_fork` hook will be run in the child process and is passed
+  # the current job. Any changes you make, therefor, will only live as
+  # long as the job currently being processes.
+  #
+  # Call with a block to set the hook.
+  # Call with no arguments to return the hook.
+  def after_fork(&block)
+    block ? (@after_fork = block) : @after_fork
+  end
+  # Set the after_fork proc.
+  def after_fork=(after_fork)
+    @after_fork = after_fork
+  end
+  def to_s
+    "Resque Client connected to #{redis.server}"
+  end
+  #
+  # queue manipulation
+  #
+  # Pushes a job onto a queue. Queue name should be a string and the
+  # item should be any JSON-able Ruby object.
+  def push(queue, item)
+    watch_queue(queue)
+    redis.rpush "queue:#{queue}", encode(item)
+  end
+  # Pops a job off a queue. Queue name should be a string.
+  #
+  # Returns a Ruby object.
+  def pop(queue)
+    decode redis.lpop("queue:#{queue}")
+  end
+  # Returns an int representing the size of a queue.
+  # Queue name should be a string.
+  def size(queue)
+    redis.llen("queue:#{queue}").to_i
+  end
+  # Returns an array of items currently queued. Queue name should be
+  # a string.
+  #
+  # start and count should be integer and can be used for pagination.
+  # start is the item to begin, count is how many items to return.
+  #
+  # To get the 3rd page of a 30 item, paginatied list one would use:
+  #   Resque.peek('my_list', 59, 30)
+  def peek(queue, start = 0, count = 1)
+    list_range("queue:#{queue}", start, count)
+  end
+  # Does the dirty work of fetching a range of items from a Redis list
+  # and converting them into Ruby objects.
+  def list_range(key, start = 0, count = 1)
+    if count == 1
+      decode redis.lindex(key, start)
+    else
+      Array(redis.lrange(key, start, start+count-1)).map do |item|
+        decode item
+      end
+    end
+  end
+  # Returns an array of all known Resque queues as strings.
+  def queues
+    redis.smembers(:queues)
+  end
+  # Given a queue name, completely deletes the queue.
+  def remove_queue(queue)
+    redis.srem(:queues, queue.to_s)
+    redis.del("queue:#{queue}")
+  end
+  # Used internally to keep track of which queues we've created.
+  # Don't call this directly.
+  def watch_queue(queue)
+    redis.sadd(:queues, queue.to_s)
+  end
+  #
+  # job shortcuts
+  #
+  # This method can be used to conveniently add a job to a queue.
+  # It assumes the class you're passing it is a real Ruby class (not
+  # a string or reference) which either:
+  #
+  #   a) has a @queue ivar set
+  #   b) responds to `queue`
+  #
+  # If either of those conditions are met, it will use the value obtained
+  # from performing one of the above operations to determine the queue.
+  #
+  # If no queue can be inferred this method will raise a `Resque::NoQueueError`
+  #
+  # This method is considered part of the `stable` API.
+  def enqueue(klass, *args)
+    Job.create(queue_from_class(klass), klass, *args)
+  end
+  # This method can be used to conveniently remove a job from a queue.
+  # It assumes the class you're passing it is a real Ruby class (not
+  # a string or reference) which either:
+  #
+  #   a) has a @queue ivar set
+  #   b) responds to `queue`
+  #
+  # If either of those conditions are met, it will use the value obtained
+  # from performing one of the above operations to determine the queue.
+  #
+  # If no queue can be inferred this method will raise a `Resque::NoQueueError`
+  #
+  # If no args are given, this method will dequeue *all* jobs matching
+  # the provided class. See `Resque::Job.destroy` for more
+  # information.
+  #
+  # Returns the number of jobs destroyed.
+  #
+  # Example:
+  #
+  #   # Removes all jobs of class `UpdateNetworkGraph`
+  #   Resque.dequeue(GitHub::Jobs::UpdateNetworkGraph)
+  #
+  #   # Removes all jobs of class `UpdateNetworkGraph` with matching args.
+  #   Resque.dequeue(GitHub::Jobs::UpdateNetworkGraph, 'repo:135325')
+  #
+  # This method is considered part of the `stable` API.
+  def dequeue(klass, *args)
+    Job.destroy(queue_from_class(klass), klass, *args)
+  end
+  # Given a class, try to extrapolate an appropriate queue based on a
+  # class instance variable or `queue` method.
+  def queue_from_class(klass)
+    klass.instance_variable_get(:@queue) ||
+      (klass.respond_to?(:queue) and klass.queue)
+  end
+  # This method will return a `Resque::Job` object or a non-true value
+  # depending on whether a job can be obtained. You should pass it the
+  # precise name of a queue: case matters.
+  #
+  # This method is considered part of the `stable` API.
+  def reserve(queue)
+    Job.reserve(queue)
+  end
+  #
+  # worker shortcuts
+  #
+  # A shortcut to Worker.all
+  def workers
+    Worker.all
+  end
+  # A shortcut to Worker.working
+  def working
+    Worker.working
+  end
+  # A shortcut to unregister_worker
+  # useful for command line tool
+  def remove_worker(worker_id)
+    worker = Resque::Worker.find(worker_id)
+    worker.unregister_worker
+  end
+  #
+  # stats
+  #
+  # Returns a hash, similar to redis-rb's #info, of interesting stats.
+  def info
+    return {
+      :pending   => queues.inject(0) { |m,k| m + size(k) },
+      :processed => Stat[:processed],
+      :queues    => queues.size,
+      :workers   => workers.size.to_i,
+      :working   => working.size,
+      :failed    => Stat[:failed],
+      :servers   => [redis.server]
+    }
+  end
+  # Returns an array of all known Resque keys in Redis. Redis' KEYS operation
+  # is O(N) for the keyspace, so be careful - this can be slow for big databases.
+  def keys
+    redis.keys("*").map do |key|
+      key.sub("#{redis.namespace}:", '')
+    end
+  end
+end