resque-ar-failover 1.0.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZTc0MjhmMTZiZTQxODZiMWVhMzZjNGI2ZTk3NTc5Nzc0ZjAwMmE2Yw==
+  data.tar.gz: !binary |-
+    YzBkN2FlNDdlODBiYWNjNmMyN2NjNTQzMTEzMjFiNjQzM2Q2ODFhOA==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    NWM1NDc2ZWVlYjU4MmE3NWQzZWZiYWFiZjk4NDRhMWY3YzlmNWNmNDJiM2Uw
+    OWJiODk0NWI3NzUzZTFkNTY2ODRlMDEyNmUxN2JhMTU3N2Y2ZTBlNWIzYzY3
+    N2RhM2ZmMGE5MzgxZTcxZTEwZDFkNmVjNzQ2YTRjZjBjOWM2YmQ=
+  data.tar.gz: !binary |-
+    OGE3YWQ4MmNhNDg3NjM1ZmJhNjA1ZjM3ZjFkNDA0ZWI2NWNlNDkyNjdmMjkw
+    M2JmOGE4Yjk4NGIxNzYxYjcyZjBiNGUwNmMyYTE2M2U1NmZiODVkN2JjOWE2
+    ZWMwMjczNjdmMzJiODFiMDFkMGI4OGQyYWM3OTE2YWE0MDZmZDE=

data/.gitignore

@@ -0,0 +1,3 @@
+spec/database.sqlite3
+spec/dump.rdb
+Gemfile.lock

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,10 @@
+source 'http://rubygems.org'
+gemspec
+
+group :development do
+  gem 'debugger'
+end
+
+group :test do
+  gem "rake"
+end

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Hotel Tonight Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,42 @@
+<img src="https://raw.github.com/hoteltonight/HTDelegateProxy/master/ht-logo-black.png" alt="HotelTonight" title="HotelTonight" style="display:block; margin: 10px auto 30px auto;">
+
+# resque-ar-failover
+
+Resque plugin to fail over to ActiveRecord backend for Resque job storage, when Redis is unavailable.
+
+## Background
+
+At HotelTonight we use Redis. We love it. We also love Resque. But every once in a while,
+Redis goes away for a split second, causing issues in our app and making some jobs to
+fall through the cracks.
+
+## resque-ar-failover
+
+Resque-ar-failover deals with this issue by rescueing the exceptions that signal that `Resque.enqueue` has failed
+before being able to contact redis. When that happens, we persist the job to our MySQL backend (which is HA ready).
+
+We use MySQL; you don't have to. Any AR adapter will do.
+
+## Installation
+
+Resque-AR-failover is distributed as a gem. Just drop it in your `Gemfile`
+
+```
+gem 'resque-ar-failover'
+```
+
+Then make sure you add [this migration][migration] to your project or run the following command to create the migration automatically.
+
+```
+rails generate resque_ar_failover install
+rake db:migrate
+```
+
+And that's it! If redis goes down a new `ResqueBackup` entry will be added and it will automatically be picked up by your resque workers.
+
+## License
+
+Check out the [LICENSE][license] file.
+
+ [migration]: https://github.com/hoteltonight/resque-ar-failover/blob/master/lib/generators/templates/migration.rb
+ [license]: https://github.com/hoteltonight/resque-ar-failover/blob/master/LICENSE

data/Rakefile

@@ -0,0 +1,33 @@
+require 'rubygems'
+require 'bundler/setup'
+
+require 'active_record'
+require 'yaml'
+
+namespace :db do
+  desc "Migrate the db"
+  task :migrate do
+    connection_details = YAML::load(File.open('spec/database.yml'))
+    ActiveRecord::Base.establish_connection(connection_details)
+    ActiveRecord::Migrator.migrate("db/migrate/")
+  end
+end
+
+task :load_path do
+  %w(lib).each do |path|
+    $LOAD_PATH.unshift(File.expand_path("../#{path}", __FILE__))
+  end
+end
+
+task gem: :build
+task :build do
+  system "gem build resque-ar-failover.gemspec"
+end
+
+task release: :build do
+  version = Resque::Plugins::ArFailover::VERSION
+  system "git tag -a v#{version} -m 'Tagging #{version}'"
+  system "git push --tags"
+  system "gem push resque-ar-failover-#{version}.gem"
+  system "rm resque-ar-failover-#{version}.gem"
+end

data/lib/generators/resque_ar_failover_generator.rb

@@ -0,0 +1,18 @@
+require 'rails/generators'
+require "rails/generators/active_record"
+
+class ResqueArFailoverGenerator < ActiveRecord::Generators::Base
+  include Rails::Generators::Migration
+
+  source_root File.expand_path('../', __FILE__)
+
+  desc <<DESC
+Description:
+  Copies over migrations for the resque AR failover.
+DESC
+
+  # Copies the migration template to db/migrate.
+  def copy_migration
+    migration_template 'templates/migration.rb', 'db/migrate/create_resque_backup.rb'
+  end
+end

data/lib/generators/templates/migration.rb

@@ -0,0 +1,15 @@
+class CreateResqueBackup < ActiveRecord::Migration
+  def up
+    create_table :resque_backups do |t|
+      t.column :queue, :string
+      t.column :klass, :string
+      t.column :payload, :string
+    end
+
+    add_index :resque_backups, :queue
+  end
+
+  def down
+    remove_table :resque_backups
+  end
+end

data/lib/resque-ar-failover.rb

@@ -0,0 +1,15 @@
+require 'rubygems'
+require 'resque'
+require "active_record"
+require 'resque-ar-failover/resque_backup'
+require 'resque-ext/enqueue'
+require 'resque-ext/size'
+require 'resque-ext/pop'
+
+module Resque
+  module Plugins
+    module ArFailover
+      THROTTLE = 1.0
+    end
+  end
+end

data/lib/resque-ar-failover/resque_backup.rb

@@ -0,0 +1,14 @@
+class ResqueBackup < ActiveRecord::Base
+  def self.pop(queue)
+    3.times do
+      @job = where(queue: queue).first
+      break if !@job || delete_all(id: @job.id) == 1
+    end
+
+    @job.as_job if @job
+  end
+
+  def as_job
+    { "class" => klass, "args" => payload }
+  end
+end

data/lib/resque-ar-failover/version.rb

@@ -0,0 +1,7 @@
+module Resque
+  module Plugins
+    module ArFailover
+      VERSION = '1.0.0'
+    end
+  end
+end

data/lib/resque-ext/enqueue.rb

@@ -0,0 +1,12 @@
+module Resque
+  def self.enqueue_to_with_failover(queue, klass, *args)
+    self.enqueue_to_without_failover queue, klass, *args
+  rescue Redis::CannotConnectError, Redis::TimeoutError
+    ResqueBackup.create!(queue: queue, klass: klass, payload: args.to_json)
+  end
+
+  class << self
+    alias_method :enqueue_to_without_failover, :enqueue_to
+    alias_method :enqueue_to, :enqueue_to_with_failover
+  end
+end

data/lib/resque-ext/pop.rb

@@ -0,0 +1,15 @@
+module Resque
+  def self.pop_with_failover(queue)
+    should_check_backup = (rand <= Resque::Plugins::ArFailover::THROTTLE)
+
+    job  = ResqueBackup.pop(queue) if should_check_backup
+    job || self.pop_without_failover(queue)
+  rescue Redis::CannotConnectError, Redis::TimeoutError
+    ResqueBackup.pop(queue) unless should_check_backup
+  end
+
+  class << self
+    alias_method :pop_without_failover, :pop
+    alias_method :pop, :pop_with_failover
+  end
+end

data/lib/resque-ext/size.rb

@@ -0,0 +1,16 @@
+module Resque
+  def self.size_with_failover(queue)
+    size_in_redis =  begin
+      self.size_without_failover(:queue)
+    rescue Redis::CannotConnectError, Redis::TimeoutError
+      0
+    end
+
+    size_in_redis + ResqueBackup.where(queue: queue).count
+  end
+
+  class << self
+    alias_method :size_without_failover, :size
+    alias_method :size, :size_with_failover
+  end
+end

data/resque-ar-failover.gemspec

@@ -0,0 +1,28 @@
+lib = File.expand_path('../lib/', __FILE__)
+$:.unshift lib unless $:.include?(lib)
+
+require 'resque-ar-failover/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'resque-ar-failover'
+  s.version     = Resque::Plugins::ArFailover::VERSION
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ['Pablo Fernandez']
+  s.email       = ['heelhook@littleq.net']
+  s.homepage    = 'http://github.com/hoteltonight/resque-ar-failover'
+  s.summary     = 'Resque plugin to fail over to ActiveRecord backend for Resque job storage, when Redis is unavailable.'
+  s.has_rdoc    = false
+
+  s.rubyforge_project = 'resque-ar-failover'
+
+  s.add_dependency 'resque', '~>1.0'
+  s.add_dependency 'activerecord', '~>3.2'
+  s.add_development_dependency 'rake'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'sqlite3'
+  s.add_development_dependency 'debugger'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- spec/*`.split("\n")
+  s.require_paths = ["lib"]
+end

data/spec/ar-failover_spec.rb

@@ -0,0 +1,102 @@
+require 'spec_helper'
+
+class JobOnQueue
+  @queue = :queue
+
+  def self.perform(id=nil)
+    puts "Processing job #{id}" if id
+  end
+end
+
+describe 'Resque' do
+  let (:worker) { Resque::Worker.new(:queue) }
+
+  before :each do
+    make_redis_available
+    cleanup
+  end
+
+  context 'when redis is accessible' do
+    before do
+      start_redis
+      cleanup
+    end
+
+    describe '.enqueue' do
+      it 'enqueues successfully' do
+        expect { Resque.enqueue(JobOnQueue) }.to change { Resque.size(:queue) }.by(1)
+      end
+    end
+
+    describe '.pop' do
+      it 'pops queued items from redis' do
+        2.times { Resque.enqueue(JobOnQueue) }
+        expect { Resque.pop(:queue) }.to change { Resque.size_without_failover(:queue) }.from(2).to(1)
+      end
+    end
+  end
+
+  context 'when redis is inaccessible' do
+    before do
+      ensure_redis_is_dead
+    end
+
+    describe '.enqueue' do
+      it 'enqueues successfully' do
+        expect { Resque.enqueue(JobOnQueue) }.to change { Resque.size(:queue) }.by(1)
+      end
+    end
+
+    describe '.pop' do
+      it 'pops queued items from redis' do
+        2.times { Resque.enqueue(JobOnQueue) }
+        expect { Resque.pop(:queue) }.to change { Resque.size(:queue) }.from(2).to(1)
+      end
+    end
+  end
+
+  context 'when redis is unresponsive' do
+    before do
+      make_redis_unavailable
+    end
+
+    describe '.enqueue' do
+      it 'enqueues successfully' do
+        expect { Resque.enqueue(JobOnQueue) }.to change { Resque.size(:queue) }.by(1)
+      end
+    end
+
+    describe '.pop' do
+    end
+  end
+
+  it 'handles jobs that are in redis and the AR backend' do
+    ensure_redis_is_running
+    cleanup
+    Resque.enqueue(JobOnQueue, 'one')
+    make_redis_unavailable
+    Resque.enqueue(JobOnQueue, 'two')
+    make_redis_available
+    Resque.size(:queue).should == 2
+
+    job = Resque.pop(:queue)
+    job['args'].should include 'two'
+    Resque.size(:queue).should == 1
+
+    job = Resque.pop(:queue)
+    job['args'].should include 'one'
+    Resque.size(:queue).should == 0
+
+    Resque.pop(:queue).should be_nil
+  end
+end
+
+describe 'ResqueBackup' do
+  describe '#as_job' do
+    let(:job) { ResqueBackup.new(queue: 'queue', klass: 'MyClass', payload: ['one', 'two', 'three'])}
+
+    it 'converts an AR model into a Resque structure' do
+      job.as_job.should == { 'class' => 'MyClass', 'args' => %w(one two three) }
+    end
+  end
+end

data/spec/database.yml

@@ -0,0 +1,2 @@
+adapter: sqlite3
+database: spec/database.sqlite3

data/spec/redis-test.conf

@@ -0,0 +1,111 @@
+# Redis configuration file example
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
+# You can specify a custom pid file location here.
+pidfile ./spec/redis-test.pid
+
+# Accept connections on the specified port, default is 6379
+port 9736
+
+# If you want you can bind a single interface, if the bind option is not
+# specified all the interfaces will listen for connections.
+#
+# bind 127.0.0.1
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 300
+
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+save 900 1
+save 300 10
+save 60 10000
+
+# The filename where to dump the DB
+dbfilename dump.rdb
+
+# For default save/load DB in/from the working directory
+# Note that you must specify a directory not a file name.
+dir ./spec/
+
+# Set server verbosity to 'debug'
+# it can be one of:
+# debug (a lot of information, useful for development/testing)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel debug
+
+# Specify the log file name. Also 'stdout' can be used to force
+# the demon to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile stdout
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+
+# slaveof <masterip> <masterport>
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+
+# requirepass foobared
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default there
+# is no limit, and it's up to the number of file descriptors the Redis process
+# is able to open. The special value '0' means no limts.
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+
+# maxclients 128
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys with an
+# EXPIRE set. It will try to start freeing keys that are going to expire
+# in little time and preserve keys with a longer time to live.
+# Redis will also try to remove objects from free lists if possible.
+#
+# If all this fails, Redis will start to reply with errors to commands
+# that will use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to most read-only commands like GET.
+#
+# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
+# 'state' server or cache, not as a real DB. When Redis is used as a real
+# database the memory usage will grow over the weeks, it will be obvious if
+# it is going to use too much memory in the long run, and you'll have the time
+# to upgrade. With maxmemory after the limit is reached you'll start to get
+# errors for write operations, and this may even lead to DB inconsistency.
+
+# maxmemory <bytes>
+
+############################### ADVANCED CONFIG ###############################
+

data/spec/spec_helper.rb

@@ -0,0 +1,72 @@
+require 'rubygems'
+require 'bundler/setup'
+require 'rspec'
+
+require 'ruby-debug'
+require 'resque'
+require 'resque-ar-failover'
+
+Bundler.require(:default, :test)
+
+$TESTING = true
+
+RSpec.configure do |config|
+  config.before(:suite) do
+    $redis_path = `which redis-server`.chomp
+    if $redis_path.empty?
+      abort "redis-server couldn't be found on your system. Check that redis is installed and available in your $PATH"
+    end
+
+    $base_path = File.dirname(File.expand_path(__FILE__))
+
+    Resque.redis = 'localhost:9736'
+
+    connection_details = YAML::load(File.open(File.join($base_path, 'database.yml')))
+    ActiveRecord::Base.establish_connection(connection_details)
+  end
+
+  config.after(:suite) do
+    kill_redis
+  end
+end
+
+def start_redis
+  system("#{$redis_path} #{File.join($base_path, 'redis-test.conf')}") or abort 'Unable to starte redis'
+end
+
+def cleanup
+  ResqueBackup.delete_all
+  Resque.redis.flushall if is_redis_alive?
+end
+
+def ensure_redis_is_running
+  return if is_redis_alive?
+
+  start_redis
+end
+
+def ensure_redis_is_dead
+  while is_redis_alive? do
+    kill_redis
+  end
+end
+
+def make_redis_unavailable
+  Resque.redis = 'google.com:9736'
+end
+
+def make_redis_available
+  Resque.redis = 'localhost:9736'
+end
+
+def redis_pid
+  pid = `ps -e -o pid,command | grep [r]edis-test`.split(" ")[0].to_i
+end
+
+def kill_redis(signal='KILL')
+  Process.kill(signal, redis_pid) if is_redis_alive?
+end
+
+def is_redis_alive?
+  redis_pid != 0
+end

metadata

@@ -0,0 +1,151 @@
+--- !ruby/object:Gem::Specification
+name: resque-ar-failover
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Pablo Fernandez
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-10-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: resque
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: activerecord
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: debugger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: 
+email:
+- heelhook@littleq.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/generators/resque_ar_failover_generator.rb
+- lib/generators/templates/migration.rb
+- lib/resque-ar-failover.rb
+- lib/resque-ar-failover/resque_backup.rb
+- lib/resque-ar-failover/version.rb
+- lib/resque-ext/enqueue.rb
+- lib/resque-ext/pop.rb
+- lib/resque-ext/size.rb
+- resque-ar-failover.gemspec
+- spec/ar-failover_spec.rb
+- spec/database.yml
+- spec/redis-test.conf
+- spec/spec_helper.rb
+homepage: http://github.com/hoteltonight/resque-ar-failover
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: resque-ar-failover
+rubygems_version: 2.0.4
+signing_key: 
+specification_version: 4
+summary: Resque plugin to fail over to ActiveRecord backend for Resque job storage,
+  when Redis is unavailable.
+test_files:
+- spec/ar-failover_spec.rb
+- spec/database.yml
+- spec/redis-test.conf
+- spec/spec_helper.rb