resque-bus 0.3.7 → 0.5.7

data/.gitignore

@@ -2,3 +2,4 @@
 .bundle
 Gemfile.lock
 pkg/*
+.DS_Store

data/Gemfile

@@ -2,5 +2,5 @@ source "http://rubygems.org"
 
 gemspec
 
-gem "debugger"
-gem "rake"
+gem "rake"
+# gem "queue-bus", path: "../queue-bus"

data/README.mdown

@@ -1,6 +1,7 @@
 ## Resque Bus
 
-This gem uses Redis and Resque to allow simple asynchronous communication between apps.
+This gem provides an adapter for Resque for use in the [queue-bus](https://github.com/queue-bus/queue-bus) system.
+It uses Redis and the Resque that you are already using to allow simple asynchronous communication between apps.
 
 ### Install
 
@@ -10,35 +11,36 @@ To install, include the 'resque-bus' gem and add the following to your Rakefile:
 require "resque_bus/tasks"
 ```
 
+
 ### Example
 
 Application A can publish an event
 
 ```ruby
-# config
-ResqueBus.redis = "192.168.1.1:6379"
+# pick an adapter
+require 'resque-bus' # (or other adapter)
 
 # business logic
-ResqueBus.publish("user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
+QueueBus.publish("user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
 
 # or do it later
-ResqueBus.publish_at(1.hour.from_now, "user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
+QueueBus.publish_at(1.hour.from_now, "user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
 ```
 
 Application B is subscribed to events
 
 ```ruby
-# config
-ResqueBus.redis = "192.168.1.1:6379"
+# pick an adapter
+require 'resque-bus' # (or other adapter)
 
 # initializer
-ResqueBus.dispatch("app_b") do
+QueueBus.dispatch("app_b") do
   # processes event on app_b_default queue
   # subscribe is short-hand to subscribe to your 'default' queue and this block with process events with the name "user_created"
   subscribe "user_created" do |attributes|
     NameCount.find_or_create_by_name(attributes["last_name"]).increment!
   end
-  
+
   # processes event on app_b_critical queue
   # critical is short-hand to subscribe to your 'critical' queue and this block with process events with the name "user_paid"
   critical "user_paid" do |attributes|
@@ -46,13 +48,13 @@ ResqueBus.dispatch("app_b") do
   end
 
   # you can pass any queue name you would like to process from as well IE: `banana "peeled" do |attributes|`
-  
+
   # and regexes work as well. note that with the above configuration along with this regex,
   # the following as well as the corresponding block above would both be executed
   subscribe /^user_/ do |attributes|
     Metrics.record_user_action(attributes["bus_event_type"], attributes["id"])
   end
-  
+
   # the above all filter on just the event_type, but you can filter on anything
   # this would be _any_ event that has a user_id and the page value of homepage regardless of bus_event_type
   subscribe "my_key", { "user_id" => :present, "page" => "homepage"} do
@@ -61,11 +63,11 @@ ResqueBus.dispatch("app_b") do
 end
 ```
 
-Applications can also subscribe within classes using the provided `Subscriber` module. 
+Applications can also subscribe within classes using the provided `Subscriber` module.
 
 ```ruby
 class SimpleSubscriber
-  include ResqueBus::Subscriber
+  include QueueBus::Subscriber
   subscribe :my_method
 
   def my_method(attributes)
@@ -78,7 +80,7 @@ The following is equivalent to the original initializer and shows more options:
 
 ```ruby
 class OtherSubscriber
-  include ResqueBus::Subscriber
+  include QueueBus::Subscriber
   application :app_b
 
   subscribe :user_created
@@ -110,20 +112,21 @@ Note: This subscribes when this class is loaded, so it needs to be in your load
 
 Each app needs to tell Redis about its subscriptions:
 
-    $ rake resquebus:subscribe
+    $ rake queuebus:subscribe
 
 The subscription block is run inside a Resque worker which needs to be started for each app.
 
-    $ rake resquebus:setup resque:work
-    
+    $ rake queuebus:setup resque:work
+
 The incoming queue also needs to be processed on a dedicated or all the app servers.
 
-    $ rake resquebus:driver resque:work
-    
+    $ rake queuebus:driver resque:work
+
 If you want retry to work for subscribing apps, you should run resque-scheduler
-    
+
     $ rake resque:scheduler
 
+
 ### Heartbeat
 
 We've found it useful to have the bus act like `cron`, triggering timed jobs throughout the system. Resque Bus calls this a heartbeat.
@@ -133,7 +136,7 @@ It uses resque-scheduler to trigger the events. You can enable it in your Rakefi
 # resque.rake
 namespace :resque do
   task :setup => [:environment] do
-    ResqueBus.heartbeat!
+    QueueBus.heartbeat!
   end
 end
 ```
@@ -143,8 +146,8 @@ Or add it to your `schedule.yml` directly
 ```yaml
 resquebus_heartbeat:
   cron: "* * * * *"
-  class: "::ResqueBus::Heartbeat"
-  queue: resquebus_incoming
+  class: "::QueueBus::Heartbeat"
+  queue: bus_incoming
   description: "I publish a heartbeat_minutes event every minute"
 ```
 
@@ -161,7 +164,7 @@ attributes = {}
 
 now = Time.now
 seconds = now.to_i
-ResqueBus.publish("hearbeat_minutes", {
+QueueBus.publish("hearbeat_minutes", {
   "epoch_seconds" => seconds,
   "epoch_minutes" => seconds / 1.minute,
   "epoch_hours"   => seconds / 1.hour,
@@ -179,18 +182,18 @@ ResqueBus.publish("hearbeat_minutes", {
 This allows you do something like this:
 
 ```ruby
-ResqueBus.dispatch("app_c") do
+QueueBus.dispatch("app_c") do
   # runs at 10:20, 11:20, etc
   subscribe "once_an_hour", 'bus_event_type' => 'heartbeat_minutes', 'minute' => 20 do |attributes|
     Sitemap.generate!
   end
-  
+
   # runs every five minutes
   subscribe "every_five_minutes", 'bus_event_type' => 'heartbeat_minutes' do |attributes|
     next unless attributes["epoch_minutes"] % 5 == 0
     HealthCheck.run!
   end
-  
+
   # runs at 8am on the first of every month
   subscribe "new_month_morning", 'bus_event_type' => 'heartbeat_minutes', 'day' => 1, hour' => 8, 'minute' => 0,  do |attributes|
     next unless attributes["epoch_minutes"] % 5 == 0
@@ -199,57 +202,32 @@ ResqueBus.dispatch("app_c") do
 end
 ```
 
-### Compatibility
-
-ResqueBus can live along side another instance of Resque that points at a different Redis server.
-
-```ruby
-# config
-Resque.redis = "192.168.1.0:6379"
-ResqueBus.redis = "192.168.1.1:6379"
-```
-
-If no Redis instance is given specifically, ResqueBus will use the Resque one.
-
-```ruby
-# config
-Resque.redis = "192.168.1.0:6379"
-```
-
-That will use the default (resque) namespace which can be helpful for using the tooling. Conflict with queue names are unlikely. You can change the namespace if you like though.
-
-```ruby
-# config
-Resque.redis = "192.168.1.0:6379"
-ResqueBus.redis.namespace = :get_on_the_bus
-```
-
 ### Local Mode
 
-For development, a local mode is also provided and is specified in the configuration.
+For development, a local mode is provided and is specified in the configuration.
 
 ```ruby
 # config
-ResqueBus.local_mode = :standalone
-or 
-ResqueBus.local_mode = :inline
+QueueBus.local_mode = :standalone
+or
+QueueBus.local_mode = :inline
 ```
 
-Standalone mode does not require a separate resquebus:driver task to be running to process the
-incoming queue. Simply publishing to the bus will distribute the incoming events 
-to the appropriate application specific queue.  A separate resquebus:work task does 
+Standalone mode does not require a separate queuebus:driver task to be running to process the
+incoming queue. Simply publishing to the bus will distribute the incoming events
+to the appropriate application specific queue.  A separate queuebus:work task does
 still need to be run to process these events
 
 Inline mode skips queue processing entirely and directly dispatches the
-event to the appropriate code block. 
+event to the appropriate code block.
 
+You can also say `QueueBus.local_mode = :suppress` to turn off publishing altogether.
+This can be helpful inside some sort of migration, for example.
 
 ### TODO
 
-* There are a few spots in the code with TODO notes
+* Replace local modes with adapters
 * Make this not freak out in development without Redis or when Redis is down
 * We might not actually need to publish in tests
 * Add some rspec helpers for the apps to use: should_ post an event_publish or something along those lines
-* Allow calling resquebus:setup and resquebus:driver together (append to ENV['QUEUES'], don't replace it)
-    
-Copyright (c) 2011 Brian Leonard, released under the MIT license
+* Allow calling queuebus:setup and queuebus:driver together (append to ENV['QUEUES'], don't replace it)

data/Rakefile

@@ -1,3 +1,2 @@
 $LOAD_PATH.unshift 'lib'
-require 'resque/tasks'
 require 'resque_bus/tasks'

data/lib/resque-bus.rb

@@ -1,311 +1,19 @@
+require "queue-bus"
+require "resque_bus/adapter"
 require "resque_bus/version"
 
-require 'redis/namespace'
-require 'resque'
-
-module QueueBus
-  autoload :Worker,          'resque_bus/compatibility'
-end
-
 module ResqueBus
-  
-  autoload :Application,      'resque_bus/application'
-  autoload :Dispatch,         'resque_bus/dispatch'
-  autoload :Driver,           'resque_bus/driver'
-  autoload :Heartbeat,        'resque_bus/heartbeat'
-  autoload :Local,            'resque_bus/local'
-  autoload :Matcher,          'resque_bus/matcher'
-  autoload :Publisher,        'resque_bus/publisher'
-  autoload :Rider,            'resque_bus/rider'
-  autoload :Subscriber,       'resque_bus/subscriber'
-  autoload :Subscription,     'resque_bus/subscription'
-  autoload :SubscriptionList, 'resque_bus/subscription_list'
-  autoload :TaskManager,      'resque_bus/task_manager'
-  autoload :Util,             'resque_bus/util'
-
-  class << self
-    
-    def default_app_key=val
-      @default_app_key = Application.normalize(val)
-    end
-    
-    def default_app_key
-      @default_app_key
-    end
-    
-    def default_queue=val
-      @default_queue = val
-    end
-    
-    def default_queue
-      @default_queue
-    end
-
-    def hostname
-      @hostname ||= `hostname 2>&1`.strip.sub(/.local/,'')
-    end
-    
-    def dispatch(app_key=nil, &block)
-      dispatcher = dispatcher_by_key(app_key)
-      dispatcher.instance_eval(&block)
-      dispatcher
-    end
-    
-    def dispatchers
-      @dispatchers ||= {}
-      @dispatchers.values
-    end
-    
-    def dispatcher_by_key(app_key)
-      app_key = Application.normalize(app_key || default_app_key)
-      @dispatchers ||= {}
-      @dispatchers[app_key] ||= Dispatch.new(app_key)
-    end
-    
-    def dispatcher_execute(app_key, key, attributes)
-      @dispatchers ||= {}
-      dispatcher = @dispatchers[app_key]
-      dispatcher.execute(key, attributes) if dispatcher
-    end
-
-    def local_mode=value
-      @local_mode = value
-    end
-
-    def local_mode
-      @local_mode
-    end
-    
-    def heartbeat!
-      # turn on the heartbeat
-      # should be down after loading scheduler yml if you do that
-      # otherwise, anytime
-      require 'resque/scheduler'
-      name     = 'resquebus_hearbeat'
-      schedule = { 'class' => '::ResqueBus::Heartbeat',
-                   'cron'  => '* * * * *',   # every minute
-                   'queue' => incoming_queue,
-                   'description' => 'I publish a heartbeat_minutes event every minute'
-                 }
-      if Resque::Scheduler.dynamic
-        Resque.set_schedule(name, schedule)
-      end
-      Resque.schedule[name] = schedule
-    end
+  # TODO: all of this will be removed
 
-    # Accepts:
-    #   1. A 'hostname:port' String
-    #   2. A 'hostname:port:db' String (to select the Redis db)
-    #   3. A 'hostname:port/namespace' String (to set the Redis namespace)
-    #   4. A Redis URL String 'redis://host:port'
-    #   5. An instance of `Redis`, `Redis::Client`, `Redis::DistRedis`,
-    #      or `Redis::Namespace`.
-    def redis=(server)
-      case server
-      when String
-        if server =~ /redis\:\/\//
-          redis = Redis.connect(:url => server, :thread_safe => true)
-        else
-          server, namespace = server.split('/', 2)
-          host, port, db = server.split(':')
-          redis = Redis.new(:host => host, :port => port,
-            :thread_safe => true, :db => db)
-        end
-        namespace ||= default_namespace
+  autoload :Deprecated,  'resque_bus/compatibility/deprecated'
+  autoload :Subscriber,  'resque_bus/compatibility/subscriber'
+  autoload :TaskManager, 'resque_bus/compatibility/task_manager'
+  autoload :Driver,      'resque_bus/compatibility/driver'
+  autoload :Rider,       'resque_bus/compatibility/rider'
+  autoload :Publisher,   'resque_bus/compatibility/publisher'
+  autoload :Heartbeat,   'resque_bus/compatibility/heartbeat'
 
-        @redis = Redis::Namespace.new(namespace, :redis => redis)
-      when Redis::Namespace
-        @redis = server
-      else
-        @redis = Redis::Namespace.new(default_namespace, :redis => server)
-      end
-    end
-
-    # Returns the current Redis connection. If none has been created, will
-    # create a new one from the Reqsue one (with a different namespace)
-    def redis
-      return @redis if @redis
-      copy = Resque.redis.clone
-      copy.namespace = default_namespace
-      self.redis = copy
-      self.redis
-    end
-    
-    def original_redis=(server)
-      @original_redis = server
-    end
-    def original_redis
-      @original_redis
-    end
-    
-    def with_global_attributes(attributes)
-      original_timezone = false
-      original_locale   = false
-      
-      if attributes["bus_locale"] && defined?(I18n) && I18n.respond_to?(:locale=)
-        original_locale = I18n.locale if I18n.respond_to?(:locale)
-        I18n.locale = attributes["bus_locale"]
-      end
-      
-      if attributes["bus_timezone"] && defined?(Time) && Time.respond_to?(:zone=)
-        original_timezone = Time.zone if Time.respond_to?(:zone)
-        Time.zone = attributes["bus_timezone"]
-      end
-
-      yield
-    ensure
-      I18n.locale = original_locale   unless original_locale   == false
-      Time.zone   = original_timezone unless original_timezone == false
-    end
-
-    def before_publish=(proc)
-      @before_publish_callback = proc
-    end
-
-    def before_publish_callback(attributes)
-      if @before_publish_callback
-        @before_publish_callback.call(attributes)
-      end
-    end
-
-    
-    def publish_metadata(event_type, attributes={})
-      # TODO: "bus_app_key" => application.app_key ?
-      bus_attr = {"bus_published_at" => Time.now.to_i, "bus_event_type" => event_type}
-      bus_attr["bus_id"]           = "#{Time.now.to_i}-#{generate_uuid}"
-      bus_attr["bus_app_hostname"] = hostname
-      bus_attr["bus_locale"]       = I18n.locale.to_s if defined?(I18n) && I18n.respond_to?(:locale) && I18n.locale
-      bus_attr["bus_timezone"]     = Time.zone.name   if defined?(Time) && Time.respond_to?(:zone)   && Time.zone
-      out = bus_attr.merge(attributes || {})
-      ResqueBus.before_publish_callback(out)
-      out
-    end
-    
-    def generate_uuid
-      require 'securerandom' unless defined?(SecureRandom)
-      return SecureRandom.uuid
-      
-      rescue Exception => e
-        # secure random not there
-        # big random number a few times
-        n_bytes = [42].pack('i').size
-        n_bits = n_bytes * 8
-        max = 2 ** (n_bits - 2) - 1
-        return "#{rand(max)}-#{rand(max)}-#{rand(max)}"
-    end
-    
-    def publish(event_type, attributes = {})
-      to_publish = publish_metadata(event_type, attributes)
-      ResqueBus.log_application("Event published: #{event_type} #{to_publish.inspect}")
-      if local_mode
-        ResqueBus::Local.perform(to_publish)
-      else
-        enqueue_to(incoming_queue, Driver, to_publish)
-      end
-    end
-    
-    def publish_at(timestamp_or_epoch, event_type, attributes = {})
-      to_publish = publish_metadata(event_type, attributes)
-      to_publish["bus_delayed_until"] ||= timestamp_or_epoch.to_i
-      to_publish.delete("bus_published_at") unless attributes["bus_published_at"] # will be put on when it actually does it
-      
-      ResqueBus.log_application("Event published:#{event_type} #{to_publish.inspect} publish_at: #{timestamp_or_epoch.to_i}")
-      item = delayed_job_to_hash_with_queue(incoming_queue, Publisher, [event_type, to_publish])
-      delayed_push(timestamp_or_epoch, item)
-    end
-    
-    def enqueue_to(queue, klass, *args)
-      push(queue, :class => klass.to_s, :args => args)
-    end
-    
-    def logger
-      @logger
-    end
-    
-    def logger=val
-      @logger = val
-    end
-    
-    def log_application(message)
-      if logger
-        time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
-        logger.info("** [#{time}] #$$: ResqueBus #{message}")
-      end
-    end
-    
-    def log_worker(message)
-      if ENV['LOGGING'] || ENV['VERBOSE'] || ENV['VVERBOSE']
-        time = Time.now.strftime('%H:%M:%S %Y-%m-%d')
-        puts "** [#{time}] #$$: #{message}"
-      end
-    end
-    
-    protected
-    
-    def reset
-      # used by tests
-      @redis = nil # clear instance of redis
-      @dispatcher = nil
-      @default_app_key = nil
-      @default_queue = nil
-      @before_publish_callback = nil
-    end
-    
-    def incoming_queue
-      "resquebus_incoming"
-    end
-
-    def default_namespace
-      # It might play better on the same server, but overall life is more complicated
-      :resque
-    end
-    
-    ## From Resque, but using a (possibly) different instance of Redis
-    
-    # Pushes a job onto a queue. Queue name should be a string and the
-    # item should be any JSON-able Ruby object.
-    #
-    # Resque works generally expect the `item` to be a hash with the following
-    # keys:
-    #
-    #   class - The String name of the job to run.
-    #    args - An Array of arguments to pass the job. Usually passed
-    #           via `class.to_class.perform(*args)`.
-    #
-    # Example
-    #
-    #   Resque.push('archive', :class => 'Archive', :args => [ 35, 'tar' ])
-    #
-    # Returns nothing
-    def push(queue, item)
-      watch_queue(queue)
-      redis.rpush "queue:#{queue}", Resque.encode(item)
-    end
-    
-    # Used internally to keep track of which queues we've created.
-    # Don't call this directly.
-    def watch_queue(queue)
-      redis.sadd(:queues, queue.to_s)
-    end
-    
-    ### From Resque Scheduler
-    # Used internally to stuff the item into the schedule sorted list.
-    # +timestamp+ can be either in seconds or a datetime object
-    # Insertion if O(log(n)).
-    # Returns true if it's the first job to be scheduled at that time, else false
-    def delayed_push(timestamp, item)
-      # First add this item to the list for this timestamp
-      redis.rpush("delayed:#{timestamp.to_i}", Resque.encode(item))
+  extend ::ResqueBus::Deprecated
+end
 
-      # Now, add this timestamp to the zsets.  The score and the value are
-      # the same since we'll be querying by timestamp, and we don't have
-      # anything else to store.
-      redis.zadd :delayed_queue_schedule, timestamp.to_i, timestamp.to_i
-    end
-    
-    def delayed_job_to_hash_with_queue(queue, klass, args)
-      {:class => klass.to_s, :args => args, :queue => queue}
-    end
-  end
-  
-end
+QueueBus.adapter = QueueBus::Adapters::Resque.new

data/lib/resque_bus/adapter.rb

@@ -0,0 +1,65 @@
+module QueueBus
+  module Adapters
+    class Resque < QueueBus::Adapters::Base
+      def enabled!
+        # know we are using it
+        require 'resque'
+        require 'resque/scheduler'
+        require 'resque-retry'
+
+        QueueBus::Worker.extend(::Resque::Plugins::ExponentialBackoff)
+        QueueBus::Worker.extend(::QueueBus::Adapters::Resque::RetryHandlers)
+      end
+
+      def redis(&block)
+        block.call(::Resque.redis)
+      end
+
+      def enqueue(queue_name, klass, json)
+        ::Resque.enqueue_to(queue_name, klass, json)
+      end
+
+      def enqueue_at(epoch_seconds, queue_name, klass, json)
+        ::Resque.enqueue_at_with_queue(queue_name, epoch_seconds, klass, json)
+      end
+
+      def setup_heartbeat!(queue_name)
+        # turn on the heartbeat
+        # should be down after loading scheduler yml if you do that
+        # otherwise, anytime
+        name     = 'resquebus_heartbeat'
+        schedule = { 'class' => '::QueueBus::Heartbeat',
+                     'cron'  => '* * * * *',   # every minute
+                     'queue' => queue_name,
+                     'description' => 'I publish a heartbeat_minutes event every minute'
+                   }
+        if ::Resque::Scheduler.dynamic
+          ::Resque.set_schedule(name, schedule)
+        end
+        ::Resque.schedule[name] = schedule
+      end
+
+      private
+
+      module RetryHandlers
+        # @failure_hooks_already_ran on https://github.com/defunkt/resque/tree/1-x-stable
+        # to prevent running twice
+        def queue
+          @my_queue
+        end
+
+        def on_failure_aaa(exception, *args)
+          # note: sorted alphabetically
+          # queue needs to be set for rety to work (know what queue in Requeue.class_to_queue)
+          hash = ::QueueBus::Util.decode(args[0])
+          @my_queue = hash["bus_rider_queue"]
+        end
+
+        def on_failure_zzz(exception, *args)
+          # note: sorted alphabetically
+          @my_queue = nil
+        end
+      end
+    end
+  end
+end

data/lib/resque_bus/compatibility/deprecated.rb

@@ -0,0 +1,38 @@
+module ResqueBus
+  module Deprecated
+    def show_deprecations=val
+      @show_deprecations = val
+    end
+
+    def show_deprecations?
+      return @show_deprecations if defined?(@show_deprecations)
+      return true if !ENV['QUEUES'] && !ENV['QUEUE'] # not in background, probably test
+      return true if ENV['VVERBOSE'] || ENV['LOGGING'] || ENV['VERBOSE']
+      false
+    end
+    
+    def note_deprecation(message)
+      @noted_deprecations ||= {}
+      if @noted_deprecations[message]
+        @noted_deprecations[message] += 1
+      else
+        warn(message) if show_deprecations?
+        @noted_deprecations[message] = 1
+      end
+    end
+
+    def redis
+      ResqueBus.note_deprecation "[DEPRECATION] ResqueBus direct usage is deprecated. Use `QueueBus.redis` instead. Note that it also requires block usage now."
+      ::Resque.redis
+    end
+
+    def redis=val
+      ResqueBus.note_deprecation "[DEPRECATION] ResqueBus can no longer set redis directly. It will use Resque's instance of redis."
+    end
+
+    def method_missing(method_name, *args, &block)
+      ResqueBus.note_deprecation "[DEPRECATION] ResqueBus direct usage is deprecated. Use `QueueBus.#{method_name}` instead."
+      ::QueueBus.send(method_name, *args, &block)
+    end
+  end
+end