sidekiq-bus 0.5.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8c073296475ba07b08f7f92b019d091b38b4038e
+  data.tar.gz: cb95f95a5ec099cc471dcd8695cbf2ec478a9b6a
+SHA512:
+  metadata.gz: 846ac47ae7600da4249d89e55e81e9e16f4de85e768697017f343b5f3952683232b29975055feddf2dcea83dc15a48e25c1e2f6d4ceee452b94f9bf32d4d590f
+  data.tar.gz: a5a9156736395bf84e9acbebbd533ce27a71436d74bc10923c893e06cf07c734e9756ddc2e42bfd64822823e322491a57b5cdb4facacbb71a7c09b17135f1715

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*
+.DS_Store

data/.rbenv-version

@@ -0,0 +1 @@
+2.1.5

data/.rspec

@@ -0,0 +1 @@
+--color

data/Gemfile

@@ -0,0 +1,5 @@
+source "http://rubygems.org"
+
+gemspec
+
+# gem "queue-bus", path: "../queue-bus"

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2012 Brian Leonard
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.mdown

@@ -0,0 +1,264 @@
+## Resque Bus
+
+This gem uses Redis and Resque to allow simple asynchronous communication between apps.
+
+### Install
+
+To install, include the 'resque-bus' gem and add the following to your Rakefile:
+
+```ruby
+require "sidekiq_bus/tasks"
+```
+
+### Example
+
+Application A can publish an event
+
+```ruby
+# config
+Resque.redis = "192.168.1.1:6379"
+
+# business logic
+ResqueBus.publish("user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
+
+# or do it later
+ResqueBus.publish_at(1.hour.from_now, "user_created", "id" => 42, "first_name" => "John", "last_name" => "Smith")
+```
+
+Application B is subscribed to events
+
+```ruby
+# config
+Resque.redis = "192.168.1.1:6379"
+
+# initializer
+ResqueBus.dispatch("app_b") do
+  # processes event on app_b_default queue
+  # subscribe is short-hand to subscribe to your 'default' queue and this block with process events with the name "user_created"
+  subscribe "user_created" do |attributes|
+    NameCount.find_or_create_by_name(attributes["last_name"]).increment!
+  end
+
+  # processes event on app_b_critical queue
+  # critical is short-hand to subscribe to your 'critical' queue and this block with process events with the name "user_paid"
+  critical "user_paid" do |attributes|
+    CreditCard.charge!(attributes)
+  end
+
+  # you can pass any queue name you would like to process from as well IE: `banana "peeled" do |attributes|`
+
+  # and regexes work as well. note that with the above configuration along with this regex,
+  # the following as well as the corresponding block above would both be executed
+  subscribe /^user_/ do |attributes|
+    Metrics.record_user_action(attributes["bus_event_type"], attributes["id"])
+  end
+
+  # the above all filter on just the event_type, but you can filter on anything
+  # this would be _any_ event that has a user_id and the page value of homepage regardless of bus_event_type
+  subscribe "my_key", { "user_id" => :present, "page" => "homepage"} do
+    Mixpanel.homepage_action!(attributes["action"])
+  end
+end
+```
+
+Applications can also subscribe within classes using the provided `Subscriber` module.
+
+```ruby
+class SimpleSubscriber
+  include ResqueBus::Subscriber
+  subscribe :my_method
+
+  def my_method(attributes)
+    # heavy lifting
+  end
+end
+```
+
+The following is equivalent to the original initializer and shows more options:
+
+```ruby
+class OtherSubscriber
+  include ResqueBus::Subscriber
+  application :app_b
+
+  subscribe :user_created
+  subscribe_queue :app_b_critical, :user_paid
+  subscribe_queue :app_b_default, :user_action, :bus_event_type => /^user_/
+  subscribe :homepage_method, :user_id => :present, :page => "homepage"
+
+  def user_created(attributes)
+    NameCount.find_or_create_by_name(attributes["last_name"]).increment!
+  end
+
+  def user_paid(attributes)
+    CreditCard.charge!(attributes)
+  end
+
+  def user_action(attributes)
+    Metrics.record_user_action(attributes["bus_event_type"], attributes["id"])
+  end
+
+  def homepage_method
+    Mixpanel.homepage_action!(attributes["action"])
+  end
+end
+```
+
+Note: This subscribes when this class is loaded, so it needs to be in your load or otherwise referenced/required during app initialization to work properly.
+
+### Commands
+
+Each app needs to tell Redis about its subscriptions:
+
+    $ rake resquebus:subscribe
+
+The subscription block is run inside a Resque worker which needs to be started for each app.
+
+    $ rake resquebus:setup resque:work
+
+The incoming queue also needs to be processed on a dedicated or all the app servers.
+
+    $ rake resquebus:driver resque:work
+
+If you want retry to work for subscribing apps, you should run resque-scheduler
+
+    $ rake resque:scheduler
+
+### Adapters
+
+ResqueBus now supports multiple adapters! By default ResqueBus uses Resque but you can now configure your application to use Sidekiq to drive and subscribe the bus.
+
+First be sure to configure ResqueBus to use Sidekiq early in your applications' initialization cycle:
+```
+ResqueBus.adapter = 'Sidekiq'
+```
+You will be responsible for setting up the queues for your Sidekiq clients however you can get the appropriate queue names with the following tasks:
+For driving applications:
+```
+$ rake resquebus:driver:sidekiq
+```
+For subscribing applications:
+```
+$ rake resquebus:setup:sidekiq
+```
+These tasks will provide the queue_names and some minimal suggestions for starting the client.
+
+Your subscribing applications will still need to also use the appropriate rake task:
+```
+$ rake resquebus:subscribe:sidekiq
+```
+
+At the moment you are expected to include the Sidekiq gem in your own applications.
+
+And yes we are planning on renaming and restructuring the project! Please contact the maintainer if you would like to add a different adapter.
+
+### Heartbeat
+
+We've found it useful to have the bus act like `cron`, triggering timed jobs throughout the system. Resque Bus calls this a heartbeat.
+It uses resque-scheduler to trigger the events. You can enable it in your Rakefile.
+
+```ruby
+# resque.rake
+namespace :resque do
+  task :setup => [:environment] do
+    ResqueBus.heartbeat!
+  end
+end
+```
+
+Or add it to your `schedule.yml` directly
+
+```yaml
+resquebus_heartbeat:
+  cron: "* * * * *"
+  class: "::ResqueBus::Heartbeat"
+  queue: resquebus_incoming
+  description: "I publish a heartbeat_minutes event every minute"
+```
+
+It is the equivalent of doing this every minute
+
+```ruby
+seconds = minutes * (60)
+hours   = minutes / (60)
+days    = minutes / (60*24)
+
+now     = Time.at(seconds)
+
+attributes = {}
+
+now = Time.now
+seconds = now.to_i
+ResqueBus.publish("hearbeat_minutes", {
+  "epoch_seconds" => seconds,
+  "epoch_minutes" => seconds / 1.minute,
+  "epoch_hours"   => seconds / 1.hour,
+  "epoch_days"    => seconds / 1.day,
+  "minute"        => now.min
+  "hour"          => now.hour
+  "day"           => now.day
+  "month"         => now.month
+  "year"          => now.year
+  "yday"          => now.yday
+  "wday"          => now.wday
+})
+```
+
+This allows you do something like this:
+
+```ruby
+ResqueBus.dispatch("app_c") do
+  # runs at 10:20, 11:20, etc
+  subscribe "once_an_hour", 'bus_event_type' => 'heartbeat_minutes', 'minute' => 20 do |attributes|
+    Sitemap.generate!
+  end
+
+  # runs every five minutes
+  subscribe "every_five_minutes", 'bus_event_type' => 'heartbeat_minutes' do |attributes|
+    next unless attributes["epoch_minutes"] % 5 == 0
+    HealthCheck.run!
+  end
+
+  # runs at 8am on the first of every month
+  subscribe "new_month_morning", 'bus_event_type' => 'heartbeat_minutes', 'day' => 1, hour' => 8, 'minute' => 0,  do |attributes|
+    next unless attributes["epoch_minutes"] % 5 == 0
+    Token.old.expire!
+  end
+end
+```
+
+### Local Mode
+
+For development, a local mode is provided and is specified in the configuration.
+
+```ruby
+# config
+ResqueBus.local_mode = :standalone
+or
+ResqueBus.local_mode = :inline
+```
+
+Standalone mode does not require a separate resquebus:driver task to be running to process the
+incoming queue. Simply publishing to the bus will distribute the incoming events
+to the appropriate application specific queue.  A separate resquebus:work task does
+still need to be run to process these events
+
+Inline mode skips queue processing entirely and directly dispatches the
+event to the appropriate code block.
+
+You can also say `ResqueBus.local_mode = :suppress` to turn off publishing altogether.
+This can be helpful inside some sort of migration, for example.
+
+### TODO
+
+* Sidekiq adapter
+* Refactor rake tasks for resque/sidekiq
+* Refactor to a storage adapter for Redis, so we can store subscription info in MySQL or something else
+* Replace local modes with adapters
+* There are a few spots in the code with TODO notes
+* Make this not freak out in development without Redis or when Redis is down
+* We might not actually need to publish in tests
+* Add some rspec helpers for the apps to use: should_ post an event_publish or something along those lines
+* Allow calling resquebus:setup and resquebus:driver together (append to ENV['QUEUES'], don't replace it)
+
+Copyright (c) 2011 Brian Leonard, released under the MIT license

data/Rakefile

@@ -0,0 +1,2 @@
+$LOAD_PATH.unshift 'lib'
+require 'sidekiq_bus/tasks'

data/lib/sidekiq-bus.rb

@@ -0,0 +1,9 @@
+require "queue-bus"
+require "sidekiq_bus/adapter"
+require "sidekiq_bus/version"
+
+module ResqueBus
+
+end
+
+QueueBus.adapter = QueueBus::Adapters::Sidekiq.new

data/lib/sidekiq_bus/adapter.rb

@@ -0,0 +1,28 @@
+module QueueBus
+  module Adapters
+    class Sidekiq < QueueBus::Adapters::Base
+      def enabled!
+        # know we are using it
+        require 'sidekiq'
+        ::QueueBus::Worker.include ::Sidekiq::Worker
+      end
+
+      def redis(&block)
+        ::Sidekiq.redis(&block)
+      end
+
+      def enqueue(queue_name, klass, hash)
+        ::Sidekiq::Client.push('queue' => queue_name, 'class' => klass, 'args' => [hash])
+      end
+
+      def enqueue_at(epoch_seconds, queue_name, klass, hash)
+        ::Sidekiq::Client.push('queue' => queue_name, 'class' => klass, 'args' => [hash], 'at' => epoch_seconds)
+      end
+
+      def setup_heartbeat!(queue_name)
+        # TODO: not sure how to do this or what is means to set this up in Sidekiq
+        raise NotImplementedError
+      end
+    end
+  end
+end

data/lib/sidekiq_bus/tasks.rb

@@ -0,0 +1,93 @@
+# require 'sidekiq_bus/tasks'
+# will give you these tasks
+
+
+require "resque/tasks"
+namespace :sidekiqbus do
+
+  desc "Setup will configure a resque task to run before resque:work"
+  task :setup => [ :preload ] do
+
+    if ENV['QUEUES'].nil?
+      manager = ::QueueBus::TaskManager.new(true)
+      queues = manager.queue_names
+      ENV['QUEUES'] = queues.join(",")
+    else
+      queues = ENV['QUEUES'].split(",")
+    end
+
+    if queues.size == 1
+      puts "  >>  Working Queue : #{queues.first}"
+    else
+      puts "  >>  Working Queues: #{queues.join(", ")}"
+    end
+  end
+
+  desc "Subscribes this application to QueueBus events"
+  task :subscribe => [ :preload ] do
+    manager = ::QueueBus::TaskManager.new(true)
+    count = manager.subscribe!
+    raise "No subscriptions created" if count == 0
+  end
+
+  desc "Unsubscribes this application from QueueBus events"
+  task :unsubscribe => [ :preload ] do
+    require 'resque-bus'
+    manager = ::QueueBus::TaskManager.new(true)
+    count = manager.unsubscribe!
+    puts "No subscriptions unsubscribed" if count == 0
+  end
+
+  desc "Sets the queue to work the driver  Use: `rake sidekiqbus:driver resque:work`"
+  task :driver => [ :preload ] do
+    ENV['QUEUES'] = ::QueueBus.incoming_queue
+  end
+
+  # Preload app files if this is Rails
+  task :preload do
+    require "sidekiq"
+  end
+
+
+  # examples to test out the system
+  namespace :example do
+    desc "Publishes events to example applications"
+    task :publish => [ "sidekiqbus:preload", "sidekiqbus:setup" ] do
+      which = ["one", "two", "three", "other"][rand(4)]
+      QueueBus.publish("event_#{which}", { "rand" => rand(99999)})
+      QueueBus.publish("event_all", { "rand" => rand(99999)})
+      QueueBus.publish("none_subscribed", { "rand" => rand(99999)})
+      puts "published event_#{which}, event_all, none_subscribed"
+    end
+
+    desc "Sets up an example config"
+    task :register => [ "sidekiqbus:preload"] do
+      QueueBus.dispatch("example") do
+        subscribe "event_one" do
+          puts "event1 happened"
+        end
+
+        subscribe "event_two" do
+          puts "event2 happened"
+        end
+
+        high "event_three" do
+          puts "event3 happened (high)"
+        end
+
+        low "event_.*" do |attributes|
+          puts "LOG ALL: #{attributes.inspect}"
+        end
+      end
+    end
+
+    desc "Subscribes this application to QueueBus example events"
+    task :subscribe => [ :register, "sidekiqbus:subscribe" ]
+
+    desc "Start a QueueBus example worker"
+    task :work => [ :register, "sidekiqbus:setup", "resque:work" ]
+
+    desc "Start a QueueBus example worker"
+    task :driver => [ :register, "sidekiqbus:driver", "resque:work" ]
+  end
+end