resque-pubsub 0.1.1

data/HISTORY.md

@@ -0,0 +1,7 @@
+## 0.1.0 (2011-03-14)
+
+* Initial version.
+
+## 0.1.1 (2011-03-14)
+
+* Bump because of a publishing error.

data/MIT-LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2011 [name of plugin creator]
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,82 @@
+ResquePubsub
+============
+
+A [Resque][rq] plugin. Requires Resque 1.9.10.
+
+A lightweight publish/subscribe messaging system, with message persistence when clients are down, written on top of Redis and Resque.
+
+
+Usage / Examples
+================
+
+A simple class that can publish a message:
+
+  class TestPublisher
+    require 'resque-pubsub'
+    
+    def some_method
+      self.publish(topic, message)
+    end
+  end
+
+
+A simple class that subscribes to messages on a particular topic:
+
+  class TestSubscriber
+    require 'resque-pubsub'
+  
+    subscribe 'test_topic'
+    
+    def self.read_test_topic_message(message)
+      # Do something with the message
+    end
+  end
+
+
+Customize & Extend
+==================
+
+The method that the is called when the subscriber is sent a message defaults to read_<topic_name>_message, 
+but can be customized with the option :reader_method, e.g.,
+
+    subscribe 'test_topic', :reader_message => :custom_message_method
+    
+The namespace that pubsub uses in Resque defaults to 'resque:pubsub' but can be configured by setting the constant 
+Resque::Plugins::Pubsub::Exchange::PUBSUB_NAMESPACE.
+
+
+Install
+=======
+
+### As a gem
+
+    $ gem install resque-pubsub
+
+### In a Rails app, as a plugin
+
+    $ rails plugin install git://github.com/Mechaferret/resque-pubsub.git
+
+
+Running Resque
+==============
+
+A sample config file is provided in examples/resque-pubsub.rb. If you put this in config/initializers for a Rails app,
+then Resque will default to the app namespace but will take an override on namespace from the environment variable RESQUE_NAMESPACE. Thus
+
+  QUEUE=* RESQUE_NAMESPACE="resque:pubsub" rake environment resque:work
+
+will run resque jobs against the default pubsub namespace (i.e., will be the pubsub server)
+
+while 
+
+  QUEUE=* rake environment resque:work
+
+will run resque in an app as normal.
+
+
+Acknowledgements
+================
+
+Copyright (c) 2011 Monica McArthur, released under the MIT license.
+
+[rq]: http://github.com/defunkt/resque

data/Rakefile

@@ -0,0 +1,23 @@
+require 'rake'
+require 'rake/testtask'
+require 'rake/rdoctask'
+
+desc 'Default: run unit tests.'
+task :default => :test
+
+desc 'Test the resque_pubsub plugin.'
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = true
+end
+
+desc 'Generate documentation for the resque_pubsub plugin.'
+Rake::RDocTask.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ResquePubsub'
+  rdoc.options << '--line-numbers' << '--inline-source'
+  rdoc.rdoc_files.include('README')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end

data/lib/resque-pubsub.rb

@@ -0,0 +1,6 @@
+require 'resque/plugins/pubsub/subscriber'
+require 'resque/plugins/pubsub/publisher'
+require 'resque/plugins/pubsub/broker'
+self.send(:include, Resque::Plugins::Pubsub::Subscriber)
+self.send(:include, Resque::Plugins::Pubsub::Publisher)
+

data/lib/resque/plugins/pubsub/broker.rb

@@ -0,0 +1,31 @@
+module Resque
+  module Plugins
+    #
+    # pubsub broker
+    #
+    module Pubsub
+      class Broker
+        @queue = :messages
+        # Returns a top-level Redis connection so that the broker can distribute messages
+        # across namespaces. If none has been created, will
+        # create a new one using information from the Resque connection.
+        def self.redis
+          return @redis if @redis
+          client_to_copy = Resque.redis.client
+          @redis = Redis.new(:host => client_to_copy.host, :port => client_to_copy.port,
+            :thread_safe => true, :db => client_to_copy.db)
+        end
+        
+        def self.perform(topic, message)
+          subscribers = Exchange.redis.smembers("#{topic}_subscribers")
+          subscribers.each {|s|
+            sinfo = JSON.parse(s)
+            puts "distributing to #{sinfo.inspect}"
+            Broker.redis.sadd("#{sinfo['namespace']}:queues", "fanout:#{topic}")
+            Broker.redis.rpush("#{sinfo['namespace']}:queue:fanout:#{topic}", {:class=> sinfo["class"], :args=>[message]}.to_json)
+          }
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/pubsub/exchange.rb

@@ -0,0 +1,43 @@
+require 'resque'
+module Resque
+  module Plugins
+    #
+    # pubsub exchange manager
+    #
+    module Pubsub
+      class Exchange
+        @@pubsub_namespace = nil
+        # Returns the current Redis connection. If none has been created, will
+        # create a new one using information from the Resque connection and our config.
+        def self.redis
+          return @redis if @redis
+          client_to_copy = Resque.redis.client
+          redis_new = Redis.new(:host => client_to_copy.host, :port => client_to_copy.port,
+            :thread_safe => true, :db => client_to_copy.db)
+          puts "making a redis in exchange, namespace will be #{@@pubsub_namespace}"
+          @redis = Redis::Namespace.new(@@pubsub_namespace || "resque:pubsub", :redis => redis_new)
+        end
+        
+        @queue = :subscription_requests
+        
+        def self.perform(subscription_info)
+          puts "handling a subscription on the exchange"
+          puts "requested subscription is #{subscription_info.inspect}"
+          puts "namespace is #{Exchange.redis.namespace}"
+          redis = Exchange.redis
+          redis.sadd("#{subscription_info["topic"]}_subscribers", {:class=>subscription_info["class"], :namespace=>subscription_info["namespace"]}.to_json)
+        end
+        
+        def self.pubsub_namespace
+          @@pubsub_namespace
+        end
+        
+        def self.pubsub_namespace=(n)
+          @@pubsub_namespace = n
+          @redis.client.disconnect
+          @redis = nil
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/pubsub/publisher.rb

@@ -0,0 +1,24 @@
+require 'resque/plugins/pubsub/exchange'
+module Resque
+  module Plugins
+    #
+    # pubsub publisher
+    #
+    module Pubsub
+      module Publisher
+      	def self.included(base)
+      	  puts 'including publisher'
+          base.send(:include, InstanceMethods)
+        end
+
+        module InstanceMethods
+          def publish(topic, message)
+            puts "Publisher publishing #{message} in #{topic}"
+            Exchange.redis.sadd(:queues, "messages")
+            Exchange.redis.rpush("queue:messages", {:class=>'Resque::Plugins::Pubsub::Broker', :args=>[topic, message]}.to_json)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/resque/plugins/pubsub/subscriber.rb

@@ -0,0 +1,34 @@
+require 'resque/plugins/pubsub/exchange'
+module Resque
+  module Plugins
+    #
+    # pubsub subscriber
+    #
+    module Pubsub
+      module Subscriber
+      	def self.included(base)
+          base.extend ClassMethods
+        end
+      
+        module ClassMethods
+          def subscribe(topic, options={})
+            @queue = "fanout:#{topic}"
+            reader_method = options[:reader_method] || "read_#{topic}_message"
+            module_eval <<-"end_eval"
+              def self.perform(message)
+                self.send("#{reader_method}", message)
+      				end
+            end_eval
+            options[:namespace] = Resque.redis.namespace
+            options[:topic] = topic
+            options[:class] = self.to_s
+            puts "Subscriber subscribing with #{options.inspect}"
+            Exchange.redis.sadd(:queues, :subscription_requests)
+            Exchange.redis.rpush("queue:subscription_requests", {:class=>'Resque::Plugins::Pubsub::Exchange', :args=>[options]}.to_json)      
+          end
+          
+        end
+      end
+    end
+  end
+end

data/test/debug.log

@@ -0,0 +1,16 @@
+# Logfile created on Thu Jan 20 18:20:02 -0800 2011 by logger.rb/22285
+  SQL (0.2ms)   SET SQL_AUTO_IS_NULL=0
+  SQL (0.3ms)   SHOW TABLES
+  SQL (430.6ms)   CREATE TABLE `transactionals` (`id` int(11) DEFAULT NULL auto_increment PRIMARY KEY, `name` varchar(255), `description` varchar(255), `other_id` int(11)) ENGINE=InnoDB
+  SQL (0.4ms)   SHOW TABLES
+  SQL (114.4ms)   CREATE TABLE `transactional_facts` (`id` int(11) DEFAULT NULL auto_increment PRIMARY KEY, `name` varchar(255), `description` varchar(255), `other_id` int(11)) ENGINE=InnoDB
+  SQL (0.4ms)   SHOW TABLES
+  SQL (102.1ms)   CREATE TABLE `schema_migrations` (`version` varchar(255) NOT NULL) ENGINE=InnoDB
+  SQL (172.5ms)   CREATE UNIQUE INDEX `unique_schema_migrations` ON `schema_migrations` (`version`)
+  SQL (0.4ms)   SHOW TABLES
+  SQL (0.1ms)   SELECT version FROM `schema_migrations`
+  SQL (0.4ms)   INSERT INTO `schema_migrations` (version) VALUES ('3')
+  Transactional Columns (38.0ms)   SHOW FIELDS FROM `transactionals`
+  SQL (0.3ms)   BEGIN
+  Transactional Create (0.3ms)   INSERT INTO `transactionals` (`name`, `other_id`, `description`) VALUES(NULL, NULL, NULL)
+  SQL (0.7ms)   COMMIT

data/test/pubsub_test.rb

@@ -0,0 +1,83 @@
+require File.dirname(__FILE__) + '/test_helper'
+
+class PubsubTest < Test::Unit::TestCase
+  def setup
+    $success = $lock_failed = $lock_expired = 0
+    Resque.redis.flushall
+    Resque.redis.namespace = 'test_pubsub'
+  end
+
+  def test_lint
+    assert_nothing_raised do
+      Resque::Plugin.lint(Resque::Plugins::Pubsub)
+    end
+  end
+  
+  def test_all
+    # Just loading the "TestSubscriber" class is enough to subscribe
+    require 'test_publisher'
+    require 'test_subscriber'
+    # Now process the subscription
+    Resque.redis.namespace = 'resque:pubsub'
+    @worker = Resque::Worker.new(:subscription_requests)
+    @worker.process
+    # Check that the subscription is in the subscribers list
+    assert check_subscription(Resque.redis.smembers("test_topic_subscribers"), "TestSubscriber", "test_pubsub")
+    p = TestPublisher.new
+    p.publish("test_topic", "Test message")
+    # Run Resque for the broker
+    Resque.redis.namespace = 'resque:pubsub'
+    @worker = Resque::Worker.new(:messages)
+    @worker.process
+    # Check that the message queue has been populated
+    Resque.redis.namespace = 'test_pubsub'
+    assert Resque.redis.keys.include?('queue:fanout:test_topic')
+    assert Resque.redis.llen('queue:fanout:test_topic')==1
+    # Now run the subscriber Resque
+    @worker = Resque::Worker.new('fanout:test_topic')
+    @worker.process
+    assert Resque.redis.llen('fanout:test_topic')==0
+    assert TestSubscriber.last_message=='Test message'
+  end
+  
+  def test_configuration_options
+    # Configure the pubsub namespace
+    require 'resque-pubsub'
+    Resque::Plugins::Pubsub::Exchange.pubsub_namespace = 'resque:custom_space'
+    puts "namespace is set to #{Resque::Plugins::Pubsub::Exchange.pubsub_namespace}"
+    require 'test_publisher'
+    require 'test_subscriber'
+    require 'test_subscriber_custom'
+    # Now process the subscription
+    Resque.redis.namespace = 'resque:custom_space'
+    @worker = Resque::Worker.new(:subscription_requests)
+    @worker.process
+    # Check that the subscription is in the subscribers list
+    assert check_subscription(Resque.redis.smembers("test_custom_topic_subscribers"), "TestSubscriberCustom", "test_pubsub")
+    p = TestPublisher.new
+    p.publish("test_custom_topic", "Test custom message")
+    # Run Resque for the broker
+    Resque.redis.namespace = 'resque:custom_space'
+    @worker = Resque::Worker.new(:messages)
+    @worker.process
+    # Check that the message queue has been populated
+    Resque.redis.namespace = 'test_pubsub'
+    assert Resque.redis.keys.include?('queue:fanout:test_custom_topic')
+    assert Resque.redis.llen('queue:fanout:test_custom_topic')==1
+    # Now run the subscriber Resque
+    @worker = Resque::Worker.new('fanout:test_custom_topic')
+    @worker.process
+    assert Resque.redis.llen('fanout:test_custom_topic')==0
+    assert TestSubscriberCustom.last_message=='Test custom message'
+    # Also make sure TestSubscriber DIDN'T get the message
+    assert TestSubscriber.last_message!='Test custom message'
+  end
+  
+  def check_subscription(subscribers, klass, namespace)
+    subscribers.inject(false) {|result, s|
+      sinfo = JSON.parse(s)
+      result = result || (sinfo["class"]==klass && sinfo['namespace']==namespace)
+    }
+  end
+
+end

data/test/redis-test.conf

@@ -0,0 +1,132 @@
+# Redis configuration file example
+
+# By default Redis does not run as a daemon. Use 'yes' if you need it.
+# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
+daemonize yes
+
+# When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
+# You can specify a custom pid file location here.
+pidfile ./test/redis-test.pid
+
+# Accept connections on the specified port, default is 6379
+port 9736
+
+# If you want you can bind a single interface, if the bind option is not
+# specified all the interfaces will listen for connections.
+#
+# bind 127.0.0.1
+
+# Close the connection after a client is idle for N seconds (0 to disable)
+timeout 300
+
+# Save the DB on disk:
+#
+#   save <seconds> <changes>
+#
+#   Will save the DB if both the given number of seconds and the given
+#   number of write operations against the DB occurred.
+#
+#   In the example below the behaviour will be to save:
+#   after 900 sec (15 min) if at least 1 key changed
+#   after 300 sec (5 min) if at least 10 keys changed
+#   after 60 sec if at least 10000 keys changed
+save 900 1
+save 300 10
+save 60 10000
+
+# The filename where to dump the DB
+dbfilename dump.rdb
+
+# For default save/load DB in/from the working directory
+# Note that you must specify a directory not a file name.
+dir ./test/
+
+# Set server verbosity to 'debug'
+# it can be one of:
+# debug (a lot of information, useful for development/testing)
+# notice (moderately verbose, what you want in production probably)
+# warning (only very important / critical messages are logged)
+loglevel debug
+
+# Specify the log file name. Also 'stdout' can be used to force
+# the demon to log on the standard output. Note that if you use standard
+# output for logging but daemonize, logs will be sent to /dev/null
+logfile stdout
+
+# Set the number of databases. The default database is DB 0, you can select
+# a different one on a per-connection basis using SELECT <dbid> where
+# dbid is a number between 0 and 'databases'-1
+databases 16
+
+################################# REPLICATION #################################
+
+# Master-Slave replication. Use slaveof to make a Redis instance a copy of
+# another Redis server. Note that the configuration is local to the slave
+# so for example it is possible to configure the slave to save the DB with a
+# different interval, or to listen to another port, and so on.
+
+# slaveof <masterip> <masterport>
+
+################################## SECURITY ###################################
+
+# Require clients to issue AUTH <PASSWORD> before processing any other
+# commands.  This might be useful in environments in which you do not trust
+# others with access to the host running redis-server.
+#
+# This should stay commented out for backward compatibility and because most
+# people do not need auth (e.g. they run their own servers).
+
+# requirepass foobared
+
+################################### LIMITS ####################################
+
+# Set the max number of connected clients at the same time. By default there
+# is no limit, and it's up to the number of file descriptors the Redis process
+# is able to open. The special value '0' means no limts.
+# Once the limit is reached Redis will close all the new connections sending
+# an error 'max number of clients reached'.
+
+# maxclients 128
+
+# Don't use more memory than the specified amount of bytes.
+# When the memory limit is reached Redis will try to remove keys with an
+# EXPIRE set. It will try to start freeing keys that are going to expire
+# in little time and preserve keys with a longer time to live.
+# Redis will also try to remove objects from free lists if possible.
+#
+# If all this fails, Redis will start to reply with errors to commands
+# that will use more memory, like SET, LPUSH, and so on, and will continue
+# to reply to most read-only commands like GET.
+#
+# WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
+# 'state' server or cache, not as a real DB. When Redis is used as a real
+# database the memory usage will grow over the weeks, it will be obvious if
+# it is going to use too much memory in the long run, and you'll have the time
+# to upgrade. With maxmemory after the limit is reached you'll start to get
+# errors for write operations, and this may even lead to DB inconsistency.
+
+# maxmemory <bytes>
+
+############################### ADVANCED CONFIG ###############################
+
+# Glue small output buffers together in order to send small replies in a
+# single TCP packet. Uses a bit more CPU but most of the times it is a win
+# in terms of number of queries per second. Use 'yes' if unsure.
+glueoutputbuf yes
+
+# Use object sharing. Can save a lot of memory if you have many common
+# string in your dataset, but performs lookups against the shared objects
+# pool so it uses more CPU and can be a bit slower. Usually it's a good
+# idea.
+#
+# When object sharing is enabled (shareobjects yes) you can use
+# shareobjectspoolsize to control the size of the pool used in order to try
+# object sharing. A bigger pool size will lead to better sharing capabilities.
+# In general you want this value to be at least the double of the number of
+# very common strings you have in your dataset.
+#
+# WARNING: object sharing is experimental, don't enable this feature
+# in production before of Redis 1.0-stable. Still please try this feature in
+# your development environment so that we can test it better.
+#shareobjects no
+#shareobjectspoolsize 1024

data/test/test_helper.rb

@@ -0,0 +1,42 @@
+dir = File.dirname(File.expand_path(__FILE__))
+$LOAD_PATH.unshift dir + '/../lib'
+$TESTING = true
+
+require 'rubygems'
+require 'test/unit'
+require 'resque'
+require 'active_support'
+require 'active_support/test_case'
+
+require 'resque-pubsub'
+
+##
+# make sure we can run redis
+if !system("which redis-server")
+  puts '', "** can't find `redis-server` in your path"
+  puts "** try running `sudo rake install`"
+  abort ''
+end
+
+##
+# start our own redis when the tests start,
+# kill it when they end
+at_exit do
+  next if $!
+
+  if defined?(MiniTest)
+    exit_code = MiniTest::Unit.new.run(ARGV)
+  else
+    exit_code = Test::Unit::AutoRunner.run
+  end
+
+  pid = `ps -e -o pid,command | grep [r]edis-test`.split(" ")[0]
+  puts "Killing test redis server..."
+  `rm -f #{dir}/dump.rdb`
+  Process.kill("KILL", pid.to_i)
+  exit exit_code
+end
+
+puts "Starting redis for testing at localhost:9736..."
+`redis-server #{dir}/redis-test.conf`
+Resque.redis = '127.0.0.1:9736'

data/test/test_publisher.rb

@@ -0,0 +1,4 @@
+class TestPublisher
+  require 'resque-pubsub'
+  
+end

data/test/test_subscriber.rb

@@ -0,0 +1,17 @@
+class TestSubscriber
+  require 'resque-pubsub'
+  
+  subscribe 'test_topic'
+  
+  @@last_message = nil
+  
+  def self.read_test_topic_message(message)
+    puts "got test topic message: #{message.inspect}"
+    @@last_message = message
+  end
+  
+  def self.last_message
+    @@last_message
+  end
+  
+end

data/test/test_subscriber_custom.rb

@@ -0,0 +1,17 @@
+class TestSubscriberCustom
+  require 'resque-pubsub'
+  
+  subscribe 'test_custom_topic', :reader_method => 'simple'
+  
+  @@last_message = nil
+  
+  def self.simple(message)
+    puts "in simple, got test custom topic message: #{message.inspect}"
+    @@last_message = message
+  end
+  
+  def self.last_message
+    @@last_message
+  end
+  
+end

metadata

@@ -0,0 +1,97 @@
+--- !ruby/object:Gem::Specification 
+name: resque-pubsub
+version: !ruby/object:Gem::Version 
+  hash: 25
+  prerelease: 
+  segments: 
+  - 0
+  - 1
+  - 1
+  version: 0.1.1
+platform: ruby
+authors: 
+- Monica McArthur
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-14 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: resque
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        hash: 39
+        segments: 
+        - 1
+        - 9
+        - 10
+        version: 1.9.10
+  type: :runtime
+  version_requirements: *id001
+description: "  A Resque plugin. Provides a lightweight publish/subscribe messaging system, with message persistence when clients are down.\n"
+email: mechaferret@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- Rakefile
+- MIT-LICENSE
+- HISTORY.md
+- lib/resque/plugins/pubsub/broker.rb
+- lib/resque/plugins/pubsub/exchange.rb
+- lib/resque/plugins/pubsub/publisher.rb
+- lib/resque/plugins/pubsub/subscriber.rb
+- lib/resque-pubsub.rb
+- test/debug.log
+- test/pubsub_test.rb
+- test/redis-test.conf
+- test/test_helper.rb
+- test/test_publisher.rb
+- test/test_subscriber.rb
+- test/test_subscriber_custom.rb
+has_rdoc: true
+homepage: http://github.com/mechaferret/resque-pubsub
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.5.2
+signing_key: 
+specification_version: 3
+summary: A Resque plugin that provides a lightweight publish/subscribe messaging system.
+test_files: []
+