remq 0.0.3 → 0.0.4

data/.gitignore

@@ -1,5 +1,4 @@
 *.gem
 .bundle
 .config
-Gemfile
 Gemfile.lock

data/.gitmodules

@@ -1,3 +1,3 @@
 [submodule "vendor/remq"]
 	path = vendor/remq
-	url = git@github.com:kainosnoema/remq.git
+	url = git://github.com/kainosnoema/remq.git

data/.travis.yml

@@ -0,0 +1,9 @@
+language: ruby
+rvm:
+  - 1.9.3
+  - 1.9.2
+  - 1.8.7
+  - jruby-19mode
+  - jruby-18mode
+services:
+  - redis-server

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown

data/Gemfile

@@ -0,0 +1,2 @@
+source :rubygems
+gemspec

data/README.md

@@ -0,0 +1,66 @@
+# remq-rb [![Build Status][travis-image]][travis-link]
+
+[travis-image]: https://secure.travis-ci.org/kainosnoema/remq-rb.png?branch=master
+[travis-link]: http://travis-ci.org/kainosnoema/remq-rb
+
+A Ruby client for [Remq](https://github.com/kainosnoema/remq), a
+[Redis](http://redis.io)-based protocol for building fast, durable message
+queues.
+
+**WARNING**: In early-stage development, API not stable. If you've used a
+previous version, you'll most likely have to clear all previously published
+messages in order to upgrade to the latest version.
+
+## Installation
+
+``` sh
+gem install remq
+```
+
+## Usage
+
+**Producer:**
+
+``` rb
+require 'json'
+require 'remq'
+
+$remq = Remq.new
+
+message = { event: 'signup', account_id: 694 }
+
+id = $remq.publish('events.accounts', JSON.dump(message))
+```
+
+**Consumer:**
+
+``` rb
+require 'json'
+require 'remq'
+
+$remq = Remq.new
+
+last_id_key = $remq.key('cursor', 'consumer-1')
+last_id = $remq.redis.get(last_id_key) || 0
+
+$remq.subscribe('events.*', from_id: last_id) do |channel, message|
+  last_id = message.id
+
+  # by persisting the last_id every 10 messages, a maximum of
+  # 10 messages will be replayed in the case of consumer failure
+  if last_id % 10 == 0
+    $remq.redis.set(last_id_key, last_id)
+  end
+
+  message.body = JSON.parse(message.body)
+
+  puts "Received message on '#{channel}' with id: #{message.id}"
+  puts "Account signed up with id: #{message.body['account_id']}"
+end
+```
+
+**Flush:**
+
+``` rb
+# TODO: not implemented yet
+```

data/Rakefile

@@ -1,8 +1,7 @@
-require 'rubygems'
 require 'bundler'
 Bundler.setup :default, :test, :development
 
 require 'rspec/core/rake_task'
 RSpec::Core::RakeTask.new(:spec)
 
-task default: :spec
+task :default => :spec

data/examples/consumer.rb

@@ -0,0 +1,26 @@
+$LOAD_PATH.unshift 'lib'
+
+require 'json'
+require 'remq'
+
+$remq = Remq.new
+
+last_id_key = $remq.key('cursor', 'consumer-1')
+last_id = $remq.redis.get(last_id_key) || 0
+
+$remq.on(:message) do |channel, message|
+  last_id = message.id
+
+  # by persisting the last_id every 10 messages, a maximum of
+  # 10 messages will be replayed in the case of consumer failure
+  if last_id % 10 == 0
+    $remq.redis.set last_id_key, last_id
+  end
+
+  message.body = JSON.parse(message.body)
+  puts message.inspect
+end
+
+thread = $remq.subscribe('events.*', from_id: last_id)
+
+thread.join

data/examples/producer.rb

@@ -0,0 +1,19 @@
+$LOAD_PATH.unshift 'lib'
+$LOAD_PATH.unshift 'examples/shared'
+
+require 'json'
+require 'remq'
+require 'message'
+
+$remq = Remq.new
+
+def publish message
+  channel = "events.#{message.type.downcase}.#{message.event}"
+  id = $remq.publish(channel, JSON.dump(message.attributes))
+  puts "Published ##{id} to channel '#{channel}'"
+end
+
+loop do
+  publish Message.new
+  sleep 0.15
+end

data/examples/shared/message.rb

@@ -0,0 +1,22 @@
+class Message
+  EVENTS = %w(create update delete)
+  TYPES = %w(Account Subscription Transaction)
+
+  def self.id
+    @id ||= 0
+    @id += 1
+  end
+
+  attr :event, :type, :attributes
+
+  def initialize
+    @event = EVENTS.sample
+    @type  = TYPES.sample
+    @attributes = {
+      account_id: self.class.id,
+      first_name: 'Evan',
+      last_name: 'Owen',
+      state: 'active'
+    }
+  end
+end

data/lib/remq.rb

@@ -15,8 +15,12 @@ class Remq
 
   include MonitorMixin
 
-  attr :redis, :predis
+  attr :redis
+  attr :predis
 
+  # Create a `Remq` client with the given `options`, which are passed to redis.
+  #
+  # @param [Hash] options
   def initialize(options = {})
     @redis     = Redis.new(options)
     @predis    = Redis.new(options) # seperate connection for pub/sub
@@ -25,6 +29,14 @@ class Remq
     super() # Monitor#initialize
   end
 
+  # Publish a `message` to the given `channel`. The `message` must be a string,
+  # but objects can easily be serialized using JSON, etc. The id of the
+  # published message will be returned as an integer.
+  #
+  # @param [String] channel
+  # @param [String] message
+  #
+  # @return [Integer] id
   def publish(channel, message)
     synchronize do
       id = call(:publish, channel, message)
@@ -32,12 +44,30 @@ class Remq
     end
   end
 
+  # Subscribe to the channels matching given `pattern`. If no initial `from_id`
+  # is provided, Remq subscribes using vanilla Redis pub/sub. Any Redis pub/sub
+  # pattern will work. If `from_id` is provided, Remq replays messages after the
+  # given id until its caught up and able to switch to pub/sub.
+  #
+  # Remq-rb subscribes to pub/sub on another thread, which is returned so you
+  # can handle it and call `Thread#join` when ready to block.
+  #
+  # @param [String] pattern
+  # @param [Hash] options
+  #   - `:from_id => Integer`: The message id to replay from (usually the last)
+  #
+  # @yield a block to add as a listener to the `message` event
+  # @yieldparam [Remq::Message] received message
+  #
+  # @return [Thread] thread where messages will be received
   def subscribe(pattern, options = {}, &block)
     synchronize do
       return if @subscription
 
       on(:message, &block) if block
 
+      @subscription = true
+
       if cursor = options[:from_id]
         @subscription = _subscribe_from_cursor(pattern, cursor)
       else
@@ -46,14 +76,25 @@ class Remq
     end
   end
 
+  # Unsubscribe. No more `message` events will be emitted after this is called.
   def unsubscribe
     synchronize do
       return unless @subscription
-      @subscription.exit
+      @subscription.exit if @subscription.is_a?(Thread)
       @subscription = nil
     end
   end
 
+  # Consume persisted messages from channels matching the given `pattern`,
+  # starting with the `cursor` if provided, or the first message. `limit`
+  # determines how many messages will be return each time `consume` is called.
+  #
+  # @param [String] pattern
+  # @param [Hash] options
+  #   - `:cursor => Integer`: id of the first message to return
+  #   - `:limit => Integer`: maximum number of messages to return
+  #
+  # @return [Array<Remq::Message>] array of parsed messages
   def consume(pattern, options = {})
     synchronize do
       cursor, limit = options.fetch(:cursor, 0), options.fetch(:limit, LIMIT)
@@ -62,8 +103,24 @@ class Remq
     end
   end
 
-  def on(event, proc=nil, &block)
-    listener = proc || block
+  # Forcibly close the connections to the Redis server.
+  def quit
+    synchronize do
+      redis.quit
+      predis.quit
+    end
+  end
+
+  # Add a listener to the given event.
+  #
+  # @param [String|Symbol] event
+  # @param [Proc] listener
+  #
+  # @yield a block to be called when the event is emitted
+  #
+  # @return [Remq] self
+  def on(event, listener=nil, &block)
+    listener ||= block
     unless listener.respond_to?(:call)
       raise ArgumentError.new('Listener must respond to #call')
     end
@@ -75,6 +132,12 @@ class Remq
     self
   end
 
+  # Remove a listener from the given event.
+  #
+  # @param [String|Symbol] event
+  # @param [Proc] listener
+  #
+  # @return [Remq] self
   def off(event, listener)
     synchronize do
       @listeners[event.to_sym].delete(listener)
@@ -83,6 +146,12 @@ class Remq
     self
   end
 
+  # Build a key from the given `name` and `channel`.
+  #
+  # @param [String] name
+  # @param [String] channel
+  #
+  # @return [String] key
   def key(*args)
     (['remq'] + args).join(':')
   end
@@ -118,7 +187,7 @@ class Remq
 
   def _subscribe_from_cursor(pattern, cursor)
     begin
-      msgs = consume(pattern, { cursor: cursor })
+      msgs = consume(pattern, { :cursor => cursor })
       cursor = msgs.last.id unless msgs.empty?
     end while msgs.length == LIMIT
 
@@ -126,13 +195,13 @@ class Remq
   end
 
   def _subscribe_to_pubsub(pattern)
-    subscription = nil
+    subscribed_thread = nil
 
     Thread.new do
       begin
         predis.client.connect
         predis.psubscribe(key('channel', pattern)) do |on|
-          on.psubscribe { subscription = Thread.current }
+          on.psubscribe { subscribed_thread = Thread.current }
           on.pmessage { |_, _, msg| _handle_raw_message(msg) }
         end
       rescue => e
@@ -140,14 +209,14 @@ class Remq
       end
     end
 
-    Thread.pass while !subscription
+    Thread.pass while !subscribed_thread
 
-    subscription
+    subscribed_thread
   end
 
   def _handle_raw_message(raw)
     msg = Message.parse raw
-    emit(:message, msg.channel, msg)
+    emit(:message, msg.channel, msg) if @subscription
     msg
   end
 

data/lib/remq/version.rb

@@ -1,3 +1,3 @@
 class Remq
-  VERSION = "0.0.3"
+  VERSION = "0.0.4"
 end

data/spec/remq_spec.rb

@@ -8,7 +8,7 @@ RSpec::Matchers.define :be_ordered_from do |cursor|
 end
 
 describe Remq do
-  subject { Remq.new(db: 4) }
+  subject { Remq.new(:db => 4) }
 
   let(:channel) { 'events.things' }
 
@@ -31,7 +31,7 @@ describe Remq do
   end
 
   describe '#subscribe' do
-    let(:producer) { Remq.new(db: 4) }
+    let(:producer) { Remq.new(:db => 4) }
 
     it 'receives messages on a separate thread' do
       msgs = []
@@ -42,27 +42,42 @@ describe Remq do
 
       publish 3, producer
 
-      thread.join(1)
+      thread.join(0.01)
 
       msgs.should have(3).items
       msgs.should be_ordered_from 0
     end
 
     context 'with :from_id option' do
-      it 'catches up to pub/sub from cursor using consume' do
+      it 'replays any missed messages' do
         publish 3, producer
 
         msgs = []
-        thread = subject.subscribe(channel, from_id: 0) do |channel, msg|
+        thread = subject.subscribe(channel, :from_id => 0) do |channel, msg|
           msgs << msg
-          subject.unsubscribe if msgs.length == 6
+          subject.unsubscribe if msgs.length == 3
         end
 
-        publish 3, producer
+        thread.join(0.01)
+
+        msgs.should have(3).items
+        msgs.should be_ordered_from 0
+      end
+
+      it 'doesn\'t miss or repeat when switching to pub/sub' do
+        publish 100, producer
+
+        msgs = []
+        thread = subject.subscribe(channel, :from_id => 0) do |channel, msg|
+          msgs << msg
+          subject.unsubscribe if msgs.length == 200
+        end
+
+        publish 100, producer
 
-        thread.join(1)
+        thread.join(0.1)
 
-        msgs.should have(6).items
+        msgs.should have(200).items
         msgs.should be_ordered_from 0
       end
     end
@@ -78,14 +93,14 @@ describe Remq do
 
     it 'limits the messages returned to value given in the :limit option' do
       publish 3
-      msgs = subject.consume(channel, limit: 2)
+      msgs = subject.consume(channel, :limit => 2)
       msgs.should have(2).items
       msgs.should be_ordered_from 0
     end
 
     it 'returns messages published since the id given in the :cursor option' do
       cursor = publish(3).first
-      msgs = subject.consume(channel, cursor: cursor)
+      msgs = subject.consume(channel, :cursor => cursor)
       msgs.should have(2).items
       msgs.should be_ordered_from cursor
     end

data/vendor/bundle/gems/multi_json-1.6.1/Gemfile

@@ -0,0 +1,31 @@
+source 'https://rubygems.org'
+
+gem 'rake', '>= 0.9'
+gem 'yard', '>= 0.8'
+
+platforms :ruby_18 do
+  gem 'json', '~> 1.4', :require => nil
+end
+
+gem 'json_pure', '~> 1.4', :require => nil
+
+platforms :ruby, :mswin, :mingw do
+  gem 'oj', '~> 2.0', :require => nil
+  gem 'yajl-ruby', '~> 1.0', :require => nil
+end
+platforms :jruby do
+  gem 'gson', '>= 0.6', :require => nil
+end
+
+group :development do
+  gem 'kramdown', '>= 0.14'
+  gem 'pry'
+  gem 'pry-debugger', :platforms => :mri_19
+end
+
+group :test do
+  gem 'rspec', '>= 2.11'
+  gem 'simplecov', :require => false
+end
+
+gemspec

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: remq
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
   prerelease: 
 platform: ruby
 authors:
@@ -72,9 +72,15 @@ extra_rdoc_files: []
 files:
 - .gitignore
 - .gitmodules
+- .travis.yml
+- .yardopts
+- Gemfile
 - LICENSE
+- README.md
 - Rakefile
-- Readme.md
+- examples/consumer.rb
+- examples/producer.rb
+- examples/shared/message.rb
 - lib/remq.rb
 - lib/remq/script.rb
 - lib/remq/version.rb

data/Readme.md

@@ -1,63 +0,0 @@
-# Remq-rb
-
-A Ruby client library for [Remq](https://github.com/kainosnoema/remq),
-a [Redis](http://redis.io)-based protocol for building fast, durable
-message queues.
-
-NOTE: In early-stage development, API not stable. If you've used a previous
-version, you'll most likely have to clear all previously published messages
-in order to upgrade to the latest version.
-
-## Installation
-
-``` sh
-gem install remq --pre
-```
-
-## Usage
-
-**Producer:**
-
-``` rb
-require 'json'
-require 'remq'
-
-remq = Remq.new
-
-message = { event: 'signup', account_id: 694 }
-
-id = remq.publish('events.accounts', JSON.dump(message))
-```
-
-**Consumer:**
-
-``` rb
-require 'json'
-require 'remq'
-
-remq = Remq.new
-
-last_id_key = remq.key('cursor', 'consumer-1')
-last_id = remq.redis.get(last_id_key) || 0
-
-remq.subscribe('events.*', from_id: last_id) do |channel, message|
-  last_id = message.id
-
-  # by persisting the last_id every 10 messages, a maximum of
-  # 10 messages will be replayed in the case of consumer failure
-  if last_id % 10 == 0
-    remq.redis.set(last_id_key, last_id)
-  end
-
-  message.body = JSON.parse(message.body)
-
-  puts "Received message on '#{channel}' with id: #{message.id}"
-  puts "Account signed up with id: #{message.body['account_id']}"
-end
-```
-
-**Flush:**
-
-``` rb
-# TODO: not implemented yet
-```