remq 0.0.1a → 0.0.3

data/Rakefile

@@ -0,0 +1,8 @@
+require 'rubygems'
+require 'bundler'
+Bundler.setup :default, :test, :development
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new(:spec)
+
+task default: :spec

data/Readme.md

@@ -1,49 +1,63 @@
 # Remq-rb
 
-A Ruby client library for [Remq](https://github.com/kainosnoema/remq), a [Redis](http://redis.io)-based protocol for building fast, persistent pub/sub message queues.
+A Ruby client library for [Remq](https://github.com/kainosnoema/remq),
+a [Redis](http://redis.io)-based protocol for building fast, durable
+message queues.
 
-NOTE: In early-stage development, API not locked.
+NOTE: In early-stage development, API not stable. If you've used a previous
+version, you'll most likely have to clear all previously published messages
+in order to upgrade to the latest version.
+
+## Installation
+
+``` sh
+gem install remq --pre
+```
 
 ## Usage
 
 **Producer:**
 
 ``` rb
+require 'json'
 require 'remq'
 
 remq = Remq.new
 
 message = { event: 'signup', account_id: 694 }
-id = remq.publish('events.accounts', message)
+
+id = remq.publish('events.accounts', JSON.dump(message))
 ```
 
-**Pub/sub consumer (messages lost during failure):**
+**Consumer:**
 
 ``` rb
+require 'json'
 require 'remq'
 
 remq = Remq.new
 
-# TODO: not implemented yet
-```
+last_id_key = remq.key('cursor', 'consumer-1')
+last_id = remq.redis.get(last_id_key) || 0
 
-**Polling consumer with cursor (resumes post-failure):**
+remq.subscribe('events.*', from_id: last_id) do |channel, message|
+  last_id = message.id
 
-``` rb
-require 'remq'
+  # by persisting the last_id every 10 messages, a maximum of
+  # 10 messages will be replayed in the case of consumer failure
+  if last_id % 10 == 0
+    remq.redis.set(last_id_key, last_id)
+  end
 
-remq = Remq.new
+  message.body = JSON.parse(message.body)
 
-# TODO: not implemented yet
+  puts "Received message on '#{channel}' with id: #{message.id}"
+  puts "Account signed up with id: #{message.body['account_id']}"
+end
 ```
 
-**Purging old messages:**
+**Flush:**
 
 ``` rb
-
-require 'remq'
-
-remq = Remq.new
-
 # TODO: not implemented yet
 ```

data/lib/remq/version.rb

@@ -1,3 +1,3 @@
 class Remq
-  VERSION = "0.0.1a"
+  VERSION = "0.0.3"
 end

data/lib/remq.rb

@@ -1,52 +1,154 @@
+require 'monitor'
 require 'redis'
 require 'remq/script'
-require 'remq/multi_json_coder'
 
 class Remq
+  Message = Struct.new(:channel, :id, :body)
 
-  attr :redis, :namespace, :scripts, :coder
+  def Message.parse(raw)
+    header, body = raw.split("\n", 2)
+    channel, id = header.split('@', 2)
+    new(channel, id.to_i, body)
+  end
+
+  LIMIT = 1000
+
+  include MonitorMixin
+
+  attr :redis, :predis
 
   def initialize(options = {})
-    @redis = options[:redis] || Redis.new(options)
-    @namespace = 'remq'
-    @scripts = {}
+    @redis     = Redis.new(options)
+    @predis    = Redis.new(options) # seperate connection for pub/sub
+    @listeners = Hash.new { |h, k| h[k] = [] }
+
+    super() # Monitor#initialize
   end
 
   def publish(channel, message)
-    channel = channel.join('.') if channel.is_a?(Array)
-    msg, utc_seconds = coder.encode(message), Time.now.to_i
-    id = eval_script(:publish, namespace, channel, msg, utc_seconds)
-    id.nil? ? nil : id.to_s
-  end
-
-  def consume(channel, options = {})
-    cursor, limit = options[:cursor] || 0, options[:limit] || 1000
-    msgs_ids = eval_script(:consume, namespace, channel, cursor, limit)
-    msgs = {}
-    msgs_ids.each_index { |i|
-      if i % 2 == 0
-        msgs[msgs_ids[i+1]] = coder.decode(msgs_ids[i])
+    synchronize do
+      id = call(:publish, channel, message)
+      id && id.to_i
+    end
+  end
+
+  def subscribe(pattern, options = {}, &block)
+    synchronize do
+      return if @subscription
+
+      on(:message, &block) if block
+
+      if cursor = options[:from_id]
+        @subscription = _subscribe_from_cursor(pattern, cursor)
+      else
+        @subscription = _subscribe_to_pubsub(pattern)
       end
-    }
-    msgs
+    end
+  end
+
+  def unsubscribe
+    synchronize do
+      return unless @subscription
+      @subscription.exit
+      @subscription = nil
+    end
+  end
+
+  def consume(pattern, options = {})
+    synchronize do
+      cursor, limit = options.fetch(:cursor, 0), options.fetch(:limit, LIMIT)
+      msgs = call(:consume, pattern, cursor, limit)
+      msgs.map { |msg| _handle_raw_message(msg) }
+    end
+  end
+
+  def on(event, proc=nil, &block)
+    listener = proc || block
+    unless listener.respond_to?(:call)
+      raise ArgumentError.new('Listener must respond to #call')
+    end
+
+    synchronize do
+      @listeners[event.to_sym] << listener
+    end
+
+    self
   end
 
-  def coder
-    @coder ||= MultiJsonCoder.new
+  def off(event, listener)
+    synchronize do
+      @listeners[event.to_sym].delete(listener)
+    end
+
+    self
+  end
+
+  def key(*args)
+    (['remq'] + args).join(':')
+  end
+
+  def inspect
+    "#<Remq client v#{Remq::VERSION} for #{redis.client.id}>"
   end
 
   protected
 
-    def eval_script(name, *args)
-      script(name).eval(@redis, *args)
+  def call(name, *args)
+    synchronize do
+      script(name).eval(redis, *args)
     end
+  end
 
-    def script(name)
-      @scripts[name.to_sym] ||= Script.new(File.read(script_path(name)))
+  def emit(event, *args)
+    synchronize do
+      @listeners[event.to_sym].each { |listener| listener.call(*args) }
     end
+  end
 
-    def script_path(name)
-      File.expand_path("../vendor/remq/scripts/#{name}.lua", File.dirname(__FILE__))
+  def script(name)
+    synchronize do
+      (@scripts ||= {})[name.to_sym] ||= begin
+        path = "../vendor/remq/scripts/#{name}.lua"
+        Script.new(File.read(File.expand_path(path, File.dirname(__FILE__))))
+      end
     end
+  end
+
+  private
+
+  def _subscribe_from_cursor(pattern, cursor)
+    begin
+      msgs = consume(pattern, { cursor: cursor })
+      cursor = msgs.last.id unless msgs.empty?
+    end while msgs.length == LIMIT
+
+    _subscribe_to_pubsub(pattern)
+  end
+
+  def _subscribe_to_pubsub(pattern)
+    subscription = nil
+
+    Thread.new do
+      begin
+        predis.client.connect
+        predis.psubscribe(key('channel', pattern)) do |on|
+          on.psubscribe { subscription = Thread.current }
+          on.pmessage { |_, _, msg| _handle_raw_message(msg) }
+        end
+      rescue => e
+        Thread.main.raise e
+      end
+    end
+
+    Thread.pass while !subscription
+
+    subscription
+  end
+
+  def _handle_raw_message(raw)
+    msg = Message.parse raw
+    emit(:message, msg.channel, msg)
+    msg
+  end
 
 end

data/remq.gemspec

@@ -16,19 +16,17 @@ Gem::Specification.new do |s|
   s.test_files        = `git ls-files -- {test,spec,features}/*`.split("\n")
 
   s.add_dependency    "redis",      "~> 3.0.1"
-  s.add_dependency    "multi_json", "~> 1.0"
 
   s.add_development_dependency "rake"
   s.add_development_dependency "rspec", "~> 2.6"
 
   s.description = <<description
-    Remq is a Redis-based protocol for building fast, persistent
-    pub/sub message queues.
+    Remq is a Redis-based protocol for building fast, durable message queues.
 
     The Remq protocol is defined by a collection of Lua scripts
     (located at https://github.com/kainosnoema/remq) which effectively
     turn Redis into a capable message queue broker for fast inter-service
     communication. The Remq Ruby client library is built on top of these
-    scripts, making it easy to build fast, persisted pub/sub message queues.
+    scripts, making it easy to build fast, durable message queues.
 description
 end

data/spec/remq_spec.rb

@@ -1,67 +1,93 @@
 require 'remq'
 
+RSpec::Matchers.define :be_ordered_from do |cursor|
+  cursor += 1
+  match do |actual|
+    actual.each_with_index.map { |msg, i| msg.id == cursor + i }.all?
+  end
+end
+
 describe Remq do
+  subject { Remq.new(db: 4) }
+
+  let(:channel) { 'events.things' }
 
-  before :all do
-    subject { Remq.new(db: 4).tap { |r| r.redis.flushdb } }
+  before { subject.redis.flushdb }
+
+  def publish n, remq = subject
+    (1..n).map { |i| remq.publish(channel, "foo #{i}") }
   end
 
-  after :each do
-    subject.redis.flushdb
+  def expect_n n, msgs, at = 0
+    msgs.should have(n).items
+    msgs.each_with_index { |msg, i| msg.body.should eql("foo #{at + i + 1}") }
   end
 
-  context "class" do
-    describe ".new" do
-      it "creates a Redis client when options hash missing" do
-        Remq.new.redis.should be_a(Redis)
-      end
+  describe '#publish' do
+    it 'publishes a message to the given channel and returns an id' do
+      id = subject.publish(channel, 'foo 1')
+      id.should be_a Integer
+    end
+  end
+
+  describe '#subscribe' do
+    let(:producer) { Remq.new(db: 4) }
 
-      it "takes a Redis client as an option" do
-        Remq.new(redis: (redis = Redis.new)).redis.should eql redis
+    it 'receives messages on a separate thread' do
+      msgs = []
+      thread = subject.subscribe(channel) do |channel, msg|
+        msgs << msg
+        subject.unsubscribe if msgs.length == 3
       end
+
+      publish 3, producer
+
+      thread.join(1)
+
+      msgs.should have(3).items
+      msgs.should be_ordered_from 0
     end
-  end
 
-  describe ".publish" do
-    it "publishes a message to the given channel and returns an id" do
-      id = subject.publish('events.things', { test: 'one' })
-      id.should be_a String
+    context 'with :from_id option' do
+      it 'catches up to pub/sub from cursor using consume' do
+        publish 3, producer
+
+        msgs = []
+        thread = subject.subscribe(channel, from_id: 0) do |channel, msg|
+          msgs << msg
+          subject.unsubscribe if msgs.length == 6
+        end
+
+        publish 3, producer
+
+        thread.join(1)
+
+        msgs.should have(6).items
+        msgs.should be_ordered_from 0
+      end
     end
   end
 
-  describe ".consume" do
-    it "consumes messages published to the given channel" do
-      subject.publish('events.things', { 'test' => 'one' })
-      subject.publish('events.things', { 'test' => 'two' })
-      subject.publish('events.things', { 'test' => 'three' })
-
-      msgs = subject.consume('events.things')
+  describe '#consume' do
+    it 'consumes messages published to the given channel' do
+      publish 3
+      msgs = subject.consume(channel)
       msgs.should have(3).items
-      msgs[msgs.keys[0]].should eql({ 'test' => 'one' })
-      msgs[msgs.keys[2]].should eql({ 'test' => 'three' })
+      msgs.should be_ordered_from 0
     end
 
-    it "limits the messages returned to value given in the :limit option" do
-      subject.publish('events.things', { 'test' => 'one' })
-      subject.publish('events.things', { 'test' => 'two' })
-      subject.publish('events.things', { 'test' => 'three' })
-
-      msgs = subject.consume('events.things', limit: 2)
+    it 'limits the messages returned to value given in the :limit option' do
+      publish 3
+      msgs = subject.consume(channel, limit: 2)
       msgs.should have(2).items
-      msgs[msgs.keys[0]].should eql({ 'test' => 'one' })
-      msgs[msgs.keys[1]].should eql({ 'test' => 'two' })
-      msgs[msgs.keys[2]].should be_nil
+      msgs.should be_ordered_from 0
     end
 
-    it "returns messages published since the id given in the :cursor option" do
-      cursor = subject.publish('events.things', { 'test' => 'one' })
-      subject.publish('events.things', { 'test' => 'two' })
-      subject.publish('events.things', { 'test' => 'three' })
-
-      msgs = subject.consume('events.things', cursor: cursor)
+    it 'returns messages published since the id given in the :cursor option' do
+      cursor = publish(3).first
+      msgs = subject.consume(channel, cursor: cursor)
       msgs.should have(2).items
-      msgs[msgs.keys[0]].should eql({ 'test' => 'two' })
-      msgs[msgs.keys[1]].should eql({ 'test' => 'three' })
+      msgs.should be_ordered_from cursor
     end
   end
 end

data/vendor/bundle/bin/autospec

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by RubyGems.
+#
+# The application 'rspec-core' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'rubygems'
+
+version = ">= 0"
+
+if ARGV.first
+  str = ARGV.first
+  str = str.dup.force_encoding("BINARY") if str.respond_to? :force_encoding
+  if str =~ /\A_(.*)_\z/
+    version = $1
+    ARGV.shift
+  end
+end
+
+gem 'rspec-core', version
+load Gem.bin_path('rspec-core', 'autospec', version)

data/vendor/bundle/bin/htmldiff

@@ -0,0 +1,25 @@
+#!/bin/sh
+'exec' "ruby" '-x' "$0" "$@"
+#!/Users/evan/.rbenv/versions/1.9.3-p194-perf/bin/ruby
+#
+# This file was generated by RubyGems.
+#
+# The application 'diff-lcs' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'rubygems'
+
+version = ">= 0"
+
+if ARGV.first
+  str = ARGV.first
+  str = str.dup.force_encoding("BINARY") if str.respond_to? :force_encoding
+  if str =~ /\A_(.*)_\z/
+    version = $1
+    ARGV.shift
+  end
+end
+
+gem 'diff-lcs', version
+load Gem.bin_path('diff-lcs', 'htmldiff', version)

data/vendor/bundle/bin/ldiff

@@ -0,0 +1,25 @@
+#!/bin/sh
+'exec' "ruby" '-x' "$0" "$@"
+#!/Users/evan/.rbenv/versions/1.9.3-p194-perf/bin/ruby
+#
+# This file was generated by RubyGems.
+#
+# The application 'diff-lcs' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'rubygems'
+
+version = ">= 0"
+
+if ARGV.first
+  str = ARGV.first
+  str = str.dup.force_encoding("BINARY") if str.respond_to? :force_encoding
+  if str =~ /\A_(.*)_\z/
+    version = $1
+    ARGV.shift
+  end
+end
+
+gem 'diff-lcs', version
+load Gem.bin_path('diff-lcs', 'ldiff', version)

data/vendor/bundle/bin/rake

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by RubyGems.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'rubygems'
+
+version = ">= 0"
+
+if ARGV.first
+  str = ARGV.first
+  str = str.dup.force_encoding("BINARY") if str.respond_to? :force_encoding
+  if str =~ /\A_(.*)_\z/
+    version = $1
+    ARGV.shift
+  end
+end
+
+gem 'rake', version
+load Gem.bin_path('rake', 'rake', version)

data/vendor/bundle/bin/rspec

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+#
+# This file was generated by RubyGems.
+#
+# The application 'rspec-core' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require 'rubygems'
+
+version = ">= 0"
+
+if ARGV.first
+  str = ARGV.first
+  str = str.dup.force_encoding("BINARY") if str.respond_to? :force_encoding
+  if str =~ /\A_(.*)_\z/
+    version = $1
+    ARGV.shift
+  end
+end
+
+gem 'rspec-core', version
+load Gem.bin_path('rspec-core', 'rspec', version)

data/vendor/bundle/gems/diff-lcs-1.1.3/History.rdoc

@@ -0,0 +1,54 @@
+== 1.1.3 / 2011-08-27
+* Converted to 'hoe' for release.
+* Converted tests to RSpec 2.
+* Extracted the body of htmldiff into a class available from
+  diff/lcs/htmldiff.
+* Migrated development and issue tracking to GitHub.
+* Bugs fixed:
+  - Eliminated the explicit use of RubyGems in both bin/htmldiff and bin/ldiff.
+    Resolves issue 4 (https://github.com/halostatue/diff-lcs/issues/4).
+  - Eliminated Ruby warnings. Resolves issue 3
+    (https://github.com/halostatue/diff-lcs/issues/3).
+
+== 1.1.2 / 2004-10-20
+* Fixed a problem reported by Mauricio Fernandez in htmldiff.
+
+== 1.1.1 / 2004-09-25
+* Fixed bug #891:
+  http://rubyforge.org/tracker/?func=detail&atid=407&aid=891&group_id=84
+* Fixed a problem with callback initialisation code (it assumed that all
+  callbacks passed as classes can be initialised; now, it rescues
+  NoMethodError in the event of private :new being called).
+* Modified the non-initialisable callbacks to have a private #new method.
+* Moved ldiff core code to Diff::LCS::Ldiff (diff/lcs/ldiff.rb).
+
+== 1.1.0 / -
+* Eliminated the need for Diff::LCS::Event and removed it.
+* Added a contextual diff callback, Diff::LCS::ContextDiffCallback.
+* Implemented patching/unpatching for standard Diff callback output formats
+  with both #diff and #sdiff.
+* Extensive documentation changes.
+
+== 1.0.4 / -
+* Fixed a problem with bin/ldiff output, especially for unified format.
+  Newlines that should have been present weren't.
+* Changed the .tar.gz installer to generate Windows batch files if ones do not
+  exist already. Removed the existing batch files as they didn't work.
+
+== 1.0.3 / -
+* Fixed a problem with #traverse_sequences where the first difference from the
+  left sequence might not be appropriately captured.
+
+== 1.0.2 / -
+* Fixed an issue with ldiff not working because actions were changed from
+  symbols to strings.
+
+== 1.0.1 / -
+* Minor modifications to the gemspec, the README.
+* Renamed the diff program to ldiff (as well as the companion batch file) so as
+  to not collide with the standard diff program.
+* Fixed issues with RubyGems. Requires RubyGems > 0.6.1 or >= 0.6.1 with the
+  latest CVS version.
+
+== 1.0 / -
+* Initial release based mostly on Perl's Algorithm::Diff.