remq 0.0.1a

data/.gitignore

@@ -0,0 +1,5 @@
+*.gem
+.bundle
+.config
+Gemfile
+Gemfile.lock

data/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "vendor/remq"]
+	path = vendor/remq
+	url = git@github.com:kainosnoema/remq.git

data/LICENSE

@@ -0,0 +1,22 @@
+Copyright (c) 2012 Evan Owen
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Readme.md

@@ -0,0 +1,49 @@
+# Remq-rb
+
+A Ruby client library for [Remq](https://github.com/kainosnoema/remq), a [Redis](http://redis.io)-based protocol for building fast, persistent pub/sub message queues.
+
+NOTE: In early-stage development, API not locked.
+
+## Usage
+
+**Producer:**
+
+``` rb
+require 'remq'
+
+remq = Remq.new
+
+message = { event: 'signup', account_id: 694 }
+id = remq.publish('events.accounts', message)
+```
+
+**Pub/sub consumer (messages lost during failure):**
+
+``` rb
+require 'remq'
+
+remq = Remq.new
+
+# TODO: not implemented yet
+```
+
+**Polling consumer with cursor (resumes post-failure):**
+
+``` rb
+require 'remq'
+
+remq = Remq.new
+
+# TODO: not implemented yet
+```
+
+**Purging old messages:**
+
+``` rb
+
+require 'remq'
+
+remq = Remq.new
+
+# TODO: not implemented yet
+```

data/lib/remq.rb

@@ -0,0 +1,52 @@
+require 'redis'
+require 'remq/script'
+require 'remq/multi_json_coder'
+
+class Remq
+
+  attr :redis, :namespace, :scripts, :coder
+
+  def initialize(options = {})
+    @redis = options[:redis] || Redis.new(options)
+    @namespace = 'remq'
+    @scripts = {}
+  end
+
+  def publish(channel, message)
+    channel = channel.join('.') if channel.is_a?(Array)
+    msg, utc_seconds = coder.encode(message), Time.now.to_i
+    id = eval_script(:publish, namespace, channel, msg, utc_seconds)
+    id.nil? ? nil : id.to_s
+  end
+
+  def consume(channel, options = {})
+    cursor, limit = options[:cursor] || 0, options[:limit] || 1000
+    msgs_ids = eval_script(:consume, namespace, channel, cursor, limit)
+    msgs = {}
+    msgs_ids.each_index { |i|
+      if i % 2 == 0
+        msgs[msgs_ids[i+1]] = coder.decode(msgs_ids[i])
+      end
+    }
+    msgs
+  end
+
+  def coder
+    @coder ||= MultiJsonCoder.new
+  end
+
+  protected
+
+    def eval_script(name, *args)
+      script(name).eval(@redis, *args)
+    end
+
+    def script(name)
+      @scripts[name.to_sym] ||= Script.new(File.read(script_path(name)))
+    end
+
+    def script_path(name)
+      File.expand_path("../vendor/remq/scripts/#{name}.lua", File.dirname(__FILE__))
+    end
+
+end

data/lib/remq/coder.rb

@@ -0,0 +1,30 @@
+
+# borrowed from Resque::Coder. Thanks @defunkt!
+
+class Remq
+  class EncodeException < StandardError; end
+  class DecodeException < StandardError; end
+
+  class Coder
+    # Given a Ruby object, returns a string suitable for storage in a
+    # queue.
+    def encode(object)
+      raise EncodeException
+    end
+
+    # alias for encode
+    def dump(object)
+      encode(object)
+    end
+
+    # Given a string, returns a Ruby object.
+    def decode(object)
+      raise DecodeException
+    end
+
+    # alias for decode
+    def load(object)
+      decode(object)
+    end
+  end
+end

data/lib/remq/multi_json_coder.rb

@@ -0,0 +1,39 @@
+require 'multi_json'
+require 'remq/coder'
+
+# borrowed from Resque::MultiJsonCoder. Thanks @defunkt!
+
+if MultiJson.respond_to?(:adapter)
+  raise "Please install the yajl-ruby or json gem" if MultiJson.adapter.to_s == 'MultiJson::Adapters::OkJson'
+elsif MultiJson.respond_to?(:engine)
+  raise "Please install the yajl-ruby or json gem" if MultiJson.engine.to_s == 'MultiJson::Engines::OkJson'
+end
+
+class Remq
+  class MultiJsonCoder < Coder
+    class EncodeException < StandardError; end
+    class DecodeException < StandardError; end
+
+    def encode(object)
+      if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+        MultiJson.dump object
+      else
+        MultiJson.encode object
+      end
+    end
+
+    def decode(object)
+      return unless object
+
+      begin
+        if MultiJson.respond_to?(:dump) && MultiJson.respond_to?(:load)
+          MultiJson.load object
+        else
+          MultiJson.decode object
+        end
+      rescue ::MultiJson::DecodeError => e
+        raise DecodeException, e.message, e.backtrace
+      end
+    end
+  end
+end

data/lib/remq/script.rb

@@ -0,0 +1,27 @@
+require 'digest/sha1'
+
+class Remq
+  class Script
+
+    attr :source, :sha
+
+    def initialize(source)
+      @source = source
+    end
+
+    def eval(redis, *args)
+      redis.evalsha(sha, [], [*args])
+    rescue => e
+      if e.message =~ /NOSCRIPT/
+        redis.eval(source, [], [*args])
+      else
+        raise
+      end
+    end
+
+    def sha
+      @sha ||= Digest::SHA1.hexdigest source
+    end
+
+  end
+end

data/lib/remq/version.rb

@@ -0,0 +1,3 @@
+class Remq
+  VERSION = "0.0.1a"
+end

data/remq.gemspec

@@ -0,0 +1,34 @@
+$LOAD_PATH.unshift 'lib'
+
+require 'remq/version'
+
+Gem::Specification.new do |s|
+  s.name              = "remq"
+  s.version           = Remq::VERSION
+  s.date              = Time.now.strftime('%Y-%m-%d')
+  s.summary           = "A Remq client library for Ruby."
+  s.homepage          = "http://github.com/kainosnoema/remq"
+  s.email             = "kainosnoema@gmail.com"
+  s.authors           = [ "Evan Owen" ]
+
+  s.files             = `git ls-files`.split("\n")
+  s.files            += Dir.glob('vendor/**/*')
+  s.test_files        = `git ls-files -- {test,spec,features}/*`.split("\n")
+
+  s.add_dependency    "redis",      "~> 3.0.1"
+  s.add_dependency    "multi_json", "~> 1.0"
+
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rspec", "~> 2.6"
+
+  s.description = <<description
+    Remq is a Redis-based protocol for building fast, persistent
+    pub/sub message queues.
+
+    The Remq protocol is defined by a collection of Lua scripts
+    (located at https://github.com/kainosnoema/remq) which effectively
+    turn Redis into a capable message queue broker for fast inter-service
+    communication. The Remq Ruby client library is built on top of these
+    scripts, making it easy to build fast, persisted pub/sub message queues.
+description
+end

data/spec/remq_spec.rb

@@ -0,0 +1,67 @@
+require 'remq'
+
+describe Remq do
+
+  before :all do
+    subject { Remq.new(db: 4).tap { |r| r.redis.flushdb } }
+  end
+
+  after :each do
+    subject.redis.flushdb
+  end
+
+  context "class" do
+    describe ".new" do
+      it "creates a Redis client when options hash missing" do
+        Remq.new.redis.should be_a(Redis)
+      end
+
+      it "takes a Redis client as an option" do
+        Remq.new(redis: (redis = Redis.new)).redis.should eql redis
+      end
+    end
+  end
+
+  describe ".publish" do
+    it "publishes a message to the given channel and returns an id" do
+      id = subject.publish('events.things', { test: 'one' })
+      id.should be_a String
+    end
+  end
+
+  describe ".consume" do
+    it "consumes messages published to the given channel" do
+      subject.publish('events.things', { 'test' => 'one' })
+      subject.publish('events.things', { 'test' => 'two' })
+      subject.publish('events.things', { 'test' => 'three' })
+
+      msgs = subject.consume('events.things')
+      msgs.should have(3).items
+      msgs[msgs.keys[0]].should eql({ 'test' => 'one' })
+      msgs[msgs.keys[2]].should eql({ 'test' => 'three' })
+    end
+
+    it "limits the messages returned to value given in the :limit option" do
+      subject.publish('events.things', { 'test' => 'one' })
+      subject.publish('events.things', { 'test' => 'two' })
+      subject.publish('events.things', { 'test' => 'three' })
+
+      msgs = subject.consume('events.things', limit: 2)
+      msgs.should have(2).items
+      msgs[msgs.keys[0]].should eql({ 'test' => 'one' })
+      msgs[msgs.keys[1]].should eql({ 'test' => 'two' })
+      msgs[msgs.keys[2]].should be_nil
+    end
+
+    it "returns messages published since the id given in the :cursor option" do
+      cursor = subject.publish('events.things', { 'test' => 'one' })
+      subject.publish('events.things', { 'test' => 'two' })
+      subject.publish('events.things', { 'test' => 'three' })
+
+      msgs = subject.consume('events.things', cursor: cursor)
+      msgs.should have(2).items
+      msgs[msgs.keys[0]].should eql({ 'test' => 'two' })
+      msgs[msgs.keys[1]].should eql({ 'test' => 'three' })
+    end
+  end
+end

data/vendor/remq/Readme.md

@@ -0,0 +1,41 @@
+# Remq
+
+Remq (pronounced 'rem-que') is two things: (1) A [Redis](http://redis.io)-based protocol defined by a collection of Lua scripts (this project) which effectively turn Redis into a capable message queue broker for fast inter-service communication. (2) Multiple client libraries using these scripts for building fast, persisted pub/sub message queues.
+
+  - Producers publish any string to a message channel and receive a unique message-id
+  - Consumers subscribe to message channels via polling with a cursor (allowing resume), or via Redis pub/sub
+  - Consumers can subscribe to multible queues at once using Redis key globbing (ie. `'events.*'`)
+  - Able to sustain ~15k messages/sec on loopback interface (1 producer -> 1 consumer)
+  - Consistent performance if Redis has enough memory (tested up to ~15m messages, 3GB in memory)
+  - Purge channels of old messages periodically to maintain performance
+
+NOTE: In early-stage development, API not locked.
+
+## Client Libraries
+
+- Node.js: [remq-node](https://github.com/kainosnoema/remq-node) (`npm install remq`)
+- Ruby: [remq-rb](https://github.com/kainosnoema/remq-rb) (`gem install remq`)
+
+## Usage
+
+This project includes just the core Lua scripts that define the Remq protocol. To use Remq to build a message queue, install Redis along with one or more of the client libraries listed above.
+
+Raw Redis syntax:
+
+**Producer:**
+``` sh
+redis> EVAL <publish.lua> 0 namespace channel message utcseconds
+# returns a unique message id
+```
+
+**Consumer:**
+``` sh
+redis> EVAL <consume.lua> 0 namespace channel cursor limit
+# returns each message followed by its id, just like ZRANGEBYSCORE
+```
+
+**Purge:**
+``` sh
+redis> EVAL <purge.lua> 0 namespace channel <BEFORE id (or) KEEP count>
+# returns the count of messages purged
+```

data/vendor/remq/scripts/consume.lua

@@ -0,0 +1,45 @@
+local namespace, channel, cursor, limit = ARGV[1], ARGV[2], ARGV[3], ARGV[4]
+
+limit = math.min(limit, 3999) -- 3999 is the limit of unpack()
+
+local channel_key = namespace .. ':channel:' .. channel
+
+-- for results from multiple channels, we'll merge them into a single set
+local union_key
+if string.find(channel_key, '*') then
+  -- if the pattern matches multiple keys, we have to merge the keys
+  -- we could use zunionstore here, but it wouldn't be optimal for very large sets
+  local matched_keys = redis.call('keys', channel_key)
+  if #matched_keys > 1 then
+    union_key = channel_key .. '@' .. redis.call('get', namespace .. ':id')
+    for i,key in ipairs(matched_keys) do
+      local msgs_ids = redis.call('zrangebyscore', key, '(' .. cursor, '+inf', 'WITHSCORES', 'LIMIT', 0, limit)
+      if #msgs_ids > 0 then
+        -- `zadd` takes scores first, so we have to reverse
+        local len, reversed = #msgs_ids, {}
+        for i = len, 1, -1 do reversed[len - i + 1] = msgs_ids[i] end
+        redis.call('zadd', union_key, unpack(reversed))
+      end
+    end
+    channel_key = union_key
+  else
+    channel_key = matched_keys[1]
+  end
+end
+
+-- as long as we have a channel key, get the messages and add wrap with them with ids
+local msgs = {}
+if channel_key ~= nil then
+  msgs = redis.call('zrangebyscore', channel_key, '(' .. cursor, '+inf', 'WITHSCORES', 'LIMIT', 0, limit)
+  -- zset decimal precision isn't great enough to retain utc seconds, so we have to round
+  for i,key in ipairs(msgs) do
+    if i % 2 == 0 then msgs[i] = string.format("%.10f", msgs[i]) end
+  end
+end
+
+-- if we've merged multiple channels, remove the union key
+if union_key ~= nil then
+  redis.call('del', union_key)
+end
+
+return msgs

data/vendor/remq/scripts/publish.lua

@@ -0,0 +1,12 @@
+local namespace, channel, msg, utc_sec = ARGV[1], ARGV[2], ARGV[3], ARGV[4]
+local channel_key = namespace .. ':channel:' .. channel
+
+-- ids are an incrementing integer followed by UTC time as a decimal value
+local id = redis.call('incr', namespace .. ':id') .. '.' .. (utc_sec or 0)
+
+redis.call('zadd', channel_key, id, msg)
+
+redis.call('publish', channel_key, msg)
+redis.call('publish', namespace .. ':stats:' .. channel, id)
+
+return id

data/vendor/remq/scripts/purge.lua

@@ -0,0 +1,18 @@
+local namespace, channel, cmd, value = ARGV[1], ARGV[2], ARGV[3], ARGV[4]
+local channel_key = namespace .. ':channel:' .. channel
+
+local matched_keys = { channel_key }
+if string.find(channel_key, '*') then
+  matched_keys = redis.call('keys', channel_key)
+end
+
+local purged = 0
+for i,key in ipairs(matched_keys) do
+  if cmd == 'BEFORE' then
+    purged = purged + redis.call('zremrangebyscore', key, '-inf', '(' .. value)
+  elseif cmd == 'KEEP' then
+    purged = purged + redis.call('zremrangebyrank', key, 0, 0 - (value - 1))
+  end
+end
+
+return purged

metadata

@@ -0,0 +1,110 @@
+--- !ruby/object:Gem::Specification
+name: remq
+version: !ruby/object:Gem::Version
+  version: 0.0.1a
+  prerelease: 5
+platform: ruby
+authors:
+- Evan Owen
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-07-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: redis
+  requirement: &70122011105460 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 3.0.1
+  type: :runtime
+  prerelease: false
+  version_requirements: *70122011105460
+- !ruby/object:Gem::Dependency
+  name: multi_json
+  requirement: &70122011104960 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: *70122011104960
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: &70122011104580 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: *70122011104580
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: &70122011103980 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '2.6'
+  type: :development
+  prerelease: false
+  version_requirements: *70122011103980
+description: ! "    Remq is a Redis-based protocol for building fast, persistent\n
+  \   pub/sub message queues.\n\n    The Remq protocol is defined by a collection
+  of Lua scripts\n    (located at https://github.com/kainosnoema/remq) which effectively\n
+  \   turn Redis into a capable message queue broker for fast inter-service\n    communication.
+  The Remq Ruby client library is built on top of these\n    scripts, making it easy
+  to build fast, persisted pub/sub message queues.\n"
+email: kainosnoema@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .gitmodules
+- LICENSE
+- Readme.md
+- lib/remq.rb
+- lib/remq/coder.rb
+- lib/remq/multi_json_coder.rb
+- lib/remq/script.rb
+- lib/remq/version.rb
+- remq.gemspec
+- spec/remq_spec.rb
+- vendor/remq/Readme.md
+- vendor/remq/scripts/consume.lua
+- vendor/remq/scripts/publish.lua
+- vendor/remq/scripts/purge.lua
+homepage: http://github.com/kainosnoema/remq
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>'
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.11
+signing_key: 
+specification_version: 3
+summary: A Remq client library for Ruby.
+test_files:
+- spec/remq_spec.rb
+has_rdoc: