remq 0.0.1a → 0.0.3

data/vendor/remq/Readme.md

@@ -1,41 +1,55 @@
 # Remq
 
-Remq (pronounced 'rem-que') is two things: (1) A [Redis](http://redis.io)-based protocol defined by a collection of Lua scripts (this project) which effectively turn Redis into a capable message queue broker for fast inter-service communication. (2) Multiple client libraries using these scripts for building fast, persisted pub/sub message queues.
+Remq (pronounced 'rem-que') is two things: (1) A [Redis](http://redis.io)-based
+protocol defined by a collection of Lua scripts (this project) which effectively
+turn Redis into a durable message queue broker for fast, reliable inter-service
+communication. (2) Multiple client libraries using these scripts for building
+fast, persisted pub/sub message queues.
 
   - Producers publish any string to a message channel and receive a unique message-id
-  - Consumers subscribe to message channels via polling with a cursor (allowing resume), or via Redis pub/sub
-  - Consumers can subscribe to multible queues at once using Redis key globbing (ie. `'events.*'`)
-  - Able to sustain ~15k messages/sec on loopback interface (1 producer -> 1 consumer)
-  - Consistent performance if Redis has enough memory (tested up to ~15m messages, 3GB in memory)
-  - Purge channels of old messages periodically to maintain performance
+  - Consumers subscribe to message channels via vanilla Redis pub/sub for instant delivery
+  - Subscribe to multiple channels using Redis key globbing (ie. `'events.*'`)
+  - Replay archived messages from a given message-id for failure recovery
+  - Able to sustain ~35k messages/sec on loopback interface (1 producer -> 1 consumer)
+  - Consistent performance up to system memory limit (tested to ~25m messages in 4GB memory)
+  - Channels may be flushed of old messages periodically to reduce memory footprint
 
-NOTE: In early-stage development, API not locked.
+**WARNING**: In early-stage development, API not stable. If you've used a previous
+version of these scripts, you'll most likely have to clear all previously
+published messages in order to upgrade to the latest version.
 
 ## Client Libraries
 
 - Node.js: [remq-node](https://github.com/kainosnoema/remq-node) (`npm install remq`)
-- Ruby: [remq-rb](https://github.com/kainosnoema/remq-rb) (`gem install remq`)
+- Ruby: [remq-rb](https://github.com/kainosnoema/remq-rb) (`gem install remq --pre`)
 
 ## Usage
 
-This project includes just the core Lua scripts that define the Remq protocol. To use Remq to build a message queue, install Redis along with one or more of the client libraries listed above.
+This project includes just the core Lua scripts that define the Remq protocol.
+To use Remq to build a message queue, install Redis along with one or more of
+the client libraries listed above.
 
 Raw Redis syntax:
 
 **Producer:**
 ``` sh
-redis> EVAL <publish.lua> 0 namespace channel message utcseconds
-# returns a unique message id
+redis> EVAL <publish.lua> 0 <channel> <message>
+# returns a unique message-id
 ```
 
 **Consumer:**
 ``` sh
-redis> EVAL <consume.lua> 0 namespace channel cursor limit
-# returns each message followed by its id, just like ZRANGEBYSCORE
+redis> PSUBSCRIBE remq:channel:<pattern>
+# messages are published in the format "<channel>@<id>\n<message>"
+```
+or
+``` sh
+redis> EVAL <consume.lua> 0 <pattern> <cursor> <limit>
+# messages are returned in the format "<channel>@<id>\n<message>"
 ```
 
-**Purge:**
+**Flush:**
 ``` sh
-redis> EVAL <purge.lua> 0 namespace channel <BEFORE id (or) KEEP count>
-# returns the count of messages purged
+redis> EVAL <flush.lua> 0 <pattern> <before-message-id>
+# returns the count of messages flushed
 ```

data/vendor/remq/VERSION

@@ -0,0 +1 @@
+0.0.3

data/vendor/remq/scripts/consume.lua

@@ -1,45 +1,31 @@
-local namespace, channel, cursor, limit = ARGV[1], ARGV[2], ARGV[3], ARGV[4]
+local pattern, cursor, limit = ARGV[1], ARGV[2], ARGV[3]
 
-limit = math.min(limit, 3999) -- 3999 is the limit of unpack()
+-- convert Redis-style globbing to a Lua Pattern
+pattern = '^' .. pattern:gsub('%.', '%%.'):gsub('%*', '.*') .. '@\%d+\n'
 
-local channel_key = namespace .. ':channel:' .. channel
+cursor = math.max(cursor or 0, 0)
+limit = math.min(math.max(limit or 1000, 0), 1000)
 
--- for results from multiple channels, we'll merge them into a single set
-local union_key
-if string.find(channel_key, '*') then
-  -- if the pattern matches multiple keys, we have to merge the keys
-  -- we could use zunionstore here, but it wouldn't be optimal for very large sets
-  local matched_keys = redis.call('keys', channel_key)
-  if #matched_keys > 1 then
-    union_key = channel_key .. '@' .. redis.call('get', namespace .. ':id')
-    for i,key in ipairs(matched_keys) do
-      local msgs_ids = redis.call('zrangebyscore', key, '(' .. cursor, '+inf', 'WITHSCORES', 'LIMIT', 0, limit)
-      if #msgs_ids > 0 then
-        -- `zadd` takes scores first, so we have to reverse
-        local len, reversed = #msgs_ids, {}
-        for i = len, 1, -1 do reversed[len - i + 1] = msgs_ids[i] end
-        redis.call('zadd', union_key, unpack(reversed))
-      end
-    end
-    channel_key = union_key
-  else
-    channel_key = matched_keys[1]
-  end
-end
+local matched, per_loop, per_bucket = {}, limit, 100000
+while true do
+  local bucket = math.floor(cursor / per_bucket) * per_bucket
+  local unfiltered = redis.call(
+    'zrangebyscore', 'remq:archive:' .. bucket,
+    '(' .. cursor, '+inf', 'LIMIT', 0, per_loop
+  )
 
--- as long as we have a channel key, get the messages and add wrap with them with ids
-local msgs = {}
-if channel_key ~= nil then
-  msgs = redis.call('zrangebyscore', channel_key, '(' .. cursor, '+inf', 'WITHSCORES', 'LIMIT', 0, limit)
-  -- zset decimal precision isn't great enough to retain utc seconds, so we have to round
-  for i,key in ipairs(msgs) do
-    if i % 2 == 0 then msgs[i] = string.format("%.10f", msgs[i]) end
+  if #unfiltered == 0 then
+    return matched -- end of the timeline
   end
-end
 
--- if we've merged multiple channels, remove the union key
-if union_key ~= nil then
-  redis.call('del', union_key)
-end
+  for i=1, #unfiltered do
+    if unfiltered[i]:match(pattern) then
+      matched[#matched + 1] = unfiltered[i]
+      if #matched == limit then
+        return matched -- reached the limit
+      end
+    end
+  end
 
-return msgs
+  cursor = cursor + per_loop
+end

data/vendor/remq/scripts/flush.lua

@@ -0,0 +1,33 @@
+local pattern, cursor = ARGV[1], ARGV[2]
+
+-- convert Redis-style globbing to a Lua Pattern
+pattern = '^' .. pattern:gsub('%.', '%%.'):gsub('%*', '.*') .. '@\%d+\n'
+
+local flushed, per_loop, per_bucket = 0, 1000, 100000
+while true do
+  local bucket = math.floor((cursor - 1) / per_bucket) * per_bucket
+  local prev_cursor = 0 + cursor - math.min(cursor - bucket, per_loop)
+
+  local unfiltered = redis.call(
+    'zrangebyscore', 'remq:archive:' .. bucket,
+    prev_cursor, '(' .. cursor
+  )
+
+  if #unfiltered == 0 then
+    return flushed -- end of the timeline
+  end
+
+  local matched = {}
+  for i=1, #unfiltered do
+    if unfiltered[i]:match(pattern) then
+      matched[#matched + 1] = unfiltered[i]
+    end
+  end
+
+  if #matched > 0 then
+    redis.call('zrem', 'remq:archive:' .. bucket, unpack(matched))
+    flushed = flushed + #matched
+  end
+
+  cursor = prev_cursor
+end

data/vendor/remq/scripts/publish.lua

@@ -1,12 +1,15 @@
-local namespace, channel, msg, utc_sec = ARGV[1], ARGV[2], ARGV[3], ARGV[4]
-local channel_key = namespace .. ':channel:' .. channel
+local channel, msg = ARGV[1], ARGV[2]
 
--- ids are an incrementing integer followed by UTC time as a decimal value
-local id = redis.call('incr', namespace .. ':id') .. '.' .. (utc_sec or 0)
+-- ids are an incrementing double precision integer
+local id, per_bucket = redis.call('incr', 'remq:message-id'), 100000
 
-redis.call('zadd', channel_key, id, msg)
+-- prefix message with header in the format: "<channel>@<id>\n<message>"
+msg = channel .. '@' .. id .. '\n' .. msg
 
-redis.call('publish', channel_key, msg)
-redis.call('publish', namespace .. ':stats:' .. channel, id)
+-- split into buckets every 100,000 to allow 4x10^14 (400 trillion) messages
+local bucket = 'remq:archive:' .. math.floor(id / per_bucket) * per_bucket
+
+redis.call('zadd', bucket, id, msg) -- add to bucket
+redis.call('publish', 'remq:channel:' .. channel, msg) --  publish to pub/sub
 
 return id