disque 0.0.1.alpha → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6753b7f8705cec0eefb2a66dbf7a4fe7db031912
-  data.tar.gz: b25be6917de9eff8d7a85a955774d4ef91851541
+  metadata.gz: a89227c0d69455fc6897e54b94b06239d0e0ad56
+  data.tar.gz: a913c4b33ba3bee489f1a87a442675cb0b8d7128
 SHA512:
-  metadata.gz: 301fac4892ef2d456611401f668301c7536a94428e0e0e708bdb4671dc236a3c947ce6b091ebcdf167722944bc08b94f6ecdaa372b9ddbec423e3556fc57e19f
-  data.tar.gz: 01141bcf43fe5c08a4325631d2932766ec71910b006f75d9d3a78ee65139ea7913ec4d6226a4b6d2b01f68b29038ed9ff6895f921ac228c08bf8f7a18c42d669
+  metadata.gz: a64e05ec0ffe98863874e148b7ccad5ab34587b5edcf10a5f8dcef1149f67f0a595d96b2f25b61f472e50d53783c80eb66bdc29806280a07b53ac9f42e55a512
+  data.tar.gz: ff5b61bd64b033056a42d5f5d162fa7fb8c8e4a57e484f1aa7f91a344ad4a74941a0b8484618b4a8a39d4a47c6875b89c674e46e07667bb6799d5fe62ae8ba87

data/.gitignore

@@ -0,0 +1 @@
+/tmp

data/AUTHORS

@@ -0,0 +1,2 @@
+Michel Martens
+Damian Janowski

data/README.md

@@ -3,18 +3,39 @@ Disque.rb
 
 Client for Disque, an in-memory, distributed job queue.
 
-Status
-------
+Usage
+-----
 
-Disque is expected to be RESP compatible, and that means most Redis
-transport clients will be able to connect to it and run any command.
-This library is thus an imaginary client for Disque, but one that
-should work at least at a very basic level and connect to one node.
+Create a new Disque client by passing a list of nodes:
+
+```ruby
+client = Disque.new(["127.0.0.1:7711", "127.0.0.1:7712", "127.0.0.1:7713"])
+```
+
+Now you can add jobs:
+
+```ruby
+client.push("foo", "bar", 100)
+```
+
+It will push the job "bar" to the queue "foo" with a timeout of 100
+ms, and return the id of the job if it was received and replicated
+in time.
+
+Then, your workers will do something like this:
+
+```ruby
+loop do
+  client.fetch(from: ["foo"]) do |job|
+    # Do something with `job`
+  end
+end
+```
 
 Installation
 ------------
 
-You will be able to install it with RubyGems.
+You can install it using rubygems.
 
 ```
 $ gem install disque

data/disque.gemspec

@@ -2,12 +2,12 @@
 
 Gem::Specification.new do |s|
   s.name              = "disque"
-  s.version           = "0.0.1.alpha"
+  s.version           = "0.0.2"
   s.summary           = "Client for Disque"
-  s.description       = "Client for Disque"
-  s.authors           = ["Michel Martens"]
-  s.email             = ["michel@soveran.com"]
-  s.homepage          = "https://github.com/soveran/disque.rb"
+  s.description       = "Disque for Ruby"
+  s.authors           = ["Michel Martens", "Damian Janowski"]
+  s.email             = ["michel@soveran.com", "damian.janowski@gmail.com"]
+  s.homepage          = "https://github.com/soveran/disque-rb"
   s.files             = `git ls-files`.split("\n")
   s.license           = "MIT"
 

data/lib/disque.rb

@@ -1,7 +1,200 @@
 require "redic"
 
 class Disque
-  def initialize(url)
-    Redic.new(url)
+  ECONN = [
+    Errno::ECONNREFUSED,
+    Errno::EINVAL,
+  ]
+
+  attr :stats
+  attr :nodes
+  attr :prefix
+
+  # Create a new Disque client by passing a list of nodes.
+  #
+  #   Disque.new(["127.0.0.1:7711", "127.0.0.1:7712", "127.0.0.1:7713"])
+  #
+  # For each operation, a counter is updated to signal which node was
+  # the originator of the message. Based on that information, after
+  # a full cycle (1000 operations, but configurable on initialization)
+  # the stats are checked to see what is the most convenient node
+  # to connect to in order to avoid extra jumps.
+  #
+  # TODO Account for authentication
+  # TODO Account for timeout
+  def initialize(hosts, cycle: 1000)
+
+    # Cycle length
+    @cycle = cycle
+
+    # Operations counter
+    @count = 0
+
+    # Known nodes
+    @nodes = Hash.new
+
+    # Connection stats
+    @stats = Hash.new(0)
+
+    # Main client
+    @client = Redic.new
+
+    # Scout client
+    @scout = Redic.new
+
+    # Preferred client prefix
+    @prefix = nil
+
+    explore!(hosts)
+  end
+
+  def url(host)
+    sprintf("disque://%s", host)
+  end
+
+  # Collect the list of nodes by means of `CLUSTER NODES` and
+  # keep a connection to the node that provided that information.
+  def explore!(hosts)
+
+    # Reset nodes
+    @nodes.clear
+
+    hosts.each do |host|
+      begin
+        @scout.configure(url(host))
+
+        @scout.call("CLUSTER", "NODES").lines do |line|
+          id, host, flag = line.split
+
+          prefix = id[0,8]
+
+          if flag == "myself"
+
+            # Configure main client
+            @client.configure(@scout.url)
+
+            # Keep track of selected node
+            @prefix = prefix
+          end
+
+          @nodes[prefix] = host
+        end
+
+        @scout.quit
+
+        break
+
+      rescue *ECONN
+        $stderr.puts($!.inspect)
+      end
+    end
+
+    if @nodes.empty?
+      raise ArgumentError, "nodes unavailable"
+    end
+  end
+
+  def pick_client!
+    if @count == @cycle
+      @count = 0
+      prefix, _ = @stats.max { |a, b| a[1] <=> b[1] }
+
+      if prefix != @prefix
+        host = @nodes[prefix]
+
+        if host
+
+          # Reconfigure main client
+          @client.configure(url(host))
+          @prefix = prefix
+
+          # Reset stats for this new connection
+          @stats.clear
+        end
+      end
+    end
+  end
+
+  # Run commands on the active connection. If the
+  # connection is lost, new connections are tried
+  # until all nodes become unavailable.
+  def call(*args)
+    @client.call(*args)
+  rescue *ECONN
+    explore!(@nodes.values)
+    retry
+  end
+
+  # Disque's ADDJOB signature is as follows:
+  #
+  #     ADDJOB queue_name job <ms-timeout>
+  #       [REPLICATE <count>]
+  #       [DELAY <sec>]
+  #       [RETRY <sec>]
+  #       [TTL <sec>]
+  #       [MAXLEN <count>]
+  #       [ASYNC]
+  #
+  # You can pass any optional arguments as a hash,
+  # for example:
+  #
+  #     disque.push("foo", "myjob", 1000, ttl: 1, async: true)
+  #
+  # Note that `async` is a special case because it's just a
+  # flag. That's why `true` must be passed as its value.
+  def push(queue_name, job, ms_timeout, options = {})
+    command = ["ADDJOB", queue_name, job, ms_timeout]
+    command += options_to_arguments(options)
+
+    call(*command)
+  end
+
+  def fetch(from: [], count: 1, timeout: 0)
+    pick_client!
+
+    jobs = call(
+      "GETJOB",
+        "TIMEOUT", timeout,
+        "COUNT", count,
+        "FROM", *from)
+
+    if jobs then
+      @count += 1
+
+      jobs.each do |queue, msgid, job|
+
+        # Update stats
+        @stats[msgid[2,8]] += 1
+
+        if block_given?
+
+          # Process job
+          yield(job, queue)
+
+          # Remove job
+          call("ACKJOB", msgid)
+        end
+      end
+    end
+
+    return jobs
+  end
+
+  def options_to_arguments(options)
+    arguments = []
+
+    options.each do |key, value|
+      if value == true
+        arguments.push(key)
+      else
+        arguments.push(key, value)
+      end
+    end
+
+    return arguments
+  end
+
+  def quit
+    @client.quit
   end
 end

data/makefile

@@ -1,2 +1,29 @@
+DIR?=./tmp
+
+all: start test stop
+
 test:
 	RUBYLIB=./lib cutest tests/*.rb
+
+default:
+	@echo make \<port\>
+	@echo make \[start\|meet\|stop\|list\]
+
+start: 7711 7712 7713
+	@disque -p 7712 CLUSTER MEET 127.0.0.1 7711 > /dev/null
+	@disque -p 7713 CLUSTER MEET 127.0.0.1 7712 > /dev/null
+
+stop:
+	@kill `cat $(DIR)/disque.*.pid`
+
+%:
+	@disque-server \
+		--port $@ \
+		--dir $(DIR) \
+		--daemonize yes \
+		--bind 127.0.0.1 \
+		--loglevel notice \
+		--pidfile disque.$@.pid \
+		--appendfilename disque.$@.aof \
+		--cluster-config-file disque.$@.nodes \
+		--logfile disque.$@.log

data/tests/disque_test.rb

@@ -0,0 +1,237 @@
+require_relative "../lib/disque"
+require "stringio"
+require "fileutils"
+
+module Silencer
+  @output = nil
+
+  def self.start
+    $olderr = $stderr
+    $stderr = StringIO.new
+  end
+
+  def self.stop
+    @output = $stderr.string
+    $stderr = $olderr
+  end
+
+  def self.output
+    @output
+  end
+end
+
+DISQUE_NODES = [
+  "127.0.0.1:7710",
+  "127.0.0.1:7711",
+  "127.0.0.1:7712",
+  "127.0.0.1:7713",
+]
+
+DISQUE_BAD_NODES  = DISQUE_NODES[0,1]
+DISQUE_GOOD_NODES = DISQUE_NODES[1,3]
+
+test "raise if connection is not possible" do
+  Silencer.start
+  assert_raise(ArgumentError) do
+    c = Disque.new(DISQUE_BAD_NODES)
+  end
+  Silencer.stop
+
+  assert_equal "#<Errno::ECONNREFUSED: Can't connect to: disque://127.0.0.1:7710>\n", Silencer.output
+end
+
+test "retry until a connection is reached" do
+  Silencer.start
+  c = Disque.new(DISQUE_NODES)
+  Silencer.stop
+
+  assert_equal "#<Errno::ECONNREFUSED: Can't connect to: disque://127.0.0.1:7710>\n", Silencer.output
+  assert_equal "PONG", c.call("PING")
+end
+
+test "lack of jobs" do
+  c = Disque.new(DISQUE_GOOD_NODES)
+  reached = false
+
+  c.fetch(from: ["foo"], timeout: 1) do |job|
+    reached = true
+  end
+
+  assert_equal false, reached
+end
+
+test "one job" do
+  c = Disque.new(DISQUE_GOOD_NODES)
+
+  c.push("foo", "bar", 1000)
+
+  c.fetch(from: ["foo"], count: 10) do |job, queue|
+    assert_equal "bar", job
+  end
+end
+
+test "multiple jobs" do
+  c = Disque.new(DISQUE_GOOD_NODES)
+
+  c.push("foo", "bar", 1000)
+  c.push("foo", "baz", 1000)
+
+  jobs = ["baz", "bar"]
+
+  c.fetch(from: ["foo"], count: 10) do |job, queue|
+    assert_equal jobs.pop, job
+    assert_equal "foo", queue
+  end
+
+  assert jobs.empty?
+end
+
+test "multiple queues" do
+  c = Disque.new(DISQUE_GOOD_NODES)
+
+  c.push("foo", "bar", 1000)
+  c.push("qux", "baz", 1000)
+
+  queues = ["qux", "foo"]
+  jobs = ["baz", "bar"]
+
+  result = c.fetch(from: ["foo", "qux"], count: 10) do |job, queue|
+    assert_equal jobs.pop, job
+    assert_equal queues.pop, queue
+  end
+
+  assert jobs.empty?
+  assert queues.empty?
+end
+
+test "add jobs with other parameters" do
+  c = Disque.new(DISQUE_GOOD_NODES)
+
+  c.push("foo", "bar", 1000, async: true, ttl: 0)
+
+  sleep 0.1
+
+  queues = ["foo"]
+  jobs = ["bar"]
+
+  result = c.fetch(from: ["foo"], count: 10, timeout: 1) do |job, queue|
+    assert_equal jobs.pop, job
+    assert_equal queues.pop, queue
+  end
+
+  assert_equal ["bar"], jobs
+  assert_equal ["foo"], queues
+end
+
+test "connect to the best node" do
+  c1 = Disque.new([DISQUE_GOOD_NODES[0]], cycle: 2)
+  c2 = Disque.new([DISQUE_GOOD_NODES[1]], cycle: 2)
+
+  assert c1.prefix != c2.prefix
+
+  # Tamper stats to trigger a reconnection
+  c1.stats[c2.prefix] = 10
+
+  c1.push("q1", "j1", 1000)
+  c1.push("q1", "j2", 1000)
+
+  c2.push("q1", "j3", 1000)
+
+  c1.fetch(from: ["q1"])
+  c1.fetch(from: ["q1"])
+  c1.fetch(from: ["q1"])
+
+  # Client should have reconnected
+  assert c1.prefix == c2.prefix
+end
+
+test "connect to the best node, part 2" do
+  c1 = Disque.new([DISQUE_GOOD_NODES[0]], cycle: 2)
+  c2 = Disque.new([DISQUE_GOOD_NODES[1]], cycle: 2)
+
+  assert c1.prefix != c2.prefix
+
+  c1.push("q1", "j1", 0)
+  c1.push("q1", "j2", 0)
+  c1.push("q1", "j3", 0)
+
+  c2.fetch(from: ["q1"])
+  c2.fetch(from: ["q1"])
+  c2.fetch(from: ["q1"])
+
+  # Client should have reconnected
+  assert c1.prefix == c2.prefix
+end
+
+test "recover after node disconnection" do
+  c1 = Disque.new([DISQUE_GOOD_NODES[0]], cycle: 2)
+
+  prefix = c1.prefix
+
+  # Tamper stats to trigger a reconnection to a bad node
+  c1.stats["fake"] = 10
+  c1.nodes["fake"] = DISQUE_BAD_NODES[0]
+
+  # Delete the other nodes just in case
+  c1.nodes.delete_if do |key, val|
+    key != prefix &&
+    key != "fake"
+  end
+
+  c1.push("q1", "j1", 1000)
+  c1.push("q1", "j2", 1000)
+  c1.push("q1", "j3", 1000)
+
+  c1.fetch(from: ["q1"])
+  c1.fetch(from: ["q1"])
+  c1.fetch(from: ["q1"])
+
+  # Prefix should stay the same
+  assert prefix == c1.prefix
+end
+
+test "federation" do
+  c1 = Disque.new([DISQUE_GOOD_NODES[0]], cycle: 2)
+  c2 = Disque.new([DISQUE_GOOD_NODES[1]], cycle: 2)
+
+  c1.push("q1", "j1", 0)
+
+  c2.fetch(from: ["q1"], count: 10) do |job, queue|
+    assert_equal "j1", job
+  end
+end
+
+test "ack jobs when block is given" do
+  c = Disque.new(DISQUE_GOOD_NODES)
+
+  c.push("q1", "j1", 1000)
+
+  id = nil
+
+  _, id, _ = c.fetch(from: ["q1"]) { |*a| }[0]
+
+  assert id
+
+  info = Hash[*c.call("SHOW", id)]
+
+  if info.any?
+
+    # If the test runs too fast, we may get the job
+    # with the status set to "acked"
+    assert_equal "acked", info.fetch("state")
+  end
+end
+
+test "don't ack jobs when no block is given" do
+  c = Disque.new(DISQUE_GOOD_NODES)
+
+  c.push("q1", "j1", 1000)
+
+  _, id, _ = c.fetch(from: ["q1"])[0]
+
+  assert id
+
+  info = Hash[*c.call("SHOW", id)]
+
+  assert_equal info.fetch("state"), "active"
+end

metadata

@@ -1,14 +1,15 @@
 --- !ruby/object:Gem::Specification
 name: disque
 version: !ruby/object:Gem::Version
-  version: 0.0.1.alpha
+  version: 0.0.2
 platform: ruby
 authors:
 - Michel Martens
+- Damian Janowski
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-16 00:00:00.000000000 Z
+date: 2015-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: redic
@@ -24,20 +25,24 @@ dependencies:
     - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: Client for Disque
+description: Disque for Ruby
 email:
 - michel@soveran.com
+- damian.janowski@gmail.com
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - .gems
+- .gitignore
+- AUTHORS
 - LICENSE
 - README.md
 - disque.gemspec
 - lib/disque.rb
 - makefile
-homepage: https://github.com/soveran/disque.rb
+- tests/disque_test.rb
+homepage: https://github.com/soveran/disque-rb
 licenses:
 - MIT
 metadata: {}
@@ -52,12 +57,12 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>'
+  - - '>='
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.0.3
 signing_key: 
 specification_version: 4
 summary: Client for Disque