queue_classic 1.0.0 → 1.0.1

data/lib/queue_classic.rb

@@ -1,17 +1,17 @@
-require 'pg'
+require "pg"
 
-require 'logger'
-require 'json'
-require 'uri'
+require "logger"
+require "uri"
 
-$: << File.expand_path(__FILE__, 'lib')
+$: << File.expand_path(__FILE__, "lib")
 
-require 'queue_classic/durable_array'
-require 'queue_classic/database'
-require 'queue_classic/worker'
-require 'queue_classic/logger'
-require 'queue_classic/queue'
-require 'queue_classic/job'
+require "queue_classic/okjson"
+require "queue_classic/durable_array"
+require "queue_classic/database"
+require "queue_classic/worker"
+require "queue_classic/logger"
+require "queue_classic/queue"
+require "queue_classic/job"
 
 module QC
   VERBOSE = ENV["VERBOSE"] || ENV["QC_VERBOSE"]

data/lib/queue_classic/database.rb

@@ -3,39 +3,47 @@ module QC
 
     @@connection = nil
 
-    DATABASE_URL        = (ENV["QC_DATABASE_URL"] || ENV["DATABASE_URL"])
-    MAX_TOP_BOUND       = (ENV["QC_TOP_BOUND"] || 9).to_i
-    NOTIFY_TIMEOUT      = (ENV["QC_NOTIFY_TIMEOUT"] || 10).to_i
-    DEFAULT_QUEUE_NAME  = "queue_classic_jobs"
-
     attr_reader :table_name
+    attr_reader :top_boundary
 
     def initialize(queue_name=nil)
       log("initialized")
 
-      @top_boundry = MAX_TOP_BOUND
-      log("top_boundry=#{@top_boundry}")
+      @top_boundary = (ENV["QC_TOP_BOUND"] || 9).to_i
+      log("top_boundary=#{@top_boundary}")
 
-      @table_name = queue_name || DEFAULT_QUEUE_NAME
+      @table_name = queue_name || "queue_classic_jobs"
       log("table_name=#{@table_name}")
 
-      @db_params = URI.parse(DATABASE_URL)
-      log("uri=#{DATABASE_URL}")
+      @channel_name = @table_name
+      log("channel_name=#{@channel_name}")
+
+      db_url = (ENV["QC_DATABASE_URL"] || ENV["DATABASE_URL"])
+      @db_params = URI.parse(db_url)
+      log("uri=#{db_url}")
+    end
+
+    def set_application_name
+      execute("SET application_name = 'queue_classic'")
+    end
+
+    def escape(string)
+      connection.escape(string)
     end
 
     def notify
       log("NOTIFY")
-      execute("NOTIFY queue_classic_jobs")
+      execute("NOTIFY #{@channel_name}")
     end
 
     def listen
       log("LISTEN")
-      execute("LISTEN queue_classic_jobs")
+      execute("LISTEN #{@channel_name}")
     end
 
     def unlisten
       log("UNLISTEN")
-      execute("UNLISTEN queue_classic_jobs")
+      execute("UNLISTEN #{@channel_name}")
     end
 
     def drain_notify
@@ -50,12 +58,29 @@ module QC
       log("done waiting for notify")
     end
 
-    def execute(sql)
-      log("executing=#{sql}")
+    def transaction
+      begin
+        execute 'BEGIN'
+        yield
+        execute 'COMMIT'
+      rescue Exception
+        execute 'ROLLBACK'
+        raise
+      end
+    end
+
+    def transaction_idle?
+      connection.transaction_status == PGconn::PQTRANS_IDLE
+    end
+
+    def execute(sql, *params)
+      log("executing #{sql.inspect}, #{params.inspect}")
       begin
-        connection.exec(sql)
+        params = nil if params.empty?
+        connection.exec(sql, params)
       rescue PGError => e
         log("execute exception=#{e.inspect}")
+        raise
       end
     end
 
@@ -81,8 +106,6 @@ module QC
       if conn.status != PGconn::CONNECTION_OK
         log("connection error=#{conn.error}")
       end
-      log("setting application name to queue_classic")
-      conn.exec("SET application_name = 'queue_classic'")
       conn
     end
 
@@ -93,7 +116,7 @@ module QC
         -- have identical columns to queue_classic_jobs.
         -- When QC supports queues with columns other than the default, we will have to change this.
 
-        CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$
+        CREATE OR REPLACE FUNCTION lock_head(tname name, top_boundary integer) RETURNS SETOF queue_classic_jobs AS $$
         DECLARE
           unlocked integer;
           relative_top integer;
@@ -103,20 +126,24 @@ module QC
           -- The select count(*) is going to slow down dequeue performance but allow
           -- for more workers. Would love to see some optimization here...
 
-          SELECT TRUNC(random() * #{@top_boundry} + 1) INTO relative_top;
-          EXECUTE 'SELECT count(*) FROM' || tname || '' INTO job_count;
-          IF job_count < 10 THEN
+          EXECUTE 'SELECT count(*) FROM ' ||
+            '(SELECT * FROM ' || quote_ident(tname) ||
+            ' LIMIT ' || quote_literal(top_boundary) || ') limited'
+            INTO job_count;
+
+          SELECT TRUNC(random() * top_boundary + 1) INTO relative_top;
+          IF job_count < top_boundary THEN
             relative_top = 0;
           END IF;
 
           LOOP
             BEGIN
               EXECUTE 'SELECT id FROM '
-                || tname::regclass
+                || quote_ident(tname)
                 || ' WHERE locked_at IS NULL'
                 || ' ORDER BY id ASC'
                 || ' LIMIT 1'
-                || ' OFFSET ' || relative_top
+                || ' OFFSET ' || quote_literal(relative_top)
                 || ' FOR UPDATE NOWAIT'
               INTO unlocked;
               EXIT;
@@ -127,7 +154,7 @@ module QC
           END LOOP;
 
           RETURN QUERY EXECUTE 'UPDATE '
-            || tname::regclass
+            || quote_ident(tname)
             || ' SET locked_at = (CURRENT_TIMESTAMP)'
             || ' WHERE id = $1'
             || ' AND locked_at is NULL'
@@ -137,6 +164,19 @@ module QC
           RETURN;
         END;
         $$ LANGUAGE plpgsql;
+
+        CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$
+        BEGIN
+          RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
+        END;
+        $$ LANGUAGE plpgsql;
+     EOD
+    end
+
+    def unload_functions
+      execute(<<-EOD)
+        DROP FUNCTION IF EXISTS lock_head(tname varchar);
+        DROP FUNCTION IF EXISTS lock_head(tname name, top_boundary integer);
       EOD
     end
 

data/lib/queue_classic/durable_array.rb

@@ -4,10 +4,11 @@ module QC
     def initialize(database)
       @database = database
       @table_name = @database.table_name
+      @top_boundary = @database.top_boundary
     end
 
     def <<(details)
-      execute("INSERT INTO #{@table_name} (details) VALUES ('#{JSON.dump(details)}')")
+      execute("INSERT INTO #{@table_name} (details) VALUES ($1)", OkJson.encode(details))
       @database.notify if ENV["QC_LISTENING_WORKER"] == "true"
     end
 
@@ -16,24 +17,20 @@ module QC
     end
 
     def delete(job)
-      execute("DELETE FROM #{@table_name} WHERE id = #{job.id}")
+      execute("DELETE FROM #{@table_name} WHERE id = $1;", job.id)
       job
     end
 
-    def find(job)
-      find_one {"SELECT * FROM #{@table_name} WHERE id = #{job.id}"}
-    end
-
     def search_details_column(q)
-      find_many { "SELECT * FROM #{@table_name} WHERE details LIKE '%#{q}%'" }
+      find_many { ["SELECT * FROM #{@table_name} WHERE details LIKE $1;", "%#{q}%"] }
     end
 
     def first
-      find_one { "SELECT * FROM lock_head('#{@table_name}')" }
+      find_one { ["SELECT * FROM lock_head($1, $2);", @table_name, @top_boundary] }
     end
 
     def each
-      execute("SELECT * FROM #{@table_name} ORDER BY id ASC").each do |r|
+      execute("SELECT * FROM #{@table_name} ORDER BY id ASC;").each do |r|
         yield Job.new(r)
       end
     end
@@ -43,11 +40,11 @@ module QC
     end
 
     def find_many
-      execute(yield).map {|r| Job.new(r)}
+      execute(*yield).map { |r| Job.new(r) }
     end
 
-    def execute(sql)
-      @database.execute(sql)
+    def execute(sql, *params)
+      @database.execute(sql, *params)
     end
 
   end

data/lib/queue_classic/job.rb

@@ -4,7 +4,7 @@ module QC
 
     def initialize(args={})
       @id        = args["id"]
-      @details   = JSON.parse(args["details"])
+      @details   = OkJson.decode(args["details"])
       @locked_at = args["locked_at"]
     end
 

data/lib/queue_classic/okjson.rb

@@ -0,0 +1,583 @@
+module QC
+  # Copyright 2011 Keith Rarick
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in
+# all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+# THE SOFTWARE.
+
+# See https://github.com/kr/okjson for updates.
+
+require 'stringio'
+
+# Some parts adapted from
+# http://golang.org/src/pkg/json/decode.go and
+# http://golang.org/src/pkg/utf8/utf8.go
+module OkJson
+  extend self
+
+
+  # Decodes a json document in string s and
+  # returns the corresponding ruby value.
+  # String s must be valid UTF-8. If you have
+  # a string in some other encoding, convert
+  # it first.
+  #
+  # String values in the resulting structure
+  # will be UTF-8.
+  def decode(s)
+    ts = lex(s)
+    v, ts = textparse(ts)
+    if ts.length > 0
+      raise Error, 'trailing garbage'
+    end
+    v
+  end
+
+
+  # Parses a "json text" in the sense of RFC 4627.
+  # Returns the parsed value and any trailing tokens.
+  # Note: this is almost the same as valparse,
+  # except that it does not accept atomic values.
+  def textparse(ts)
+    if ts.length < 0
+      raise Error, 'empty'
+    end
+
+    typ, _, val = ts[0]
+    case typ
+    when '{' then objparse(ts)
+    when '[' then arrparse(ts)
+    else
+      raise Error, "unexpected #{val.inspect}"
+    end
+  end
+
+
+  # Parses a "value" in the sense of RFC 4627.
+  # Returns the parsed value and any trailing tokens.
+  def valparse(ts)
+    if ts.length < 0
+      raise Error, 'empty'
+    end
+
+    typ, _, val = ts[0]
+    case typ
+    when '{' then objparse(ts)
+    when '[' then arrparse(ts)
+    when :val,:str then [val, ts[1..-1]]
+    else
+      raise Error, "unexpected #{val.inspect}"
+    end
+  end
+
+
+  # Parses an "object" in the sense of RFC 4627.
+  # Returns the parsed value and any trailing tokens.
+  def objparse(ts)
+    ts = eat('{', ts)
+    obj = {}
+
+    if ts[0][0] == '}'
+      return obj, ts[1..-1]
+    end
+
+    k, v, ts = pairparse(ts)
+    obj[k] = v
+
+    if ts[0][0] == '}'
+      return obj, ts[1..-1]
+    end
+
+    loop do
+      ts = eat(',', ts)
+
+      k, v, ts = pairparse(ts)
+      obj[k] = v
+
+      if ts[0][0] == '}'
+        return obj, ts[1..-1]
+      end
+    end
+  end
+
+
+  # Parses a "member" in the sense of RFC 4627.
+  # Returns the parsed values and any trailing tokens.
+  def pairparse(ts)
+    (typ, _, k), ts = ts[0], ts[1..-1]
+    if typ != :str
+      raise Error, "unexpected #{k.inspect}"
+    end
+    ts = eat(':', ts)
+    v, ts = valparse(ts)
+    [k, v, ts]
+  end
+
+
+  # Parses an "array" in the sense of RFC 4627.
+  # Returns the parsed value and any trailing tokens.
+  def arrparse(ts)
+    ts = eat('[', ts)
+    arr = []
+
+    if ts[0][0] == ']'
+      return arr, ts[1..-1]
+    end
+
+    v, ts = valparse(ts)
+    arr << v
+
+    if ts[0][0] == ']'
+      return arr, ts[1..-1]
+    end
+
+    loop do
+      ts = eat(',', ts)
+
+      v, ts = valparse(ts)
+      arr << v
+
+      if ts[0][0] == ']'
+        return arr, ts[1..-1]
+      end
+    end
+  end
+
+
+  def eat(typ, ts)
+    if ts[0][0] != typ
+      raise Error, "expected #{typ} (got #{ts[0].inspect})"
+    end
+    ts[1..-1]
+  end
+
+
+  # Scans s and returns a list of json tokens,
+  # excluding white space (as defined in RFC 4627).
+  def lex(s)
+    ts = []
+    while s.length > 0
+      typ, lexeme, val = tok(s)
+      if typ == nil
+        raise Error, "invalid character at #{s[0,10].inspect}"
+      end
+      if typ != :space
+        ts << [typ, lexeme, val]
+      end
+      s = s[lexeme.length..-1]
+    end
+    ts
+  end
+
+
+  # Scans the first token in s and
+  # returns a 3-element list, or nil
+  # if s does not begin with a valid token.
+  #
+  # The first list element is one of
+  # '{', '}', ':', ',', '[', ']',
+  # :val, :str, and :space.
+  #
+  # The second element is the lexeme.
+  #
+  # The third element is the value of the
+  # token for :val and :str, otherwise
+  # it is the lexeme.
+  def tok(s)
+    case s[0]
+    when ?{  then ['{', s[0,1], s[0,1]]
+    when ?}  then ['}', s[0,1], s[0,1]]
+    when ?:  then [':', s[0,1], s[0,1]]
+    when ?,  then [',', s[0,1], s[0,1]]
+    when ?[  then ['[', s[0,1], s[0,1]]
+    when ?]  then [']', s[0,1], s[0,1]]
+    when ?n  then nulltok(s)
+    when ?t  then truetok(s)
+    when ?f  then falsetok(s)
+    when ?"  then strtok(s)
+    when Spc then [:space, s[0,1], s[0,1]]
+    when ?\t then [:space, s[0,1], s[0,1]]
+    when ?\n then [:space, s[0,1], s[0,1]]
+    when ?\r then [:space, s[0,1], s[0,1]]
+    else          numtok(s)
+    end
+  end
+
+
+  def nulltok(s);  s[0,4] == 'null'  && [:val, 'null',  nil]   end
+  def truetok(s);  s[0,4] == 'true'  && [:val, 'true',  true]  end
+  def falsetok(s); s[0,5] == 'false' && [:val, 'false', false] end
+
+
+  def numtok(s)
+    m = /-?([1-9][0-9]+|[0-9])([.][0-9]+)?([eE][+-]?[0-9]+)?/.match(s)
+    if m && m.begin(0) == 0
+      if m[3] && !m[2]
+        [:val, m[0], Integer(m[1])*(10**Integer(m[3][1..-1]))]
+      elsif m[2]
+        [:val, m[0], Float(m[0])]
+      else
+        [:val, m[0], Integer(m[0])]
+      end
+    end
+  end
+
+
+  def strtok(s)
+    m = /"([^"\\]|\\["\/\\bfnrt]|\\u[0-9a-fA-F]{4})*"/.match(s)
+    if ! m
+      raise Error, "invalid string literal at #{abbrev(s)}"
+    end
+    [:str, m[0], unquote(m[0])]
+  end
+
+
+  def abbrev(s)
+    t = s[0,10]
+    p = t['`']
+    t = t[0,p] if p
+    t = t + '...' if t.length < s.length
+    '`' + t + '`'
+  end
+
+
+  # Converts a quoted json string literal q into a UTF-8-encoded string.
+  # The rules are different than for Ruby, so we cannot use eval.
+  # Unquote will raise an error if q contains control characters.
+  def unquote(q)
+    q = q[1...-1]
+    a = q.dup # allocate a big enough string
+    r, w = 0, 0
+    while r < q.length
+      c = q[r]
+      case true
+      when c == ?\\
+        r += 1
+        if r >= q.length
+          raise Error, "string literal ends with a \"\\\": \"#{q}\""
+        end
+
+        case q[r]
+        when ?",?\\,?/,?'
+          a[w] = q[r]
+          r += 1
+          w += 1
+        when ?b,?f,?n,?r,?t
+          a[w] = Unesc[q[r]]
+          r += 1
+          w += 1
+        when ?u
+          r += 1
+          uchar = begin
+            hexdec4(q[r,4])
+          rescue RuntimeError => e
+            raise Error, "invalid escape sequence \\u#{q[r,4]}: #{e}"
+          end
+          r += 4
+          if surrogate? uchar
+            if q.length >= r+6
+              uchar1 = hexdec4(q[r+2,4])
+              uchar = subst(uchar, uchar1)
+              if uchar != Ucharerr
+                # A valid pair; consume.
+                r += 6
+              end
+            end
+          end
+          w += ucharenc(a, w, uchar)
+        else
+          raise Error, "invalid escape char #{q[r]} in \"#{q}\""
+        end
+      when c == ?", c < Spc
+        raise Error, "invalid character in string literal \"#{q}\""
+      else
+        # Copy anything else byte-for-byte.
+        # Valid UTF-8 will remain valid UTF-8.
+        # Invalid UTF-8 will remain invalid UTF-8.
+        a[w] = c
+        r += 1
+        w += 1
+      end
+    end
+    a[0,w]
+  end
+
+
+  # Encodes unicode character u as UTF-8
+  # bytes in string a at position i.
+  # Returns the number of bytes written.
+  def ucharenc(a, i, u)
+    case true
+    when u <= Uchar1max
+      a[i] = (u & 0xff).chr
+      1
+    when u <= Uchar2max
+      a[i+0] = (Utag2 | ((u>>6)&0xff)).chr
+      a[i+1] = (Utagx | (u&Umaskx)).chr
+      2
+    when u <= Uchar3max
+      a[i+0] = (Utag3 | ((u>>12)&0xff)).chr
+      a[i+1] = (Utagx | ((u>>6)&Umaskx)).chr
+      a[i+2] = (Utagx | (u&Umaskx)).chr
+      3
+    else
+      a[i+0] = (Utag4 | ((u>>18)&0xff)).chr
+      a[i+1] = (Utagx | ((u>>12)&Umaskx)).chr
+      a[i+2] = (Utagx | ((u>>6)&Umaskx)).chr
+      a[i+3] = (Utagx | (u&Umaskx)).chr
+      4
+    end
+  end
+
+
+  def hexdec4(s)
+    if s.length != 4
+      raise Error, 'short'
+    end
+    (nibble(s[0])<<12) | (nibble(s[1])<<8) | (nibble(s[2])<<4) | nibble(s[3])
+  end
+
+
+  def subst(u1, u2)
+    if Usurr1 <= u1 && u1 < Usurr2 && Usurr2 <= u2 && u2 < Usurr3
+      return ((u1-Usurr1)<<10) | (u2-Usurr2) + Usurrself
+    end
+    return Ucharerr
+  end
+
+
+  def unsubst(u)
+    if u < Usurrself || u > Umax || surrogate?(u)
+      return Ucharerr, Ucharerr
+    end
+    u -= Usurrself
+    [Usurr1 + ((u>>10)&0x3ff), Usurr2 + (u&0x3ff)]
+  end
+
+
+  def surrogate?(u)
+    Usurr1 <= u && u < Usurr3
+  end
+
+
+  def nibble(c)
+    case true
+    when ?0 <= c && c <= ?9 then c.ord - ?0.ord
+    when ?a <= c && c <= ?z then c.ord - ?a.ord + 10
+    when ?A <= c && c <= ?Z then c.ord - ?A.ord + 10
+    else
+      raise Error, "invalid hex code #{c}"
+    end
+  end
+
+
+  # Encodes x into a json text. It may contain only
+  # Array, Hash, String, Numeric, true, false, nil.
+  # (Note, this list excludes Symbol.)
+  # X itself must be an Array or a Hash.
+  # No other value can be encoded, and an error will
+  # be raised if x contains any other value, such as
+  # Nan, Infinity, Symbol, and Proc, or if a Hash key
+  # is not a String.
+  # Strings contained in x must be valid UTF-8.
+  def encode(x)
+    case x
+    when Hash    then objenc(x)
+    when Array   then arrenc(x)
+    else
+      raise Error, 'root value must be an Array or a Hash'
+    end
+  end
+
+
+  def valenc(x)
+    case x
+    when Hash    then objenc(x)
+    when Array   then arrenc(x)
+    when String  then strenc(x)
+    when Numeric then numenc(x)
+    when true    then "true"
+    when false   then "false"
+    when nil     then "null"
+    else
+      raise Error, "cannot encode #{x.class}: #{x.inspect}"
+    end
+  end
+
+
+  def objenc(x)
+    '{' + x.map{|k,v| keyenc(k) + ':' + valenc(v)}.join(',') + '}'
+  end
+
+
+  def arrenc(a)
+    '[' + a.map{|x| valenc(x)}.join(',') + ']'
+  end
+
+
+  def keyenc(k)
+    case k
+    when String then strenc(k)
+    else
+      raise Error, "Hash key is not a string: #{k.inspect}"
+    end
+  end
+
+
+  def strenc(s)
+    t = StringIO.new
+    t.putc(?")
+    r = 0
+    while r < s.length
+      case s[r]
+      when ?"  then t.print('\\"')
+      when ?\\ then t.print('\\\\')
+      when ?\b then t.print('\\b')
+      when ?\f then t.print('\\f')
+      when ?\n then t.print('\\n')
+      when ?\r then t.print('\\r')
+      when ?\t then t.print('\\t')
+      else
+        c = s[r]
+        case true
+        when Spc <= c && c <= ?~
+          t.putc(c)
+        when true
+          u, size = uchardec(s, r)
+          r += size - 1 # we add one more at the bottom of the loop
+          if u < 0x10000
+            t.print('\\u')
+            hexenc4(t, u)
+          else
+            u1, u2 = unsubst(u)
+            t.print('\\u')
+            hexenc4(t, u1)
+            t.print('\\u')
+            hexenc4(t, u2)
+          end
+        else
+          # invalid byte; skip it
+        end
+      end
+      r += 1
+    end
+    t.putc(?")
+    t.string
+  end
+
+
+  def hexenc4(t, u)
+    t.putc(Hex[(u>>12)&0xf])
+    t.putc(Hex[(u>>8)&0xf])
+    t.putc(Hex[(u>>4)&0xf])
+    t.putc(Hex[u&0xf])
+  end
+
+
+  def numenc(x)
+    if ((x.nan? || x.infinite?) rescue false)
+      raise Error, "Numeric cannot be represented: #{x}"
+    end
+    "#{x}"
+  end
+
+
+  # Decodes unicode character u from UTF-8
+  # bytes in string s at position i.
+  # Returns u and the number of bytes read.
+  def uchardec(s, i)
+    n = s.length - i
+    return [Ucharerr, 1] if n < 1
+
+    c0 = s[i].ord
+
+    # 1-byte, 7-bit sequence?
+    if c0 < Utagx
+      return [c0, 1]
+    end
+
+    # unexpected continuation byte?
+    return [Ucharerr, 1] if c0 < Utag2
+
+    # need continuation byte
+    return [Ucharerr, 1] if n < 2
+    c1 = s[i+1].ord
+    return [Ucharerr, 1] if c1 < Utagx || Utag2 <= c1
+
+    # 2-byte, 11-bit sequence?
+    if c0 < Utag3
+      u = (c0&Umask2)<<6 | (c1&Umaskx)
+      return [Ucharerr, 1] if u <= Uchar1max
+      return [u, 2]
+    end
+
+    # need second continuation byte
+    return [Ucharerr, 1] if n < 3
+    c2 = s[i+2].ord
+    return [Ucharerr, 1] if c2 < Utagx || Utag2 <= c2
+
+    # 3-byte, 16-bit sequence?
+    if c0 < Utag4
+      u = (c0&Umask3)<<12 | (c1&Umaskx)<<6 | (c2&Umaskx)
+      return [Ucharerr, 1] if u <= Uchar2max
+      return [u, 3]
+    end
+
+    # need third continuation byte
+    return [Ucharerr, 1] if n < 4
+    c3 = s[i+3].ord
+    return [Ucharerr, 1] if c3 < Utagx || Utag2 <= c3
+
+    # 4-byte, 21-bit sequence?
+    if c0 < Utag5
+      u = (c0&Umask4)<<18 | (c1&Umaskx)<<12 | (c2&Umaskx)<<6 | (c3&Umaskx)
+      return [Ucharerr, 1] if u <= Uchar3max
+      return [u, 4]
+    end
+
+    return [Ucharerr, 1]
+  end
+
+
+  class Error < ::StandardError
+  end
+
+
+  Utagx = 0x80 # 1000 0000
+  Utag2 = 0xc0 # 1100 0000
+  Utag3 = 0xe0 # 1110 0000
+  Utag4 = 0xf0 # 1111 0000
+  Utag5 = 0xF8 # 1111 1000
+  Umaskx = 0x3f # 0011 1111
+  Umask2 = 0x1f # 0001 1111
+  Umask3 = 0x0f # 0000 1111
+  Umask4 = 0x07 # 0000 0111
+  Uchar1max = (1<<7) - 1
+  Uchar2max = (1<<11) - 1
+  Uchar3max = (1<<16) - 1
+  Ucharerr = 0xFFFD # unicode "replacement char"
+  Usurrself = 0x10000
+  Usurr1 = 0xd800
+  Usurr2 = 0xdc00
+  Usurr3 = 0xe000
+  Umax = 0x10ffff
+
+  Spc = ' '[0]
+  Unesc = {?b=>?\b, ?f=>?\f, ?n=>?\n, ?r=>?\r, ?t=>?\t}
+  Hex = '0123456789abcdef'
+end
+end

data/lib/queue_classic/queue.rb

@@ -2,9 +2,9 @@ module QC
   module AbstractQueue
 
     def enqueue(job,*params)
-      if job.respond_to?(:details) and job.respond_to?(:params)
-        job = job.signature
+      if job.respond_to?(:signature) and job.respond_to?(:params)
         params = *job.params
+        job = job.signature
       end
       array << {"job" => job, "params" => params}
     end
@@ -39,16 +39,15 @@ module  QC
     extend AbstractQueue
 
     def self.array
-      if defined? @@array
-        @@array
-      else
-        @@database = Database.new
-        @@array = DurableArray.new(@@database)
-      end
+      default_queue.array
     end
 
     def self.database
-      @@database
+      default_queue.database
+    end
+
+    def self.default_queue
+      @queue ||= new(nil)
     end
 
     def initialize(queue_name)

data/lib/queue_classic/tasks.rb

@@ -1,21 +1,23 @@
 namespace :jobs do
 
-  task :work  => :environment do
-    QC::Worker.new.start
-  end
+  desc 'Alias for qc:work'
+  task :work  => 'qc:work'
 
 end
 
 namespace :qc do
 
+  desc 'Start a new worker for the (default or QUEUE) queue'
   task :work  => :environment do
     QC::Worker.new.start
   end
 
+  desc 'Returns the number of jobs in the (default or QUEUE) queue'
   task :jobs => :environment do
-    QC.queue_length
+    puts QC::Queue.new(ENV['QUEUE']).length
   end
 
+  desc 'Ensure the database has the necessary functions for QC'
   task :load_functions => :environment do
     db = QC::Database.new
     db.load_functions

data/readme.md

@@ -12,51 +12,98 @@ queue_classic features:
 * JSON encoding for jobs
 * Forking workers
 * Postgres' rock-solid locking mechanism
+* Fuzzy-FIFO support (1)
 * Long term support
 
+1.Theory found here: http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf
+
+## Proven
+
+I wrote queue_classic to solve a production problem.  My problem was that I needed a
+queueing system that wouldn't fall over should I decide to press it nor should it freak out
+if I attached 100 workers to it. However, my problem didn't warrant adding an additional service.
+I was already using PostgreSQL to manage my application's data, why not use PostgreSQL to pass some messages?
+PostgreSQL was already handling thousands of reads and writes per second anyways. Why not add 35 more
+reads/writes per second to my established performance metric?
+
+queue_classic handles over **3,000,000** jobs per day. It does this on Heroku's Ronin Database.
+
 ## Quick Start
 
 See doc/installation.md for Rails instructions
 
 ```bash
-  $ createdb queue_classic_test
-  $ psql queue_classic_test
-  psql=# CREATE TABLE queue_classic_jobs (id serial, details text, locked_at timestamp);
-  psql=# CREATE INDEX queue_classic_jobs_id_idx ON queue_classic_jobs (id);
-  $ export QC_DATABASE_URL="postgres://username:password@localhost/queue_classic_test"
-  $ gem install queue_classic
-  $ ruby -r queue_classic -e "QC::Database.new.load_functions"
-  $ ruby -r queue_classic -e "QC.enqueue("Kernel.puts", "hello world")"
-  $ ruby -r queue_classic -e "QC::Worker.new.start"
+$ createdb queue_classic_test
+$ psql queue_classic_test
+psql- CREATE TABLE queue_classic_jobs (id serial, details text, locked_at timestamp);
+psql- CREATE INDEX queue_classic_jobs_id_idx ON queue_classic_jobs (id);
+$ export QC_DATABASE_URL="postgres://username:password@localhost/queue_classic_test"
+$ gem install queue_classic
+$ ruby -r queue_classic -e "QC::Database.new.load_functions"
+$ ruby -r queue_classic -e "QC.enqueue('Kernel.puts', 'hello world')"
+$ ruby -r queue_classic -e "QC::Worker.new.start"
+```
+
+## Configure
+
+```bash
+# Enable logging.
+$VERBOSE
+
+# Specifies the database that queue_classic will rely upon.
+$QC_DATABASE_URL || $DATABASE_URL
+
+# Fuzzy-FIFO
+# For strict FIFO set to 1. Otherwise, worker will
+# attempt to lock a job in this top region.
+# Default: 9
+$QC_TOP_BOUND
+
+# If you want your worker to fork a new
+# child process for each job, set this var to 'true'
+# Default: false
+$QC_FORK_WORKER
+
+# The worker uses an exp backoff algorithm
+# if you want high throughput don't use Kernel.sleep
+# use LISTEN/NOTIFY sleep. When set to true, the worker's
+# sleep will be preempted by insertion into the queue.
+# Default: false
+$QC_LISTENING_WORKER
+
+# The worker uses an exp backoff algorithm. The base of
+# the exponent is 2. This var determines the max power of the
+# exp.
+# Default: 5 which implies max sleep time of 2^(5-1) => 16 seconds
+$QC_MAX_LOCK_ATTEMPTS
+
+# This var is important for consumers of the queue.
+# If you have configured many queues, this var will
+# instruct the worker to bind to a particular queue.
+# Default: queue_classic_jobs --which is the default queue table.
+$QUEUE
 ```
 
 ## Hacking on queue_classic
 
 ### Dependencies
 
-* Ruby 1.9.2
+* Ruby 1.9.2 (tests work in 1.8.7 but compatibility is not guaranteed or supported)
 * Postgres ~> 9.0
-* Rubygems: pg ~> 0.11.0
+* Rubygem: pg ~> 0.11.0
 
 ### Running Tests
 
 ```bash
-  $ bundle
-  $ createdb queue_classic_test
-  $ export QC_DATABASE_URL="postgres://username:pass@localhost/queue_classic_test"
-  $ rake
+$ bundle
+$ createdb queue_classic_test
+$ export QC_DATABASE_URL="postgres://username:pass@localhost/queue_classic_test"
+$ rake
 ```
 
-### Building Documentation
-
-If you are adding new features, please document them in the doc directory. Also,
-once you have the markdown in place, please run: ruby doc/build.rb to make HTML
-for the docs.
-
 ## Other Resources
 
-###[Documentation](https://github.com/ryandotsmith/queue_classic/tree/master/doc)
-
-###[Example Rails App](https://github.com/ryandotsmith/queue_classic_example)
-
-###[Discussion Group](http://groups.google.com/group/queue_classic "discussion group")
+* [Discussion Group](http://groups.google.com/group/queue_classic "discussion group")
+* [Documentation](https://github.com/ryandotsmith/queue_classic/tree/master/doc)
+* [Example Rails App](https://github.com/ryandotsmith/queue_classic_example)
+* [Slide Deck](http://dl.dropbox.com/u/1579953/talks/queue_classic.pdf)

data/test/database_test.rb

@@ -25,4 +25,49 @@ context "DatabaseTest" do
     assert   @database.connection.notifies.nil?
   end
 
+  test "execute should return rows" do
+    result = @database.execute 'SELECT 11 foo, 22 bar;'
+    assert_equal [{'foo'=>'11', 'bar'=>'22'}], result.to_a
+  end
+
+  test "should raise error on failure" do
+    assert_raises PGError do
+      @database.execute 'SELECT unknown FROM missing;'
+    end
+  end
+
+  test "execute should accept parameters" do
+    result = @database.execute 'SELECT $2::int b, $1::int a, $1::int + $2::int c;', 123, '456'
+    assert_equal [{"a"=>"123", "b"=>"456", "c"=>"579"}], result.to_a
+  end
+
+  def job_count
+    @database.execute('SELECT COUNT(*) FROM queue_classic_jobs')[0].values.first.to_i
+  end
+
+  test "transaction should commit" do
+    assert_equal true, @database.transaction_idle?
+    assert_equal 0, job_count
+    @database.transaction do
+      assert_equal false, @database.transaction_idle?
+      assert_equal 0, job_count
+      @database.execute "INSERT INTO queue_classic_jobs (details) VALUES ('test');"
+      assert_equal false, @database.transaction_idle?
+      assert_equal 1, job_count
+    end
+    assert_equal true, @database.transaction_idle?
+    assert_equal 1, job_count
+  end
+
+  test "transaction should rollback if there's an error" do
+    assert_raises RuntimeError do
+      @database.transaction do
+        @database.execute "INSERT INTO queue_classic_jobs (details) VALUES ('test');"
+        assert_equal 1, job_count
+        raise "force rollback"
+      end
+    end
+    assert_equal 0, job_count
+  end
+
 end

data/test/durable_array_test.rb

@@ -29,6 +29,18 @@ context "DurableArray" do
     assert_equal job, @array.first.details
   end
 
+  test "passes through strings with quotes" do
+    job = {"foo'bar\"baz" => 'abc\\def'}
+    @array << job
+    assert_equal job, @array.first.details
+  end
+
+  test "passes through newlines" do
+    job = {"word" => "line1\nline2\nline3\n"}
+    @array << job
+    assert_equal job, @array.first.details
+  end
+
   test "first returns first job when many are in the array" do
     @array << {"job" => "one"}
     @array << {"job" => "two"}
@@ -68,13 +80,13 @@ context "DurableArray" do
     assert_equal([{"job" => "one"},{"job" => "two"}], results)
   end
 
-  test "seach" do
+  test "search" do
     @array << {"job" => "A.signature"}
     jobs = @array.search_details_column("A.signature")
     assert_equal "A.signature", jobs.first.signature
   end
 
-  test "seach when data will not match" do
+  test "search when data will not match" do
     @array << {"job" => "A.signature"}
     jobs = @array.search_details_column("B.signature")
     assert_equal [], jobs

data/test/job_test.rb

@@ -14,7 +14,7 @@ context "Job" do
   test "signature returns the class and method" do
     job = QC::Job.new(
       "id" => 1,
-      "details" => {:job => "Class.method", :params => []}.to_json,
+      "details" => QC::OkJson.encode({"job" => "Class.method", "params" => []}),
       "locked_at" => nil
     )
     assert_equal "Class.method", job.signature
@@ -23,7 +23,7 @@ context "Job" do
   test "method returns the class method" do
     job = QC::Job.new(
       "id" => 1,
-      "details" => {:job => "Class.method", :params => []}.to_json,
+      "details" => QC::OkJson.encode({"job" => "Class.method", "params" => []}),
       "locked_at" => nil
     )
     assert_equal "method", job.method
@@ -33,7 +33,7 @@ context "Job" do
     class WhoHa; end
     job = QC::Job.new(
       "id" => 1,
-      "details" => {:job => "WhoHa.method", :params => []}.to_json,
+      "details" => QC::OkJson.encode({"job" => "WhoHa.method", "params" => []}),
       "locked_at" => nil
     )
     assert_equal WhoHa, job.klass
@@ -47,7 +47,7 @@ context "Job" do
 
     job = QC::Job.new(
       "id" => 1,
-      "details" => {:job => "Mod::K.method", :params => []}.to_json,
+      "details" => QC::OkJson.encode({"job" => "Mod::K.method", "params" => []}),
       "locked_at" => nil
     )
     assert_equal Mod::K, job.klass
@@ -56,7 +56,7 @@ context "Job" do
   test "params returns empty array when nil" do
      job = QC::Job.new(
       "id" => 1,
-      "details" => {:job => "Mod::K.method", :params => nil}.to_json,
+      "details" => QC::OkJson.encode({"job" => "Mod::K.method", "params" => nil}),
       "locked_at" => nil
     )
     assert_equal [], job.params
@@ -65,7 +65,7 @@ context "Job" do
   test "params returns 1 items when there is 1 param" do
     job = QC::Job.new(
       "id" => 1,
-      "details" => {:job => "Mod::K.method", :params => ["arg"]}.to_json,
+      "details" => QC::OkJson.encode({"job" => "Mod::K.method", "params" => ["arg"]}),
       "locked_at" => nil
     )
     assert_equal "arg", job.params
@@ -74,7 +74,7 @@ context "Job" do
   test "params retuns many items when there are many params" do
     job = QC::Job.new(
       "id" => 1,
-      "details" => {:job => "Mod::K.method", :params => ["arg","arg"]}.to_json,
+      "details" => QC::OkJson.encode({"job" => "Mod::K.method", "params" => ["arg","arg"]}),
       "locked_at" => nil
     )
     assert_equal ["arg","arg"], job.params

data/test/queue_test.rb

@@ -1,4 +1,5 @@
 require File.expand_path("../helper.rb", __FILE__)
+require 'ostruct'
 
 context "Queue" do
 
@@ -9,7 +10,8 @@ context "Queue" do
   end
 
   test "Queue class has a default table name" do
-    QC::Queue.enqueue("Klass.method")
+    default_table_name = QC::Database.new.table_name
+    assert_equal default_table_name, QC::Queue.database.table_name
   end
 
   test "Queue class responds to dequeue" do
@@ -58,4 +60,19 @@ context "Queue" do
     assert_equal 1, @database.execute("SELECT count(*) from pg_stat_activity")[0]["count"].to_i
   end
 
+  test "Queue class enqueues a job" do
+    job = OpenStruct.new :signature => 'Klass.method', :params => ['param']
+    QC::Queue.enqueue(job)
+    dequeued_job = QC::Queue.dequeue
+    assert_equal "Klass.method", dequeued_job.signature
+    assert_equal 'param', dequeued_job.params
+  end
+
+  test "Queues have their own array" do
+    refute_equal(Class.new(QC::Queue).array, Class.new(QC::Queue).array)
+  end
+
+  test "Queues have their own database" do
+    refute_equal(Class.new(QC::Queue).database, Class.new(QC::Queue).database)
+  end
 end

data/test/worker_test.rb

@@ -50,7 +50,8 @@ context "Worker" do
   test "only makes one connection" do
     QC.enqueue "TestNotifier.deliver", {}
     @worker.work
-    assert_equal 1, @database.execute("SELECT count(*) from pg_stat_activity")[0]["count"].to_i
+    assert_equal 1, @database.execute("SELECT count(*) from pg_stat_activity")[0]["count"].to_i,
+      "Multiple connections -- Are there other connections in other terminals?"
   end
 
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
   prerelease: 
 platform: ruby
 authors:
@@ -13,7 +13,7 @@ date: 2011-08-22 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
-  requirement: &2153191980 !ruby/object:Gem::Requirement
+  requirement: &16027180 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,7 +21,7 @@ dependencies:
         version: 0.11.0
   type: :runtime
   prerelease: false
-  version_requirements: *2153191980
+  version_requirements: *16027180
 description: queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...)
   queue_classic features asynchronous job polling, database maintained locks and no
   ridiculous dependencies. As a matter of fact, queue_classic only requires pg.
@@ -31,21 +31,22 @@ extensions: []
 extra_rdoc_files: []
 files:
 - readme.md
+- lib/queue_classic/okjson.rb
+- lib/queue_classic/logger.rb
+- lib/queue_classic/worker.rb
 - lib/queue_classic/database.rb
-- lib/queue_classic/durable_array.rb
 - lib/queue_classic/job.rb
-- lib/queue_classic/logger.rb
 - lib/queue_classic/queue.rb
 - lib/queue_classic/tasks.rb
-- lib/queue_classic/worker.rb
+- lib/queue_classic/durable_array.rb
 - lib/queue_classic.rb
-- test/database_helpers.rb
 - test/database_test.rb
-- test/durable_array_test.rb
-- test/helper.rb
 - test/job_test.rb
-- test/queue_test.rb
+- test/database_helpers.rb
 - test/worker_test.rb
+- test/queue_test.rb
+- test/durable_array_test.rb
+- test/helper.rb
 homepage: http://github.com/ryandotsmith/queue_classic
 licenses: []
 post_install_message: 
@@ -66,13 +67,13 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.7
+rubygems_version: 1.8.10
 signing_key: 
 specification_version: 3
 summary: postgres backed queue
 test_files:
 - test/database_test.rb
-- test/durable_array_test.rb
 - test/job_test.rb
-- test/queue_test.rb
 - test/worker_test.rb
+- test/queue_test.rb
+- test/durable_array_test.rb