queue_classic 2.0.0rc1 → 2.0.0rc4

data/lib/queue_classic/okjson.rb

@@ -1,5 +1,7 @@
 module QC
-  # Copyright 2011 Keith Rarick
+# encoding: UTF-8
+#
+# Copyright 2011, 2012 Keith Rarick
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -261,6 +263,12 @@ module OkJson
   def unquote(q)
     q = q[1...-1]
     a = q.dup # allocate a big enough string
+    rubydoesenc = false
+    # In ruby >= 1.9, a[w] is a codepoint, not a byte.
+    if a.class.method_defined?(:force_encoding)
+      a.force_encoding('UTF-8')
+      rubydoesenc = true
+    end
     r, w = 0, 0
     while r < q.length
       c = q[r]
@@ -298,7 +306,12 @@ module OkJson
               end
             end
           end
-          w += ucharenc(a, w, uchar)
+          if rubydoesenc
+            a[w] = '' << uchar
+            w += 1
+          else
+            w += ucharenc(a, w, uchar)
+          end
         else
           raise Error, "invalid escape char #{q[r]} in \"#{q}\""
         end
@@ -308,6 +321,8 @@ module OkJson
         # Copy anything else byte-for-byte.
         # Valid UTF-8 will remain valid UTF-8.
         # Invalid UTF-8 will remain invalid UTF-8.
+        # In ruby >= 1.9, c is a codepoint, not a byte,
+        # in which case this is still what we want.
         a[w] = c
         r += 1
         w += 1
@@ -442,6 +457,10 @@ module OkJson
     t = StringIO.new
     t.putc(?")
     r = 0
+
+    # In ruby >= 1.9, s[r] is a codepoint, not a byte.
+    rubydoesenc = s.class.method_defined?(:encoding)
+
     while r < s.length
       case s[r]
       when ?"  then t.print('\\"')
@@ -456,21 +475,13 @@ module OkJson
         case true
         when Spc <= c && c <= ?~
           t.putc(c)
-        when true
+        when rubydoesenc
+          u = c.ord
+          surrenc(t, u)
+        else
           u, size = uchardec(s, r)
           r += size - 1 # we add one more at the bottom of the loop
-          if u < 0x10000
-            t.print('\\u')
-            hexenc4(t, u)
-          else
-            u1, u2 = unsubst(u)
-            t.print('\\u')
-            hexenc4(t, u1)
-            t.print('\\u')
-            hexenc4(t, u2)
-          end
-        else
-          # invalid byte; skip it
+          surrenc(t, u)
         end
       end
       r += 1
@@ -480,6 +491,20 @@ module OkJson
   end
 
 
+  def surrenc(t, u)
+    if u < 0x10000
+      t.print('\\u')
+      hexenc4(t, u)
+    else
+      u1, u2 = unsubst(u)
+      t.print('\\u')
+      hexenc4(t, u1)
+      t.print('\\u')
+      hexenc4(t, u2)
+    end
+  end
+
+
   def hexenc4(t, u)
     t.putc(Hex[(u>>12)&0xf])
     t.putc(Hex[(u>>8)&0xf])

data/lib/queue_classic.rb

@@ -13,24 +13,25 @@ require "queue_classic/worker"
 
 module QC
 
-  Root = File.expand_path(File.dirname(__FILE__))
+  Root = File.expand_path("..", File.dirname(__FILE__))
   SqlFunctions = File.join(QC::Root, "/sql/ddl.sql")
   DropSqlFunctions = File.join(QC::Root, "/sql/drop_ddl.sql")
 
-  Log = Logger.new($stdout)
-  Log.level = (ENV["QC_LOG_LEVEL"] || Logger::DEBUG).to_i
-  Log.info("program=queue_classic log=true")
-
   DB_URL =
     ENV["QC_DATABASE_URL"] ||
     ENV["DATABASE_URL"]    ||
     raise(ArgumentError, "missing QC_DATABASE_URL or DATABASE_URL")
 
+  # export QC_LOG_LEVEL=`ruby -r "logger" -e "puts Logger::ERROR"`
+  LOG_LEVEL = (ENV["QC_LOG_LEVEL"] || Logger::DEBUG).to_i
+
   # You can use the APP_NAME to query for
   # postgres related process information in the
   # pg_stat_activity table. Don't set this unless
   # you are using PostgreSQL > 9.0
-  APP_NAME = ENV["QC_APP_NAME"]
+  if APP_NAME = ENV["QC_APP_NAME"]
+    Conn.execute("SET application_name = '#{APP_NAME}'")
+  end
 
   # Why do you want to change the table name?
   # Just deal with the default OK?
@@ -55,20 +56,24 @@ module QC
 
   # Set this variable if you wish for
   # the worker to fork a UNIX process for
-  # each locked job. Remember to restablish
-  # any database connectoins. See the worker
+  # each locked job. Remember to re-establish
+  # any database connections. See the worker
   # for more details.
   FORK_WORKER = !ENV["QC_FORK_WORKER"].nil?
 
-  # The worker uses an exponential backoff
+  # The worker uses an exponential back-off
   # algorithm to lock a job. This value will be used
   # as the max exponent.
   MAX_LOCK_ATTEMPTS = (ENV["QC_MAX_LOCK_ATTEMPTS"] || 5).to_i
 
-  if APP_NAME
-    Conn.execute("SET application_name = '#{APP_NAME}'")
-  end
 
+  # Setup the logger
+  Log = Logger.new($stdout)
+  Log.level = LOG_LEVEL
+  Log.info("program=queue_classic log=true")
+
+  # Defer method calls on the QC module to the
+  # default queue. This facilitates QC.enqueue()
   def self.method_missing(sym, *args, &block)
     default_queue.send(sym, *args, &block)
   end

data/readme.md

@@ -1,6 +1,6 @@
 # queue_classic
 
-v2.0.0rc1
+v2.0.0rc2
 
 queue_classic is a PostgreSQL-backed queueing library that is focused on
 concurrent job locking, minimizing database load & providing a simple &
@@ -139,49 +139,9 @@ end
 
 ## Configure
 
-```bash
-# Log level.
-# export QC_LOG_LEVEL=`ruby -r "logger" -e "puts Logger::ERROR"`
-$QC_LOG_LEVEL
-
-# Specifies the database that queue_classic will rely upon.
-# queue_classic will try and use QC_DATABASE_URL before it uses DATABASE_URL.
-$QC_DATABASE_URL
-$DATABASE_URL
-
-# Fuzzy-FIFO
-# For strict FIFO set to 1. Otherwise, worker will
-# attempt to lock a job in this top region.
-# Default: 9
-$QC_TOP_BOUND
-
-# If you want your worker to fork a new
-# UNIX process for each job, set this var to 'true'
-#
-# Default: false
-$QC_FORK_WORKER
-
-# The worker uses an exp backoff algorithm
-# if you want high throughput don't use Kernel.sleep
-# use LISTEN/NOTIFY sleep. When set to true, the worker's
-# sleep will be preempted by insertion into the queue.
-#
-# Default: false
-$QC_LISTENING_WORKER
-
-# The worker uses an exp backoff algorithm. The base of
-# the exponent is 2. This var determines the max power of the exp.
-#
-# Default: 5 which implies max sleep time of 2^(5-1) => 16 seconds
-$QC_MAX_LOCK_ATTEMPTS
-
-# This var is important for consumers of the queue.
-# If you have configured many queues, this var will
-# instruct the worker to bind to a particular queue.
-#
-# Default: queue_classic_jobs
-$QUEUE
-```
+All configuration takes place in the form of environment vars.
+See [queue_classic.rb](https://github.com/ryandotsmith/queue_classic/blob/master/lib/queue_classic.rb#L29-66)
+for a list of options.
 
 ## Usage
 
@@ -217,7 +177,7 @@ QC.enqueue("Kernel.printf", "hello %s", "world")
 # This method has a hash argument.
 QC.enqueue("Kernel.puts", {"hello" => "world"})
 
-# This method has a hash argument.
+# This method has a array argument.
 QC.enqueue("Kernel.puts", ["hello", "world"])
 ```
 
@@ -258,22 +218,25 @@ p_queue.enqueue("Kernel.printf", "hello %s", "world")
 # This method has a hash argument.
 p_queue.enqueue("Kernel.puts", {"hello" => "world"})
 
-# This method has a hash argument.
+# This method has a array argument.
 p_queue.enqueue("Kernel.puts", ["hello", "world"])
 ```
 
 This code example shows how to produce jobs into a custom queue,
-to consume jobs from the customer queue be sure and set the `$QUEUE`
+to consume jobs from the custome queue be sure and set the `$QUEUE`
 var to the q_name in the worker's UNIX environment.
 
 ### Consumer
 
-Now that you have some jobs in your queue, you probably want to work them.
-Let's find out how... If you are using a Rakefile and have included `queue_classic/tasks`
-then you can enter the following command to start a worker:
+There are several approaches to working jobs. The first is to include
+a task file provided by queue_classic and the other approach is to
+write a custom bin file.
 
 #### Rake Task
 
+Be sure to include `queue_classic` and `queue_classic/tasks`
+in your primary Rakefile.
+
 To work jobs from the default queue:
 
 ```bash
@@ -287,9 +250,8 @@ $ QUEUE="p_queue" bundle exec rake qc:work
 
 #### Bin File
 
-The approach that I take when building simple ruby programs and sinatra apps is to
-create an executable file that starts the worker. Start by making a bin directory
-in your project's root directory. Then add a file called worker.
+Start by making a bin directory in your project's root directory.
+Then add an executable file called worker.
 
 **bin/worker**
 
@@ -549,6 +511,22 @@ clock: clockwork clock.rb
 
 ## Upgrading From Older Versions
 
+### 1.X to 2.X
+
+#### Database Schema Changes
+
+* all queues are in 1 table with a q_name column
+* table includes a method column and an args column
+
+#### Producer Changes
+
+* initializing a Queue instance takes a column name instead of a table name
+
+#### Consumer Changes
+
+* all of the worker configuratoin is passed in through the initializer
+* rake task uses data from env vars to initialize a worker
+
 ### 0.2.X to 0.3.X
 
 * Deprecated QC.queue_length in favor of QC.length

data/sql/ddl.sql

@@ -0,0 +1,66 @@
+-- We are declaring the return type to be queue_classic_jobs.
+-- This is ok since I am assuming that all of the users added queues will
+-- have identical columns to queue_classic_jobs.
+-- When QC supports queues with columns other than the default, we will have to change this.
+
+CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
+RETURNS SETOF queue_classic_jobs AS $$
+DECLARE
+  unlocked integer;
+  relative_top integer;
+  job_count integer;
+BEGIN
+  -- The purpose is to release contention for the first spot in the table.
+  -- The select count(*) is going to slow down dequeue performance but allow
+  -- for more workers. Would love to see some optimization here...
+
+  EXECUTE 'SELECT count(*) FROM '
+    || '(SELECT * FROM queue_classic_jobs WHERE q_name = '
+    || quote_literal(q_name)
+    || ' LIMIT '
+    || quote_literal(top_boundary)
+    || ') limited'
+  INTO job_count;
+
+  SELECT TRUNC(random() * (top_boundary - 1))
+  INTO relative_top;
+
+  IF job_count < top_boundary THEN
+    relative_top = 0;
+  END IF;
+
+  LOOP
+    BEGIN
+      EXECUTE 'SELECT id FROM queue_classic_jobs '
+        || ' WHERE locked_at IS NULL'
+        || ' AND q_name = '
+        || quote_literal(q_name)
+        || ' ORDER BY id ASC'
+        || ' LIMIT 1'
+        || ' OFFSET ' || quote_literal(relative_top)
+        || ' FOR UPDATE NOWAIT'
+      INTO unlocked;
+      EXIT;
+    EXCEPTION
+      WHEN lock_not_available THEN
+        -- do nothing. loop again and hope we get a lock
+    END;
+  END LOOP;
+
+  RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
+    || ' SET locked_at = (CURRENT_TIMESTAMP)'
+    || ' WHERE id = $1'
+    || ' AND locked_at is NULL'
+    || ' RETURNING *'
+  USING unlocked;
+
+  RETURN;
+END;
+$$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION lock_head(tname varchar)
+RETURNS SETOF queue_classic_jobs AS $$
+BEGIN
+  RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
+END;
+$$ LANGUAGE plpgsql;

data/sql/drop_ddl.sql

@@ -0,0 +1,2 @@
+DROP FUNCTION IF EXISTS lock_head(tname varchar);
+DROP FUNCTION IF EXISTS lock_head(tname name, top_boundary integer);

metadata

@@ -1,11 +1,11 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 2.0.0rc1
+  version: 2.0.0rc4
   prerelease: 5
 platform: ruby
 authors:
-- Ryan Smith
+- Ryan Smith (ace hacker)
 autorequire: 
 bindir: bin
 cert_chain: []
@@ -13,7 +13,7 @@ date: 2012-02-29 00:00:00.000000000Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: pg
-  requirement: &18736220 !ruby/object:Gem::Requirement
+  requirement: &8948440 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -21,7 +21,7 @@ dependencies:
         version: 0.13.2
   type: :runtime
   prerelease: false
-  version_requirements: *18736220
+  version_requirements: *8948440
 description: queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...)
   queue_classic features asynchronous job polling, database maintained locks and no
   ridiculous dependencies. As a matter of fact, queue_classic only requires pg.
@@ -31,6 +31,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - readme.md
+- sql/ddl.sql
+- sql/drop_ddl.sql
 - lib/queue_classic/conn.rb
 - lib/queue_classic/okjson.rb
 - lib/queue_classic/worker.rb