queue_classic 1.0.2 → 2.0.0rc1

data/lib/queue_classic.rb

@@ -6,18 +6,77 @@ require "uri"
 $: << File.expand_path(__FILE__, "lib")
 
 require "queue_classic/okjson"
-require "queue_classic/durable_array"
-require "queue_classic/database"
-require "queue_classic/worker"
-require "queue_classic/logger"
+require "queue_classic/conn"
+require "queue_classic/queries"
 require "queue_classic/queue"
-require "queue_classic/job"
+require "queue_classic/worker"
 
 module QC
-  VERBOSE = ENV["VERBOSE"] || ENV["QC_VERBOSE"]
-  Logger.puts("Logging enabled")
+
+  Root = File.expand_path(File.dirname(__FILE__))
+  SqlFunctions = File.join(QC::Root, "/sql/ddl.sql")
+  DropSqlFunctions = File.join(QC::Root, "/sql/drop_ddl.sql")
+
+  Log = Logger.new($stdout)
+  Log.level = (ENV["QC_LOG_LEVEL"] || Logger::DEBUG).to_i
+  Log.info("program=queue_classic log=true")
+
+  DB_URL =
+    ENV["QC_DATABASE_URL"] ||
+    ENV["DATABASE_URL"]    ||
+    raise(ArgumentError, "missing QC_DATABASE_URL or DATABASE_URL")
+
+  # You can use the APP_NAME to query for
+  # postgres related process information in the
+  # pg_stat_activity table. Don't set this unless
+  # you are using PostgreSQL > 9.0
+  APP_NAME = ENV["QC_APP_NAME"]
+
+  # Why do you want to change the table name?
+  # Just deal with the default OK?
+  # If you do want to change this, you will
+  # need to update the PL/pgSQL lock_head() function.
+  # Come on. Don't do it.... Just stick with the default.
+  TABLE_NAME = "queue_classic_jobs"
+
+  # Each row in the table will have a column that
+  # notes the queue. You can point your workers
+  # at different queues but only one at a time.
+  QUEUE = ENV["QUEUE"] || "default"
+
+  # Set this to 1 for strict FIFO.
+  # There is nothing special about 9....
+  TOP_BOUND = (ENV["QC_TOP_BOUND"] || 9).to_i
+
+  # If you are using PostgreSQL > 9
+  # then you will have access to listen/notify with payload.
+  # Set this value if you wish to make your worker more efficient.
+  LISTENING_WORKER = !ENV["QC_LISTENING_WORKER"].nil?
+
+  # Set this variable if you wish for
+  # the worker to fork a UNIX process for
+  # each locked job. Remember to restablish
+  # any database connectoins. See the worker
+  # for more details.
+  FORK_WORKER = !ENV["QC_FORK_WORKER"].nil?
+
+  # The worker uses an exponential backoff
+  # algorithm to lock a job. This value will be used
+  # as the max exponent.
+  MAX_LOCK_ATTEMPTS = (ENV["QC_MAX_LOCK_ATTEMPTS"] || 5).to_i
+
+  if APP_NAME
+    Conn.execute("SET application_name = '#{APP_NAME}'")
+  end
 
   def self.method_missing(sym, *args, &block)
-    Queue.send(sym, *args, &block)
+    default_queue.send(sym, *args, &block)
+  end
+
+  def self.default_queue
+    @default_queue ||= begin
+      Queue.new(QUEUE, LISTENING_WORKER)
+    end
   end
+
 end

data/lib/queue_classic/conn.rb

@@ -0,0 +1,97 @@
+module QC
+  module Conn
+    extend self
+
+    def execute(stmt, *params)
+      log("executing #{stmt.inspect}, #{params.inspect}")
+      begin
+        params = nil if params.empty?
+        r = connection.exec(stmt, params)
+        result = []
+        r.each {|t| result << t}
+        result.length > 1 ? result : result.pop
+      rescue PGError => e
+        log("execute exception=#{e.inspect}")
+        raise
+      end
+    end
+
+    def notify(chan)
+      log("NOTIFY")
+      execute("NOTIFY #{chan}")
+    end
+
+    def listen(chan)
+      log("LISTEN")
+      execute("LISTEN #{chan}")
+    end
+
+    def unlisten(chan)
+      log("UNLISTEN")
+      execute("UNLISTEN #{chan}")
+    end
+
+    def drain_notify
+      until connection.notifies.nil?
+        log("draining notifications")
+      end
+    end
+
+    def wait_for_notify(t)
+      log("waiting for notify timeout=#{t}")
+      connection.wait_for_notify(t) do |event, pid, msg|
+        log("received notification #{event}")
+      end
+      log("done waiting for notify")
+    end
+
+    def transaction
+      begin
+        execute("BEGIN")
+        yield
+        execute("COMMIT")
+      rescue Exception
+        execute("ROLLBACK")
+        raise
+      end
+    end
+
+    def transaction_idle?
+      connection.transaction_status == PGconn::PQTRANS_IDLE
+    end
+
+    def connection
+      @connection ||= connect
+    end
+
+    def disconnect
+      connection.finish
+      @connection = nil
+    end
+
+    def connect
+      log("establishing connection")
+      conn = PGconn.connect(
+        db_url.host,
+        db_url.port || 5432,
+        nil, '', #opts, tty
+        db_url.path.gsub("/",""), # database name
+        db_url.user,
+        db_url.password
+      )
+      if conn.status != PGconn::CONNECTION_OK
+        log("connection error=#{conn.error}")
+      end
+      conn
+    end
+
+    def db_url
+      URI.parse(DB_URL)
+    end
+
+    def log(msg)
+      Log.info(msg)
+    end
+
+  end
+end

data/lib/queue_classic/okjson.rb

@@ -1,7 +1,5 @@
 module QC
-# encoding: UTF-8
-#
-# Copyright 2011, 2012 Keith Rarick
+  # Copyright 2011 Keith Rarick
 #
 # Permission is hereby granted, free of charge, to any person obtaining a copy
 # of this software and associated documentation files (the "Software"), to deal
@@ -263,12 +261,6 @@ module OkJson
   def unquote(q)
     q = q[1...-1]
     a = q.dup # allocate a big enough string
-    rubydoesenc = false
-    # In ruby >= 1.9, a[w] is a codepoint, not a byte.
-    if a.class.method_defined?(:force_encoding)
-      a.force_encoding('UTF-8')
-      rubydoesenc = true
-    end
     r, w = 0, 0
     while r < q.length
       c = q[r]
@@ -306,12 +298,7 @@ module OkJson
               end
             end
           end
-          if rubydoesenc
-            a[w] = '' << uchar
-            w += 1
-          else
-            w += ucharenc(a, w, uchar)
-          end
+          w += ucharenc(a, w, uchar)
         else
           raise Error, "invalid escape char #{q[r]} in \"#{q}\""
         end
@@ -321,8 +308,6 @@ module OkJson
         # Copy anything else byte-for-byte.
         # Valid UTF-8 will remain valid UTF-8.
         # Invalid UTF-8 will remain invalid UTF-8.
-        # In ruby >= 1.9, c is a codepoint, not a byte,
-        # in which case this is still what we want.
         a[w] = c
         r += 1
         w += 1
@@ -457,10 +442,6 @@ module OkJson
     t = StringIO.new
     t.putc(?")
     r = 0
-
-    # In ruby >= 1.9, s[r] is a codepoint, not a byte.
-    rubydoesenc = s.class.method_defined?(:encoding)
-
     while r < s.length
       case s[r]
       when ?"  then t.print('\\"')
@@ -475,13 +456,21 @@ module OkJson
         case true
         when Spc <= c && c <= ?~
           t.putc(c)
-        when rubydoesenc
-          u = c.ord
-          surrenc(t, u)
-        else
+        when true
           u, size = uchardec(s, r)
           r += size - 1 # we add one more at the bottom of the loop
-          surrenc(t, u)
+          if u < 0x10000
+            t.print('\\u')
+            hexenc4(t, u)
+          else
+            u1, u2 = unsubst(u)
+            t.print('\\u')
+            hexenc4(t, u1)
+            t.print('\\u')
+            hexenc4(t, u2)
+          end
+        else
+          # invalid byte; skip it
         end
       end
       r += 1
@@ -491,20 +480,6 @@ module OkJson
   end
 
 
-  def surrenc(t, u)
-    if u < 0x10000
-      t.print('\\u')
-      hexenc4(t, u)
-    else
-      u1, u2 = unsubst(u)
-      t.print('\\u')
-      hexenc4(t, u1)
-      t.print('\\u')
-      hexenc4(t, u2)
-    end
-  end
-
-
   def hexenc4(t, u)
     t.putc(Hex[(u>>12)&0xf])
     t.putc(Hex[(u>>8)&0xf])

data/lib/queue_classic/queries.rb

@@ -0,0 +1,56 @@
+module QC
+  module Queries
+    extend self
+
+    def insert(q_name, method, args, chan=nil)
+      s = "INSERT INTO #{TABLE_NAME} (q_name, method, args) VALUES ($1, $2, $3)"
+      res = Conn.execute(s, q_name, method, OkJson.encode(args))
+      Conn.notify(chan) if chan
+    end
+
+    def lock_head(q_name, top_bound)
+      s = "SELECT * FROM lock_head($1, $2)"
+      if r = Conn.execute(s, q_name, top_bound)
+        {
+          :id     => r["id"],
+          :method => r["method"],
+          :args   => OkJson.decode(r["args"])
+        }
+      end
+    end
+
+    def count(q_name=nil)
+      s = "SELECT COUNT(*) FROM #{TABLE_NAME}"
+      s << " WHERE q_name = $1" if q_name
+      r = Conn.execute(*[s, q_name].compact)
+      r["count"].to_i
+    end
+
+    def delete(id)
+      Conn.execute("DELETE FROM #{TABLE_NAME} where id = $1", id)
+    end
+
+    def delete_all(q_name=nil)
+      s = "DELETE FROM #{TABLE_NAME}"
+      s << "WHERE q_name = $1" if q_name
+      Conn.execute(*[s, q_name].compact)
+    end
+
+    def load_functions
+      file = File.open(SqlFunctions)
+      Conn.transaction do
+        Conn.execute(file.read)
+      end
+      file.close
+    end
+
+    def drop_functions
+      file = File.open(DropSqlFunctions)
+      Conn.transaction do
+        Conn.execute(file.read)
+      end
+      file.close
+    end
+
+  end
+end

data/lib/queue_classic/queue.rb

@@ -1,66 +1,31 @@
 module QC
-  module AbstractQueue
-
-    def enqueue(job,*params)
-      if job.respond_to?(:signature) and job.respond_to?(:params)
-        params = *job.params
-        job = job.signature
-      end
-      array << {"job" => job, "params" => params}
-    end
-
-    def dequeue
-      array.first
-    end
-
-    def query(signature)
-      array.search_details_column(signature)
-    end
-
-    def delete(job)
-      array.delete(job)
-    end
-
-    def delete_all
-      array.each {|j| delete(j) }
-    end
-
-    def length
-      array.count
-    end
-
-  end
-end
-
-module  QC
   class Queue
 
-    include AbstractQueue
-    extend AbstractQueue
+    attr_reader :name, :chan
 
-    def self.array
-      default_queue.array
+    def initialize(name, notify=false)
+      @name = name
+      @chan = @name if notify
     end
 
-    def self.database
-      default_queue.database
+    def enqueue(method, *args)
+      Queries.insert(name, method, args, chan)
     end
 
-    def self.default_queue
-      @queue ||= new(nil)
+    def lock(top_bound=TOP_BOUND)
+      Queries.lock_head(name, top_bound)
     end
 
-    def initialize(queue_name)
-      @database = Database.new(queue_name)
-      @array = DurableArray.new(@database)
+    def delete(id)
+      Queries.delete(id)
     end
 
-    def array
-      @array
+    def delete_all(q_name=nil)
+      Queries.delete_all(q_name)
     end
 
-    def database
-      @database
+    def count(q_name=nil)
+      Queries.count(q_name)
     end
 
   end

data/lib/queue_classic/tasks.rb

@@ -1,27 +1,38 @@
 namespace :jobs do
-
-  desc 'Alias for qc:work'
-  task :work  => 'qc:work'
-
+  desc "Alias for qc:work"
+  task :work  => "qc:work"
 end
 
 namespace :qc do
-
-  desc 'Start a new worker for the (default or QUEUE) queue'
+  desc "Start a new worker for the (default or $QUEUE) queue"
   task :work  => :environment do
-    QC::Worker.new.start
+    QC::Worker.new(
+      QC::TABLE_NAME,
+      QC::TOP_BOUND,
+      QC::FORK_WORKER,
+      QC::LISTENING_WORKER,
+      QC::MAX_LOCK_ATTEMPTS
+    ).start
   end
 
-  desc 'Returns the number of jobs in the (default or QUEUE) queue'
-  task :jobs => :environment do
-    puts QC::Queue.new(ENV['QUEUE']).length
+  desc "Returns the number of jobs in the (default or QUEUE) queue"
+  task :length => :environment do
+    puts QC::Worker.new(
+      QC::TABLE_NAME,
+      QC::TOP_BOUND,
+      QC::FORK_WORKER,
+      QC::LISTENING_WORKER,
+      QC::MAX_LOCK_ATTEMPTS
+    ).length
   end
 
-  desc 'Ensure the database has the necessary functions for QC'
+  desc "Ensure the database has the necessary functions for QC"
   task :load_functions => :environment do
-    db = QC::Database.new
-    db.load_functions
-    db.disconnect
+    QC::Queries.load_functions
   end
 
+  desc "Remove queue_classic functions from database."
+  task :load_functions => :environment do
+    QC::Queries.drop_functions
+  end
 end