queue_classic_pg2 3.2.0.RC1

data/lib/queue_classic/config.rb

@@ -0,0 +1,85 @@
+module QC
+  module Config
+    # You can use the APP_NAME to query for
+    # postgres related process information in the
+    # pg_stat_activity table.
+    def app_name
+      @app_name ||= ENV["QC_APP_NAME"] || "queue_classic"
+    end
+
+    # Number of seconds to block on the listen chanel for new jobs.
+    def wait_time
+      @wait_time ||= (ENV["QC_LISTEN_TIME"] || 5).to_i
+    end
+
+    # Why do you want to change the table name?
+    # Just deal with the default OK?
+    # If you do want to change this, you will
+    # need to update the PL/pgSQL lock_head() function.
+    # Come on. Don't do it.... Just stick with the default.
+    def table_name
+      @table_name ||= "queue_classic_jobs"
+    end
+
+    def queue
+      @queue = ENV["QUEUE"] || "default"
+    end
+
+    # The default queue used by `QC.enqueue`.
+    def default_queue
+      @default_queue ||= Queue.new(QC.queue)
+    end
+
+    def default_queue=(queue)
+      @default_queue = queue
+    end
+
+    # Each row in the table will have a column that
+    # notes the queue. You can point your workers
+    # at different queues.
+    def queues
+      @queues ||= (ENV["QUEUES"] && ENV["QUEUES"].split(",").map(&:strip)) || []
+    end
+
+    # Set this to 1 for strict FIFO.
+    # There is nothing special about 9....
+    def top_bound
+      @top_bound ||= (ENV["QC_TOP_BOUND"] || 9).to_i
+    end
+
+    # Set this variable if you wish for
+    # the worker to fork a UNIX process for
+    # each locked job. Remember to re-establish
+    # any database connections. See the worker
+    # for more details.
+    def fork_worker?
+      @fork_worker ||= (!ENV["QC_FORK_WORKER"].nil?)
+    end
+
+    # The worker class instantiated by QC's rake tasks.
+    def default_worker_class
+
+      @worker_class ||= (ENV["QC_DEFAULT_WORKER_CLASS"] && Kernel.const_get(ENV["QC_DEFAULT_WORKER_CLASS"]) ||
+                         QC::Worker)
+
+    end
+
+    def default_worker_class=(worker_class)
+      @worker_class = worker_class
+    end
+
+    # reset memoized configuration
+    def reset_config
+      # TODO: we might want to think about storing these in a Hash.
+      @app_name = nil
+      @wait_time = nil
+      @table_name = nil
+      @queue = nil
+      @default_queue = nil
+      @queues = nil
+      @top_bound = nil
+      @fork_worker = nil
+      @worker_class = nil
+    end
+  end
+end

data/lib/queue_classic/conn_adapter.rb

@@ -0,0 +1,111 @@
+require 'uri'
+require 'pg'
+
+module QC
+  class ConnAdapter
+
+    attr_accessor :connection
+    def initialize(c=nil)
+      @connection = c.nil? ? establish_new : validate!(c)
+      @mutex = Mutex.new
+    end
+
+    def execute(stmt, *params)
+      @mutex.synchronize do
+        QC.log(:at => "exec_sql", :sql => stmt.inspect)
+        begin
+          params = nil if params.empty?
+          r = @connection.exec(stmt, params)
+          result = []
+          r.each {|t| result << t}
+          result.length > 1 ? result : result.pop
+        rescue PG::Error => e
+          QC.log(:error => e.inspect)
+          @connection.reset
+          raise
+        end
+      end
+    end
+
+    def wait(time, *channels)
+      @mutex.synchronize do
+        listen_cmds = channels.map {|c| 'LISTEN "' + c.to_s + '"'}
+        @connection.exec(listen_cmds.join(';'))
+        wait_for_notify(time)
+        unlisten_cmds = channels.map {|c| 'UNLISTEN "' + c.to_s + '"'}
+        @connection.exec(unlisten_cmds.join(';'))
+        drain_notify
+      end
+    end
+
+    def disconnect
+      @mutex.synchronize do
+        begin
+          @connection.close
+        rescue => e
+          QC.log(:at => 'disconnect', :error => e.message)
+        end
+      end
+    end
+
+    def server_version
+      @server_version ||= begin
+                            version = execute("SHOW server_version_num;")["server_version_num"]
+                            version && version.to_i
+                          end
+    end
+
+    private
+
+    def wait_for_notify(t)
+      Array.new.tap do |msgs|
+        @connection.wait_for_notify(t) {|event, pid, msg| msgs << msg}
+      end
+    end
+
+    def drain_notify
+      until @connection.notifies.nil?
+        QC.log(:at => "drain_notifications")
+      end
+    end
+
+    def validate!(c)
+      return c if c.is_a?(PG::Connection)
+      err = "connection must be an instance of PG::Connection, but was #{c.class}"
+      raise(ArgumentError, err)
+    end
+
+    def establish_new
+      QC.log(:at => "establish_conn")
+      conn = PG.connect(*normalize_db_url(db_url))
+      if conn.status != PG::CONNECTION_OK
+        QC.log(:error => conn.error)
+      end
+      conn.exec("SET application_name = '#{QC.app_name}'")
+      conn
+    end
+
+    def normalize_db_url(url)
+      host = url.host
+      host = host.gsub(/%2F/i, '/') if host
+
+      [
+       host, # host or percent-encoded socket path
+       url.port || 5432,
+       nil, '', #opts, tty
+       url.path.gsub("/",""), # database name
+       url.user,
+       url.password
+      ]
+    end
+
+    def db_url
+      return @db_url if defined?(@db_url) && @db_url
+      url = ENV["QC_DATABASE_URL"] ||
+            ENV["DATABASE_URL"]    ||
+            raise(ArgumentError, "missing QC_DATABASE_URL or DATABASE_URL")
+      @db_url = URI.parse(url)
+    end
+
+  end
+end

data/lib/queue_classic/queue.rb

@@ -0,0 +1,119 @@
+require_relative 'conn_adapter'
+require 'json'
+require 'time'
+
+module QC
+  # The queue class maps a queue abstraction onto a database table.
+  class Queue
+
+    attr_reader :name, :top_bound
+    def initialize(name, top_bound=nil)
+      @name = name
+      @top_bound = top_bound || QC.top_bound
+    end
+
+    def conn_adapter=(a)
+      @adapter = a
+    end
+
+    def conn_adapter
+      @adapter ||= QC.default_conn_adapter
+    end
+
+    # enqueue(m,a) inserts a row into the jobs table and trigger a notification.
+    # The job's queue is represented by a name column in the row.
+    # There is a trigger on the table which will send a NOTIFY event
+    # on a channel which corresponds to the name of the queue.
+    # The method argument is a string encoded ruby expression. The expression
+    # will be separated by a `.` character and then `eval`d.
+    # Examples of the method argument include: `puts`, `Kernel.puts`,
+    # `MyObject.new.puts`.
+    # The args argument will be encoded as JSON and stored as a JSON datatype
+    # in the row. (If the version of PG does not support JSON,
+    # then the args will be stored as text.
+    # The args are stored as a collection and then splatted inside the worker.
+    # Examples of args include: `'hello world'`, `['hello world']`,
+    # `'hello', 'world'`.
+    # This method returns a hash with the id of the enqueued job.
+    def enqueue(method, *args)
+      QC.log_yield(:measure => 'queue.enqueue') do
+        s = "INSERT INTO #{QC.table_name} (q_name, method, args) VALUES ($1, $2, $3) RETURNING id"
+        conn_adapter.execute(s, name, method, JSON.dump(args))
+      end
+    end
+
+    # enqueue_at(t,m,a) inserts a row into the jobs table representing a job
+    # to be executed not before the specified time.
+    # The time argument must be a Time object or a float timestamp. The method
+    # and args argument must be in the form described in the documentation for
+    # the #enqueue method.
+    # This method returns a hash with the id of the enqueued job.
+    def enqueue_at(timestamp, method, *args)
+      offset = Time.at(timestamp).to_i - Time.now.to_i
+      enqueue_in(offset, method, *args)
+    end
+
+    # enqueue_in(t,m,a) inserts a row into the jobs table representing a job
+    # to be executed not before the specified time offset.
+    # The seconds argument must be an integer. The method and args argument
+    # must be in the form described in the documentation for the #enqueue
+    # method.
+    # This method returns a hash with the id of the enqueued job.
+    def enqueue_in(seconds, method, *args)
+      QC.log_yield(:measure => 'queue.enqueue') do
+        s = "INSERT INTO #{QC.table_name} (q_name, method, args, scheduled_at)
+             VALUES ($1, $2, $3, now() + interval '#{seconds.to_i} seconds')
+             RETURNING id"
+        conn_adapter.execute(s, name, method, JSON.dump(args))
+      end
+    end
+
+    def lock
+      QC.log_yield(:measure => 'queue.lock') do
+        s = "SELECT * FROM lock_head($1, $2)"
+        if r = conn_adapter.execute(s, name, top_bound)
+          {}.tap do |job|
+            job[:id] = r["id"]
+            job[:q_name] = r["q_name"]
+            job[:method] = r["method"]
+            job[:args] = JSON.parse(r["args"])
+            if r["scheduled_at"]
+              job[:scheduled_at] = Time.parse(r["scheduled_at"])
+              ttl = Integer((Time.now - job[:scheduled_at]) * 1000)
+              QC.measure("time-to-lock=#{ttl}ms source=#{name}")
+            end
+          end
+        end
+      end
+    end
+
+    def unlock(id)
+      QC.log_yield(:measure => 'queue.unlock') do
+        s = "UPDATE #{QC.table_name} SET locked_at = NULL WHERE id = $1"
+        conn_adapter.execute(s, id)
+      end
+    end
+
+    def delete(id)
+      QC.log_yield(:measure => 'queue.delete') do
+        conn_adapter.execute("DELETE FROM #{QC.table_name} WHERE id = $1", id)
+      end
+    end
+
+    def delete_all
+      QC.log_yield(:measure => 'queue.delete_all') do
+        s = "DELETE FROM #{QC.table_name} WHERE q_name = $1"
+        conn_adapter.execute(s, name)
+      end
+    end
+
+    def count
+      QC.log_yield(:measure => 'queue.count') do
+        s = "SELECT COUNT(*) FROM #{QC.table_name} WHERE q_name = $1"
+        r = conn_adapter.execute(s, name)
+        r["count"].to_i
+      end
+    end
+
+  end
+end

data/lib/queue_classic/railtie.rb

@@ -0,0 +1,9 @@
+require 'rails/railtie'
+
+module QC
+  class Railtie < ::Rails::Railtie
+    rake_tasks do
+      load 'queue_classic/tasks.rb'
+    end
+  end
+end

data/lib/queue_classic/setup.rb

@@ -0,0 +1,58 @@
+module QC
+  module Setup
+    Root = File.expand_path("../..", File.dirname(__FILE__))
+    SqlFunctions = File.join(Root, "/sql/ddl.sql")
+    CreateTable = File.join(Root, "/sql/create_table.sql")
+    DropSqlFunctions = File.join(Root, "/sql/drop_ddl.sql")
+    UpgradeTo_3_0_0 = File.join(Root, "/sql/update_to_3_0_0.sql")
+    DowngradeFrom_3_0_0 = File.join(Root, "/sql/downgrade_from_3_0_0.sql")
+    UpgradeTo_3_1_0 = File.join(Root, "/sql/update_to_3_1_0.sql")
+    DowngradeFrom_3_1_0 = File.join(Root, "/sql/downgrade_from_3_1_0.sql")
+
+    def self.create(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(CreateTable))
+      conn.execute(File.read(SqlFunctions))
+      conn.disconnect if c.nil? #Don't close a conn we didn't create.
+    end
+
+    def self.drop(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute("DROP TABLE IF EXISTS queue_classic_jobs CASCADE")
+      conn.execute(File.read(DropSqlFunctions))
+      conn.disconnect if c.nil? #Don't close a conn we didn't create.
+    end
+
+    def self.update(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(UpgradeTo_3_0_0))
+      conn.execute(File.read(UpgradeTo_3_1_0))
+      conn.execute(File.read(DropSqlFunctions))
+      conn.execute(File.read(SqlFunctions))
+    end
+
+    def self.update_to_3_0_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(UpgradeTo_3_0_0))
+      conn.execute(File.read(DropSqlFunctions))
+      conn.execute(File.read(SqlFunctions))
+    end
+
+    def self.downgrade_from_3_0_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(DowngradeFrom_3_0_0))
+    end
+
+    def self.update_to_3_1_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(UpgradeTo_3_1_0))
+      conn.execute(File.read(DropSqlFunctions))
+      conn.execute(File.read(SqlFunctions))
+    end
+
+    def self.downgrade_from_3_1_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(DowngradeFrom_3_1_0))
+    end
+  end
+end

data/lib/queue_classic/tasks.rb

@@ -0,0 +1,49 @@
+task :environment
+
+namespace :jobs do
+  desc "Alias for qc:work"
+  task :work  => "qc:work"
+end
+
+namespace :qc do
+  desc "Start a new worker for the (default or $QUEUE / $QUEUES) queue"
+  task :work  => :environment do
+    @worker = QC.default_worker_class.new
+
+    trap('INT') do
+      $stderr.puts("Received INT. Shutting down.")
+      if !@worker.running
+        $stderr.puts("Worker has stopped running. Exit.")
+        exit(1)
+      end
+      @worker.stop
+    end
+
+    trap('TERM') do
+      $stderr.puts("Received Term. Shutting down.")
+      @worker.stop
+    end
+
+    @worker.start
+  end
+
+  desc "Returns the number of jobs in the (default or $QUEUE / $QUEUES) queue"
+  task :count => :environment do
+    puts QC.default_queue.count
+  end
+
+  desc "Setup queue_classic tables and functions in database"
+  task :create => :environment do
+    QC::Setup.create
+  end
+
+  desc "Remove queue_classic tables and functions from database."
+  task :drop => :environment do
+    QC::Setup.drop
+  end
+
+  desc "Update queue_classic tables and functions in database"
+  task :update => :environment do
+    QC::Setup.update
+  end
+end

data/lib/queue_classic/version.rb

@@ -0,0 +1,3 @@
+module QC
+  VERSION = "3.2.0.RC1"
+end

data/lib/queue_classic/worker.rb

@@ -0,0 +1,166 @@
+# -*- coding: utf-8 -*-
+require_relative 'queue'
+require_relative 'conn_adapter'
+
+module QC
+  # A Worker object can process jobs from one or many queues.
+  class Worker
+
+    attr_accessor :queues, :running
+
+    # Creates a new worker but does not start the worker. See Worker#start.
+    # This method takes a single hash argument. The following keys are read:
+    # fork_worker:: Worker forks each job execution.
+    # wait_interval:: Time to wait between failed lock attempts
+    # connection:: PG::Connection object.
+    # q_name:: Name of a single queue to process.
+    # q_names:: Names of queues to process. Will process left to right.
+    # top_bound:: Offset to the head of the queue. 1 == strict FIFO.
+    def initialize(args={})
+      @fork_worker = args[:fork_worker] || QC.fork_worker?
+      @wait_interval = args[:wait_interval] || QC.wait_time
+
+      if args[:connection]
+        @conn_adapter = ConnAdapter.new(args[:connection])
+      else
+        @conn_adapter = QC.default_conn_adapter
+      end
+
+      @queues = setup_queues(@conn_adapter,
+        (args[:q_name] || QC.queue),
+        (args[:q_names] || QC.queues),
+        (args[:top_bound] || QC.top_bound))
+      log(args.merge(:at => "worker_initialized"))
+      @running = true
+    end
+
+    # Commences the working of jobs.
+    # start() spins on @running –which is initialized as true.
+    # This method is the primary entry point to starting the worker.
+    # The canonical example of starting a worker is as follows:
+    # QC::Worker.new.start
+    def start
+      QC.unlock_jobs_of_dead_workers
+
+      while @running
+        @fork_worker ? fork_and_work : work
+      end
+    end
+
+    # Signals the worker to stop taking new work.
+    # This method has no immediate effect. However, there are
+    # two loops in the worker (one in #start and another in #lock_job)
+    # which check the @running variable to determine if further progress
+    # is desirable. In the case that @running is false, the aforementioned
+    # methods will short circuit and cause the blocking call to #start
+    # to unblock.
+    def stop
+      @running = false
+    end
+
+    # Calls Worker#work but after the current process is forked.
+    # The parent process will wait on the child process to exit.
+    def fork_and_work
+      cpid = fork {setup_child; work}
+      log(:at => :fork, :pid => cpid)
+      Process.wait(cpid)
+    end
+
+    # Blocks on locking a job, and once a job is locked,
+    # it will process the job.
+    def work
+      queue, job = lock_job
+      if queue && job
+        QC.log_yield(:at => "work", :job => job[:id]) do
+          process(queue, job)
+        end
+      end
+    end
+
+    # Attempt to lock a job in the queue's table.
+    # If a job can be locked, this method returns an array with
+    # 2 elements. The first element is the queue from which the job was locked
+    # and the second is a hash representation of the job.
+    # If a job is returned, its locked_at column has been set in the
+    # job's row. It is the caller's responsibility to delete the job row
+    # from the table when the job is complete.
+    def lock_job
+      log(:at => "lock_job")
+      job = nil
+      while @running
+        @queues.each do |queue|
+          if job = queue.lock
+            return [queue, job]
+          end
+        end
+        @conn_adapter.wait(@wait_interval, *@queues.map {|q| q.name})
+      end
+    end
+
+    # A job is processed by evaluating the target code.
+    # if the job is evaluated with no exceptions
+    # then it is deleted from the queue.
+    # If the job has raised an exception the responsibility of what
+    # to do with the job is delegated to Worker#handle_failure.
+    # If the job is not finished and an INT signal is trapped,
+    # this method will unlock the job in the queue.
+    def process(queue, job)
+      start = Time.now
+      finished = false
+      begin
+        call(job).tap do
+          queue.delete(job[:id])
+          finished = true
+        end
+      rescue => e
+        handle_failure(job, e)
+        finished = true
+      ensure
+        if !finished
+          queue.unlock(job[:id])
+        end
+        ttp = Integer((Time.now - start) * 1000)
+        QC.measure("time-to-process=#{ttp} source=#{queue.name}")
+      end
+    end
+
+    # Each job includes a method column. We will use ruby's eval
+    # to grab the ruby object from memory. We send the method to
+    # the object and pass the args.
+    def call(job)
+      args = job[:args]
+      receiver_str, _, message = job[:method].rpartition('.')
+      receiver = eval(receiver_str)
+      receiver.send(message, *args)
+    end
+
+    # This method will be called when an exception
+    # is raised during the execution of the job.
+    def handle_failure(job,e)
+      $stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect} at=#{e.backtrace.first}")
+    end
+
+    # This method should be overriden if
+    # your worker is forking and you need to
+    # re-establish database connections
+    def setup_child
+      log(:at => "setup_child")
+    end
+
+    def log(data)
+      QC.log(data)
+    end
+
+    private
+
+    def setup_queues(adapter, queue, queues, top_bound)
+      names = queues.length > 0 ? queues : [queue]
+      names.map do |name|
+        QC::Queue.new(name, top_bound).tap do |q|
+          q.conn_adapter = adapter
+        end
+      end
+    end
+
+  end
+end