queue_classic_pg2 3.2.0.RC1

data/lib/queue_classic.rb

@@ -0,0 +1,122 @@
+require_relative "queue_classic/config"
+
+module QC
+  extend QC::Config
+
+  # Assign constants for backwards compatibility.
+  # They should no longer be used. Prefer the corresponding methods.
+  # See +QC::Config+ for more details.
+  DEPRECATED_CONSTANTS = {
+    :APP_NAME => :app_name,
+    :WAIT_TIME => :wait_time,
+    :TABLE_NAME => :table_name,
+    :QUEUE => :queue,
+    :QUEUES => :queues,
+    :TOP_BOUND => :top_bound,
+    :FORK_WORKER => :fork_worker?,
+  }
+
+  def self.const_missing(const_name)
+    if DEPRECATED_CONSTANTS.key? const_name
+      config_method = DEPRECATED_CONSTANTS[const_name]
+      $stderr.puts <<-MSG
+The constant QC::#{const_name} is deprecated and will be removed in the future.
+Please use the method QC.#{config_method} instead.
+      MSG
+      QC.public_send config_method
+    else
+      super
+    end
+  end
+
+  # Defer method calls on the QC module to the
+  # default queue. This facilitates QC.enqueue()
+  def self.method_missing(sym, *args, &block)
+    if default_queue.respond_to? sym
+      default_queue.public_send(sym, *args, &block)
+    else
+      super
+    end
+  end
+
+  # Ensure QC.respond_to?(:enqueue) equals true (ruby 1.9 only)
+  def self.respond_to_missing?(method_name, include_private = false)
+    default_queue.respond_to?(method_name)
+  end
+
+  def self.has_connection?
+    !default_conn_adapter.nil?
+  end
+
+  def self.default_conn_adapter
+    t = Thread.current
+    return t[:qc_conn_adapter] if t[:qc_conn_adapter]
+    adapter = if rails_connection_sharing_enabled?
+      ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
+    else
+      ConnAdapter.new
+    end
+
+    t[:qc_conn_adapter] = adapter
+  end
+
+  def self.default_conn_adapter=(conn)
+    Thread.current[:qc_conn_adapter] = conn
+  end
+
+  def self.log_yield(data)
+    t0 = Time.now
+    begin
+      yield
+    rescue => e
+      log({:at => "error", :error => e.inspect}.merge(data))
+      raise
+    ensure
+      t = Integer((Time.now - t0)*1000)
+      log(data.merge(:elapsed => t)) unless e
+    end
+  end
+
+  def self.log(data)
+    result = nil
+    data = {:lib => "queue-classic"}.merge(data)
+    if block_given?
+      result = yield
+      data.merge(:elapsed => Integer((Time.now - t0)*1000))
+    end
+    data.reduce(out=String.new) do |s, tup|
+      s << [tup.first, tup.last].join("=") << " "
+    end
+    puts(out) if ENV["DEBUG"]
+    return result
+  end
+
+  def self.measure(data)
+    if ENV['QC_MEASURE']
+      $stdout.puts("measure#qc.#{data}")
+    end
+  end
+
+  # This will unlock all jobs any postgres' PID that is not existing anymore
+  # to prevent any infinitely locked jobs
+  def self.unlock_jobs_of_dead_workers
+    pid_column = default_conn_adapter.server_version < 90200 ? "procpid" : "pid"
+    default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT #{pid_column} FROM pg_stat_activity);")
+  end
+
+  # private class methods
+  class << self
+    private
+
+    def rails_connection_sharing_enabled?
+      enabled = ENV.fetch('QC_RAILS_DATABASE', 'true') != 'false'
+      return false unless enabled
+      return Object.const_defined?("ActiveRecord") && ActiveRecord::Base.respond_to?("connection")
+    end
+  end
+end
+
+require_relative "queue_classic/queue"
+require_relative "queue_classic/worker"
+require_relative "queue_classic/setup"
+require_relative "queue_classic/railtie" if defined?(Rails)

data/queue_classic.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'queue_classic/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "queue_classic_pg2"
+  spec.email         = "r@32k.io"
+  spec.version       = QC::VERSION
+  spec.description   = "Support for Postgres2 : queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...) queue_classic features asynchronous job polling, database maintained locks and no ridiculous dependencies. As a matter of fact, queue_classic only requires pg."
+  spec.summary       = "Simple, efficient worker queue for Ruby & PostgreSQL."
+  spec.authors       = ["Ryan Smith (♠ ace hacker)"]
+  spec.homepage      = "https://github.com/PatrickLef/queue_classic"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.require_paths = %w[lib]
+
+  spec.add_dependency "pg", ">= 0.17", "< 2.0"
+end

data/sql/create_table.sql

@@ -0,0 +1,25 @@
+DO $$ BEGIN
+
+CREATE TABLE queue_classic_jobs (
+  id bigserial PRIMARY KEY,
+  q_name text NOT NULL CHECK (length(q_name) > 0),
+  method text NOT NULL CHECK (length(method) > 0),
+  args   text NOT NULL,
+  locked_at timestamptz,
+  locked_by integer,
+  created_at timestamptz DEFAULT now(),
+  scheduled_at timestamptz DEFAULT now()
+);
+
+-- If jsonb type is available, use it for the args column
+IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'jsonb') THEN
+  ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE jsonb USING args::jsonb;
+-- Otherwise, use json type for the args column if available
+ELSIF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'json') THEN
+  ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE json USING args::json;
+END IF;
+
+END $$ LANGUAGE plpgsql;
+
+CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
+CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;

data/sql/ddl.sql

@@ -0,0 +1,78 @@
+-- We are declaring the return type to be queue_classic_jobs.
+-- This is ok since I am assuming that all of the users added queues will
+-- have identical columns to queue_classic_jobs.
+-- When QC supports queues with columns other than the default, we will have to change this.
+
+CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
+RETURNS SETOF queue_classic_jobs AS $$
+DECLARE
+  unlocked bigint;
+  relative_top integer;
+  job_count integer;
+BEGIN
+  -- The purpose is to release contention for the first spot in the table.
+  -- The select count(*) is going to slow down dequeue performance but allow
+  -- for more workers. Would love to see some optimization here...
+
+  EXECUTE 'SELECT count(*) FROM '
+    || '(SELECT * FROM queue_classic_jobs '
+    || ' WHERE locked_at IS NULL'
+    || ' AND q_name = '
+    || quote_literal(q_name)
+    || ' AND scheduled_at <= '
+    || quote_literal(now())
+    || ' LIMIT '
+    || quote_literal(top_boundary)
+    || ') limited'
+  INTO job_count;
+
+  SELECT TRUNC(random() * (top_boundary - 1))
+  INTO relative_top;
+
+  IF job_count < top_boundary THEN
+    relative_top = 0;
+  END IF;
+
+  LOOP
+    BEGIN
+      EXECUTE 'SELECT id FROM queue_classic_jobs '
+        || ' WHERE locked_at IS NULL'
+        || ' AND q_name = '
+        || quote_literal(q_name)
+        || ' AND scheduled_at <= '
+        || quote_literal(now())
+        || ' ORDER BY id ASC'
+        || ' LIMIT 1'
+        || ' OFFSET ' || quote_literal(relative_top)
+        || ' FOR UPDATE NOWAIT'
+      INTO unlocked;
+      EXIT;
+    EXCEPTION
+      WHEN lock_not_available THEN
+        -- do nothing. loop again and hope we get a lock
+    END;
+  END LOOP;
+
+  RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
+    || ' SET locked_at = (CURRENT_TIMESTAMP),'
+    || ' locked_by = (select pg_backend_pid())'
+    || ' WHERE id = $1'
+    || ' AND locked_at is NULL'
+    || ' RETURNING *'
+  USING unlocked;
+
+  RETURN;
+END $$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$ BEGIN
+  RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
+END $$ LANGUAGE plpgsql;
+
+-- queue_classic_notify function and trigger
+CREATE FUNCTION queue_classic_notify() RETURNS TRIGGER AS $$ BEGIN
+  perform pg_notify(new.q_name, ''); RETURN NULL;
+END $$ LANGUAGE plpgsql;
+
+CREATE TRIGGER queue_classic_notify
+AFTER INSERT ON queue_classic_jobs FOR EACH ROW
+EXECUTE PROCEDURE queue_classic_notify();

data/sql/downgrade_from_3_0_0.sql

@@ -0,0 +1,2 @@
+ALTER TABLE queue_classic_jobs DROP COLUMN locked_by;
+ALTER TABLE queue_classic_jobs DROP COLUMN created_at;

data/sql/downgrade_from_3_1_0.sql

@@ -0,0 +1 @@
+ALTER TABLE queue_classic_jobs DROP COLUMN scheduled_at;

data/sql/drop_ddl.sql

@@ -0,0 +1,3 @@
+DROP FUNCTION IF EXISTS lock_head(tname varchar);
+DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer);
+DROP FUNCTION IF EXISTS queue_classic_notify() cascade;

data/sql/update_to_3_0_0.sql

@@ -0,0 +1,17 @@
+DO $$DECLARE r record;
+BEGIN
+  BEGIN
+    ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz DEFAULT now();
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
+  END;
+END$$;
+
+DO $$DECLARE r record;
+BEGIN
+  BEGIN
+    ALTER TABLE queue_classic_jobs ADD COLUMN locked_by integer;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column locked_by already exists in queue_classic_jobs.';
+  END;
+END$$;

data/sql/update_to_3_1_0.sql

@@ -0,0 +1,9 @@
+DO $$DECLARE r record;
+BEGIN
+  BEGIN
+    ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz DEFAULT now();
+    CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
+  END;
+END$$;

data/test/benchmark_test.rb

@@ -0,0 +1,39 @@
+require_relative 'helper'
+
+if ENV["QC_BENCHMARK"]
+  class BenchmarkTest < QCTest
+    BENCHMARK_SIZE = Integer(ENV.fetch("QC_BENCHMARK_SIZE", 10_000))
+    BENCHMARK_MAX_TIME_DEQUEUE = Integer(ENV.fetch("QC_BENCHMARK_MAX_TIME_DEQUEUE", 30))
+    BENCHMARK_MAX_TIME_ENQUEUE = Integer(ENV.fetch("QC_BENCHMARK_MAX_TIME_ENQUEUE", 5))
+
+    def test_enqueue
+      start = Time.now
+      BENCHMARK_SIZE.times do
+        QC.enqueue("1.odd?")
+      end
+      assert_equal(BENCHMARK_SIZE, QC.count)
+
+      elapsed = Time.now - start
+      assert_operator(elapsed, :<, BENCHMARK_MAX_TIME_ENQUEUE)
+    end
+
+    def test_dequeue
+      worker = QC::Worker.new
+      worker.running = true
+      BENCHMARK_SIZE.times do
+        QC.enqueue("1.odd?")
+      end
+      assert_equal(BENCHMARK_SIZE, QC.count)
+
+      start = Time.now
+      BENCHMARK_SIZE.times do
+        worker.work
+      end
+      elapsed = Time.now - start
+
+      assert_equal(0, QC.count)
+      assert_operator(elapsed, :<, BENCHMARK_MAX_TIME_DEQUEUE)
+    end
+
+  end
+end

data/test/config_test.rb

@@ -0,0 +1,121 @@
+require_relative 'helper'
+
+class ConfigTest < QCTest
+  def setup
+    QC.reset_config
+  end
+
+  def teardown
+    QC.reset_config
+  end
+
+  def test_app_name_default
+    assert_equal "queue_classic", QC.app_name
+  end
+
+  def test_configure_app_name_with_env_var
+    with_env "QC_APP_NAME" => "zomg_qc" do
+      assert_equal "zomg_qc", QC.app_name
+    end
+  end
+
+  def test_wait_time_default
+    assert_equal 5, QC.wait_time
+  end
+
+  def test_configure_wait_time_with_env_var
+    with_env "QC_LISTEN_TIME" => "7" do
+      assert_equal 7, QC.wait_time
+    end
+  end
+
+  def test_table_name_default
+    assert_equal "queue_classic_jobs", QC.table_name
+  end
+
+  def test_queue_default
+    assert_equal "default", QC.queue
+    assert_equal "default", QC.default_queue.name
+  end
+
+  def test_configure_queue_with_env_var
+    with_env "QUEUE" => "priority" do
+      assert_equal "priority", QC.queue
+      assert_equal "priority", QC.default_queue.name
+    end
+  end
+
+  def test_assign_default_queue
+    QC.default_queue = QC::Queue.new "dispensable"
+    assert_equal "default", QC.queue
+    assert_equal "dispensable", QC.default_queue.name
+  end
+
+  def test_queues_default
+    assert_equal [], QC.queues
+  end
+
+  def test_configure_queues_with_env_var
+    with_env "QUEUES" => "first,second,third" do
+      assert_equal %w(first second third), QC.queues
+    end
+  end
+
+  def test_configure_queues_with_whitespace
+    with_env "QUEUES" => " one, two, three " do
+      assert_equal %w(one two three), QC.queues
+    end
+  end
+
+  def test_top_bound_default
+    assert_equal 9, QC.top_bound
+  end
+
+  def test_configure_top_bound_with_env_var
+    with_env "QC_TOP_BOUND" => "5" do
+      assert_equal 5, QC.top_bound
+    end
+  end
+
+  def test_fork_worker_default
+    refute QC.fork_worker?
+  end
+
+  def test_configure_fork_worker_with_env_var
+    with_env "QC_FORK_WORKER" => "yo" do
+      assert QC.fork_worker?
+    end
+  end
+
+  def test_configuration_constants_are_deprecated
+    warning = capture_stderr_output do
+      QC::FORK_WORKER
+    end
+    assert_match "QC::FORK_WORKER is deprecated", warning
+    assert_match "QC.fork_worker? instead", warning
+  end
+
+  class TestWorker < QC::Worker; end
+
+  def test_default_worker_class
+    assert_equal QC::Worker, QC.default_worker_class
+  end
+
+  def test_configure_default_worker_class_with_env_var
+    if RUBY_VERSION =~ /^1\.9\./
+        skip "Kernel.const_get in Ruby 1.9.x does not perform recursive lookups"
+    end
+    with_env "QC_DEFAULT_WORKER_CLASS" => "ConfigTest::TestWorker" do
+      assert_equal TestWorker, QC.default_worker_class
+    end
+  end
+
+  def test_assign_default_worker_class
+    original_worker = QC.default_worker_class
+    QC.default_worker_class = TestWorker
+
+    assert_equal TestWorker, QC.default_worker_class
+  ensure
+    QC.default_worker_class = original_worker
+  end
+end

data/test/helper.rb

@@ -0,0 +1,61 @@
+require "bundler"
+Bundler.setup :default, :test
+
+ENV["DATABASE_URL"] ||= "postgres:///queue_classic_test"
+
+require_relative '../lib/queue_classic'
+require "stringio"
+require "minitest/autorun"
+
+class QCTest < Minitest::Test
+
+  def setup
+    init_db
+  end
+
+  def teardown
+    QC.delete_all
+  end
+
+  def init_db
+    c = QC::ConnAdapter.new
+    c.execute("SET client_min_messages TO 'warning'")
+    QC::Setup.drop(c.connection)
+    QC::Setup.create(c.connection)
+    c.execute(File.read('./test/helper.sql'))
+    c.disconnect
+  end
+
+  def capture_stderr_output
+    original_stderr = $stderr
+    $stderr = StringIO.new
+    yield
+    $stderr.string
+  ensure
+    $stderr = original_stderr
+  end
+
+  def capture_debug_output
+    original_debug = ENV['DEBUG']
+    original_stdout = $stdout
+
+    ENV['DEBUG'] = "true"
+    $stdout = StringIO.new
+    yield
+    $stdout.string
+  ensure
+    ENV['DEBUG'] = original_debug
+    $stdout = original_stdout
+  end
+
+  def with_env(temporary_environment)
+    original_environment = {}
+    temporary_environment.each do |name, value|
+      original_environment[name] = ENV[name]
+      ENV[name] = value
+    end
+    yield
+  ensure
+    original_environment.each { |name, value| ENV[name] = value }
+  end
+end

data/test/helper.sql

@@ -0,0 +1,25 @@
+DO $$
+-- Set initial sequence to a large number to test the entire toolchain
+-- works on integers with higher bits set.
+DECLARE
+    quoted_name text;
+    quoted_size text;
+BEGIN
+    -- Find the name of the relevant sequence.
+    --
+    -- pg_get_serial_sequence quotes identifiers as part of its
+    -- behavior.
+    SELECT name
+    INTO STRICT quoted_name
+    FROM pg_get_serial_sequence('queue_classic_jobs', 'id') AS name;
+
+    -- Don't quote, because ALTER SEQUENCE RESTART doesn't like
+    -- general literals, only unquoted numeric literals.
+    SELECT pow(2, 34)::text AS size
+    INTO STRICT quoted_size;
+
+    EXECUTE 'ALTER SEQUENCE ' || quoted_name ||
+        ' RESTART ' || quoted_size || ';';
+END;
+$$;
+

data/test/lib/queue_classic_rails_connection_test.rb

@@ -0,0 +1,43 @@
+require File.expand_path("../../helper.rb", __FILE__)
+
+class QueueClassicRailsConnectionTest < QCTest
+  def before_setup
+    Object.send :const_set, :ActiveRecord, Module.new
+    ActiveRecord.const_set :Base, Module.new
+
+    @original_conn_adapter = QC.default_conn_adapter
+    QC.default_conn_adapter = nil
+  end
+
+  def after_teardown
+    ActiveRecord.send :remove_const, :Base
+    Object.send :remove_const, :ActiveRecord
+
+    QC.default_conn_adapter = @original_conn_adapter
+  end
+
+  def test_uses_active_record_connection_if_exists
+    connection = get_connection
+    assert connection.verify
+  end
+
+  def test_does_not_use_active_record_connection_if_env_var_set
+    with_env 'QC_RAILS_DATABASE' => 'false' do
+      connection = get_connection
+      assert_raises(MockExpectationError) { connection.verify }
+    end
+  end
+
+  private
+  def get_connection
+    connection = Minitest::Mock.new
+    connection.expect(:raw_connection, QC::ConnAdapter.new.connection)
+
+    ActiveRecord::Base.define_singleton_method(:connection) do
+      connection
+    end
+
+    QC.default_conn_adapter
+    connection
+  end
+end

data/test/lib/queue_classic_test.rb

@@ -0,0 +1,42 @@
+require File.expand_path("../../helper.rb", __FILE__)
+
+class QueueClassicTest < QCTest
+  def test_only_delegate_calls_to_queue_it_understands
+    e = assert_raises(NoMethodError) do
+      QC.probably_not
+    end
+    assert_match "undefined method `probably_not' for QC:Module", e.message
+  end
+
+  def test_default_conn_adapter_default_value
+    assert(QC.default_conn_adapter.is_a?(QC::ConnAdapter))
+  end
+
+  def test_assigning_a_default_conn_adapter
+    original_conn_adapter = QC.default_conn_adapter
+    connection = QC::ConnAdapter.new
+    QC.default_conn_adapter = connection
+    assert_equal(QC.default_conn_adapter, connection)
+  ensure
+    QC.default_conn_adapter = original_conn_adapter
+  end
+
+  def test_unlock_jobs_of_dead_workers
+    # Insert a locked job
+    adapter = QC::ConnAdapter.new
+    query = "INSERT INTO #{QC.table_name} (q_name, method, args, locked_by, locked_at) VALUES ('whatever', 'Kernel.puts', '[\"ok?\"]', 0, (CURRENT_TIMESTAMP))"
+    adapter.execute(query)
+
+    # We should have no unlocked jobs
+    query_locked_jobs = "SELECT * FROM #{QC.table_name} WHERE locked_at IS NULL"
+    res = adapter.connection.exec(query_locked_jobs)
+    assert_equal(0, res.count)
+
+    # Unlock the job
+    QC.unlock_jobs_of_dead_workers
+
+    # We should have an unlocked job now
+    res = adapter.connection.exec(query_locked_jobs)
+    assert_equal(1, res.count)
+  end
+end