queue_classic 3.1.0.RC1 → 4.0.0.pre.beta1

data/sql/downgrade_from_4_0_0.sql

@@ -0,0 +1,88 @@
+DO $$DECLARE r record;
+BEGIN
+  -- If jsonb type is available, do nothing as we're downgrading from 4.0.0
+  IF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'jsonb') THEN
+   -- do nothing - it should already be already jsonb
+  -- Otherwise, use json type for the args column if available
+  ELSIF EXISTS (SELECT 1 FROM pg_type WHERE typname = 'json') THEN
+    -- this should only happen if someone downgrades QC and their database < pg 9.4
+    ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE json USING args::json;
+  END IF;
+
+
+END$$;
+
+
+--
+-- Re install the lock_head function
+--
+
+-- We are declaring the return type to be queue_classic_jobs.
+-- This is ok since I am assuming that all of the users added queues will
+-- have identical columns to queue_classic_jobs.
+-- When QC supports queues with columns other than the default, we will have to change this.
+
+CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
+RETURNS SETOF queue_classic_jobs AS $$
+DECLARE
+  unlocked bigint;
+  relative_top integer;
+  job_count integer;
+BEGIN
+  -- The purpose is to release contention for the first spot in the table.
+  -- The select count(*) is going to slow down dequeue performance but allow
+  -- for more workers. Would love to see some optimization here...
+
+  EXECUTE 'SELECT count(*) FROM '
+    || '(SELECT * FROM queue_classic_jobs '
+    || ' WHERE locked_at IS NULL'
+    || ' AND q_name = '
+    || quote_literal(q_name)
+    || ' AND scheduled_at <= '
+    || quote_literal(now())
+    || ' LIMIT '
+    || quote_literal(top_boundary)
+    || ') limited'
+  INTO job_count;
+
+  SELECT TRUNC(random() * (top_boundary - 1))
+  INTO relative_top;
+
+  IF job_count < top_boundary THEN
+    relative_top = 0;
+  END IF;
+
+  LOOP
+    BEGIN
+      EXECUTE 'SELECT id FROM queue_classic_jobs '
+        || ' WHERE locked_at IS NULL'
+        || ' AND q_name = '
+        || quote_literal(q_name)
+        || ' AND scheduled_at <= '
+        || quote_literal(now())
+        || ' ORDER BY id ASC'
+        || ' LIMIT 1'
+        || ' OFFSET ' || quote_literal(relative_top)
+        || ' FOR UPDATE NOWAIT'
+      INTO unlocked;
+      EXIT;
+    EXCEPTION
+      WHEN lock_not_available THEN
+        -- do nothing. loop again and hope we get a lock
+    END;
+  END LOOP;
+
+  RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
+    || ' SET locked_at = (CURRENT_TIMESTAMP),'
+    || ' locked_by = (select pg_backend_pid())'
+    || ' WHERE id = $1'
+    || ' AND locked_at is NULL'
+    || ' RETURNING *'
+  USING unlocked;
+
+  RETURN;
+END $$ LANGUAGE plpgsql;
+
+CREATE OR REPLACE FUNCTION lock_head(tname varchar) RETURNS SETOF queue_classic_jobs AS $$ BEGIN
+  RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
+END $$ LANGUAGE plpgsql;

data/sql/update_to_3_0_0.sql

@@ -1,10 +1,10 @@
 DO $$DECLARE r record;
 BEGIN
-	BEGIN
-		ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz default now();
-	EXCEPTION
-		WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
-	END;
+  BEGIN
+    ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz DEFAULT now();
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
+  END;
 END$$;
 
 DO $$DECLARE r record;

data/sql/update_to_3_1_0.sql

@@ -1,9 +1,9 @@
 DO $$DECLARE r record;
 BEGIN
-	BEGIN
-		ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz default now();
-		CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
-	EXCEPTION
-		WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
-	END;
+  BEGIN
+    ALTER TABLE queue_classic_jobs ADD COLUMN scheduled_at timestamptz DEFAULT now();
+    CREATE INDEX idx_qc_on_scheduled_at_only_unlocked ON queue_classic_jobs (scheduled_at, id) WHERE locked_at IS NULL;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column scheduled_at already exists in queue_classic_jobs.';
+  END;
 END$$;

data/sql/update_to_4_0_0.sql

@@ -0,0 +1,6 @@
+DO $$DECLARE r record;
+BEGIN
+  ALTER TABLE queue_classic_jobs ALTER COLUMN args TYPE jsonb USING args::jsonb;
+  DROP FUNCTION IF EXISTS lock_head(tname varchar);
+  DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer);
+END$$;

data/test/benchmark_test.rb

@@ -1,37 +1,40 @@
-require File.expand_path("../helper.rb", __FILE__)
+# frozen_string_literal: true
+
+require_relative 'helper'
 
 if ENV["QC_BENCHMARK"]
   class BenchmarkTest < QCTest
+    BENCHMARK_SIZE = Integer(ENV.fetch("QC_BENCHMARK_SIZE", 10_000))
+    BENCHMARK_MAX_TIME_DEQUEUE = Integer(ENV.fetch("QC_BENCHMARK_MAX_TIME_DEQUEUE", 30))
+    BENCHMARK_MAX_TIME_ENQUEUE = Integer(ENV.fetch("QC_BENCHMARK_MAX_TIME_ENQUEUE", 5))
 
     def test_enqueue
-      n = 10_000
       start = Time.now
-      n.times do
-        QC.enqueue("1.odd?", [])
+      BENCHMARK_SIZE.times do
+        QC.enqueue("1.odd?")
       end
-      assert_equal(n, QC.count)
+      assert_equal(BENCHMARK_SIZE, QC.count)
 
       elapsed = Time.now - start
-      assert_in_delta(4, elapsed, 1)
+      assert_operator(elapsed, :<, BENCHMARK_MAX_TIME_ENQUEUE)
     end
 
     def test_dequeue
       worker = QC::Worker.new
       worker.running = true
-      n = 10_000
-      n.times do
-        QC.enqueue("1.odd?", [])
+      BENCHMARK_SIZE.times do
+        QC.enqueue("1.odd?")
       end
-      assert_equal(n, QC.count)
+      assert_equal(BENCHMARK_SIZE, QC.count)
 
       start = Time.now
-      n.times do
+      BENCHMARK_SIZE.times do
         worker.work
       end
       elapsed = Time.now - start
 
       assert_equal(0, QC.count)
-      assert_in_delta(10, elapsed, 3)
+      assert_operator(elapsed, :<, BENCHMARK_MAX_TIME_DEQUEUE)
     end
 
   end

data/test/config_test.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require_relative 'helper'
+
+class ConfigTest < QCTest
+  def setup
+    QC.reset_config
+  end
+
+  def teardown
+    QC.reset_config
+  end
+
+  def test_app_name_default
+    assert_equal "queue_classic", QC.app_name
+  end
+
+  def test_configure_app_name_with_env_var
+    with_env "QC_APP_NAME" => "zomg_qc" do
+      assert_equal "zomg_qc", QC.app_name
+    end
+  end
+
+  def test_wait_time_default
+    assert_equal 5, QC.wait_time
+  end
+
+  def test_configure_wait_time_with_env_var
+    with_env "QC_LISTEN_TIME" => "7" do
+      assert_equal 7, QC.wait_time
+    end
+  end
+
+  def test_table_name_default
+    assert_equal "queue_classic_jobs", QC.table_name
+  end
+
+  def test_queue_default
+    assert_equal "default", QC.queue
+    assert_equal "default", QC.default_queue.name
+  end
+
+  def test_configure_queue_with_env_var
+    with_env "QUEUE" => "priority" do
+      assert_equal "priority", QC.queue
+      assert_equal "priority", QC.default_queue.name
+    end
+  end
+
+  def test_assign_default_queue
+    QC.default_queue = QC::Queue.new "dispensable"
+    assert_equal "default", QC.queue
+    assert_equal "dispensable", QC.default_queue.name
+  end
+
+  def test_queues_default
+    assert_equal [], QC.queues
+  end
+
+  def test_configure_queues_with_env_var
+    with_env "QUEUES" => "first,second,third" do
+      assert_equal %w(first second third), QC.queues
+    end
+  end
+
+  def test_configure_queues_with_whitespace
+    with_env "QUEUES" => " one, two, three " do
+      assert_equal %w(one two three), QC.queues
+    end
+  end
+
+  def test_top_bound_default
+    assert_equal 9, QC.top_bound
+  end
+
+  def test_configure_top_bound_with_env_var
+    with_env "QC_TOP_BOUND" => "5" do
+      assert_equal 5, QC.top_bound
+    end
+  end
+
+  def test_fork_worker_default
+    refute QC.fork_worker?
+  end
+
+  def test_configure_fork_worker_with_env_var
+    with_env "QC_FORK_WORKER" => "yo" do
+      assert QC.fork_worker?
+    end
+  end
+
+  def test_configuration_constants_are_deprecated
+    warning = capture_stderr_output do
+      QC::FORK_WORKER
+    end
+    assert_match "QC::FORK_WORKER is deprecated", warning
+    assert_match "QC.fork_worker? instead", warning
+  end
+
+  class TestWorker < QC::Worker; end
+
+  def test_default_worker_class
+    assert_equal QC::Worker, QC.default_worker_class
+  end
+
+  def test_configure_default_worker_class_with_env_var
+    if RUBY_VERSION =~ /^1\.9\./
+        skip "Kernel.const_get in Ruby 1.9.x does not perform recursive lookups"
+    end
+    with_env "QC_DEFAULT_WORKER_CLASS" => "ConfigTest::TestWorker" do
+      assert_equal TestWorker, QC.default_worker_class
+    end
+  end
+
+  def test_assign_default_worker_class
+    original_worker = QC.default_worker_class
+    QC.default_worker_class = TestWorker
+
+    assert_equal TestWorker, QC.default_worker_class
+  ensure
+    QC.default_worker_class = original_worker
+  end
+end

data/test/helper.rb

@@ -1,9 +1,19 @@
-$: << File.expand_path("lib")
-$: << File.expand_path("test")
+# frozen_string_literal: true
+
+require "bundler"
+require "minitest/reporters"
+
+Bundler.setup :default, :test
+
+if ENV['CIRCLECI'] == "true"
+  Minitest::Reporters.use! Minitest::Reporters::JUnitReporter.new
+else
+  Minitest::Reporters.use! Minitest::Reporters::SpecReporter.new
+end
 
 ENV["DATABASE_URL"] ||= "postgres:///queue_classic_test"
 
-require "queue_classic"
+require_relative '../lib/queue_classic'
 require "stringio"
 require "minitest/autorun"
 
@@ -48,4 +58,38 @@ class QCTest < Minitest::Test
     $stdout = original_stdout
   end
 
+  def with_env(temporary_environment)
+    original_environment = {}
+    temporary_environment.each do |name, value|
+      original_environment[name] = ENV[name]
+      ENV[name] = value
+    end
+    yield
+  ensure
+    original_environment.each { |name, value| ENV[name] = value }
+  end
+
+  def stub_any_instance(class_name, method_name, definition)
+    new_method_name = "new_#{method_name}"
+    original_method_name = "original_#{method_name}"
+
+    method_present = class_name.instance_methods(false).include? method_name
+
+    if method_present
+      class_name.send(:alias_method, original_method_name, method_name)
+      class_name.send(:define_method, new_method_name, definition)
+      class_name.send(:alias_method, method_name, new_method_name)
+
+      yield
+    else
+      message = "#{class_name} does not have method #{method_name}."
+      message << "\nAvailable methods: #{class_name.instance_methods(false)}"
+      raise ArgumentError.new message
+    end
+  ensure
+    if method_present
+      class_name.send(:alias_method, method_name, original_method_name)
+      class_name.send(:undef_method, new_method_name)
+    end
+  end
 end

data/test/helper.sql

@@ -0,0 +1,25 @@
+DO $$
+-- Set initial sequence to a large number to test the entire toolchain
+-- works on integers with higher bits set.
+DECLARE
+    quoted_name text;
+    quoted_size text;
+BEGIN
+    -- Find the name of the relevant sequence.
+    --
+    -- pg_get_serial_sequence quotes identifiers as part of its
+    -- behavior.
+    SELECT name
+    INTO STRICT quoted_name
+    FROM pg_get_serial_sequence('queue_classic_jobs', 'id') AS name;
+
+    -- Don't quote, because ALTER SEQUENCE RESTART doesn't like
+    -- general literals, only unquoted numeric literals.
+    SELECT pow(2, 34)::text AS size
+    INTO STRICT quoted_size;
+
+    EXECUTE 'ALTER SEQUENCE ' || quoted_name ||
+        ' RESTART ' || quoted_size || ';';
+END;
+$$;
+

data/test/lib/queue_classic_rails_connection_test.rb

@@ -1,35 +1,41 @@
+# frozen_string_literal: true
+
 require File.expand_path("../../helper.rb", __FILE__)
 
 class QueueClassicRailsConnectionTest < QCTest
   def before_setup
-    Object.send :const_set, :ActiveRecord, Module.new
-    ActiveRecord.const_set :Base, Module.new
-
+    @original_conn_adapter = QC.default_conn_adapter
     QC.default_conn_adapter = nil
   end
 
-  def after_teardown
+  def before_teardown
     ActiveRecord.send :remove_const, :Base
     Object.send :remove_const, :ActiveRecord
+
+    QC.default_conn_adapter = @original_conn_adapter
   end
 
   def test_uses_active_record_connection_if_exists
     connection = get_connection
-    assert connection.verify
+    QC.default_conn_adapter.execute('SELECT 1;')
+    connection.verify
   end
 
   def test_does_not_use_active_record_connection_if_env_var_set
-    ENV['QC_RAILS_DATABASE'] = 'false'
-    connection = get_connection
-    assert_raises(MockExpectationError) { connection.verify }
-    ENV['QC_RAILS_DATABASE'] = 'true'
+    with_env 'QC_RAILS_DATABASE' => 'false' do
+      connection = get_connection
+      QC.default_conn_adapter.execute('SELECT 1;')
+      assert_raises(MockExpectationError) { connection.verify }
+    end
   end
 
   private
   def get_connection
     connection = Minitest::Mock.new
-    connection.expect(:raw_connection, QC::ConnAdapter.new.connection)
+    connection.expect(:raw_connection, QC::ConnAdapter.new(active_record_connection_share: true).connection)
 
+    Object.send :const_set, :ActiveRecord, Module.new
+    ActiveRecord.const_set :Base, Module.new
     ActiveRecord::Base.define_singleton_method(:connection) do
       connection
     end

data/test/lib/queue_classic_test.rb

@@ -1,24 +1,36 @@
+# frozen_string_literal: true
+
 require File.expand_path("../../helper.rb", __FILE__)
 
 class QueueClassicTest < QCTest
+  def test_only_delegate_calls_to_queue_it_understands
+    e = assert_raises(NoMethodError) do
+      QC.probably_not
+    end
+    assert_match "undefined method `probably_not' for QC:Module", e.message
+  end
+
   def test_default_conn_adapter_default_value
     assert(QC.default_conn_adapter.is_a?(QC::ConnAdapter))
   end
 
-  def test_default_conn_adapter=
+  def test_assigning_a_default_conn_adapter
+    original_conn_adapter = QC.default_conn_adapter
     connection = QC::ConnAdapter.new
     QC.default_conn_adapter = connection
     assert_equal(QC.default_conn_adapter, connection)
+  ensure
+    QC.default_conn_adapter = original_conn_adapter
   end
 
   def test_unlock_jobs_of_dead_workers
     # Insert a locked job
     adapter = QC::ConnAdapter.new
-    query = "INSERT INTO #{QC::TABLE_NAME} (q_name, method, args, locked_by, locked_at) VALUES ('whatever', 'Kernel.puts', '[\"ok?\"]', 0, (CURRENT_TIMESTAMP))"
+    query = "INSERT INTO #{QC.table_name} (q_name, method, args, locked_by, locked_at) VALUES ('whatever', 'Kernel.puts', '[\"ok?\"]', 0, (CURRENT_TIMESTAMP))"
     adapter.execute(query)
 
     # We should have no unlocked jobs
-    query_locked_jobs = "SELECT * FROM #{QC::TABLE_NAME} WHERE locked_at IS NULL"
+    query_locked_jobs = "SELECT * FROM #{QC.table_name} WHERE locked_at IS NULL"
     res = adapter.connection.exec(query_locked_jobs)
     assert_equal(0, res.count)
 

data/test/lib/queue_classic_test_with_activerecord_typecast.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require File.expand_path("../../helper.rb", __FILE__)
+
+class QueueClassicTest < QCTest
+  def before_teardown
+    ActiveRecord.send :remove_const, :Base
+    Object.send :remove_const, :ActiveRecord
+
+    QC.default_conn_adapter = @original_conn_adapter
+  end
+
+  def test_lock_with_active_record_timestamp_type_cast
+    # Insert an unlocked job
+    p_queue = QC::Queue.new("priority_queue")
+    conn_adapter = Minitest::Mock.new
+    conn_adapter.expect(:execute, {"id" => '1', "q_name" => 'test', "method" => "Kernel.puts", "args" => "[]", "scheduled_at" => Time.now}, [String, String])
+    QC.default_conn_adapter = conn_adapter
+    assert_equal(p_queue.lock, {})
+  end
+end