que 0.1.0 → 0.2.0

data/lib/que/job.rb

@@ -1,4 +1,4 @@
-require 'json'
+require 'multi_json'
 
 module Que
   class Job
@@ -36,7 +36,7 @@ module Que
           args << options if options.any?
         end
 
-        attrs = {:job_class => to_s, :args => JSON.dump(args)}
+        attrs = {:job_class => to_s, :args => MultiJson.dump(args)}
 
         if t = run_at || @default_run_at && @default_run_at.call
           attrs[:run_at] = t
@@ -46,7 +46,7 @@ module Que
           attrs[:priority] = p
         end
 
-        if Que.mode == :sync
+        if Que.mode == :sync && !attrs[:run_at]
           run_job(attrs)
         else
           Que.execute *insert_sql(attrs)
@@ -134,8 +134,9 @@ module Que
 
       def run_job(attrs)
         attrs = indifferentiate(attrs)
-        attrs[:args] = indifferentiate(JSON.load(attrs[:args]))
-        const_get("::#{attrs[:job_class]}").new(attrs).tap(&:_run)
+        attrs[:args] = indifferentiate(MultiJson.load(attrs[:args]))
+        klass = attrs[:job_class].split('::').inject(Object, &:const_get)
+        klass.new(attrs).tap(&:_run)
       end
 
       def indifferentiate(input)

data/lib/que/version.rb

@@ -1,3 +1,3 @@
 module Que
-  Version = '0.1.0'
+  Version = '0.2.0'
 end

data/lib/que/worker.rb

@@ -89,11 +89,16 @@ module Que
     # This has to be called when trapping a SIGTERM, so it can't lock the monitor.
     def stop!
       @thread[:directive] = :stop
-      @thread.wakeup
     end
 
     def wait_until_stopped
-      @thread.join
+      loop do
+        case @thread.status
+          when false then break
+          when 'sleep' then @thread.wakeup
+        end
+        sleep 0.0001
+      end
     end
 
     private

data/que.gemspec

@@ -19,12 +19,6 @@ Gem::Specification.new do |spec|
   spec.require_paths = ['lib']
 
   spec.add_development_dependency 'bundler', '~> 1.3'
-  spec.add_development_dependency 'rake'
-  spec.add_development_dependency 'rspec', '~> 2.14.1'
-  spec.add_development_dependency 'pry'
 
-  spec.add_development_dependency 'sequel'
-  spec.add_development_dependency 'activerecord'
-  spec.add_development_dependency 'pg'
-  spec.add_development_dependency 'connection_pool'
+  spec.add_dependency 'multi_json', '~> 1.0'
 end

data/spec/spec_helper.rb

@@ -1,4 +1,5 @@
 require 'que'
+require 'json'
 
 Dir["./spec/support/**/*.rb"].sort.each &method(:require)
 

data/spec/{helper_spec.rb → unit/helper_spec.rb}

@@ -1,6 +1,10 @@
 require 'spec_helper'
 
 describe Que, 'helpers' do
+  it "should be able to run arbitrary SQL" do
+    Que.execute("SELECT 1 AS one").to_a.should == [{'one' => '1'}]
+  end
+
   it "should be able to drop and create the jobs table" do
     DB.table_exists?(:que_jobs).should be true
     Que.drop!

data/spec/{pool_spec.rb → unit/pool_spec.rb}

@@ -6,16 +6,25 @@ describe "Managing the Worker pool" do
     $logger.messages.should == ["[Que] Set mode to :off"]
   end
 
-  it "Que.mode = :sync should make jobs run in the same thread as they are queued" do
-    Que.mode = :sync
+  describe "Que.mode = :sync" do
+    it "should make jobs run in the same thread as they are queued" do
+      Que.mode = :sync
 
-    ArgsJob.queue(5, :testing => "synchronous").should be_an_instance_of ArgsJob
-    $passed_args.should == [5, {'testing' => "synchronous"}]
-    DB[:que_jobs].count.should be 0
+      ArgsJob.queue(5, :testing => "synchronous").should be_an_instance_of ArgsJob
+      $passed_args.should == [5, {'testing' => "synchronous"}]
+      DB[:que_jobs].count.should be 0
+
+      $logger.messages.length.should be 2
+      $logger.messages[0].should == "[Que] Set mode to :sync"
+      $logger.messages[1].should =~ /\A\[Que\] Worked job in/
+    end
 
-    $logger.messages.length.should be 2
-    $logger.messages[0].should == "[Que] Set mode to :sync"
-    $logger.messages[1].should =~ /\A\[Que\] Worked job in/
+    it "should not affect jobs that are queued with specific run_ats" do
+      Que.mode = :sync
+
+      ArgsJob.queue(5, :testing => "synchronous", :run_at => Time.now + 60)
+      DB[:que_jobs].select_map(:job_class).should == ["ArgsJob"]
+    end
   end
 
   describe "Que.mode = :async" do
@@ -103,9 +112,10 @@ describe "Managing the Worker pool" do
 
     it "should poke a worker every Que.sleep_period seconds" do
       begin
-        Que.sleep_period = 0.001 # 1 ms
         Que.mode = :async
+
         sleep_until { Que::Worker.workers.all? &:sleeping? }
+        Que.sleep_period = 0.01 # 10 ms
         Que::Job.queue
         sleep_until { DB[:que_jobs].count == 0 }
       ensure

data/spec/{work_spec.rb → unit/work_spec.rb}

@@ -240,7 +240,7 @@ describe Que::Job, '.work' do
       DB[:que_jobs].count.should be 1
       job = DB[:que_jobs].first
       job[:error_count].should be 1
-      job[:last_error].should =~ /\Auninitialized constant NonexistentClass/
+      job[:last_error].should =~ /uninitialized constant:? NonexistentClass/
       job[:run_at].should be_within(3).of Time.now + 4
     end
   end

data/tasks/benchmark.rb

@@ -0,0 +1,93 @@
+task :benchmark do
+  # The following benchmark is meant to test Que's scalability by having it
+  # bombard Postgres from many, many processes. Note that this benchmark tests
+  # the entire Que stack, not just the locking queries.
+
+  JOB_COUNT          = (ENV['JOB_COUNT'] || 1000).to_i
+  PROCESS_COUNT      = (ENV['PROCESS_COUNT'] || 1).to_i
+  WORKER_COUNT       = (ENV['WORKER_COUNT']  || 4).to_i
+  SYNCHRONOUS_COMMIT = ENV['SYNCHRONOUS_COMMIT'] || 'on'
+
+  require 'que'
+  require 'uri'
+  require 'pg'
+  require 'connection_pool'
+
+  uri = URI.parse ENV["DATABASE_URL"] || "postgres://postgres:@localhost/que-test"
+
+  new_connection = proc do
+    conn = PG::Connection.open :host     => uri.host,
+                               :user     => uri.user,
+                               :password => uri.password,
+                               :port     => uri.port || 5432,
+                               :dbname   => uri.path[1..-1]
+
+    conn.async_exec "SET SESSION synchronous_commit = #{SYNCHRONOUS_COMMIT}"
+    conn
+  end
+
+  Que.connection = pg = new_connection.call
+  Que.drop! rescue nil
+  Que.create!
+
+  # Stock table with jobs and analyze.
+  pg.async_exec <<-SQL
+    INSERT INTO que_jobs (job_class, args, priority)
+    SELECT 'Que::Job', ('[' || i || ',{}]')::json, 1
+    FROM generate_Series(1,#{JOB_COUNT}) AS i;
+    ANALYZE;
+  SQL
+
+  # Fork!
+  $parent_pid = Process.pid
+  def parent?
+    Process.pid == $parent_pid
+  end
+
+  # Synchronize all workers to start at the same time using, what else?
+  # Advisory locks. I am such a one-trick pony.
+  pg.async_exec("SELECT pg_advisory_lock(0)")
+
+  PROCESS_COUNT.times { Process.fork if parent? }
+
+  if parent?
+    # This is the main process, get ready to start monitoring the queues.
+
+    # First hold until all the children are ready.
+    sleep 0.1 until pg.async_exec("select count(*) from pg_locks where locktype = 'advisory' and objid = 0").first['count'].to_i == PROCESS_COUNT + 1
+
+    puts "Benchmarking: #{JOB_COUNT} jobs, #{PROCESS_COUNT} processes with #{WORKER_COUNT} workers each, synchronous_commit = #{SYNCHRONOUS_COMMIT}..."
+    pg.async_exec("select pg_advisory_unlock_all()") # Go!
+    start = Time.now
+
+    loop do
+      sleep 0.01
+      break if pg.async_exec("SELECT 1 AS one FROM que_jobs LIMIT 1").none? # There must be a better way to do this?
+    end
+    time = Time.now - start
+
+    locks = pg.async_exec("SELECT * FROM pg_locks WHERE locktype = 'advisory'").to_a
+    puts "Advisory locks left over! #{locks.inspect}" if locks.any?
+    puts "#{JOB_COUNT} jobs in #{time} seconds = #{(JOB_COUNT / time).round} jobs per second"
+  else
+    # This is a child, get ready to start hitting the queues.
+    pool = ConnectionPool.new :size => WORKER_COUNT, &new_connection
+
+    Que.connection = pool
+
+    pool.with do |conn|
+      # Block here until the advisory lock is released, which is our start pistol.
+      conn.async_exec "SELECT pg_advisory_lock(0); SELECT pg_advisory_unlock_all();"
+    end
+
+    Que.mode = :async
+    Que.worker_count = WORKER_COUNT
+
+    loop do
+      sleep 1
+      break if pool.with { |pg| pg.async_exec("SELECT 1 AS one FROM que_jobs LIMIT 1").none? }
+    end
+  end
+
+  Process.waitall
+end

data/tasks/benchmark_queues.rb

@@ -0,0 +1,398 @@
+task :benchmark_queues do
+  # The following is a somewhat simplistic benchmark (aren't they all) meant
+  # to compare the speed and concurrency of the locking mechanisms used by Que
+  # (standard and lateral queries), DelayedJob and QueueClassic - it does this
+  # by simply sending the raw queries that each system sends during operation.
+
+  # It is NOT meant to benchmark the overall performance of each system (which
+  # would include the time each spends working in Ruby), but to see which one
+  # supports the highest concurrency under load, assuming that there will be
+  # many workers and that Postgres will be the bottleneck. I'm unsure how
+  # useful it is for this, but it's a start.
+
+  JOB_COUNT          = (ENV['JOB_COUNT'] || 1000).to_i
+  WORKER_COUNT       = (ENV['WORKER_COUNT'] || 10).to_i
+  SYNCHRONOUS_COMMIT = ENV['SYNCHRONOUS_COMMIT'] || 'on'
+
+  require 'uri'
+  require 'pg'
+  require 'connection_pool'
+
+  uri = URI.parse ENV["DATABASE_URL"] || "postgres://postgres:@localhost/que-test"
+
+  new_connection = proc do
+    PG::Connection.open :host     => uri.host,
+                        :user     => uri.user,
+                        :password => uri.password,
+                        :port     => uri.port || 5432,
+                        :dbname   => uri.path[1..-1]
+  end
+
+  pg = new_connection.call
+
+
+
+  # Necessary setup, mostly for QueueClassic. I apologize for this - I hope your editor supports code folding.
+  pg.async_exec <<-SQL
+    SET SESSION client_min_messages = 'WARNING';
+
+    -- Que table.
+    DROP TABLE IF EXISTS que_jobs;
+    CREATE TABLE que_jobs
+    (
+      priority    integer     NOT NULL DEFAULT 1,
+      run_at      timestamptz NOT NULL DEFAULT now(),
+      job_id      bigserial   NOT NULL,
+      job_class   text        NOT NULL,
+      args        json        NOT NULL DEFAULT '[]'::json,
+      error_count integer     NOT NULL DEFAULT 0,
+      last_error  text,
+
+      CONSTRAINT que_jobs_pkey PRIMARY KEY (priority, run_at, job_id)
+    );
+
+    DROP TABLE IF EXISTS que_lateral_jobs;
+    CREATE TABLE que_lateral_jobs
+    (
+      priority    integer     NOT NULL DEFAULT 1,
+      run_at      timestamptz NOT NULL DEFAULT now(),
+      job_id      bigserial   NOT NULL,
+      job_class   text        NOT NULL,
+      args        json        NOT NULL DEFAULT '[]'::json,
+      error_count integer     NOT NULL DEFAULT 0,
+      last_error  text,
+
+      CONSTRAINT que_lateral_jobs_pkey PRIMARY KEY (priority, run_at, job_id)
+    );
+
+    DROP TABLE IF EXISTS delayed_jobs;
+    -- DelayedJob table.
+    CREATE TABLE delayed_jobs
+    (
+      id serial NOT NULL,
+      priority integer NOT NULL DEFAULT 0,
+      attempts integer NOT NULL DEFAULT 0,
+      handler text NOT NULL,
+      last_error text,
+      run_at timestamp without time zone,
+      locked_at timestamp without time zone,
+      failed_at timestamp without time zone,
+      locked_by character varying(255),
+      queue character varying(255),
+      created_at timestamp without time zone,
+      updated_at timestamp without time zone,
+      CONSTRAINT delayed_jobs_pkey PRIMARY KEY (id)
+    );
+    ALTER TABLE delayed_jobs
+      OWNER TO postgres;
+
+    CREATE INDEX delayed_jobs_priority
+      ON delayed_jobs
+      USING btree
+      (priority, run_at);
+
+
+
+    -- QueueClassic table and functions.
+    DROP FUNCTION IF EXISTS lock_head(tname varchar);
+    DROP FUNCTION IF EXISTS lock_head(q_name varchar, top_boundary integer);
+    DROP FUNCTION IF EXISTS queue_classic_notify() cascade;
+    DROP TABLE IF EXISTS queue_classic_jobs;
+
+    CREATE TABLE queue_classic_jobs (
+      id bigserial PRIMARY KEY,
+      q_name text not null check (length(q_name) > 0),
+      method text not null check (length(method) > 0),
+      args   text not null,
+      locked_at timestamptz
+    );
+
+    alter table queue_classic_jobs alter column args type json using (args::json);
+
+    create function queue_classic_notify() returns trigger as $$ begin
+      perform pg_notify(new.q_name, '');
+      return null;
+    end $$ language plpgsql;
+
+    create trigger queue_classic_notify
+    after insert on queue_classic_jobs
+    for each row
+    execute procedure queue_classic_notify();
+
+    CREATE INDEX idx_qc_on_name_only_unlocked ON queue_classic_jobs (q_name, id) WHERE locked_at IS NULL;
+
+    CREATE OR REPLACE FUNCTION lock_head(q_name varchar, top_boundary integer)
+    RETURNS SETOF queue_classic_jobs AS $$
+    DECLARE
+      unlocked bigint;
+      relative_top integer;
+      job_count integer;
+    BEGIN
+      -- The purpose is to release contention for the first spot in the table.
+      -- The select count(*) is going to slow down dequeue performance but allow
+      -- for more workers. Would love to see some optimization here...
+
+      EXECUTE 'SELECT count(*) FROM '
+        || '(SELECT * FROM queue_classic_jobs WHERE q_name = '
+        || quote_literal(q_name)
+        || ' LIMIT '
+        || quote_literal(top_boundary)
+        || ') limited'
+      INTO job_count;
+
+      SELECT TRUNC(random() * (top_boundary - 1))
+      INTO relative_top;
+
+      IF job_count < top_boundary THEN
+        relative_top = 0;
+      END IF;
+
+      LOOP
+        BEGIN
+          EXECUTE 'SELECT id FROM queue_classic_jobs '
+            || ' WHERE locked_at IS NULL'
+            || ' AND q_name = '
+            || quote_literal(q_name)
+            || ' ORDER BY id ASC'
+            || ' LIMIT 1'
+            || ' OFFSET ' || quote_literal(relative_top)
+            || ' FOR UPDATE NOWAIT'
+          INTO unlocked;
+          EXIT;
+        EXCEPTION
+          WHEN lock_not_available THEN
+            -- do nothing. loop again and hope we get a lock
+        END;
+      END LOOP;
+
+      RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
+        || ' SET locked_at = (CURRENT_TIMESTAMP)'
+        || ' WHERE id = $1'
+        || ' AND locked_at is NULL'
+        || ' RETURNING *'
+      USING unlocked;
+
+      RETURN;
+    END;
+    $$ LANGUAGE plpgsql;
+
+    CREATE OR REPLACE FUNCTION lock_head(tname varchar)
+    RETURNS SETOF queue_classic_jobs AS $$
+    BEGIN
+      RETURN QUERY EXECUTE 'SELECT * FROM lock_head($1,10)' USING tname;
+    END;
+    $$ LANGUAGE plpgsql;
+
+
+
+
+
+    INSERT INTO que_jobs (job_class, args, priority)
+    SELECT 'Que::Job', ('[' || i || ',{}]')::json, 1
+    FROM generate_Series(1,#{JOB_COUNT}) AS i;
+
+    INSERT INTO que_lateral_jobs (job_class, args, priority)
+    SELECT 'Que::Job', ('[' || i || ',{}]')::json, 1
+    FROM generate_Series(1,#{JOB_COUNT}) AS i;
+
+    INSERT INTO delayed_jobs (handler, run_at, created_at, updated_at)
+    SELECT '--- !ruby/struct:NewsletterJob\ntext: lorem ipsum...\nemails: blah@blah.com\n', now(), now(), now()
+    FROM generate_Series(1,#{JOB_COUNT}) AS i;
+
+    INSERT INTO queue_classic_jobs (q_name, method, args)
+    SELECT 'default', 'Kernel.puts', '["hello world"]'
+    FROM generate_Series(1,#{JOB_COUNT}) AS i;
+
+
+
+
+    -- Necessary tables and functions made, now stock them with jobs and analyze.
+    ANALYZE;
+  SQL
+
+
+  queries = {
+    :que => (
+      <<-SQL
+        WITH RECURSIVE cte AS (
+          SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
+          FROM (
+            SELECT job
+            FROM que_jobs AS job
+            WHERE run_at <= now()
+            ORDER BY priority, run_at, job_id
+            LIMIT 1
+          ) AS t1
+          UNION ALL (
+            SELECT (job).*, pg_try_advisory_lock((job).job_id) AS locked
+            FROM (
+              SELECT (
+               SELECT job
+                FROM que_jobs AS job
+                WHERE run_at <= now() AND (priority, run_at, job_id) > (cte.priority, cte.run_at, cte.job_id)
+                ORDER BY priority, run_at, job_id
+                LIMIT 1
+              ) AS job
+              FROM cte
+              WHERE NOT cte.locked
+              LIMIT 1
+            ) AS t1
+          )
+        )
+        SELECT job_id, priority, run_at, args, job_class, error_count
+        FROM cte
+        WHERE locked
+        LIMIT 1
+      SQL
+    ),
+    :que_lateral => (
+      <<-SQL
+        WITH RECURSIVE cte AS (
+          SELECT *, pg_try_advisory_lock(s.job_id) AS locked
+          FROM (
+            SELECT *
+            FROM que_lateral_jobs
+            WHERE run_at <= now()
+            ORDER BY priority, run_at, job_id
+            LIMIT 1
+          ) s
+          UNION ALL (
+            SELECT j.*, pg_try_advisory_lock(j.job_id) AS locked
+            FROM (
+              SELECT *
+              FROM cte
+              WHERE NOT locked
+            ) t,
+            LATERAL (
+              SELECT *
+              FROM que_lateral_jobs
+              WHERE run_at <= now()
+              AND (priority, run_at, job_id) > (t.priority, t.run_at, t.job_id)
+              ORDER BY priority, run_at, job_id
+              LIMIT 1
+            ) j
+          )
+        )
+        SELECT *
+        FROM cte
+        WHERE locked
+        LIMIT 1
+      SQL
+    ),
+    :delayed_job => (
+      # From delayed_job_active_record
+      <<-SQL
+        UPDATE delayed_jobs
+        SET locked_at = now(),
+            locked_by = $1::text
+        WHERE id IN (
+          SELECT id
+          FROM delayed_jobs
+          WHERE (
+            (run_at <= now() AND (locked_at IS NULL OR locked_at < now()) OR locked_by = $1) AND failed_at IS NULL
+          )
+          ORDER BY priority ASC, run_at ASC
+          LIMIT 1
+          FOR UPDATE
+        )
+        RETURNING *
+      SQL
+    )
+  }
+
+  connections = WORKER_COUNT.times.map do
+    conn = new_connection.call
+    conn.async_exec "SET SESSION synchronous_commit = #{SYNCHRONOUS_COMMIT}"
+    queries.each do |name, sql|
+      conn.prepare(name.to_s, sql)
+    end
+    conn
+  end
+
+
+
+  # Track the ids that are worked, to make sure they're all hit.
+  $results = {
+    :delayed_job   => [],
+    :queue_classic => [],
+    :que           => [],
+    :que_lateral   => []
+  }
+
+  def work_job(type, conn)
+    case type
+    when :delayed_job
+      return unless r = conn.exec_prepared("delayed_job", [conn.object_id]).first
+      $results[type] << r['id']
+      conn.async_exec "DELETE FROM delayed_jobs WHERE id = $1", [r['id']]
+
+    when :queue_classic
+      return unless r = conn.async_exec("SELECT * FROM lock_head($1, $2)", ['default', 9]).first
+      $results[type] << r['id']
+      conn.async_exec "DELETE FROM queue_classic_jobs WHERE id = $1", [r['id']]
+
+    when :que
+      begin
+        return unless r = conn.exec_prepared("que").first
+        # Have to double-check that the job is valid, as explained at length in Que::Job.work.
+        return true unless conn.async_exec("SELECT * FROM que_jobs WHERE priority = $1 AND run_at = $2 AND job_id = $3", [r['priority'], r['run_at'], r['job_id']]).first
+        conn.async_exec "DELETE FROM que_jobs WHERE priority = $1 AND run_at = $2 AND job_id = $3", [r['priority'], r['run_at'], r['job_id']]
+        $results[type] << r['job_id']
+      ensure
+        conn.async_exec "SELECT pg_advisory_unlock_all()" if r
+      end
+
+    when :que_lateral
+      begin
+        return unless r = conn.exec_prepared("que_lateral").first
+        return true unless conn.async_exec("SELECT * FROM que_lateral_jobs WHERE priority = $1 AND run_at = $2 AND job_id = $3", [r['priority'], r['run_at'], r['job_id']]).first
+        conn.async_exec "DELETE FROM que_lateral_jobs WHERE priority = $1 AND run_at = $2 AND job_id = $3", [r['priority'], r['run_at'], r['job_id']]
+        $results[type] << r['job_id']
+      ensure
+        conn.async_exec "SELECT pg_advisory_unlock_all()" if r
+      end
+
+    end
+  end
+
+  puts "Benchmarking #{JOB_COUNT} jobs, #{WORKER_COUNT} workers and synchronous_commit = #{SYNCHRONOUS_COMMIT}..."
+
+  {
+    :delayed_job   => :delayed_jobs,
+    :queue_classic => :queue_classic_jobs,
+    :que           => :que_jobs,
+    :que_lateral   => :que_lateral_jobs
+  }.each do |type, table|
+    print "Benchmarking #{type}... "
+    start = Time.now
+
+    threads = connections.map do |conn|
+      Thread.new do
+        loop do
+          begin
+            break unless work_job(type, conn)
+          rescue
+            # DelayedJob deadlocks sometimes.
+          end
+        end
+      end
+    end
+
+    threads.each &:join
+    time = Time.now - start
+    puts "#{JOB_COUNT} jobs in #{time} seconds = #{(JOB_COUNT / time).round} jobs per second"
+
+
+    # These checks are commented out because I can't seem to get DelayedJob to
+    # pass them (Que and QueueClassic don't have the same problem). It seems
+    # to repeat some jobs multiple times on every run, and its run times are
+    # also highly variable.
+
+    # worked = $results[type].map(&:to_i).sort
+    # puts "Jobs worked more than once! #{worked.inspect}" unless worked == worked.uniq
+    # puts "Jobs worked less than once! #{worked.inspect}" unless worked.length == JOB_COUNT
+
+    puts "Jobs left in DB" unless pg.async_exec("SELECT count(*) FROM #{table}").first['count'].to_i == 0
+    puts "Advisory locks left over!" if pg.async_exec("SELECT * FROM pg_locks WHERE locktype = 'advisory'").first
+  end
+end