qc-additions 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ZjQ0ZWQ3OGM2OWY2ZjVkYmVmZThkNmM4ZTg0NTE1OTMzZDBjNzQ1OQ==
+  data.tar.gz: !binary |-
+    YzUxYmNmNWJmNTU0Y2U2NWQ1ZmI0ZWRhNGIwNWFiY2YxY2QyMWM4ZQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MzczZTc2MTE2ZTM0ZTlmMzAyMjc3MjMwMTkwNGUzMzcxOTU0ZTIxYWNhODNj
+    OTk2ZWNkNmU3ODdmYWY0NDZjYmMwNjcyYTU0N2U3YmEzNmJmMWY4NTE0MWUy
+    NWZjODMyOGM0NjMxZDhmOTVmMGY4ZjVkNzdiMzI2YmY3NmExYWI=
+  data.tar.gz: !binary |-
+    ZDEwYTAyN2NkYWI1ZmU1ZTM5ZjRjZmRhNmNmM2E3NzUxMTRkMDA5ZDIzOThi
+    NmUwYzBmNjE4YWMyZmVhODM0ODRiYjJiNjIyYTkyMDIwMTg1MzExZjYzZTEx
+    MTc1MzI0OTYzZWJhZDQwOTBkZDgxZDk1YmY5MGU2ZTNkNjJhMjY=

data/README.md

@@ -0,0 +1,30 @@
+# qc-additions
+
+
+This gem adds some methods to [queue_classic](https://github.com/ryandotsmith/queue_classic) queues that I found helpful:
+
+* `enqueue_if_not_queued(method, *args)`
+* `job_count(method, *args)`
+* `job_exists?(method, *args)`
+
+
+An index might help to speed up the `job_count` and `job_exists?` queries.  Although, I haven't really tested if there is much to gain by adding this:
+
+```SQL
+CREATE INDEX idx_qc_unlocked_job_count ON queue_classic_jobs (q_name, method, args) WHERE locked_at IS NULL;
+```
+
+
+## Caveats when comparing args column
+
+The method arguments are serialized to JSON.  However, the comparison performed when looking for jobs in the database is a string comparison.  Results might be incorrect if there is more than one way to serialize the arguments to a JSON string.  It should be safe for simple things like passing a numeric id.
+
+
+---
+
+
+I have some of the original code from there:
+
+* https://github.com/ryandotsmith/queue_classic/pull/92
+
+* https://github.com/GreenplumChorus/queue_classic/commit/2719301c2813717692169c1eeab42d317df0ac59

data/lib/qc-additions.rb

@@ -0,0 +1,4 @@
+require "queue_classic"
+
+require "qc-additions/queries"
+require "qc-additions/queue"

data/lib/qc-additions/queries.rb

@@ -0,0 +1,22 @@
+module QC
+  module Queries
+    extend self
+
+    # TODO:
+    # Once PostgreSQL supports JSON comparison, use it.
+    # But older versions should be supported too (at least >= 9.0).
+
+    def job_count(q_name, method, args)
+      s = "SELECT COUNT(*) FROM #{TABLE_NAME} WHERE q_name = $1 AND method = $2 AND args::text = $3 AND locked_at IS NULL"
+      r = Conn.execute(s, q_name, method, JSON.dump(args))
+      r["count"].to_i
+    end
+
+    def job_exists?(q_name, method, args)
+      s = "SELECT 1 AS one FROM #{TABLE_NAME} WHERE q_name = $1 AND method = $2 AND args::text = $3 AND locked_at IS NULL LIMIT 1"
+      r = Conn.execute(s, q_name, method, JSON.dump(args))
+      !r.nil?
+    end
+
+  end
+end

data/lib/qc-additions/queue.rb

@@ -0,0 +1,17 @@
+module QC
+  class Queue
+
+    def enqueue_if_not_queued(method, *args)
+      enqueue(method, *args) unless job_exists?(method, *args)
+    end
+
+    def job_count(method, *args)
+      Queries.job_count(name, method, args)
+    end
+
+    def job_exists?(method, *args)
+      Queries.job_exists?(name, method, args)
+    end
+
+  end
+end

data/test/additions_test.rb

@@ -0,0 +1,36 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+class QueueTest < QCTest
+
+  def test_enqueue_if_not_queued
+    QC.enqueue_if_not_queued("Klass.method", "arg1", "arg2")
+    QC.enqueue_if_not_queued("Klass.method", "arg1", "arg2")
+    QC.lock
+    QC.enqueue_if_not_queued("Klass.method", "arg1", "arg2")
+    assert_equal(1, QC.job_count("Klass.method", "arg1", "arg2"))
+  end
+
+  def test_job_count
+    #Should return the count of unstarted jobs that match both method and arguments
+    QC.enqueue("Klass.method", "arg1", "arg2")
+    QC.enqueue("Klass.method", "arg1", "arg2")
+    QC.enqueue("Klass.method", "arg1", "arg2")
+    QC.enqueue("Klass.method", "arg3", "arg4")
+    QC.enqueue("Klass.other_method", "arg1", "arg2")
+    QC.lock  #start the first job
+    assert_equal(2, QC.job_count("Klass.method", "arg1", "arg2"))
+  end
+
+  def test_job_exists
+    # Should return true if an unstarted job with same method and arguments exists
+    assert_equal(false, QC.job_exists?("Klass.method"))
+    QC.enqueue("Klass.method")
+    assert_equal(true, QC.job_exists?("Klass.method"))
+    assert_equal(false, QC.job_exists?("Klass.method", "arg1"))
+    assert_equal(false, QC.job_exists?("Klass.other_method"))
+    assert_equal(false, QC.job_exists?("Klass.other_method", "arg1"))
+    QC.lock # start the job
+    assert_equal(false, QC.job_exists?("Klass.method"))
+  end
+
+end

data/test/benchmark_test.rb

@@ -0,0 +1,38 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+if ENV["QC_BENCHMARK"]
+  class BenchmarkTest < QCTest
+
+    def test_enqueue
+      n = 10_000
+      start = Time.now
+      n.times do
+        QC.enqueue("1.odd?", [])
+      end
+      assert_equal(n, QC.count)
+
+      elapsed = Time.now - start
+      assert_in_delta(4, elapsed, 1)
+    end
+
+    def test_dequeue
+      worker = QC::Worker.new
+      worker.running = true
+      n = 10_000
+      n.times do
+        QC.enqueue("1.odd?", [])
+      end
+      assert_equal(n, QC.count)
+
+      start = Time.now
+      n.times do
+        worker.work
+      end
+      elapsed = Time.now - start
+
+      assert_equal(0, QC.count)
+      assert_in_delta(10, elapsed, 3)
+    end
+
+  end
+end

data/test/conn_test.rb

@@ -0,0 +1,22 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+class ConnTest < QCTest
+
+  def test_extracts_the_segemnts_to_connect
+    database_url = "postgres://ryan:secret@localhost:1234/application_db"
+    normalized = QC::Conn.normalize_db_url(URI.parse(database_url))
+    assert_equal ["localhost",
+                  1234,
+                  nil, "",
+                  "application_db",
+                  "ryan",
+                  "secret"], normalized
+  end
+
+  def test_regression_database_url_without_host
+    database_url = "postgres:///my_db"
+    normalized = QC::Conn.normalize_db_url(URI.parse(database_url))
+    assert_equal [nil, 5432, nil, "", "my_db", nil, nil], normalized
+  end
+
+end

data/test/helper.rb

@@ -0,0 +1,66 @@
+$: << File.expand_path("lib")
+$: << File.expand_path("test")
+
+ENV["DATABASE_URL"] ||= "postgres:///queue_classic_test"
+
+require "queue_classic"
+require "qc-additions"
+require "stringio"
+require "minitest/autorun"
+
+class QCTest < Minitest::Test
+
+  def setup
+    init_db
+  end
+
+  def teardown
+    QC.delete_all
+  end
+
+  def init_db(table_name="queue_classic_jobs")
+    QC::Conn.execute("SET client_min_messages TO 'warning'")
+    QC::Setup.drop
+    QC::Setup.create
+    QC::Conn.execute(<<EOS)
+DO $$
+-- Set initial sequence to a large number to test the entire toolchain
+-- works on integers with higher bits set.
+DECLARE
+    quoted_name text;
+    quoted_size text;
+BEGIN
+    -- Find the name of the relevant sequence.
+    --
+    -- pg_get_serial_sequence quotes identifiers as part of its
+    -- behavior.
+    SELECT name
+    INTO STRICT quoted_name
+    FROM pg_get_serial_sequence('queue_classic_jobs', 'id') AS name;
+
+    -- Don't quote, because ALTER SEQUENCE RESTART doesn't like
+    -- general literals, only unquoted numeric literals.
+    SELECT pow(2, 34)::text AS size
+    INTO STRICT quoted_size;
+
+    EXECUTE 'ALTER SEQUENCE ' || quoted_name ||
+        ' RESTART ' || quoted_size || ';';
+END;
+$$;
+EOS
+  end
+
+  def capture_debug_output
+    original_debug = ENV['DEBUG']
+    original_stdout = $stdout
+
+    ENV['DEBUG'] = "true"
+    $stdout = StringIO.new
+    yield
+    $stdout.string
+  ensure
+    ENV['DEBUG'] = original_debug
+    $stdout = original_stdout
+  end
+
+end

data/test/queue_test.rb

@@ -0,0 +1,103 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+class QueueTest < QCTest
+
+  def test_enqueue
+    QC.enqueue("Klass.method")
+  end
+
+  def test_respond_to
+    assert_equal(true, QC.respond_to?(:enqueue))
+  end
+
+  def test_lock
+    QC.enqueue("Klass.method")
+
+    # See helper.rb for more information about the large initial id
+    # number.
+    expected = {:id=>(2**34).to_s, :method=>"Klass.method", :args=>[]}
+    assert_equal(expected, QC.lock)
+  end
+
+  def test_lock_when_empty
+    assert_nil(QC.lock)
+  end
+
+  def test_count
+    QC.enqueue("Klass.method")
+    assert_equal(1, QC.count)
+  end
+
+  def test_delete
+    QC.enqueue("Klass.method")
+    assert_equal(1, QC.count)
+    QC.delete(QC.lock[:id])
+    assert_equal(0, QC.count)
+  end
+
+  def test_delete_all
+    QC.enqueue("Klass.method")
+    QC.enqueue("Klass.method")
+    assert_equal(2, QC.count)
+    QC.delete_all
+    assert_equal(0, QC.count)
+  end
+
+  def test_delete_all_by_queue_name
+    p_queue = QC::Queue.new("priority_queue")
+    s_queue = QC::Queue.new("secondary_queue")
+    p_queue.enqueue("Klass.method")
+    s_queue.enqueue("Klass.method")
+    assert_equal(1, p_queue.count)
+    assert_equal(1, s_queue.count)
+    p_queue.delete_all
+    assert_equal(0, p_queue.count)
+    assert_equal(1, s_queue.count)
+  end
+
+  def test_queue_instance
+    queue = QC::Queue.new("queue_classic_jobs", false)
+    queue.enqueue("Klass.method")
+    assert_equal(1, queue.count)
+    queue.delete(queue.lock[:id])
+    assert_equal(0, queue.count)
+  end
+
+  def test_repair_after_error
+    queue = QC::Queue.new("queue_classic_jobs", false)
+    queue.enqueue("Klass.method")
+    assert_equal(1, queue.count)
+    connection = QC::Conn.connection
+    saved_method = connection.method(:exec)
+    def connection.exec(*args)
+      raise PGError
+    end
+    assert_raises(PG::Error) { queue.enqueue("Klass.other_method") }
+    assert_equal(1, queue.count)
+    queue.enqueue("Klass.other_method")
+    assert_equal(2, queue.count)
+  rescue PG::Error
+    QC::Conn.disconnect
+    assert false, "Expected to QC repair after connection error"
+  end
+
+  def test_custom_default_queue
+    queue_class = Class.new do
+      attr_accessor :jobs
+      def enqueue(method, *args)
+        @jobs ||= []
+        @jobs << method
+      end
+    end
+
+    queue_instance = queue_class.new
+    QC.default_queue = queue_instance
+
+    QC.enqueue("Klass.method1")
+    QC.enqueue("Klass.method2")
+
+    assert_equal ["Klass.method1", "Klass.method2"], queue_instance.jobs
+  ensure
+    QC.default_queue = nil
+  end
+end

data/test/worker_test.rb

@@ -0,0 +1,125 @@
+require File.expand_path("../helper.rb", __FILE__)
+
+module TestObject
+  extend self
+  def no_args; return nil; end
+  def one_arg(a); return a; end
+  def two_args(a,b); return [a,b]; end
+end
+
+# This not only allows me to test what happens
+# when a failure occurs but it also demonstrates
+# how to override the worker to handle failures the way
+# you want.
+class TestWorker < QC::Worker
+  attr_accessor :failed_count
+
+  def initialize(*args)
+    super(*args)
+    @failed_count = 0
+  end
+
+  def handle_failure(job,e)
+    @failed_count += 1
+  end
+end
+
+class WorkerTest < QCTest
+
+  def test_work
+    QC.enqueue("TestObject.no_args")
+    worker = TestWorker.new
+    assert_equal(1, QC.count)
+    worker.work
+    assert_equal(0, QC.count)
+    assert_equal(0, worker.failed_count)
+  end
+
+  def test_failed_job
+    QC.enqueue("TestObject.not_a_method")
+    worker = TestWorker.new
+    worker.work
+    assert_equal(1, worker.failed_count)
+  end
+
+  def test_failed_job_is_logged
+    output = capture_debug_output do
+      QC.enqueue("TestObject.not_a_method")
+      QC::Worker.new.work
+    end
+    expected_output = /lib=queue-classic at=handle_failure job={:id=>"\d+", :method=>"TestObject.not_a_method", :args=>\[\]} error=#<NoMethodError: undefined method `not_a_method' for TestObject:Module>/
+    assert_match(expected_output, output, "=== debug output ===\n #{output}")
+  end
+
+  def test_log_yield
+    output = capture_debug_output do
+      QC.log_yield(:action => "test") do
+        0 == 1
+      end
+    end
+    expected_output = /lib=queue-classic action=test elapsed=\d*/
+    assert_match(expected_output, output, "=== debug output ===\n #{output}")
+  end
+
+  def test_log
+    output = capture_debug_output do
+      QC.log(:action => "test")
+    end
+    expected_output = /lib=queue-classic action=test/
+    assert_match(expected_output, output, "=== debug output ===\n #{output}")
+  end
+
+  def test_work_with_no_args
+    QC.enqueue("TestObject.no_args")
+    worker = TestWorker.new
+    r = worker.work
+    assert_nil(r)
+    assert_equal(0, worker.failed_count)
+  end
+
+  def test_work_with_one_arg
+    QC.enqueue("TestObject.one_arg", "1")
+    worker = TestWorker.new
+    r = worker.work
+    assert_equal("1", r)
+    assert_equal(0, worker.failed_count)
+  end
+
+  def test_work_with_two_args
+    QC.enqueue("TestObject.two_args", "1", 2)
+    worker = TestWorker.new
+    r = worker.work
+    assert_equal(["1", 2], r)
+    assert_equal(0, worker.failed_count)
+  end
+
+  def test_work_custom_queue
+    p_queue = QC::Queue.new("priority_queue")
+    p_queue.enqueue("TestObject.two_args", "1", 2)
+    worker = TestWorker.new(q_name: "priority_queue")
+    r = worker.work
+    assert_equal(["1", 2], r)
+    assert_equal(0, worker.failed_count)
+  end
+
+  def test_worker_listens_on_chan
+    p_queue = QC::Queue.new("priority_queue")
+    p_queue.enqueue("TestObject.two_args", "1", 2)
+    worker = TestWorker.new(q_name: "priority_queue", listening_worker: true)
+    r = worker.work
+    assert_equal(["1", 2], r)
+    assert_equal(0, worker.failed_count)
+  end
+
+  def test_worker_ueses_one_conn
+    QC.enqueue("TestObject.no_args")
+    worker = TestWorker.new
+    worker.work
+    assert_equal(
+      1,
+      QC::Conn.execute("SELECT count(*) from pg_stat_activity where datname = current_database()")["count"].to_i,
+      "Multiple connections found -- are there open connections to #{ QC::Conn.db_url } in other terminals?"
+    )
+  end
+
+end

metadata

@@ -0,0 +1,73 @@
+--- !ruby/object:Gem::Specification
+name: qc-additions
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Jakob Rath
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-07-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: queue_classic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 2.2.0
+description: Add some methods to queue_classic to determine whether a job exists or
+  how many times a given job is already queued.
+email: mail@jakobrath.eu
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- lib/qc-additions/queries.rb
+- lib/qc-additions/queue.rb
+- lib/qc-additions.rb
+- test/additions_test.rb
+- test/benchmark_test.rb
+- test/conn_test.rb
+- test/helper.rb
+- test/queue_test.rb
+- test/worker_test.rb
+homepage: https://github.com/JakobR/qc-additions
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Add some methods to queue_classic.
+test_files:
+- test/additions_test.rb
+- test/benchmark_test.rb
+- test/conn_test.rb
+- test/queue_test.rb
+- test/worker_test.rb
+has_rdoc: