queue_classic 3.0.0rc → 3.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 549f3341849e84bc2068f82448e39e98c0ef2051
-  data.tar.gz: 0746db47bbe386bd31e348b9d925985e8b185c04
+  metadata.gz: 3558784567b56e6d75e1c872aaecf392e2bbfcf7
+  data.tar.gz: e4f6f925b2db3333b93216b4caee0ebe76ee0662
 SHA512:
-  metadata.gz: 5d9d9f8e660d294671ef3fd6d99f96fa2e092049920f72b754c62fa59857466978e6a7f10544fd35de9e52e6d72d99fb51dadb08cc2c0b8b002906f6628fe672
-  data.tar.gz: 3174b46493945c6b126323fa20dbaa1bf53bfd40a6ff07926a934ea4820e5e67276331d872d77e182fc57127e590f916a2042b48df9e9bee63965cd4d0f20f26
+  metadata.gz: c82dbc205598a39eb6d0d35dec8accaeb5a7cf3b0dea45cd76a9a9b55c4fab6802b0eb86720b9805f0a37abe23919873706386063b4ec52f3050ea2d22c9ed1f
+  data.tar.gz: d14075365a1c02a36350ce9c6fc986772425abda40e8ce45bdf97eb55e91979d970b57339c735cf135ec6878aaa353790b7fda9c1cebca2e9124ec9c1000bed4

data/lib/generators/queue_classic/install_generator.rb

@@ -16,7 +16,13 @@ module QC
     end
 
     def create_migration_file
-      migration_template 'add_queue_classic.rb', 'db/migrate/add_queue_classic.rb'
+      if self.class.migration_exists?('db/migrate', 'add_queue_classic').nil?
+        migration_template 'add_queue_classic.rb', 'db/migrate/add_queue_classic.rb'
+      end
+
+      if self.class.migration_exists?('db/migrate', 'update_queue_classic_3_0_0').nil?
+        migration_template 'update_queue_classic_3_0_0.rb', 'db/migrate/update_queue_classic_3_0_0.rb'
+      end
     end
   end
 end

data/lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb

@@ -0,0 +1,9 @@
+class UpdateQueueClassic300 < ActiveRecord::Migration
+  def self.up
+    QC::Setup.update_to_3_0_0
+  end
+
+  def self.down
+    QC::Setup.downgrade_from_3_0_0
+  end
+end

data/lib/queue_classic.rb

@@ -52,10 +52,18 @@ module QC
     end
   end
 
+  def self.has_connection?
+    !@conn_adapter.nil?
+  end
+
   def self.default_conn_adapter
     @conn_adapter ||= ConnAdapter.new
   end
 
+  def self.default_conn_adapter=(conn)
+    @conn_adapter = conn
+  end
+
   def self.log_yield(data)
     begin
       t0 = Time.now
@@ -83,9 +91,21 @@ module QC
     puts(out) if ENV["DEBUG"]
     return result
   end
+
+  def self.measure(data)
+    if ENV['QC_MEASURE']
+      $stdout.puts("measure#qc.#{data}")
+    end
+  end
+
+  # This will unlock all jobs any postgres' PID that is not existing anymore
+  # to prevent any infinitely locked jobs
+  def self.unlock_jobs_of_dead_workers
+    @conn_adapter.execute("UPDATE #{QC::TABLE_NAME} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT pid FROM pg_stat_activity);")
+  end
 end
 
-require "queue_classic/queue"
-require "queue_classic/worker"
-require "queue_classic/setup"
-require "queue_classic/railtie" if defined?(Rails)
+require_relative "queue_classic/queue"
+require_relative "queue_classic/worker"
+require_relative "queue_classic/setup"
+require_relative "queue_classic/railtie" if defined?(Rails)

data/lib/queue_classic/conn_adapter.rb

@@ -30,10 +30,10 @@ module QC
 
     def wait(time, *channels)
       @mutex.synchronize do
-        listen_cmds = channels.map {|c| 'LISTEN "' + c + '"'}
+        listen_cmds = channels.map {|c| 'LISTEN "' + c.to_s + '"'}
         @connection.exec(listen_cmds.join(';'))
         wait_for_notify(time)
-        unlisten_cmds = channels.map {|c| 'UNLISTEN "' + c +'"'}
+        unlisten_cmds = channels.map {|c| 'UNLISTEN "' + c.to_s + '"'}
         @connection.exec(unlisten_cmds.join(';'))
         drain_notify
       end

data/lib/queue_classic/queue.rb

@@ -1,6 +1,6 @@
-require 'queue_classic'
-require 'queue_classic/conn_adapter'
+require_relative 'conn_adapter'
 require 'json'
+require 'time'
 
 module QC
   # The queue class maps a queue abstraction onto a database table.
@@ -45,9 +45,17 @@ module QC
       QC.log_yield(:measure => 'queue.lock') do
         s = "SELECT * FROM lock_head($1, $2)"
         if r = conn_adapter.execute(s, name, top_bound)
-          {:id => r["id"],
-            :method => r["method"],
-            :args => JSON.parse(r["args"])}
+          {}.tap do |job|
+            job[:id] = r["id"]
+            job[:q_name] = r["q_name"]
+            job[:method] = r["method"]
+            job[:args] = JSON.parse(r["args"])
+            if r["created_at"]
+              job[:created_at] = Time.parse(r["created_at"])
+              ttl = Integer((Time.now - job[:created_at]) * 1000)
+              QC.measure("time-to-lock=#{ttl}ms source=#{name}")
+            end
+          end
         end
       end
     end

data/lib/queue_classic/setup.rb

@@ -4,19 +4,40 @@ module QC
     SqlFunctions = File.join(Root, "/sql/ddl.sql")
     CreateTable = File.join(Root, "/sql/create_table.sql")
     DropSqlFunctions = File.join(Root, "/sql/drop_ddl.sql")
+    UpgradeTo_3_0_0 = File.join(Root, "/sql/update_to_3_0_0.sql")
+    DowngradeTo_3_0_0 = File.join(Root, "/sql/downgrade_from_3_0_0.sql")
 
-    def self.create(c=nil)
+    def self.create(c = QC::default_conn_adapter.connection)
       conn = QC::ConnAdapter.new(c)
       conn.execute(File.read(CreateTable))
       conn.execute(File.read(SqlFunctions))
       conn.disconnect if c.nil? #Don't close a conn we didn't create.
     end
 
-    def self.drop(c=nil)
+    def self.drop(c = QC::default_conn_adapter.connection)
       conn = QC::ConnAdapter.new(c)
       conn.execute("DROP TABLE IF EXISTS queue_classic_jobs CASCADE")
       conn.execute(File.read(DropSqlFunctions))
       conn.disconnect if c.nil? #Don't close a conn we didn't create.
     end
+
+    def self.update(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(UpgradeTo_3_0_0))
+      conn.execute(File.read(DropSqlFunctions))
+      conn.execute(File.read(SqlFunctions))
+    end
+
+    def self.update_to_3_0_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(UpgradeTo_3_0_0))
+      conn.execute(File.read(DropSqlFunctions))
+      conn.execute(File.read(SqlFunctions))
+    end
+
+    def self.downgrade_from_3_0_0(c = QC::default_conn_adapter.connection)
+      conn = QC::ConnAdapter.new(c)
+      conn.execute(File.read(DowngradeTo_3_0_0))
+    end
   end
 end

data/lib/queue_classic/tasks.rb

@@ -41,4 +41,9 @@ namespace :qc do
   task :drop => :environment do
     QC::Setup.drop
   end
+
+  desc "Update queue_classic tables and functions in database"
+  task :update => :environment do
+    QC::Setup.update
+  end
 end

data/lib/queue_classic/worker.rb

@@ -1,6 +1,5 @@
-require 'queue_classic'
-require 'queue_classic/queue'
-require 'queue_classic/conn_adapter'
+require_relative 'queue'
+require_relative 'conn_adapter'
 
 module QC
   # A Worker object can process jobs from one or many queues.
@@ -19,7 +18,13 @@ module QC
     def initialize(args={})
       @fork_worker = args[:fork_worker] || QC::FORK_WORKER
       @wait_interval = args[:wait_interval] || QC::WAIT_TIME
-      @conn_adapter = ConnAdapter.new(args[:connection])
+
+      if args[:connection]
+        @conn_adapter = ConnAdapter.new(args[:connection])
+      elsif QC.has_connection?
+        @conn_adapter = QC.default_conn_adapter
+      end
+
       @queues = setup_queues(@conn_adapter,
         (args[:q_name] || QC::QUEUE),
         (args[:q_names] || QC::QUEUES),
@@ -34,6 +39,8 @@ module QC
     # The canonical example of starting a worker is as follows:
     # QC::Worker.new.start
     def start
+      QC.unlock_jobs_of_dead_workers
+
       while @running
         @fork_worker ? fork_and_work : work
       end
@@ -97,6 +104,7 @@ module QC
     # If the job is not finished and an INT signal is traped,
     # this method will unlock the job in the queue.
     def process(queue, job)
+      start = Time.now
       finished = false
       begin
         call(job).tap do
@@ -110,6 +118,8 @@ module QC
         if !finished
           queue.unlock(job[:id])
         end
+        ttp = Integer((Time.now - start) * 1000)
+        QC.measure("time-to-process=#{ttp} source=#{queue.name}")
       end
     end
 
@@ -126,7 +136,7 @@ module QC
     # This method will be called when an exception
     # is raised during the execution of the job.
     def handle_failure(job,e)
-      log(:at => "handle_failure", :job => job, :error => e.inspect)
+      $stderr.puts("count#qc.job-error=1 job=#{job} error=#{e.inspect}")
     end
 
     # This method should be overriden if

data/readme.md

@@ -1,7 +1,10 @@
 # queue_classic
 
+[![Build Status](https://travis-ci.org/ryandotsmith/queue_classic.svg?branch=master)](https://travis-ci.org/ryandotsmith/queue_classic)
+[![Code Climate](https://codeclimate.com/github/ryandotsmith/queue_classic.png)](https://codeclimate.com/github/ryandotsmith/queue_classic)
+
 Stable: [v2.2.3](https://github.com/ryandotsmith/queue_classic/tree/v2.2.3)
-Latest: v3.0.0beta
+Latest: v3.0.0rc
 
 queue_classic provides a simple interface to a PostgreSQL-backed message queue. queue_classic specializes in concurrent locking and minimizing database load while providing a simple, intuitive developer experience. queue_classic assumes that you are already using PostgreSQL in your production environment and that adding another dependency (e.g. redis, beanstalkd, 0mq) is undesirable.
 
@@ -19,7 +22,10 @@ Contents:
 * [Documentation](http://rubydoc.info/gems/queue_classic/2.2.3/frames)
 * [Usage](#usage)
 * [Setup](#setup)
+* [Upgrade from V2 to V3](#upgrade-from-v2-to-v3)
 * [Configuration](#configuration)
+  * [JSON](#json)
+  * [Logging](#logging)
 * [Support](#support)
 * [Hacking](#hacking-on-queue_classic)
 * [License](#license)
@@ -165,6 +171,25 @@ $ bundle exec rake qc:create
 $ bundle exec rake qc:drop
 ```
 
+## Upgrade from V2 to V3
+If you are upgrading from a previous version of queue_classic, you might need some new database columns and/or functions. Luckily enough for you, it is easy to do so.
+
+### Ruby on Rails
+
+You just need to run those lines, which will copy the new required migrations:
+
+```
+rails generate queue_classic:install
+rake db:migrate
+```
+### Rake Task
+
+This rake task will get you covered:
+```bash
+# Updating the table and functions
+$ bundle exec rake qc:update
+```
+
 ## Configuration
 
 All configuration takes place in the form of environment vars. See [queue_classic.rb](https://github.com/ryandotsmith/queue_classic/blob/master/lib/queue_classic.rb#L23-62) for a list of options.
@@ -179,8 +204,17 @@ alter table queue_classic_jobs alter column args type json using (args::json);
 ## Logging
 
 By default queue_classic does not talk very much.
-If you find yourself in a situation where you need to know what's happening inside QC,
-you can enable the debug output by setting the `DEBUG` environment variable:
+If you find yourself in a situation where you need to know what's happening inside QC, there are two different kind of logging you can enable: DEBUG and MEASURE.
+
+### Measure
+This will output the time to process and that kind of thing. To enable it, set the `QC_MEASURE`:
+
+```
+export QC_MEASURE="true"
+```
+
+### Debug
+You can enable the debug output by setting the `DEBUG` environment variable:
 
 ```
 export DEBUG="true"
@@ -201,7 +235,8 @@ https://groups.google.com/d/forum/queue_classic
 
 ## Hacking on queue_classic
 
-[![Build Status](https://drone.io/github.com/ryandotsmith/queue_classic/status.png)](https://drone.io/github.com/ryandotsmith/queue_classic/latest)
+[![Build Status](https://travis-ci.org/ryandotsmith/queue_classic.svg?branch=master)](https://travis-ci.org/ryandotsmith/queue_classic)
+[![Code Climate](https://codeclimate.com/github/ryandotsmith/queue_classic.png)](https://codeclimate.com/github/ryandotsmith/queue_classic)
 
 ### Dependencies
 

data/sql/create_table.sql

@@ -5,7 +5,9 @@ CREATE TABLE queue_classic_jobs (
   q_name text not null check (length(q_name) > 0),
   method text not null check (length(method) > 0),
   args   text not null,
-  locked_at timestamptz
+  locked_at timestamptz,
+  locked_by integer,
+  created_at timestamptz default now()
 );
 
 -- If json type is available, use it for the args column.

data/sql/ddl.sql

@@ -15,7 +15,9 @@ BEGIN
   -- for more workers. Would love to see some optimization here...
 
   EXECUTE 'SELECT count(*) FROM '
-    || '(SELECT * FROM queue_classic_jobs WHERE q_name = '
+    || '(SELECT * FROM queue_classic_jobs '
+    || ' WHERE locked_at IS NULL'
+    || ' AND q_name = '
     || quote_literal(q_name)
     || ' LIMIT '
     || quote_literal(top_boundary)
@@ -48,7 +50,8 @@ BEGIN
   END LOOP;
 
   RETURN QUERY EXECUTE 'UPDATE queue_classic_jobs '
-    || ' SET locked_at = (CURRENT_TIMESTAMP)'
+    || ' SET locked_at = (CURRENT_TIMESTAMP),'
+    || ' locked_by = (select pg_backend_pid())'
     || ' WHERE id = $1'
     || ' AND locked_at is NULL'
     || ' RETURNING *'

data/sql/downgrade_from_3_0_0.sql

@@ -0,0 +1,2 @@
+ALTER TABLE queue_classic_jobs DROP COLUMN locked_by;
+ALTER TABLE queue_classic_jobs DROP COLUMN created_at;

data/sql/update_to_3_0_0.sql

@@ -0,0 +1,17 @@
+DO $$DECLARE r record;
+BEGIN
+	BEGIN
+		ALTER TABLE queue_classic_jobs ADD COLUMN created_at timestamptz default now();
+	EXCEPTION
+		WHEN duplicate_column THEN RAISE NOTICE 'column created_at already exists in queue_classic_jobs.';
+	END;
+END$$;
+
+DO $$DECLARE r record;
+BEGIN
+  BEGIN
+    ALTER TABLE queue_classic_jobs ADD COLUMN locked_by integer;
+  EXCEPTION
+    WHEN duplicate_column THEN RAISE NOTICE 'column locked_by already exists in queue_classic_jobs.';
+  END;
+END$$;

data/test/helper.rb

@@ -26,6 +26,15 @@ class QCTest < Minitest::Test
     c.disconnect
   end
 
+  def capture_stderr_output
+    original_stderr = $stderr
+    $stderr = StringIO.new
+    yield
+    $stderr.string
+  ensure
+    $stderr = original_stderr
+  end
+
   def capture_debug_output
     original_debug = ENV['DEBUG']
     original_stdout = $stdout

data/test/lib/queue_classic_test.rb

@@ -0,0 +1,32 @@
+require File.expand_path("../../helper.rb", __FILE__)
+
+class QueueClassicTest < QCTest
+  def test_default_conn_adapter_default_value
+    assert(QC.default_conn_adapter.is_a?(QC::ConnAdapter))
+  end
+
+  def test_default_conn_adapter=
+    connection = QC::ConnAdapter.new
+    QC.default_conn_adapter = connection
+    assert_equal(QC.default_conn_adapter, connection)
+  end
+
+  def test_unlock_jobs_of_dead_workers
+    # Insert a locked job
+    adapter = QC::ConnAdapter.new
+    query = "INSERT INTO #{QC::TABLE_NAME} (q_name, method, args, locked_by, locked_at) VALUES ('whatever', 'Kernel.puts', '[\"ok?\"]', 0, (CURRENT_TIMESTAMP))"
+    adapter.execute(query)
+
+    # We should have no unlocked jobs
+    query_locked_jobs = "SELECT * FROM #{QC::TABLE_NAME} WHERE locked_at IS NULL"
+    res = adapter.connection.exec(query_locked_jobs)
+    assert_equal(0, res.count)
+
+    # Unlock the job
+    QC.unlock_jobs_of_dead_workers
+
+    # We should have an unlocked job now
+    res = adapter.connection.exec(query_locked_jobs)
+    assert_equal(1, res.count)
+  end
+end

data/test/queue_test.rb

@@ -13,12 +13,14 @@ class QueueTest < QCTest
   end
 
   def test_lock
-    QC.enqueue("Klass.method")
-
-    # See helper.rb for more information about the large initial id
-    # number.
-    expected = {:id=>(2**34).to_s, :method=>"Klass.method", :args=>[]}
-    assert_equal(expected, QC.lock)
+    queue = QC::Queue.new("queue_classic_jobs")
+    queue.enqueue("Klass.method")
+    job = queue.lock
+    # See helper.rb for more information about the large initial id number.
+    assert_equal((2**34).to_s, job[:id])
+    assert_equal("queue_classic_jobs", job[:q_name])
+    assert_equal("Klass.method", job[:method])
+    assert_equal([], job[:args])
   end
 
   def test_lock_when_empty

data/test/worker_test.rb

@@ -45,12 +45,11 @@ class WorkerTest < QCTest
   end
 
   def test_failed_job_is_logged
-    output = capture_debug_output do
+    output = capture_stderr_output do
       QC.enqueue("TestObject.not_a_method")
       TestWorker.new.work
     end
-    expected_output = /lib=queue-classic at=handle_failure job={:id=>"\d+", :method=>"TestObject.not_a_method", :args=>\[\]} error=#<NoMethodError: undefined method `not_a_method' for TestObject:Module>/
-    assert_match(expected_output, output, "=== debug output ===\n #{output}")
+    assert(output.include?("#<NoMethodError: undefined method `not_a_method'"))
   end
 
   def test_log_yield
@@ -175,5 +174,4 @@ class WorkerTest < QCTest
     assert_equal(42, r)
     assert_equal(0, worker.failed_count)
   end
-
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: queue_classic
 version: !ruby/object:Gem::Version
-  version: 3.0.0rc
+  version: 3.0.0
 platform: ruby
 authors:
 - Ryan Smith (♠ ace hacker)
@@ -14,14 +14,14 @@ dependencies:
   name: pg
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.17.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.17.0
 description: queue_classic is a queueing library for Ruby apps. (Rails, Sinatra, Etc...)
@@ -32,21 +32,25 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- readme.md
-- sql/create_table.sql
-- sql/ddl.sql
-- sql/drop_ddl.sql
 - lib/generators/queue_classic/install_generator.rb
 - lib/generators/queue_classic/templates/add_queue_classic.rb
+- lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb
+- lib/queue_classic.rb
 - lib/queue_classic/conn_adapter.rb
 - lib/queue_classic/queue.rb
 - lib/queue_classic/railtie.rb
 - lib/queue_classic/setup.rb
 - lib/queue_classic/tasks.rb
 - lib/queue_classic/worker.rb
-- lib/queue_classic.rb
+- readme.md
+- sql/create_table.sql
+- sql/ddl.sql
+- sql/downgrade_from_3_0_0.sql
+- sql/drop_ddl.sql
+- sql/update_to_3_0_0.sql
 - test/benchmark_test.rb
 - test/helper.rb
+- test/lib/queue_classic_test.rb
 - test/queue_test.rb
 - test/worker_test.rb
 homepage: http://github.com/ryandotsmith/queue_classic
@@ -59,21 +63,22 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>'
+  - - ">="
     - !ruby/object:Gem::Version
-      version: 1.3.1
+      version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Simple, efficient worker queue for Ruby & PostgreSQL.
 test_files:
 - test/benchmark_test.rb
+- test/lib/queue_classic_test.rb
 - test/queue_test.rb
 - test/worker_test.rb