queue_classic_pg2 3.2.0.RC1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: afdc18c05f5f58df0d6faf6b8e0a993d89672ea71605d81bb472b11e24aabbaa
+  data.tar.gz: e8be23038d61da8be0e3f10f16b59b3eb9edc927da5e5eb71f6fd168fb9ca636
+SHA512:
+  metadata.gz: d2a6459471f79073ffc56cbf18c286538365d2c203adb3b7ad9352b298be236f47022e06422412d98c965edc14e7c214e3198effd3ca6a8597f7cc587b79d73c
+  data.tar.gz: ac85327e74443bdc04c8c9a579b412bbba0133dd5424294b0b7af144226f58ed97f5fbd46dbd76c79ab3464b68a0ad9056d7e73c11afd218910934d4b95bcb49

data/.gitignore

@@ -0,0 +1,9 @@
+.db
+.ruby-version
+.bundle
+.rvmrc
+etc/
+*.gem
+.env
+Gemfile.lock
+tips-and-tricks.md

data/.travis.yml

@@ -0,0 +1,19 @@
+language: ruby
+before_install:
+  - gem update bundler --no-document
+before_script:
+  - psql -c 'create database queue_classic_test;' -U postgres
+env:
+  global:
+    - QC_DATABASE_URL="postgres://postgres@localhost/queue_classic_test"
+    - QC_BENCHMARK=true
+    - QC_BENCHMARK_MAX_TIME_DEQUEUE=60
+rvm:
+  - 2.4.1
+  - 2.3.4
+  - 2.2.7
+  - 2.1.10
+  - 2.0.0
+  - 1.9.3
+addons:
+  postgresql: 9.3

data/CONTRIBUTING.md

@@ -0,0 +1,17 @@
+queue_classic is a volunteer effort. We encourage you to pitch in.
+
+1. Fork queue_classic
+2. Create a topic branch - `git checkout -b my_branch`
+3. Push to your branch - `git push origin my_branch`
+4. Send us a pull-request for your topic branch
+5. That's it!
+
+If you make code changes, please check that your patch:
+
+1. has tests
+2. works on Rails and non-Rails projects
+3. updates documentation
+
+Thanks! :heart:
+
+queue_classic Team

data/Gemfile

@@ -0,0 +1,9 @@
+source "https://rubygems.org"
+
+gem "rake"
+
+gemspec
+
+group :test do
+  gem 'minitest', '~> 5.5.1'
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,326 @@
+# queue_classic
+
+<p align="center">
+  <b>Simple, efficient worker queue for Ruby & PostgreSQL</b>
+  <br />
+  <a href="https://travis-ci.org/QueueClassic/queue_classic"><img src="http://img.shields.io/travis/QueueClassic/queue_classic/master.svg?style=flat" /></a>
+
+  <a href="https://codeclimate.com/github/QueueClassic/queue_classic"><img src="http://img.shields.io/codeclimate/github/QueueClassic/queue_classic.svg?style=flat" /></a>
+
+  <a href="http://badge.fury.io/rb/queue_classic"><img src="http://img.shields.io/gem/v/queue_classic.svg?style=flat" alt="Gem Version" height="18"></a>
+</p>
+
+
+**IMPORTANT NOTE REGARDING VERSIONS**
+
+**This README is representing the current work for queue_classic edge [unstable]. You can find the README for other versions:**
+
+- current release candidate: [v3.2.0.RC1](https://github.com/QueueClassic/queue_classic/tree/v3.2.0.RC1)
+- latest stable can be found: [v3.1.x](https://github.com/QueueClassic/queue_classic/tree/3-1-stable)
+- older stable: [v3.0.x](https://github.com/QueueClassic/queue_classic/tree/3-0-stable)
+
+
+## What is queue_classic?
+
+queue_classic provides a simple interface to a PostgreSQL-backed message queue. queue_classic specializes in concurrent locking and minimizing database load while providing a simple, intuitive developer experience. queue_classic assumes that you are already using PostgreSQL in your production environment and that adding another dependency (e.g. redis, beanstalkd, 0mq) is undesirable.
+
+## Features
+
+* Leverage of PostgreSQL's listen/notify & row locking.
+* Support for multiple queues with heterogeneous workers.
+* JSON data format.
+* Forking workers.
+* Workers can work multiple queues.
+* Reduced row contention using a [relaxed FIFO](http://www.cs.tau.ac.il/~shanir/nir-pubs-web/Papers/Lock_Free.pdf) technique.
+
+## Table of content
+
+* [Documentation](http://rubydoc.info/gems/queue_classic/2.2.3/frames)
+* [Usage](#usage)
+* [Setup](#setup)
+* [Upgrade from earlier versions to V3.1](#upgrade-from-earlier-versions)
+* [Configuration](#configuration)
+  * [JSON](#json)
+  * [Logging](#logging)
+* [Support](#support)
+* [Hacking](#hacking-on-queue_classic)
+* [License](#license)
+
+## Usage
+
+There are 2 ways to use queue_classic.
+
+* Producing Jobs
+* Working Jobs
+
+### Producing Jobs
+
+The first argument is a string which represents a ruby object and a method name. The second argument(s) will be passed along as arguments to the method invocation defined by the first argument. The set of arguments will be encoded as JSON in the database.
+
+```ruby
+# This method has no arguments.
+QC.enqueue("Time.now")
+
+# This method has 1 argument.
+QC.enqueue("Kernel.puts", "hello world")
+
+# This method has 2 arguments.
+QC.enqueue("Kernel.printf", "hello %s", "world")
+
+# This method has a hash argument.
+QC.enqueue("Kernel.puts", {"hello" => "world"})
+
+# This method has an array argument.
+QC.enqueue("Kernel.puts", ["hello", "world"])
+
+# This method uses a non-default queue.
+p_queue = QC::Queue.new("priority_queue")
+p_queue.enqueue("Kernel.puts", ["hello", "world"])
+```
+
+There is also the possibility to schedule a job at a specified time in the future. It will not be worked off before that specified time.
+
+```ruby
+# Specifying the job execution time exactly.
+QC.enqueue_at(Time.new(2024,01,02,10,00), "Kernel.puts", "hello future")
+
+# Specifying the job execution time as an offset in seconds.
+QC.enqueue_in(60, "Kernel.puts", "hello from 1 minute later")
+```
+
+### Working Jobs
+
+There are two ways to work jobs. The first approach is to use the Rake task. The second approach is to use a custom executable.
+
+#### Rake Task
+
+Require queue_classic in your Rakefile.
+
+```ruby
+require 'queue_classic'
+require 'queue_classic/tasks'
+```
+
+Start the worker via the Rakefile.
+```bash
+$ bundle exec rake qc:work
+```
+
+Setup a worker to work a non-default queue.
+```bash
+$ QUEUE="priority_queue" bundle exec rake qc:work
+```
+
+Setup a worker to work multiple queues.
+```bash
+$ QUEUES="priority_queue,secondary_queue" bundle exec rake qc:work
+```
+In this scenario, on each iteration of the worker's loop, it will look for jobs in the first queue prior to looking at the second queue. This means that the first queue must be empty before the worker will look at the second queue.
+
+#### Custom Worker
+
+This example is probably not production ready; however, it serves as an example of how to leverage the code in the Worker class to fit your non-default requirements.
+
+```ruby
+require 'timeout'
+require 'queue_classic'
+
+FailedQueue = QC::Queue.new("failed_jobs")
+
+class MyWorker < QC::Worker
+
+  # A job is a Hash containing these attributes:
+  # :id Integer, the job id
+  # :method String, containing the object and method
+  # :args String, the arguments
+  # :q_name String, the queue name
+  # :scheduled_at Time, the scheduled time if the job was scheduled
+
+  # Execute the job using the methods and arguments
+  def call(job)
+     # Do something with the job
+     ...
+  end
+
+  # This method will be called when an exception
+  # is raised during the execution of the job.
+  # First argument is the job that failed.
+  # Second argument is the exception.
+  def handle_failure(job, e)
+    FailedQueue.enqueue(job[:method], *job[:args])
+  end
+
+end
+
+worker = MyWorker.new
+
+trap('INT') { exit }
+trap('TERM') { worker.stop }
+
+loop do
+  queue, job = worker.lock_job
+  Timeout::timeout(5) { worker.process(queue, job) }
+end
+```
+
+The `qc:work` rake task uses `QC::Worker` by default. However, it's easy to
+inject your own worker class:
+
+```ruby
+QC.default_worker_class = MyWorker
+```
+
+## Setup
+
+In addition to installing the rubygem, you will need to prepare your database. Database preparation includes creating a table and loading PL/pgSQL functions. You can issue the database preparation commands using `PSQL(1)` or use a database migration script.
+
+### Quick Start
+
+```bash
+$ gem install queue_classic
+$ createdb queue_classic_test
+$ export QC_DATABASE_URL="postgres://username:password@localhost/queue_classic_test"
+$ ruby -r queue_classic -e "QC::Setup.create"
+$ ruby -r queue_classic -e "QC.enqueue('Kernel.puts', 'hello world')"
+$ ruby -r queue_classic -e "QC::Worker.new.work"
+```
+
+### Ruby on Rails Setup
+
+Declare dependencies in Gemfile.
+```ruby
+source "https://rubygems.org"
+gem "queue_classic"
+```
+
+Add the database tables and stored procedures.
+
+```
+rails generate queue_classic:install
+bundle exec rake db:migrate
+```
+
+#### Active Job
+
+If you use Rails 4.2+, all you need to do is to set `config.active_job.queue_adapter = :queue_classic` in your `application.rb`. Everything else will be taken care for you. You can now use the Active Job functionality from now.
+
+Just for your information, queue_classic detects your database connection and uses it.
+
+### Rake Task Setup
+
+Alternatively, you can use the Rake task to prepare your database.
+
+```bash
+# Creating the table and functions
+$ bundle exec rake qc:create
+
+# Dropping the table and functions
+$ bundle exec rake qc:drop
+```
+
+### Database connection
+
+#### Ruby on Rails
+
+Starting with with queue_classic 3.1, Rails is automatically detected and its connection is used.
+
+If you don't want to use the automatic database connection, set this environment variable to false: `export QC_RAILS_DATABASE=false`
+
+**Note on using ActiveRecord migrations:** If you use the migration, and you wish to use commands that reset the database from the stored schema (e.g. `rake db:reset`), your application must be configured with `config.active_record.schema_format = :sql` in `config/application.rb`.  If you don't do this, the PL/pgSQL function that queue_classic creates will be lost when you reset the database.
+
+
+#### Other Ruby apps
+
+By default, queue_classic will use the QC_DATABASE_URL falling back on DATABASE_URL. The URL must be in the following format: `postgres://username:password@localhost/database_name`.  If you use Heroku's PostgreSQL service, this will already be set. If you don't want to set this variable, you can set the connection in an initializer. **QueueClassic will maintain its own connection to the database.** This may double the number of connections to your database.
+
+## Upgrade from earlier versions
+If you are upgrading from a previous version of queue_classic, you might need some new database columns and/or functions. Luckily enough for you, it is easy to do so.
+
+### Ruby on Rails
+
+You just need to run those lines, which will copy the new required migrations:
+
+```
+rails generate queue_classic:install
+bundle exec rake db:migrate
+```
+### Rake Task
+
+This rake task will get you covered:
+```bash
+# Updating the table and functions
+$ bundle exec rake qc:update
+```
+
+## Configuration
+
+All configuration takes place in the form of environment vars. See [queue_classic.rb](https://github.com/QueueClassic/queue_classic/blob/master/lib/queue_classic.rb#L23-62) for a list of options.
+
+## JSON
+
+If you are running PostgreSQL 9.4 or higher, queue_classic will use the [jsonb](http://www.postgresql.org/docs/9.4/static/datatype-json.html) datatype for new tables. Versions 9.2 and 9.3 will use the `json` data type and versions 9.1 and lower will use the `text` data type.
+If you are updating queue_classic and are running PostgreSQL >= 9.4, run the following to switch to `jsonb`:
+```
+alter table queue_classic_jobs alter column args type jsonb using (args::jsonb);
+```
+
+## Logging
+
+By default queue_classic does not talk very much.
+If you find yourself in a situation where you need to know what's happening inside QC, there are two different kind of logging you can enable: DEBUG and MEASURE.
+
+### Measure
+This will output the time to process and that kind of thing. To enable it, set the `QC_MEASURE`:
+
+```
+export QC_MEASURE="true"
+```
+
+### Debug
+You can enable the debug output by setting the `DEBUG` environment variable:
+
+```
+export DEBUG="true"
+```
+
+## Support
+
+If you think you have found a bug, feel free to open an issue. Use the following template for the new issue:
+
+1. List Versions: Ruby, PostgreSQL, queue_classic.
+2. Define what you would have expected to happen.
+3. List what actually happened.
+4. Provide sample codes & commands which will reproduce the problem.
+
+If you have general questions about how to use queue_classic, send a message to the mailing list:
+
+https://groups.google.com/d/forum/queue_classic
+
+## Hacking on queue_classic
+
+### Dependencies
+
+* Ruby 1.9.2
+* Postgres ~> 9.0
+* Rubygem: pg ~> 0.11.0
+* For JRuby, see [queue_classic_java](https://github.com/bdon/queue_classic_java)
+
+### Running Tests
+
+```bash
+$ bundle
+$ createdb queue_classic_test
+$ export QC_DATABASE_URL="postgres://username:pass@localhost/queue_classic_test"
+$ bundle exec rake                      # run all tests
+$ bundle exec ruby test/queue_test.rb   # run a single test
+```
+
+## License
+
+Copyright (C) 2010 Ryan Smith
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,14 @@
+$:.unshift("lib")
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+require "./lib/queue_classic"
+require "./lib/queue_classic/tasks"
+
+task :default => ['test']
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+  t.warning = true
+end

data/changelog

@@ -0,0 +1,146 @@
+Unreleased
+- Fixed a bug in the offset calculation of `.enqueue_at`.
+- Use the jsonb type for the args column from now on. If not available, fall back to json or text.
+- `enqueue`, `enqueue_at`, `enqueue_in` return job hash with id.
+- Fixed unlock query for versions below Postgres 9.2
+
+Version 3.0.0rc
+- Improved signal handling
+
+Version 3.0.0beta
+- Workers can process many queues.
+
+Version 2.2.3
+- Update pg dependency to 0.17.0
+
+Version 2.3.0beta [YANKED]
+- Concurrent job processing.
+
+Version 2.2.2
+- Update pg dependency to 0.16.0
+
+Version 2.2.1
+- Force listen/notify on worker
+- Notifications happen inside PostgreSQL trigger
+- Add rake task for generating rails migrations
+- Fix bug related to listening worker
+
+Version 2.2.0
+- Use json from the stdlib in place of MultiJson.
+- Use postgresql's json type for the args column if json type is available
+- QC::Worker#handle_failure logs the job and the error
+- QC.default_queue= to set your own default queue. (can be used
+  in testing to configure a mock queue)
+- QC.log now reports time elapsed in milliseconds.
+
+Version 2.1.4
+- Update pg dependency to 0.15.1
+- Document logging behaviour
+
+Version 2.1.3
+- Use MultiJson (Ezekiel Templin: #106)
+
+Version 2.1.2
+- Use 64bit ints as default data types in PostgreSQL
+- Add process method in worker
+- Allow percent-encoded socket paths in DATABASE_URL
+
+Version 2.1.1
+- Update pg gem version
+
+Version 2.1.0
+- Wrap connection execution in mutex making it thread safe
+- Cleanup logging
+- Refactor worker class making it more extensible
+- Added rdoc style docs for worker class
+
+Version 2.0.5
+- Allow term signal to halt the lock_job function
+
+Version 2.0.4
+- Provider a connection setter.
+
+Version 2.0.3
+- Fix typo :(
+
+Version 2.0.2
+- Remove scrolls dependency
+- Fix issue with notify not working on non-default queues
+
+Version 2.0.1
+
+Version 2.0.0
+- Simpler setup via QC::Setup.create (rake qc:create) & QC::Setup.drop (rake
+qc:drop)
+- Simpler abstractions in implementation
+- Better support for instrumentation via log_yield hook in QC module
+- Multiple queues use one table with a queue_name column
+
+Version 1.0.2
+- Update to latest okjson as the current has bugs
+
+Version 1.0.1
+- Using OkJson instead of any sort of rubygem
+- Remove html from docs
+- Use parameterised queries
+- Don't set application name by default
+- Injection attack bug fixed in lock_head()
+- Notificaiton get sent on seperate chans for disjoint queues
+
+Version 1.0.0rc1
+- Removed json gem and relying on ruby 1.9.2's stdlib
+- Added better documentation
+
+Version 0.3.6pre
+- Added listen/notify support configured by $QC_LISTENING_WORKER otherwise uses Kernel.sleep()
+
+Version 0.3.5pre
+- Removed debug statement. Mistake!
+
+Version 0.3.4pre
+- Added logging configured by $VERBOSE or $QC_VERBOSE.
+- Added a method setup_child that gets called right after a worker forks.
+- Removed database helper methods: create_table, drop_table, silence_warnings.
+- Removed queue connection helper methods. Status should be discoverd by psql or the likes.
+
+Version 0.3.3pre
+- Removed PUB/SUB
+- Added GC after working a job
+- Added support for a database_url other than $DATABASE_URL. $QC_DATABASE_URL
+- Added exp backoff configured by $QC_MAX_LOCK_ATTEMPTS (default = 5)
+- Added option for forking worker configured by $QC_FORK_WORKER (default = false)
+
+Version 0.3.2
+- Fixed bug which caused workers to consume 2 connections. Now they only consume 1
+- Added a rake file for tests
+- Added support for postgres:///db_name DATABASE_URLs
+
+Version 0.3.1
+- Added query interface for introspection success
+- Moved the locking of jobs into the DB as a PG function. SELECT lock_head()
+- Added requirement for DB connection. MUST BE URI i.e. DATABASE_URL=postgres://user:pass@localhost/db_name
+- Added rake qc:create_queue. This task will add a new table. Use this for multiple queues.
+- Added a bit of randomness to the lock_head() function. Helps you scale to a hilarious number of workers.
+- Added support for trapping INT and TERM signals in the worker. ^C to stop after finished and ^C^C to kill.
+- Renamed the jobs table to queue_classic_jobs
+- Renamed the jobs channel to queue_classic_jobs
+- Added support for multiple queues
+
+Version 0.2.2
+- Fixed problems with enqueueing a list of parameters.
+
+Version 0.2.1
+- Added method for handling errors.
+- Added ability to enqueue a Job instance. Makes retrying jobs easier.
+- Added delete_all.
+- Fixed connection algorithm. 1 connection per process.
+- Fixed API for enqueue. Now accepting 1 arg or many args.
+
+Version 0.2.0
+- Beta Release
+- Added method for handling failed jobs
+- Added Benchmarks
+- Removed logging
+- Moved the Job class into it's own file
+
+0.1.6

data/lib/generators/queue_classic/install_generator.rb

@@ -0,0 +1,36 @@
+require 'rails/generators'
+require 'rails/generators/migration'
+require 'active_record'
+
+module QC
+  class InstallGenerator < Rails::Generators::Base
+    include Rails::Generators::Migration
+
+    namespace "queue_classic:install"
+    self.source_paths << File.join(File.dirname(__FILE__), 'templates')
+    desc 'Generates (but does not run) a migration to add a queue_classic table.'
+
+    def self.next_migration_number(dirname)
+      next_migration_number = current_migration_number(dirname) + 1
+      ActiveRecord::Migration.next_migration_number(next_migration_number)
+    end
+
+    def create_migration_file
+      if self.class.migration_exists?('db/migrate', 'add_queue_classic').nil?
+        migration_template 'add_queue_classic.rb', 'db/migrate/add_queue_classic.rb'
+      end
+
+      if self.class.migration_exists?('db/migrate', 'update_queue_classic_3_0_0').nil?
+        migration_template 'update_queue_classic_3_0_0.rb', 'db/migrate/update_queue_classic_3_0_0.rb'
+      end
+
+      if self.class.migration_exists?('db/migrate', 'update_queue_classic_3_0_2').nil?
+        migration_template 'update_queue_classic_3_0_2.rb', 'db/migrate/update_queue_classic_3_0_2.rb'
+      end
+
+      if self.class.migration_exists?('db/migrate', 'update_queue_classic_3_1_0').nil?
+        migration_template 'update_queue_classic_3_1_0.rb', 'db/migrate/update_queue_classic_3_1_0.rb'
+      end
+    end
+  end
+end

data/lib/generators/queue_classic/templates/add_queue_classic.rb

@@ -0,0 +1,9 @@
+class AddQueueClassic < ActiveRecord::Migration
+  def self.up
+    QC::Setup.create
+  end
+
+  def self.down
+    QC::Setup.drop
+  end
+end

data/lib/generators/queue_classic/templates/update_queue_classic_3_0_0.rb

@@ -0,0 +1,9 @@
+class UpdateQueueClassic300 < ActiveRecord::Migration
+  def self.up
+    QC::Setup.update_to_3_0_0
+  end
+
+  def self.down
+    QC::Setup.downgrade_from_3_0_0
+  end
+end

data/lib/generators/queue_classic/templates/update_queue_classic_3_0_2.rb

@@ -0,0 +1,11 @@
+class UpdateQueueClassic302 < ActiveRecord::Migration
+  def self.up
+    QC::Setup.update_to_3_0_0
+  end
+
+  def self.down
+    # This migration is fixing a bug, so we don't want to do anything here.
+    # I didn't want to make it irreversible either, as it could prevent
+    # rolling back other, unrelated, stuff.
+  end
+end

data/lib/generators/queue_classic/templates/update_queue_classic_3_1_0.rb

@@ -0,0 +1,9 @@
+class UpdateQueueClassic310 < ActiveRecord::Migration
+  def self.up
+    QC::Setup.update_to_3_1_0
+  end
+
+  def self.down
+    QC::Setup.downgrade_from_3_1_0
+  end
+end