queue_classic 3.1.0 → 3.2.0.RC1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 88e26c8a7314c7ffca4aac79bf29fac181a304f4
-  data.tar.gz: 9d92ce5e8c313ea9a81c66b769b77a1c80265745
+  metadata.gz: 173f9534b0c2b7318cccdf7de96b762ee63306d3
+  data.tar.gz: c808e6bc17371a57e997c13800ba803666be969e
 SHA512:
-  metadata.gz: 8fa62030bb04d3464d2772a6b68c2b9d0d698d2ed146125b69b2a72b2bdb5a2d79f63424e4a676f13e2f981b980e53d8534caf01f906ed575e5c31630b9da1f4
-  data.tar.gz: 8f257e023700a47ff33f4b449f5df73fdb37f06dbbf338583ba244196f1883ff1670484cd4267f3e081526a504552aba5b9aebcda72de8cc951a0d3b6f83d572
+  metadata.gz: 7065d297515c3cd9bc40ac5f93a36f164d124f812dc2c53a649b163aceecf9995ebc07bee273660a4540defd08b95afc5b0b69f7b6af1491fd25f8c61cdbb927
+  data.tar.gz: 9f4abe2f2f6c58a5bc4f049b0851d63e730c2922dcf8141a14222e01e2900c7993db939033925ebb684c88b6452e689269c3bbccb2e7386a60a4eacf24308aed

data/.gitignore

@@ -0,0 +1,9 @@
+.db
+.ruby-version
+.bundle
+.rvmrc
+etc/
+*.gem
+.env
+Gemfile.lock
+tips-and-tricks.md

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+before_script:
+  - psql -c 'create database queue_classic_test;' -U postgres
+env:
+  global:
+    - QC_DATABASE_URL="postgres://postgres@localhost/queue_classic_test"
+    - QC_BENCHMARK=true
+    - QC_BENCHMARK_MAX_TIME_DEQUEUE=60
+rvm:
+  - 2.2
+  - 2.1
+  - 2.0.0
+  - 1.9.3
+addons:
+  postgresql: 9.3

data/CONTRIBUTING.md

@@ -0,0 +1,17 @@
+queue_classic is a volunteer effort. We encourage you to pitch in.
+
+1. Fork queue_classic
+2. Create a topic branch - `git checkout -b my_branch`
+3. Push to your branch - `git push origin my_branch`
+4. Send us a pull-request for your topic branch
+5. That's it!
+
+If you make code changes, please check that your patch:
+
+1. has tests
+2. works on Rails and non-Rails projects
+3. updates documentation
+
+Thanks! :heart:
+
+queue_classic Team

data/Gemfile

@@ -0,0 +1,9 @@
+source "https://rubygems.org"
+
+gem "rake"
+
+gemspec
+
+group :test do
+  gem 'minitest', '~> 5.5.1'
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -13,8 +13,9 @@
 
 **IMPORTANT NOTE REGARDING VERSIONS**
 
-**This README is representing the current work for queue_classic 3.1. You can find the README for other versions:**
+**This README is representing the current work for queue_classic edge [unstable]. You can find the README for other versions:**
 
+- current release candidate: [v3.1.X](https://github.com/QueueClassic/queue_classic/tree/3-1-stable)
 - latest stable can be found: [v3.0.X](https://github.com/QueueClassic/queue_classic/tree/3-0-stable)
 - older stable: [v2.2.3](https://github.com/QueueClassic/queue_classic/tree/v2.2.3)
 
@@ -127,15 +128,34 @@ require 'queue_classic'
 FailedQueue = QC::Queue.new("failed_jobs")
 
 class MyWorker < QC::Worker
+
+  # A job is a Hash containing these attributes:
+  # :id Integer, the job id
+  # :method String, containing the object and method
+  # :args String, the arguments
+  # :q_name String, the queue name
+  # :scheduled_at Time, the scheduled time if the job was scheduled
+
+  # Execute the job using the methods and arguments
+  def call(job)
+     # Do something with the job
+     ...
+  end
+  
+  # This method will be called when an exception
+  # is raised during the execution of the job.
+  # First argument is the job that failed.
+  # Second argument is the exception.
   def handle_failure(job, e)
     FailedQueue.enqueue(job[:method], *job[:args])
   end
+    
 end
 
 worker = MyWorker.new
 
-trap('INT') {exit}
-trap('TERM') {worker.stop}
+trap('INT') { exit }
+trap('TERM') { worker.stop }
 
 loop do
   job = worker.lock_job
@@ -143,6 +163,13 @@ loop do
 end
 ```
 
+The `qc:work` rake task uses `QC::Worker` by default. However, it's easy to
+inject your own worker class:
+
+```ruby
+QC.default_worker_class = MyWorker
+```
+
 ## Setup
 
 In addition to installing the rubygem, you will need to prepare your database. Database preparation includes creating a table and loading PL/pgSQL functions. You can issue the database preparation commands using `PSQL(1)` or use a database migration script.
@@ -162,7 +189,7 @@ $ ruby -r queue_classic -e "QC::Worker.new.work"
 
 Declare dependencies in Gemfile.
 ```ruby
-source "http://rubygems.org"
+source "https://rubygems.org"
 gem "queue_classic", "~> 3.0.0"
 ```
 
@@ -231,9 +258,10 @@ All configuration takes place in the form of environment vars. See [queue_classi
 
 ## JSON
 
-If you are running PostgreSQL 9.2 or higher, queue_classic will use the [json](http://www.postgresql.org/docs/9.2/static/datatype-json.html) datatype for storing arguments. Versions 9.1 and lower will use the 'text' column. If you have installed queue_classic prior to version 2.1.4 and are running PostgreSQL >= 9.2, run the following to switch to using the json type:
+If you are running PostgreSQL 9.4 or higher, queue_classic will use the [jsonb](http://www.postgresql.org/docs/9.4/static/datatype-json.html) datatype for new tables. Versions 9.2 and 9.3 will use the `json` data type and versions 9.1 and lower will use the `text` data type.
+If you are updating queue_classic and are running PostgreSQL >= 9.4, run the following to switch to `jsonb`:
 ```
-alter table queue_classic_jobs alter column args type json using (args::json);
+alter table queue_classic_jobs alter column args type jsonb using (args::jsonb);
 ```
 
 ## Logging
@@ -272,7 +300,7 @@ https://groups.google.com/d/forum/queue_classic
 
 ### Dependencies
 
-* Ruby 1.9.2 (tests work in 1.8.7 but compatibility is not guaranteed or supported)
+* Ruby 1.9.2
 * Postgres ~> 9.0
 * Rubygem: pg ~> 0.11.0
 * For JRuby, see [queue_classic_java](https://github.com/bdon/queue_classic_java)

data/Rakefile

@@ -0,0 +1,14 @@
+$:.unshift("lib")
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+require "./lib/queue_classic"
+require "./lib/queue_classic/tasks"
+
+task :default => ['test']
+Rake::TestTask.new do |t|
+  t.libs << 'test'
+  t.test_files = FileList['test/**/*_test.rb']
+  t.verbose = true
+  t.warning = true
+end

data/changelog

@@ -0,0 +1,146 @@
+Unreleased
+- Fixed a bug in the offset calculation of `.enqueue_at`.
+- Use the jsonb type for the args column from now on. If not available, fall back to json or text.
+- `enqueue`, `enqueue_at`, `enqueue_in` return job hash with id.
+- Fixed unlock query for versions below Postgres 9.2
+
+Version 3.0.0rc
+- Improved signal handling
+
+Version 3.0.0beta
+- Workers can process many queues.
+
+Version 2.2.3
+- Update pg dependency to 0.17.0
+
+Version 2.3.0beta [YANKED]
+- Concurrent job processing.
+
+Version 2.2.2
+- Update pg dependency to 0.16.0
+
+Version 2.2.1
+- Force listen/notify on worker
+- Notifications happen inside PostgreSQL trigger
+- Add rake task for generating rails migrations
+- Fix bug related to listening worker
+
+Version 2.2.0
+- Use json from the stdlib in place of MultiJson.
+- Use postgresql's json type for the args column if json type is available
+- QC::Worker#handle_failure logs the job and the error
+- QC.default_queue= to set your own default queue. (can be used
+  in testing to configure a mock queue)
+- QC.log now reports time elapsed in milliseconds.
+
+Version 2.1.4
+- Update pg dependency to 0.15.1
+- Document logging behaviour
+
+Version 2.1.3
+- Use MultiJson (Ezekiel Templin: #106)
+
+Version 2.1.2
+- Use 64bit ints as default data types in PostgreSQL
+- Add process method in worker
+- Allow percent-encoded socket paths in DATABASE_URL
+
+Version 2.1.1
+- Update pg gem version
+
+Version 2.1.0
+- Wrap connection execution in mutex making it thread safe
+- Cleanup logging
+- Refactor worker class making it more extensible
+- Added rdoc style docs for worker class
+
+Version 2.0.5
+- Allow term signal to halt the lock_job function
+
+Version 2.0.4
+- Provider a connection setter.
+
+Version 2.0.3
+- Fix typo :(
+
+Version 2.0.2
+- Remove scrolls dependency
+- Fix issue with notify not working on non-default queues
+
+Version 2.0.1
+
+Version 2.0.0
+- Simpler setup via QC::Setup.create (rake qc:create) & QC::Setup.drop (rake
+qc:drop)
+- Simpler abstractions in implementation
+- Better support for instrumentation via log_yield hook in QC module
+- Multiple queues use one table with a queue_name column
+
+Version 1.0.2
+- Update to latest okjson as the current has bugs
+
+Version 1.0.1
+- Using OkJson instead of any sort of rubygem
+- Remove html from docs
+- Use parameterised queries
+- Don't set application name by default
+- Injection attack bug fixed in lock_head()
+- Notificaiton get sent on seperate chans for disjoint queues
+
+Version 1.0.0rc1
+- Removed json gem and relying on ruby 1.9.2's stdlib
+- Added better documentation
+
+Version 0.3.6pre
+- Added listen/notify support configured by $QC_LISTENING_WORKER otherwise uses Kernel.sleep()
+
+Version 0.3.5pre
+- Removed debug statement. Mistake!
+
+Version 0.3.4pre
+- Added logging configured by $VERBOSE or $QC_VERBOSE.
+- Added a method setup_child that gets called right after a worker forks.
+- Removed database helper methods: create_table, drop_table, silence_warnings.
+- Removed queue connection helper methods. Status should be discoverd by psql or the likes.
+
+Version 0.3.3pre
+- Removed PUB/SUB
+- Added GC after working a job
+- Added support for a database_url other than $DATABASE_URL. $QC_DATABASE_URL
+- Added exp backoff configured by $QC_MAX_LOCK_ATTEMPTS (default = 5)
+- Added option for forking worker configured by $QC_FORK_WORKER (default = false)
+
+Version 0.3.2
+- Fixed bug which caused workers to consume 2 connections. Now they only consume 1
+- Added a rake file for tests
+- Added support for postgres:///db_name DATABASE_URLs
+
+Version 0.3.1
+- Added query interface for introspection success
+- Moved the locking of jobs into the DB as a PG function. SELECT lock_head()
+- Added requirement for DB connection. MUST BE URI i.e. DATABASE_URL=postgres://user:pass@localhost/db_name
+- Added rake qc:create_queue. This task will add a new table. Use this for multiple queues.
+- Added a bit of randomness to the lock_head() function. Helps you scale to a hilarious number of workers.
+- Added support for trapping INT and TERM signals in the worker. ^C to stop after finished and ^C^C to kill.
+- Renamed the jobs table to queue_classic_jobs
+- Renamed the jobs channel to queue_classic_jobs
+- Added support for multiple queues
+
+Version 0.2.2
+- Fixed problems with enqueueing a list of parameters.
+
+Version 0.2.1
+- Added method for handling errors.
+- Added ability to enqueue a Job instance. Makes retrying jobs easier.
+- Added delete_all.
+- Fixed connection algorithm. 1 connection per process.
+- Fixed API for enqueue. Now accepting 1 arg or many args.
+
+Version 0.2.0
+- Beta Release
+- Added method for handling failed jobs
+- Added Benchmarks
+- Removed logging
+- Moved the Job class into it's own file
+
+0.1.6

data/lib/queue_classic.rb

@@ -1,40 +1,42 @@
+require_relative "queue_classic/config"
+
 module QC
-  # You can use the APP_NAME to query for
-  # postgres related process information in the
-  # pg_stat_activity table.
-  APP_NAME = ENV["QC_APP_NAME"] || "queue_classic"
-
-  # Number of seconds to block on the listen chanel for new jobs.
-  WAIT_TIME = (ENV["QC_LISTEN_TIME"] || 5).to_i
-
-  # Why do you want to change the table name?
-  # Just deal with the default OK?
-  # If you do want to change this, you will
-  # need to update the PL/pgSQL lock_head() function.
-  # Come on. Don't do it.... Just stick with the default.
-  TABLE_NAME = "queue_classic_jobs"
-
-  # Each row in the table will have a column that
-  # notes the queue. You can point your workers
-  # at different queues but only one at a time.
-  QUEUE = ENV["QUEUE"] || "default"
-  QUEUES = (ENV["QUEUES"] && ENV["QUEUES"].split(",")) || []
-
-  # Set this to 1 for strict FIFO.
-  # There is nothing special about 9....
-  TOP_BOUND = (ENV["QC_TOP_BOUND"] || 9).to_i
-
-  # Set this variable if you wish for
-  # the worker to fork a UNIX process for
-  # each locked job. Remember to re-establish
-  # any database connections. See the worker
-  # for more details.
-  FORK_WORKER = !ENV["QC_FORK_WORKER"].nil?
+  extend QC::Config
+
+  # Assign constants for backwards compatibility.
+  # They should no longer be used. Prefer the corresponding methods.
+  # See +QC::Config+ for more details.
+  DEPRECATED_CONSTANTS = {
+    :APP_NAME => :app_name,
+    :WAIT_TIME => :wait_time,
+    :TABLE_NAME => :table_name,
+    :QUEUE => :queue,
+    :QUEUES => :queues,
+    :TOP_BOUND => :top_bound,
+    :FORK_WORKER => :fork_worker?,
+  }
+
+  def self.const_missing(const_name)
+    if DEPRECATED_CONSTANTS.key? const_name
+      config_method = DEPRECATED_CONSTANTS[const_name]
+      $stderr.puts <<-MSG
+The constant QC::#{const_name} is deprecated and will be removed in the future.
+Please use the method QC.#{config_method} instead.
+      MSG
+      QC.public_send config_method
+    else
+      super
+    end
+  end
 
   # Defer method calls on the QC module to the
   # default queue. This facilitates QC.enqueue()
   def self.method_missing(sym, *args, &block)
-    default_queue.send(sym, *args, &block)
+    if default_queue.respond_to? sym
+      default_queue.public_send(sym, *args, &block)
+    else
+      super
+    end
   end
 
   # Ensure QC.respond_to?(:enqueue) equals true (ruby 1.9 only)
@@ -42,37 +44,29 @@ module QC
     default_queue.respond_to?(method_name)
   end
 
-  def self.default_queue=(queue)
-    @default_queue = queue
-  end
-
-  def self.default_queue
-    @default_queue ||= begin
-      Queue.new(QUEUE)
-    end
-  end
-
   def self.has_connection?
     !default_conn_adapter.nil?
   end
 
   def self.default_conn_adapter
-    return @conn_adapter if defined?(@conn_adapter) && @conn_adapter
-    if rails_connection_sharing_enabled?
-      @conn_adapter = ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
+    t = Thread.current
+    return t[:qc_conn_adapter] if t[:qc_conn_adapter]
+    adapter = if rails_connection_sharing_enabled?
+      ConnAdapter.new(ActiveRecord::Base.connection.raw_connection)
     else
-      @conn_adapter = ConnAdapter.new
+      ConnAdapter.new
     end
-    @conn_adapter
+
+    t[:qc_conn_adapter] = adapter
   end
 
   def self.default_conn_adapter=(conn)
-    @conn_adapter = conn
+    Thread.current[:qc_conn_adapter] = conn
   end
 
   def self.log_yield(data)
+    t0 = Time.now
     begin
-      t0 = Time.now
       yield
     rescue => e
       log({:at => "error", :error => e.inspect}.merge(data))
@@ -106,7 +100,8 @@ module QC
   # This will unlock all jobs any postgres' PID that is not existing anymore
   # to prevent any infinitely locked jobs
   def self.unlock_jobs_of_dead_workers
-    default_conn_adapter.execute("UPDATE #{QC::TABLE_NAME} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT pid FROM pg_stat_activity);")
+    pid_column = default_conn_adapter.server_version < 90200 ? "procpid" : "pid"
+    default_conn_adapter.execute("UPDATE #{QC.table_name} SET locked_at = NULL, locked_by = NULL WHERE locked_by NOT IN (SELECT #{pid_column} FROM pg_stat_activity);")
   end
 
   # private class methods