que 0.11.3 → 2.2.0

data/lib/que.rb

@@ -1,180 +1,124 @@
 # frozen_string_literal: true
 
-require 'socket' # For hostname
+require 'forwardable'
+require 'socket' # For Socket.gethostname
 
 module Que
-  autoload :Adapters,   'que/adapters/base'
-  autoload :Job,        'que/job'
-  autoload :Migrations, 'que/migrations'
-  autoload :SQL,        'que/sql'
-  autoload :Version,    'que/version'
-  autoload :Worker,     'que/worker'
-
-  begin
-    require 'multi_json'
-    JSON_MODULE = MultiJson
-  rescue LoadError
-    require 'json'
-    JSON_MODULE = JSON
-  end
+  CURRENT_HOSTNAME = Socket.gethostname.freeze
+  DEFAULT_QUEUE    = 'default'.freeze
+  TIME_REGEX       = /\A\d{4}\-\d{2}\-\d{2}T\d{2}:\d{2}:\d{2}.\d{6}Z\z/
+  CONFIG_MUTEX     = Mutex.new
+  MAXIMUM_PRIORITY = 32767
+
+  class Error < StandardError; end
+
+  # Store SQL strings frozen, with squashed whitespace so logs read better.
+  SQL = {}
+  def SQL.[]=(k,v); super(k, v.strip.gsub(/\s+/, ' ').freeze); end
+
+  # Load up modules that allow registration before modules that use it.
+  require_relative 'que/listener'
+
+  # Load utilities before main logic that will use them.
+  require_relative 'que/utils/assertions'
+  require_relative 'que/utils/constantization'
+  require_relative 'que/utils/error_notification'
+  require_relative 'que/utils/freeze'
+  require_relative 'que/utils/introspection'
+  require_relative 'que/utils/json_serialization'
+  require_relative 'que/utils/logging'
+  require_relative 'que/utils/middleware'
+  require_relative 'que/utils/queue_management'
+  require_relative 'que/utils/ruby2_keywords'
+  require_relative 'que/utils/transactions'
+
+  require_relative 'que/version'
+
+  require_relative 'que/connection'
+  require_relative 'que/connection_pool'
+  require_relative 'que/job_methods'
+  require_relative 'que/job'
+  require_relative 'que/job_buffer'
+  require_relative 'que/locker'
+  require_relative 'que/metajob'
+  require_relative 'que/migrations'
+  require_relative 'que/poller'
+  require_relative 'que/result_queue'
+  require_relative 'que/worker'
 
-  HASH_DEFAULT_PROC = proc { |hash, key| hash[key.to_s] if Symbol === key }
-
-  INDIFFERENTIATOR = proc do |object|
-    case object
-    when Array
-      object.each(&INDIFFERENTIATOR)
-    when Hash
-      object.default_proc = HASH_DEFAULT_PROC
-      object.each { |key, value| object[key] = INDIFFERENTIATOR.call(value) }
-      object
-    else
-      object
-    end
+  class << self
+    attr_writer :default_queue
   end
 
-  SYMBOLIZER = proc do |object|
-    case object
-    when Hash
-      object.keys.each do |key|
-        object[key.to_sym] = SYMBOLIZER.call(object.delete(key))
-      end
-      object
-    when Array
-      object.map! { |e| SYMBOLIZER.call(e) }
-    else
-      object
-    end
-  end
+  self.default_queue = nil
 
   class << self
-    attr_accessor :error_handler
-    attr_writer :logger, :adapter, :log_formatter, :disable_prepared_statements, :json_converter
-
-    def connection=(connection)
-      self.adapter =
-        if connection.to_s == 'ActiveRecord'
-          Adapters::ActiveRecord.new
+    include Utils::Assertions
+    include Utils::Constantization
+    include Utils::ErrorNotification
+    include Utils::Freeze
+    include Utils::Introspection
+    include Utils::JSONSerialization
+    include Utils::Logging
+    include Utils::Middleware
+    include Utils::QueueManagement
+    include Utils::Ruby2Keywords
+    include Utils::Transactions
+
+    extend Forwardable
+
+    # Copy some commonly-used methods here, for convenience.
+    def_delegators :pool, :execute, :checkout, :in_transaction?
+    def_delegators Job, :enqueue, :bulk_enqueue, :run_synchronously, :run_synchronously=
+    def_delegators Migrations, :db_version, :migrate!
+
+    # Global configuration logic.
+    attr_accessor :use_prepared_statements
+
+    def default_queue
+      @default_queue || DEFAULT_QUEUE
+    end
+
+    # Support simple integration with many common connection pools.
+    def connection=(conn)
+      self.connection_proc =
+        if conn.to_s == 'ActiveRecord'
+          # Load and setup AR compatibility.
+          require_relative 'que/active_record/connection'
+          m = Que::ActiveRecord::Connection::JobMiddleware
+          job_middleware << m unless job_middleware.include?(m)
+          Que::ActiveRecord::Connection.method(:checkout)
         else
-          case connection.class.to_s
-          when 'Sequel::Postgres::Database' then Adapters::Sequel.new(connection)
-          when 'ConnectionPool'             then Adapters::ConnectionPool.new(connection)
-          when 'PG::Connection'             then Adapters::PG.new(connection)
-          when 'Pond'                       then Adapters::Pond.new(connection)
-          when 'NilClass'                   then connection
-          else raise "Que connection not recognized: #{connection.inspect}"
+          case conn.class.to_s
+          when 'Sequel::Postgres::Database' then conn.method(:synchronize)
+          when 'Pond'                       then conn.method(:checkout)
+          when 'ConnectionPool'             then conn.method(:with)
+          when 'NilClass'                   then conn
+          else raise Error, "Unsupported connection: #{conn.class}"
           end
         end
     end
 
-    def adapter
-      @adapter || raise("Que connection not established!")
-    end
-
-    def execute(*args)
-      adapter.execute(*args)
-    end
-
-    def clear!
-      execute "DELETE FROM que_jobs"
-    end
-
-    def job_stats
-      execute :job_stats
-    end
-
-    def worker_states
-      execute :worker_states
-    end
-
-    # Give us a cleaner interface when specifying a job_class as a string.
-    def enqueue(*args)
-      Job.enqueue(*args)
-    end
-
-    def db_version
-      Migrations.db_version
-    end
-
-    def migrate!(version = {:version => Migrations::CURRENT_VERSION})
-      Migrations.migrate!(version)
-    end
-
-    # Have to support create! and drop! in old migrations. They just created
-    # and dropped the bare table.
-    def create!
-      migrate! :version => 1
-    end
-
-    def drop!
-      migrate! :version => 0
-    end
-
-    def log(data)
-      level = data.delete(:level) || :info
-      data = {:lib => 'que', :hostname => Socket.gethostname, :pid => Process.pid, :thread => Thread.current.object_id}.merge(data)
-
-      if (l = logger) && output = log_formatter.call(data)
-        l.send level, output
-      end
+    # Integrate Que with any connection pool by passing it a reentrant block
+    # that locks and yields a Postgres connection.
+    def connection_proc=(connection_proc)
+      @pool = connection_proc && ConnectionPool.new(&connection_proc)
     end
 
-    def logger
-      @logger.respond_to?(:call) ? @logger.call : @logger
+    # How to actually access Que's established connection pool.
+    def pool
+      @pool || raise(Error, "Que connection not established!")
     end
 
-    def log_formatter
-      @log_formatter ||= JSON_MODULE.method(:dump)
-    end
-
-    def disable_prepared_statements
-      @disable_prepared_statements || false
-    end
-
-    def constantize(camel_cased_word)
-      if camel_cased_word.respond_to?(:constantize)
-        # Use ActiveSupport's version if it exists.
-        camel_cased_word.constantize
-      else
-        camel_cased_word.split('::').inject(Object, &:const_get)
-      end
-    end
-
-    # A helper method to manage transactions, used mainly by the migration
-    # system. It's available for general use, but if you're using an ORM that
-    # provides its own transaction helper, be sure to use that instead, or the
-    # two may interfere with one another.
-    def transaction
-      adapter.checkout do
-        if adapter.in_transaction?
-          yield
-        else
-          begin
-            execute "BEGIN"
-            yield
-          rescue => error
-            raise
-          ensure
-            # Handle a raised error or a killed thread.
-            if error || Thread.current.status == 'aborting'
-              execute "ROLLBACK"
-            else
-              execute "COMMIT"
-            end
-          end
-        end
-      end
-    end
-
-    def json_converter
-      @json_converter ||= INDIFFERENTIATOR
-    end
-
-    # Copy some of the Worker class' config methods here for convenience.
-    [:mode, :mode=, :worker_count, :worker_count=, :wake_interval, :wake_interval=, :queue_name, :queue_name=, :wake!, :wake_all!].each do |meth|
-      define_method(meth) { |*args| Worker.send(meth, *args) }
-    end
+    # Set the current pool. Helpful for specs, but probably shouldn't be used
+    # generally.
+    attr_writer :pool
   end
+
+  # Set config defaults.
+  self.use_prepared_statements = true
 end
 
-require 'que/railtie' if defined? Rails::Railtie
+# Load Rails features as appropriate.
+require_relative 'que/rails/railtie'         if defined?(::Rails::Railtie)
+require_relative 'que/active_job/extensions' if defined?(::ActiveJob)

data/que.gemspec

@@ -5,18 +5,31 @@ require 'que/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'que'
-  spec.version       = Que::Version
+  spec.version       = Que::VERSION
   spec.authors       = ["Chris Hanks"]
   spec.email         = ['christopher.m.hanks@gmail.com']
   spec.description   = %q{A job queue that uses PostgreSQL's advisory locks for speed and reliability.}
   spec.summary       = %q{A PostgreSQL-based Job Queue}
-  spec.homepage      = 'https://github.com/chanks/que'
+  spec.homepage      = 'https://github.com/que-rb/que'
   spec.license       = 'MIT'
 
-  spec.files         = `git ls-files`.split($/)
+  spec.required_ruby_version = '>= 2.7.0'
+
+  files_to_exclude = [
+    /\A\.circleci/,
+    /\AGemfile/,
+    /\Aspec/,
+    /\Atasks/,
+    /spec\.rb\z/,
+  ]
+
+  spec.files = `git ls-files`.split($/).reject do |file|
+    files_to_exclude.any? { |r| r === file }
+  end
+
   spec.executables   = ['que']
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']
 
-  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'bundler'
 end

data/scripts/docker-entrypoint

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+# For using your own dotfiles within the Docker container
+if [ -f /.docker-rc.d/.docker-bashrc ]; then
+  echo "source /.docker-rc.d/.docker-bashrc" >> ~/.bashrc
+fi
+
+gem list -i -e bundler -v "$RUBY_BUNDLER_VERSION" >/dev/null || gem install bundler -v "$RUBY_BUNDLER_VERSION"
+
+bundle check --dry-run || bundle install
+
+exec "${@-bash}"

data/scripts/test

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+set -Eeuo pipefail
+
+bundle exec rake spec "$@"
+USE_RAILS=true bundle exec rake spec "$@"

metadata

@@ -1,29 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: que
 version: !ruby/object:Gem::Version
-  version: 0.11.3
+  version: 2.2.0
 platform: ruby
 authors:
 - Chris Hanks
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2016-02-26 00:00:00.000000000 Z
+date: 2022-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.3'
+        version: '0'
 description: A job queue that uses PostgreSQL's advisory locks for speed and reliability.
 email:
 - christopher.m.hanks@gmail.com
@@ -32,37 +32,35 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- ".github/workflows/tests.yml"
 - ".gitignore"
-- ".rspec"
-- ".travis.yml"
+- ".ruby-version"
 - CHANGELOG.md
-- Gemfile
+- Dockerfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- auto/dev
+- auto/pre-push-hook
+- auto/psql
+- auto/test
+- auto/test-postgres-14
 - bin/que
-- docs/advanced_setup.md
-- docs/customizing_que.md
-- docs/error_handling.md
-- docs/inspecting_the_queue.md
-- docs/logging.md
-- docs/managing_workers.md
-- docs/migrating.md
-- docs/multiple_queues.md
-- docs/shutting_down_safely.md
-- docs/using_plain_connections.md
-- docs/using_sequel.md
-- docs/writing_reliable_jobs.md
-- lib/generators/que/install_generator.rb
-- lib/generators/que/templates/add_que.rb
+- docker-compose.yml
+- docs/README.md
 - lib/que.rb
-- lib/que/adapters/active_record.rb
-- lib/que/adapters/base.rb
-- lib/que/adapters/connection_pool.rb
-- lib/que/adapters/pg.rb
-- lib/que/adapters/pond.rb
-- lib/que/adapters/sequel.rb
+- lib/que/active_job/extensions.rb
+- lib/que/active_record/connection.rb
+- lib/que/active_record/model.rb
+- lib/que/command_line_interface.rb
+- lib/que/connection.rb
+- lib/que/connection_pool.rb
 - lib/que/job.rb
+- lib/que/job_buffer.rb
+- lib/que/job_methods.rb
+- lib/que/listener.rb
+- lib/que/locker.rb
+- lib/que/metajob.rb
 - lib/que/migrations.rb
 - lib/que/migrations/1/down.sql
 - lib/que/migrations/1/up.sql
@@ -70,46 +68,39 @@ files:
 - lib/que/migrations/2/up.sql
 - lib/que/migrations/3/down.sql
 - lib/que/migrations/3/up.sql
-- lib/que/railtie.rb
-- lib/que/rake_tasks.rb
-- lib/que/sql.rb
+- lib/que/migrations/4/down.sql
+- lib/que/migrations/4/up.sql
+- lib/que/migrations/5/down.sql
+- lib/que/migrations/5/up.sql
+- lib/que/migrations/6/down.sql
+- lib/que/migrations/6/up.sql
+- lib/que/migrations/7/down.sql
+- lib/que/migrations/7/up.sql
+- lib/que/poller.rb
+- lib/que/rails/railtie.rb
+- lib/que/result_queue.rb
+- lib/que/sequel/model.rb
+- lib/que/utils/assertions.rb
+- lib/que/utils/constantization.rb
+- lib/que/utils/error_notification.rb
+- lib/que/utils/freeze.rb
+- lib/que/utils/introspection.rb
+- lib/que/utils/json_serialization.rb
+- lib/que/utils/logging.rb
+- lib/que/utils/middleware.rb
+- lib/que/utils/queue_management.rb
+- lib/que/utils/ruby2_keywords.rb
+- lib/que/utils/transactions.rb
 - lib/que/version.rb
 - lib/que/worker.rb
 - que.gemspec
-- spec/adapters/active_record_spec.rb
-- spec/adapters/connection_pool_spec.rb
-- spec/adapters/pg_spec.rb
-- spec/adapters/pond_spec.rb
-- spec/adapters/sequel_spec.rb
-- spec/gemfiles/Gemfile1
-- spec/gemfiles/Gemfile2
-- spec/spec_helper.rb
-- spec/support/helpers.rb
-- spec/support/jobs.rb
-- spec/support/shared_examples/adapter.rb
-- spec/support/shared_examples/multi_threaded_adapter.rb
-- spec/travis.rb
-- spec/unit/connection_spec.rb
-- spec/unit/customization_spec.rb
-- spec/unit/enqueue_spec.rb
-- spec/unit/helper_spec.rb
-- spec/unit/logging_spec.rb
-- spec/unit/migrations_spec.rb
-- spec/unit/pool_spec.rb
-- spec/unit/run_spec.rb
-- spec/unit/states_spec.rb
-- spec/unit/stats_spec.rb
-- spec/unit/transaction_spec.rb
-- spec/unit/work_spec.rb
-- spec/unit/worker_spec.rb
-- tasks/benchmark.rb
-- tasks/rspec.rb
-- tasks/safe_shutdown.rb
-homepage: https://github.com/chanks/que
+- scripts/docker-entrypoint
+- scripts/test
+homepage: https://github.com/que-rb/que
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -117,42 +108,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.7.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.5.1
-signing_key: 
+rubygems_version: 3.3.7
+signing_key:
 specification_version: 4
 summary: A PostgreSQL-based Job Queue
-test_files:
-- spec/adapters/active_record_spec.rb
-- spec/adapters/connection_pool_spec.rb
-- spec/adapters/pg_spec.rb
-- spec/adapters/pond_spec.rb
-- spec/adapters/sequel_spec.rb
-- spec/gemfiles/Gemfile1
-- spec/gemfiles/Gemfile2
-- spec/spec_helper.rb
-- spec/support/helpers.rb
-- spec/support/jobs.rb
-- spec/support/shared_examples/adapter.rb
-- spec/support/shared_examples/multi_threaded_adapter.rb
-- spec/travis.rb
-- spec/unit/connection_spec.rb
-- spec/unit/customization_spec.rb
-- spec/unit/enqueue_spec.rb
-- spec/unit/helper_spec.rb
-- spec/unit/logging_spec.rb
-- spec/unit/migrations_spec.rb
-- spec/unit/pool_spec.rb
-- spec/unit/run_spec.rb
-- spec/unit/states_spec.rb
-- spec/unit/stats_spec.rb
-- spec/unit/transaction_spec.rb
-- spec/unit/work_spec.rb
-- spec/unit/worker_spec.rb
+test_files: []

data/.rspec

@@ -1,2 +0,0 @@
---color
---order random

data/.travis.yml

@@ -1,17 +0,0 @@
-language: ruby
-rvm:
-  - "1.9.3"
-  - "2.0"
-  - "2.1"
-  - "2.2"
-  - "2.3"
-  - "rbx-2"
-  - "jruby-1.7.5"
-before_script:
-  - psql -c 'create database "que-test"' -U postgres
-  - bundle exec ruby -r sequel -r ./lib/que -e 'Que.connection=Sequel.connect("postgres://localhost/que-test"); Que.migrate!'
-
-script: "./spec/travis.rb"
-
-addons:
-  postgresql: 9.3

data/Gemfile

@@ -1,24 +0,0 @@
-source 'https://rubygems.org'
-
-group :development, :test do
-  gem 'rake'
-
-  gem 'activerecord',    :require => nil
-  gem 'sequel',          :require => nil
-  gem 'connection_pool', :require => nil
-  gem 'pond',            :require => nil
-  gem 'pg',              :require => nil, :platform => :ruby
-  gem 'pg_jruby',        :require => nil, :platform => :jruby
-end
-
-group :test do
-  gem 'rspec', '~> 2.14.1'
-  gem 'pry'
-end
-
-platforms :rbx do
-  gem 'rubysl', '~> 2.0'
-  gem 'json', '~> 1.8'
-end
-
-gemspec