que 0.14.3 → 1.0.0.beta

data/lib/que/utils/queue_management.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# Tools for managing the contents/state of the queue.
+
+module Que
+  module Utils
+    module QueueManagement
+      def clear!
+        execute "DELETE FROM que_jobs"
+      end
+
+      # Very old migrations may use Que.create! and Que.drop!, which just
+      # created and dropped the initial version of the jobs table.
+      def create!; migrate!(version: 1); end
+      def drop!;   migrate!(version: 0); end
+    end
+  end
+end

data/lib/que/utils/transactions.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+# A helper method to manage transactions, used mainly by the migration system.
+# It's available for general use, but if you're using an ORM that provides its
+# own transaction helper, be sure to use that instead, or the two may interfere
+# with one another.
+
+module Que
+  module Utils
+    module Transactions
+      def transaction
+        pool.checkout do
+          if pool.in_transaction?
+            yield
+          else
+            begin
+              execute "BEGIN"
+              yield
+            rescue => error
+              raise
+            ensure
+              # Handle a raised error or a killed thread.
+              if error || Thread.current.status == 'aborting'
+                execute "ROLLBACK"
+              else
+                execute "COMMIT"
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/que/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Que
-  Version = '0.14.3'
+  VERSION = '1.0.0.beta'
 end

data/lib/que/worker.rb

@@ -1,196 +1,157 @@
 # frozen_string_literal: true
 
-require 'monitor'
+# Workers wrap threads which continuously pull job pks from JobCache objects,
+# fetch and work those jobs, and export relevant data to ResultQueues.
 
 module Que
   class Worker
-    # Each worker has a thread that does the actual work of running jobs.
-    # Since both the worker's thread and whatever thread is managing the
-    # worker are capable of affecting the worker's state, we need to
-    # synchronize access to it.
-    include MonitorMixin
-
-    attr_reader :thread, :state, :queue
-
-    def initialize(queue = '')
-      super() # For MonitorMixin.
-      @queue  = queue
-      @state  = :working
-      @thread = Thread.new { work_loop }
-      @thread.abort_on_exception = true
-    end
-
-    def alive?
-      !!@thread.status
-    end
-
-    def sleeping?
-      synchronize { _sleeping? }
-    end
-
-    def working?
-      synchronize { @state == :working }
-    end
-
-    def wake!
-      synchronize do
-        if sleeping?
-          # Have to set the state here so that another thread checking
-          # immediately after this won't see the worker as asleep.
-          @state = :working
-          @thread.wakeup
-          true
-        end
+    attr_reader :thread, :priority
+
+    SQL[:check_job] =
+      %{
+        SELECT 1 AS one
+        FROM public.que_jobs
+        WHERE id = $1::bigint
+      }
+
+    def initialize(
+      job_cache:,
+      result_queue:,
+      priority: nil,
+      start_callback: nil
+    )
+
+      @priority     = Que.assert([NilClass, Integer], priority)
+      @job_cache    = Que.assert(JobCache, job_cache)
+      @result_queue = Que.assert(ResultQueue, result_queue)
+
+      Que.internal_log(:worker_instantiate, self) do
+        {
+          priority:     priority,
+          job_cache:    job_cache.object_id,
+          result_queue: result_queue.object_id,
+        }
       end
-    end
 
-    # This needs to be called when trapping a signal, so it can't lock the monitor.
-    def stop
-      @stop = true
-      @thread.wakeup if _sleeping?
+      @thread =
+        Thread.new do
+          # An error causing this thread to exit is a bug in Que, which we want
+          # to know about ASAP, so propagate the error if it happens.
+          Thread.current.abort_on_exception = true
+          start_callback.call(self) if start_callback.respond_to?(:call)
+          work_loop
+        end
     end
 
     def wait_until_stopped
-      wait while alive?
+      @thread.join
     end
 
     private
 
-    # Sleep very briefly while waiting for a thread to get somewhere.
-    def wait
-      sleep 0.0001
-    end
-
-    def _sleeping?
-      if @state == :sleeping
-        # There's a very small period of time between when the Worker marks
-        # itself as sleeping and when it actually goes to sleep. Only report
-        # true when we're certain the thread is sleeping.
-        wait until @thread.status == 'sleep'
-        true
-      end
-    end
-
     def work_loop
-      loop do
-        cycle = nil
-
-        if Que.mode == :async
-          time   = Time.now
-          result = Job.work(queue)
-
-          case result[:event]
-          when :job_unavailable
-            cycle = false
-            result[:level] = :debug
-          when :job_race_condition
-            cycle = true
-            result[:level] = :debug
-          when :job_worked
-            cycle = true
-            result[:elapsed] = (Time.now - time).round(5)
-          when :job_errored
-            # For PG::Errors, assume we had a problem reaching the database, and
-            # don't hit it again right away.
-            cycle = !result[:error].is_a?(PG::Error)
-            result[:error] = {:class => result[:error].class.to_s, :message => result[:error].message}
-          else
-            raise "Unknown Event: #{result[:event].inspect}"
-          end
-
-          Que.log(result)
+      # Blocks until a job of the appropriate priority is available. If the
+      # queue is shutting down this will return nil, which breaks the loop and
+      # lets the thread finish.
+      while metajob = fetch_next_metajob
+        id = metajob.id
+
+        Que.internal_log(:worker_received_job, self) { {id: id} }
+
+        if Que.execute(:check_job, [id]).first
+          Que.recursively_freeze(metajob.job)
+          Que.internal_log(:worker_fetched_job, self) { {id: id} }
+
+          work_job(metajob)
+        else
+          # The job was locked but doesn't exist anymore, due to a race
+          # condition that exists because advisory locks don't obey MVCC. Not
+          # necessarily a problem, but if it happens a lot it may be meaningful.
+          Que.internal_log(:worker_job_lock_race_condition, self) { {id: id} }
         end
 
-        synchronize { @state = :sleeping unless cycle || @stop }
-        sleep if @state == :sleeping
-        break if @stop
-      end
-    ensure
-      @state = :stopped
-    end
-
-    # Setting Que.wake_interval = nil should ensure that the wrangler thread
-    # doesn't wake up a worker again, even if it's currently sleeping for a
-    # set period. So, we double-check that @wake_interval is set before waking
-    # a worker, and make sure to wake up the wrangler when @wake_interval is
-    # changed in Que.wake_interval= below.
-    @wake_interval = 5
+        Que.internal_log(:worker_pushing_finished_job, self) { {id: id} }
 
-    # Four workers is a sensible default for most use cases.
-    @worker_count = 4
-
-    class << self
-      attr_reader :mode, :wake_interval, :worker_count
-      attr_accessor :queue_name
-
-      # In order to work in a forking webserver, we need to be able to accept
-      # worker_count and wake_interval settings without actually instantiating
-      # the relevant threads until the mode is actually set to :async in a
-      # post-fork hook (since forking will kill any running background threads).
-
-      def mode=(mode)
-        Que.log :event => 'mode_change', :value => mode.to_s
-        @mode = mode
-
-        if mode == :async
-          set_up_workers
-          wrangler
-        end
-      end
-
-      def worker_count=(count)
-        Que.log :event => 'worker_count_change', :value => count.to_s
-        @worker_count = count
-        set_up_workers if mode == :async
-      end
-
-      def workers
-        @workers ||= []
-      end
-
-      def wake_interval=(interval)
-        @wake_interval = interval
-        begin
-          wrangler.wakeup if mode == :async
-        rescue ThreadError # killed thread for some reason.
-          v = wrangler.value # Reraise the error that killed the thread.
-          # if that didn't raise an error, something else is wrong, so raise
-          # whatever this is:
-          raise "Dead thread!: #{v.inspect}"
-        end
+        @result_queue.push(
+          metajob: metajob,
+          message_type: :job_finished,
+        )
       end
+    end
 
-      def wake!
-        workers.find(&:wake!)
-      end
+    def fetch_next_metajob
+      @job_cache.shift(*priority)
+    end
 
-      def wake_all!
-        workers.each(&:wake!)
+    def work_job(metajob)
+      job      = metajob.job
+      start    = Time.now
+      klass    = Que.constantize(job.fetch(:job_class))
+      instance = klass.new(job)
+
+      Que.run_middleware(instance) { instance.tap(&:_run) }
+
+      log_message = {
+        level: :debug,
+        job_id: metajob.id,
+        elapsed: (Time.now - start),
+      }
+
+      if error = instance.que_error
+        log_message[:event] = :job_errored
+        log_message[:error] = "#{error.class}: #{error.message}".slice(0, 500)
+      else
+        log_message[:event] = :job_worked
       end
 
-      private
-
-      def set_up_workers
-        if worker_count > workers.count
-          workers.push(*(worker_count - workers.count).times.map{new(queue_name || '')})
-        elsif worker_count < workers.count
-          workers.pop(workers.count - worker_count).each(&:stop).each(&:wait_until_stopped)
+      Que.log(log_message)
+
+      instance
+    rescue => error
+      Que.log(
+        level: :debug,
+        event: :job_errored,
+        job_id: metajob.id,
+        error: {
+          class:   error.class.to_s,
+          message: error.message,
+        },
+      )
+
+      Que.notify_error(error)
+
+      begin
+        # If the Job class couldn't be resolved, use the default retry
+        # backoff logic in Que::Job.
+        job_class = (klass && klass <= Job) ? klass : Job
+
+        error_count = job.fetch(:error_count) + 1
+
+        max_retry_count = job_class.resolve_que_setting(:maximum_retry_count)
+
+        if max_retry_count && error_count > max_retry_count
+          Que.execute :expire_job, [job.fetch(:id)]
+        else
+          delay =
+            job_class.
+            resolve_que_setting(
+              :retry_interval,
+              error_count,
+            )
+
+          Que.execute :set_error, [
+            delay,
+            "#{error.class}: #{error.message}".slice(0, 500),
+            error.backtrace.join("\n").slice(0, 10000),
+            job.fetch(:id),
+          ]
         end
+      rescue
+        # If we can't reach the database for some reason, too bad, but
+        # don't let it crash the work loop.
       end
 
-      def wrangler
-        @wrangler ||= Thread.new do
-          loop do
-            if @wake_interval
-              sleep(@wake_interval)
-            else
-              sleep
-            end
-
-            wake! if @wake_interval && mode == :async
-          end
-        end
-      end
+      error
     end
   end
 end

data/que.gemspec

@@ -5,7 +5,7 @@ require 'que/version'
 
 Gem::Specification.new do |spec|
   spec.name          = 'que'
-  spec.version       = Que::Version
+  spec.version       = Que::VERSION
   spec.authors       = ["Chris Hanks"]
   spec.email         = ['christopher.m.hanks@gmail.com']
   spec.description   = %q{A job queue that uses PostgreSQL's advisory locks for speed and reliability.}
@@ -13,7 +13,18 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/chanks/que'
   spec.license       = 'MIT'
 
-  spec.files         = `git ls-files`.split($/)
+  files_to_exclude = [
+    /\A\.circleci/,
+    /\AGemfile/,
+    /\Aspec/,
+    /\Atasks/,
+    /spec\.rb\z/,
+  ]
+
+  spec.files = `git ls-files`.split($/).reject do |file|
+    files_to_exclude.any? { |r| r === file }
+  end
+
   spec.executables   = ['que']
   spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
   spec.require_paths = ['lib']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: que
 version: !ruby/object:Gem::Version
-  version: 0.14.3
+  version: 1.0.0.beta
 platform: ruby
 authors:
 - Chris Hanks
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-03-02 00:00:00.000000000 Z
+date: 2017-10-25 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -33,37 +33,40 @@ extensions: []
 extra_rdoc_files: []
 files:
 - ".gitignore"
-- ".rspec"
-- ".travis.yml"
 - CHANGELOG.md
-- Gemfile
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/command_line_interface.rb
 - bin/que
 - docs/README.md
+- docs/active_job.md
 - docs/advanced_setup.md
-- docs/customizing_que.md
+- docs/command_line_interface.md
 - docs/error_handling.md
 - docs/inspecting_the_queue.md
+- docs/job_helper_methods.md
 - docs/logging.md
 - docs/managing_workers.md
+- docs/middleware.md
 - docs/migrating.md
 - docs/multiple_queues.md
 - docs/shutting_down_safely.md
 - docs/using_plain_connections.md
 - docs/using_sequel.md
 - docs/writing_reliable_jobs.md
-- lib/generators/que/install_generator.rb
-- lib/generators/que/templates/add_que.rb
 - lib/que.rb
-- lib/que/adapters/active_record.rb
-- lib/que/adapters/base.rb
-- lib/que/adapters/connection_pool.rb
-- lib/que/adapters/pg.rb
-- lib/que/adapters/pond.rb
-- lib/que/adapters/sequel.rb
+- lib/que/active_job/extensions.rb
+- lib/que/active_record/connection.rb
+- lib/que/active_record/model.rb
+- lib/que/connection.rb
+- lib/que/connection_pool.rb
 - lib/que/job.rb
+- lib/que/job_cache.rb
+- lib/que/job_methods.rb
+- lib/que/listener.rb
+- lib/que/locker.rb
+- lib/que/metajob.rb
 - lib/que/migrations.rb
 - lib/que/migrations/1/down.sql
 - lib/que/migrations/1/up.sql
@@ -71,43 +74,25 @@ files:
 - lib/que/migrations/2/up.sql
 - lib/que/migrations/3/down.sql
 - lib/que/migrations/3/up.sql
-- lib/que/railtie.rb
-- lib/que/rake_tasks.rb
-- lib/que/sql.rb
+- lib/que/migrations/4/down.sql
+- lib/que/migrations/4/up.sql
+- lib/que/poller.rb
+- lib/que/rails/railtie.rb
+- lib/que/result_queue.rb
+- lib/que/sequel/model.rb
+- lib/que/utils/assertions.rb
+- lib/que/utils/constantization.rb
+- lib/que/utils/error_notification.rb
+- lib/que/utils/freeze.rb
+- lib/que/utils/introspection.rb
+- lib/que/utils/json_serialization.rb
+- lib/que/utils/logging.rb
+- lib/que/utils/middleware.rb
+- lib/que/utils/queue_management.rb
+- lib/que/utils/transactions.rb
 - lib/que/version.rb
 - lib/que/worker.rb
 - que.gemspec
-- spec/adapters/active_record_spec.rb
-- spec/adapters/connection_pool_spec.rb
-- spec/adapters/pg_spec.rb
-- spec/adapters/pond_spec.rb
-- spec/adapters/sequel_spec.rb
-- spec/gemfiles/Gemfile.current
-- spec/gemfiles/Gemfile.old
-- spec/gemfiles/Gemfile.older
-- spec/gemfiles/Gemfile.oldest
-- spec/spec_helper.rb
-- spec/support/helpers.rb
-- spec/support/jobs.rb
-- spec/support/shared_examples/adapter.rb
-- spec/support/shared_examples/multi_threaded_adapter.rb
-- spec/unit/configuration_spec.rb
-- spec/unit/connection_spec.rb
-- spec/unit/customization_spec.rb
-- spec/unit/enqueue_spec.rb
-- spec/unit/helper_spec.rb
-- spec/unit/logging_spec.rb
-- spec/unit/migrations_spec.rb
-- spec/unit/pool_spec.rb
-- spec/unit/run_spec.rb
-- spec/unit/states_spec.rb
-- spec/unit/stats_spec.rb
-- spec/unit/transaction_spec.rb
-- spec/unit/work_spec.rb
-- spec/unit/worker_spec.rb
-- tasks/benchmark.rb
-- tasks/rspec.rb
-- tasks/safe_shutdown.rb
 homepage: https://github.com/chanks/que
 licenses:
 - MIT
@@ -123,41 +108,13 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.7.3
+rubygems_version: 2.6.14
 signing_key: 
 specification_version: 4
 summary: A PostgreSQL-based Job Queue
-test_files:
-- spec/adapters/active_record_spec.rb
-- spec/adapters/connection_pool_spec.rb
-- spec/adapters/pg_spec.rb
-- spec/adapters/pond_spec.rb
-- spec/adapters/sequel_spec.rb
-- spec/gemfiles/Gemfile.current
-- spec/gemfiles/Gemfile.old
-- spec/gemfiles/Gemfile.older
-- spec/gemfiles/Gemfile.oldest
-- spec/spec_helper.rb
-- spec/support/helpers.rb
-- spec/support/jobs.rb
-- spec/support/shared_examples/adapter.rb
-- spec/support/shared_examples/multi_threaded_adapter.rb
-- spec/unit/configuration_spec.rb
-- spec/unit/connection_spec.rb
-- spec/unit/customization_spec.rb
-- spec/unit/enqueue_spec.rb
-- spec/unit/helper_spec.rb
-- spec/unit/logging_spec.rb
-- spec/unit/migrations_spec.rb
-- spec/unit/pool_spec.rb
-- spec/unit/run_spec.rb
-- spec/unit/states_spec.rb
-- spec/unit/stats_spec.rb
-- spec/unit/transaction_spec.rb
-- spec/unit/work_spec.rb
-- spec/unit/worker_spec.rb
+test_files: []