que 1.0.0.beta2 → 1.0.0.beta3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8ad84e5065bf9a35a8ed97f1b905b62935a67bf3713d537c4601c1bb6bc48e7c
-  data.tar.gz: 6ced1d272b59a81854f6794147856a1f21bfe048316d767b27ee984935fa8300
+  metadata.gz: 832beb15feb06f24511a7adec121b635639a87449f1403dcf8763cf8517ab11c
+  data.tar.gz: d39465285d79a8348753e4783a22c7432f369dc8ce0205c41872a87798cd5b48
 SHA512:
-  metadata.gz: a103cc5cd43608bc66023a16b32fe71f87b5adbaae7ba8d544f388d704d12d18fddbf2528f98d5aaddaf4e93d4c112d27f2921c3f33c4fd9dedb23410e8ff24e
-  data.tar.gz: befd6ca6e2ecd9f7f454e8a7c1789d5effb1638b4b9fb74ed5c83b07e413e7f5d54ea76338621e6a696464194a6118fe0ffcd46acc0c094b0fd57a4e8a244b85
+  metadata.gz: bb927afb7f17dd55a2207686f2707bb994860a6c4a4fa2a30e79fb763a7d298b410e3dec36d4520b7617e996e7927b8b90c8fa1ab68a3e264d96b59df9c51a4d
+  data.tar.gz: 421aaa5ce29afb83ac376966b2f859a4ba356087fad248738a57b3e9c77bb193ee8f08d58320ec10b6aa9bb3587c9c05bf6c9d7a22c6da62d908ef6876cdd173

data/CHANGELOG.1.0.beta.md

@@ -1,3 +1,13 @@
+### 1.0.0.beta3 (2018-05-18)
+
+*   Added support for customizing log levels for `job_worked` events (#217).
+
+*   Began logging all `job_errored` events at the `ERROR` log level.
+
+*   Fixed the Railtie when running in test mode (#214).
+
+*   Tweaked the meanings of worker-priorities and worker-count options in the CLI, to better support use cases with low worker counts (#216).
+
 ### 1.0.0.beta2 (2018-04-13)
 
 *   Fixed an incompatibility that caused the new locker to hang when using Rails in development mode (#213).

data/bin/command_line_interface.rb

@@ -18,12 +18,14 @@ module Que
         default_require_file: RAILS_ENVIRONMENT_FILE
       )
 
-        options        = {}
-        queues         = []
-        log_level      = 'info'
-        log_internals  = false
-        poll_interval  = 5
-        connection_url = nil
+        options           = {}
+        queues            = []
+        log_level         = 'info'
+        log_internals     = false
+        poll_interval     = 5
+        connection_url    = nil
+        worker_count      = nil
+        worker_priorities = nil
 
         parser =
           OptionParser.new do |opts|
@@ -58,6 +60,26 @@ module Que
               log_level = l
             end
 
+            opts.on(
+              '-p',
+              '--worker-priorities [LIST]',
+              Array,
+              "List of priorities to assign to workers (default: 10,30,50,any,any,any)",
+            ) do |priority_array|
+              worker_priorities =
+                priority_array.map do |p|
+                  case p
+                  when /\Aany\z/i
+                    nil
+                  when /\A\d+\z/
+                    Integer(p)
+                  else
+                    output.puts "Invalid priority option: '#{p}'. Please use an integer or the word 'any'."
+                    return 1
+                  end
+                end
+            end
+
             opts.on(
               '-q',
               '--queue-name [NAME]',
@@ -69,6 +91,15 @@ module Que
               queues << queue_name
             end
 
+            opts.on(
+              '-w',
+              '--worker-count [COUNT]',
+              Integer,
+              "Set number of workers in process (default: 6)",
+            ) do |w|
+              worker_count = w
+            end
+
             opts.on(
               '-v',
               '--version',
@@ -79,15 +110,6 @@ module Que
               return 0
             end
 
-            opts.on(
-              '-w',
-              '--worker-count [COUNT]',
-              Integer,
-              "Set number of workers in process (default: 6)",
-            ) do |w|
-              options[:worker_count] = w
-            end
-
             opts.on(
               '--connection-url [URL]',
               String,
@@ -130,19 +152,21 @@ module Que
             ) do |p|
               options[:wait_period] = p
             end
-
-            opts.on(
-              '--worker-priorities [LIST]',
-              Array,
-              "List of priorities to assign to workers, " \
-                "unspecified workers take jobs of any priority (default: 10,30,50)",
-            ) do |p|
-              options[:worker_priorities] = p.map(&:to_i)
-            end
           end
 
         parser.parse!(args)
 
+        options[:worker_priorities] =
+          if worker_count && worker_priorities
+            worker_priorities.values_at(0...worker_count)
+          elsif worker_priorities
+            worker_priorities
+          elsif worker_count
+            Array.new(worker_count) { nil }
+          else
+            [10, 30, 50, nil, nil, nil]
+          end
+
         if args.length.zero?
           if File.exist?(default_require_file)
             args << default_require_file

data/docs/README.md

@@ -20,6 +20,7 @@ TODO: Fix doc links.
   - [Worker States](inspecting_the_queue.md#worker-states)
   - [Custom Queries](inspecting_the_queue.md#custom-queries)
 - [Logging](logging.md#logging)
+  - [Logging Job Completion](logging.md#logging-job-completion)
 - [Managing Workers](managing_workers.md#managing-workers)
   - [Working Jobs Via Executable](managing_workers.md#working-jobs-via-executable)
   - [Thread-Unsafe Application Code](managing_workers.md#thread-unsafe-application-code)

data/docs/command_line_interface.md

@@ -5,24 +5,28 @@ usage: que [options] [file/to/require] ...
     -h, --help                       Show this help text.
     -i, --poll-interval [INTERVAL]   Set maximum interval between polls for available jobs, in seconds (default: 5)
     -l, --log-level [LEVEL]          Set level at which to log to STDOUT (debug, info, warn, error, fatal) (default: info)
+    -p, --worker-priorities [LIST]   List of priorities to assign to workers (default: 10,30,50,any,any,any)
     -q, --queue-name [NAME]          Set a queue name to work jobs from. Can be passed multiple times. (default: the default queue only)
-    -v, --version                    Print Que version and exit.
     -w, --worker-count [COUNT]       Set number of workers in process (default: 6)
+    -v, --version                    Print Que version and exit.
         --connection-url [URL]       Set a custom database url to connect to for locking purposes.
         --log-internals              Log verbosely about Que's internal state. Only recommended for debugging issues
         --maximum-buffer-size [SIZE] Set maximum number of jobs to be locked and held in this process awaiting a worker (default: 8)
         --minimum-buffer-size [SIZE] Set minimum number of jobs to be locked and held in this process awaiting a worker (default: 2)
         --wait-period [PERIOD]       Set maximum interval between checks of the in-memory job queue, in milliseconds (default: 50)
-        --worker-priorities [LIST]   List of priorities to assign to workers, unspecified workers take jobs of any priority (default: 10,30,50)
 ```
 
 Some explanation of the more unusual options:
 
-### worker-count and worker-priorities
+### worker-priorities and worker-count
+
+These options dictate the size and priority distribution of the worker pool. The default worker-priorities is `10,30,50,any,any,any`. This means that the default worker pool will reserve one worker to only works jobs with priorities under 10, one for priorities under 30, and one for priorities under 50. Three more workers will work any job.
+
+For example, with these defaults, you could have a large backlog of jobs of priority 100. When a more important job (priority 40) comes in, there's guaranteed to be a free worker. If the process then becomes saturated with jobs of priority 40, and then a priority 20 job comes in, there's guaranteed to be a free worker for it, and so on. You can pass a priority more than once to have multiple workers at that level (for example: `--worker-priorities=100,100,any,any`). This gives you a lot of freedom to manage your worker capacity at different priority levels.
 
-These options dictate the size and priority distribution of the worker pool. The default worker-count is 6 and the default worker-priorities is 10,30,50. This means that the default worker pool will have one worker that only works jobs with priorities under 10, one for priorities under 30, and one for priorities under 50. The leftover workers will work any job.
+Instead of passing worker-priorities, you can pass a `worker-count` - this is a shorthand for creating the given number of workers at the `any` priority level. So, `--worker-count=3` is just like passing equivalent to `worker-priorities=any,any,any`.
 
-For example, with these defaults, you could have a large backlog of jobs of priority 100. When a more important job (priority 40) comes in, there's guaranteed to be a free worker. If the process then becomes saturated with jobs of priority 40, and then a priority 20 job comes in, there's guaranteed to be a free worker for it, and so on.
+If you pass both worker-count and worker-priorities, the count will trim or pad the priorities list with `any` workers. So, `--worker-priorities=20,30,40 --worker-count=6` would be the same as passing `--worker-priorities=20,30,40,any,any,any`.
 
 ### poll-interval
 

data/docs/logging.md

@@ -29,3 +29,34 @@ Que.log_formatter = proc do |data|
   end
 end
 ```
+
+## Logging Job Completion
+
+Que logs a `job_worked` event whenever a job completes, though by default this event is logged at the `DEBUG` level. Since people often run their applications at the `INFO` level or above, this can make the logs too silent for some use cases. Similarly, you may want to log at a higher level if a time-sensitive job begins taking too long to run.
+
+You can solve these problems by configuring the level at which a job is logged on a per-job basis. Simply define a `log_level` method in your job class - it will be called with a float representing the number of seconds it took for the job to run, and it should return a symbol indicating what level to log the job at:
+
+```ruby
+class TimeSensitiveJob < Que::Job
+  def run(*args)
+    RemoteAPI.execute_important_request
+  end
+
+  def log_level(elapsed)
+    if elapsed > 60
+      # This job took over a minute! We should complain about it!
+      :warn
+    elsif elapsed > 30
+      # A little long, but no big deal!
+      :info
+    else
+      # This is fine, don't bother logging at all.
+      false
+    end
+  end
+end
+```
+
+This method should return a symbol that is a valid logging level (one of `[:debug, :info, :warn, :error, :fatal, :unknown]`). If the method returns anything other than one of these symbols, the job won't be logged.
+
+If a job errors, a `job_errored` event will be emitted at the `ERROR` log level. This is not currently configurable.

data/docs/using_sequel.md

@@ -7,6 +7,22 @@ DB = Sequel.connect(ENV['DATABASE_URL'])
 Que.connection = DB
 ```
 
+If you are using Sequel's migrator, your app initialization won't happen, so you may need to tweak your migrations to `require 'que'` and set its connection:
+
+```ruby
+require 'que'
+Sequel.migration do
+  up do
+    Que.connection = self
+    Que.migrate! :version => 3
+  end
+  down do
+    Que.connection = self
+    Que.migrate! :version => 0
+  end
+end
+```
+
 Then you can safely use the same database object to transactionally protect your jobs:
 
 ```ruby

data/lib/que/job_methods.rb

@@ -65,6 +65,10 @@ module Que
       raise error if reraise_errors
     end
 
+    def log_level(elapsed)
+      :debug
+    end
+
     private
 
     # This method defines the object on which the various job helper methods are

data/lib/que/locker.rb

@@ -47,8 +47,7 @@ module Que
     DEFAULT_WAIT_PERIOD         = 50
     DEFAULT_MINIMUM_BUFFER_SIZE = 2
     DEFAULT_MAXIMUM_BUFFER_SIZE = 8
-    DEFAULT_WORKER_COUNT        = 6
-    DEFAULT_WORKER_PRIORITIES   = [10, 30, 50].freeze
+    DEFAULT_WORKER_PRIORITIES   = [10, 30, 50, nil, nil, nil].freeze
 
     def initialize(
       queues:              [Que.default_queue],
@@ -59,7 +58,6 @@ module Que
       wait_period:         DEFAULT_WAIT_PERIOD,
       maximum_buffer_size: DEFAULT_MAXIMUM_BUFFER_SIZE,
       minimum_buffer_size: DEFAULT_MINIMUM_BUFFER_SIZE,
-      worker_count:        DEFAULT_WORKER_COUNT,
       worker_priorities:   DEFAULT_WORKER_PRIORITIES,
       on_worker_start:     nil
     )
@@ -71,19 +69,16 @@ module Que
 
       Que.assert Numeric, poll_interval
       Que.assert Numeric, wait_period
-      Que.assert Integer, worker_count
 
       Que.assert Array, worker_priorities
-      worker_priorities.each { |p| Que.assert(Integer, p) }
-
-      all_worker_priorities = worker_priorities.values_at(0...worker_count)
+      worker_priorities.each { |p| Que.assert([Integer, NilClass], p) }
 
       # We use a JobBuffer to track jobs and pass them to workers, and a
       # ResultQueue to receive messages from workers.
       @job_buffer = JobBuffer.new(
         maximum_size: maximum_buffer_size,
         minimum_size: minimum_buffer_size,
-        priorities:   all_worker_priorities.uniq,
+        priorities:   worker_priorities.uniq,
       )
 
       @result_queue = ResultQueue.new
@@ -99,7 +94,6 @@ module Que
           wait_period:         wait_period,
           maximum_buffer_size: maximum_buffer_size,
           minimum_buffer_size: minimum_buffer_size,
-          worker_count:        worker_count,
           worker_priorities:   worker_priorities,
         }
       end
@@ -110,13 +104,8 @@ module Que
       @queue_names = queues.is_a?(Hash) ? queues.keys : queues
       @wait_period = wait_period.to_f / 1000 # Milliseconds to seconds.
 
-      # If the worker_count exceeds the array of priorities it'll result in
-      # extra workers that will work jobs of any priority. For example, the
-      # default worker_count of 6 and the default worker priorities of [10, 30,
-      # 50] will result in three workers that only work jobs that meet those
-      # priorities, and three workers that will work any job.
       @workers =
-        all_worker_priorities.map do |priority|
+        worker_priorities.map do |priority|
           Worker.new(
             priority:       priority,
             job_buffer:     @job_buffer,
@@ -138,13 +127,20 @@ module Que
         if connection_url
           uri = URI.parse(connection_url)
 
-          {
-            host:     uri.host,
-            user:     uri.user,
-            password: uri.password,
-            port:     uri.port || 5432,
-            dbname:   uri.path[1..-1],
-          }.merge(Hash[uri.query.split("&").map{|s| s.split('=')}.map{|a,b| [a.to_sym, b]}])
+          opts =
+            {
+              host:     uri.host,
+              user:     uri.user,
+              password: uri.password,
+              port:     uri.port || 5432,
+              dbname:   uri.path[1..-1],
+            }
+
+          if uri.query
+            opts.merge!(Hash[uri.query.split("&").map{|s| s.split('=')}.map{|a,b| [a.to_sym, b]}])
+          end
+
+          opts
         else
           Que.pool.checkout do |conn|
             c = conn.wrapped_connection

data/lib/que/rails/railtie.rb

@@ -5,8 +5,6 @@ module Que
     class Railtie < ::Rails::Railtie
       config.que = Que
 
-      Que.run_asynchronously = true if ::Rails.env.test?
-
       Que.logger     = proc { ::Rails.logger }
       Que.connection = ::ActiveRecord if defined? ::ActiveRecord
     end

data/lib/que/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Que
-  VERSION = '1.0.0.beta2'
+  VERSION = '1.0.0.beta3'
 end

data/lib/que/worker.rb

@@ -3,10 +3,14 @@
 # Workers wrap threads which continuously pull job pks from JobBuffer objects,
 # fetch and work those jobs, and export relevant data to ResultQueues.
 
+require 'set'
+
 module Que
   class Worker
     attr_reader :thread, :priority
 
+    VALID_LOG_LEVELS = [:debug, :info, :warn, :error, :fatal, :unknown].to_set.freeze
+
     SQL[:check_job] =
       %{
         SELECT 1 AS one
@@ -91,20 +95,31 @@ module Que
 
       Que.run_job_middleware(instance) { instance.tap(&:_run) }
 
-      log_message = {
-        level: :debug,
-        job_id: metajob.id,
-        elapsed: (Time.now - start),
-      }
+      elapsed = Time.now - start
 
-      if error = instance.que_error
-        log_message[:event] = :job_errored
-        log_message[:error] = "#{error.class}: #{error.message}".slice(0, 500)
-      else
-        log_message[:event] = :job_worked
-      end
+      log_level =
+        if instance.que_error
+          :error
+        else
+          instance.log_level(elapsed)
+        end
+
+      if VALID_LOG_LEVELS.include?(log_level)
+        log_message = {
+          level: log_level,
+          job_id: metajob.id,
+          elapsed: elapsed,
+        }
 
-      Que.log(log_message)
+        if error = instance.que_error
+          log_message[:event] = :job_errored
+          log_message[:error] = "#{error.class}: #{error.message}".slice(0, 500)
+        else
+          log_message[:event] = :job_worked
+        end
+
+        Que.log(log_message)
+      end
 
       instance
     rescue => error

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: que
 version: !ruby/object:Gem::Version
-  version: 1.0.0.beta2
+  version: 1.0.0.beta3
 platform: ruby
 authors:
 - Chris Hanks
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-04-13 00:00:00.000000000 Z
+date: 2018-05-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler