canvas-jobs 0.10.5 → 0.10.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1a1dcbb68bb9d5945b79b3806d27b6a7a3c05688
-  data.tar.gz: fc3aaa3d27c040360abbcf76f4a24db2390384ec
+  metadata.gz: c17b53cc20eeba1d6def3d6defed59ec2f4e6786
+  data.tar.gz: 462039f0631c4c8446e485ff09543c2430958fd9
 SHA512:
-  metadata.gz: cdef99a4e43b2cde8ff73fdbb9b7e7f3148acac67f7016ae3fa6895dc9337ecb706afa0ace9fe92538465c3340a35ad2ed1be956c3b8e6d3fe7d69e11b4832f0
-  data.tar.gz: 8c88f6fd42e35a134c1511e2137037766642a83fd2b700ac6253344e33520801654b42306af7e8d4c7982e34fc05fd727935f7cca89f0514cfa5c69a15d2159a
+  metadata.gz: 479c5177f7f37fa4c2f84caf850f214a316def2cc182f815bb59ef63f905490912e6b87396f57df6e87eee6af77b45047f4a2051ecf1076d467ebf963a0a01a7
+  data.tar.gz: 779faa887354c56f5dcba06e16e1bc52f20d0084c0b432d59c282afc0908a74d32eb51ca83e1067cdf833daa1d5461e44c1e78607bf0d9443aa16e02f5e27ccf

data/bin/canvas_job

@@ -1,4 +1,4 @@
 #!/usr/bin/env ruby
 
 require File.expand_path('config/environment')
-Delayed::Pool.new.run()
+Delayed::CLI.new.run()

data/lib/delayed/cli.rb

@@ -0,0 +1,104 @@
+require 'optparse'
+
+module Delayed
+class CLI
+  def initialize(args = ARGV)
+    @args = args
+    # config that will be applied on Settings
+    @config = {}
+    # worker configs that will be passed to the created Pool
+    @worker_configs = []
+    # CLI options that will be kept to this class
+    @options = {
+      :config_file => Settings.default_worker_config_name,
+      :pid_folder => Settings.expand_rails_path("tmp/pids"),
+      :tail_logs => true, # only in FG mode
+    }
+  end
+
+  def run
+    parse_cli_options!
+    load_and_apply_config!
+
+    command = @args.shift
+    case command
+    when 'start'
+      exit 1 if daemon.status(print: :alive) == :running
+      daemon.daemonize!
+      start
+    when 'stop'
+      daemon.stop(kill: @options[:kill])
+    when 'run'
+      start
+    when 'status'
+      if daemon.status
+        exit 0
+      else
+        exit 1
+      end
+    when 'restart'
+      daemon.stop(kill: @options[:kill])
+      daemon.daemonize!
+      start
+    when nil
+      puts option_parser.to_s
+    else
+      raise("Unknown command: #{command.inspect}")
+    end
+  end
+
+  def parse_cli_options!
+    option_parser.parse!(@args)
+    @options
+  end
+
+  protected
+
+  def load_and_apply_config!
+    @config = Settings.worker_config(@options[:config_file])
+    @worker_configs = @config.delete(:workers)
+    Settings.apply_worker_config!(@config)
+  end
+
+  def option_parser
+    @option_parser ||= OptionParser.new do |opts|
+      opts.banner = "Usage #{$0} <command> <options>"
+      opts.separator %{\nWhere <command> is one of:
+  start      start the jobs daemon
+  stop       stop the jobs daemon
+  run        start and run in the foreground
+  restart    stop and then start the jobs daemon
+  status     show daemon status
+}
+
+      opts.separator "\n<options>"
+      opts.on("-c", "--config [CONFIG_PATH]", "Use alternate config file (default #{@options[:config_file]})") { |c| @options[:config_file] = c }
+      opts.on("-p", "--pid", "Use alternate folder for PID files (default #{@options[:pid_folder]})") { |p| @options[:pid_folder] = p }
+      opts.on("--no-tail", "Don't tail the logs (only affects non-daemon mode)") { @options[:tail_logs] = false }
+      opts.on("--with-prejudice", "When stopping, interrupt jobs in progress, instead of letting them drain") { @options[:kill] ||= true }
+      opts.on("--with-extreme-prejudice", "When stopping, immediately kill jobs in progress, instead of letting them drain") { @options[:kill] = 9 }
+      opts.on_tail("-h", "--help", "Show this message") { puts opts; exit }
+    end
+  end
+
+  def daemon
+    @daemon ||= Delayed::Daemon.new(@options[:pid_folder])
+  end
+
+  def start
+    load_rails
+    tail_rails_log unless daemon.daemonized?
+    Delayed::Pool.new(@worker_configs).start
+  end
+
+  def load_rails
+    require(Settings.expand_rails_path("config/environment.rb"))
+    Dir.chdir(Rails.root)
+  end
+
+  def tail_rails_log
+    return if !@options[:tail_logs]
+    Delayed::LogTailer.new.run
+  end
+end
+end

data/lib/delayed/daemon.rb

@@ -0,0 +1,103 @@
+require 'fileutils'
+
+module Delayed
+# Daemon controls the parent proces that runs the Pool and monitors the Worker processes.
+class Daemon
+  attr_reader :pid_folder
+
+  def initialize(pid_folder)
+    @pid_folder = pid_folder
+  end
+
+  def status(print: true, pid: self.pid)
+    alive = pid && (Process.kill(0, pid) rescue false) && :running
+    alive ||= :draining if pid && Process.kill(0, -pid) rescue false
+    if alive
+      puts "Delayed jobs #{alive}, pool PID: #{pid}" if print
+    else
+      puts "No delayed jobs pool running" if print && print != :alive
+    end
+    alive
+  end
+
+  def daemonize!
+    FileUtils.mkdir_p(pid_folder)
+    puts "Daemonizing..."
+
+    exit if fork
+    Process.setsid
+    exit if fork
+    Process.setpgrp
+
+    @daemon = true
+    lock_file = File.open(pid_file, 'wb')
+    # someone else is already running; just exit
+    unless lock_file.flock(File::LOCK_EX | File::LOCK_NB)
+      exit
+    end
+    at_exit { lock_file.flock(File::LOCK_UN) }
+    lock_file.puts(Process.pid.to_s)
+    lock_file.flush
+
+    # if we blow up so badly that we can't syslog the error, try to send
+    # it somewhere useful
+    last_ditch_logfile = Settings.last_ditch_logfile || "log/delayed_job.log"
+    if last_ditch_logfile[0] != '|'
+      last_ditch_logfile = Settings.expand_rails_path(last_ditch_logfile)
+    end
+    STDIN.reopen("/dev/null")
+    STDOUT.reopen(open(last_ditch_logfile, 'a'))
+    STDERR.reopen(STDOUT)
+    STDOUT.sync = STDERR.sync = true
+  end
+
+  # stop the currently running daemon (not this current process, the one in the pid_file)
+  def stop(kill: false, pid: self.pid)
+    alive = status(pid: pid, print: false)
+    if alive == :running || (kill && alive == :draining)
+      puts "Stopping pool #{pid}..."
+      signal = 'INT'
+      if kill
+        pid = -pid # send to the whole group
+        if kill == 9
+          signal = 'KILL'
+        else
+          signal = 'TERM'
+        end
+      end
+      begin
+        Process.kill(signal, pid)
+      rescue Errno::ESRCH
+        # ignore if the pid no longer exists
+      end
+      wait(kill)
+    else
+      status
+    end
+  end
+
+  def wait(kill)
+    if kill
+      sleep(0.5) while status(pid: pid, print: false)
+    else
+      sleep(0.5) while status(pid: pid, print: false) == :running
+    end
+  end
+
+  def pid_file
+    File.join(pid_folder, 'delayed_jobs_pool.pid')
+  end
+
+  def pid
+    if File.file?(pid_file)
+      pid = File.read(pid_file).to_i
+      pid = nil unless pid > 0
+    end
+    pid
+  end
+
+  def daemonized?
+    !!@daemon
+  end
+end
+end

data/lib/delayed/lifecycle.rb

@@ -3,11 +3,13 @@ module Delayed
 
   class Lifecycle
     EVENTS = {
-      :perform          => [:worker, :job],
-      :pop              => [:worker],
+      :error            => [:worker, :job, :exception],
       :exceptional_exit => [:worker, :exception],
+      :execute          => [:worker],
       :invoke_job       => [:job],
-      :error            => [:worker, :job, :exception],
+      :loop             => [:worker],
+      :perform          => [:worker, :job],
+      :pop              => [:worker],
     }
 
     def initialize

data/lib/delayed/log_tailer.rb

@@ -0,0 +1,22 @@
+module Delayed
+class LogTailer
+  def run
+    if Rails.logger.respond_to?(:log_path)
+      log_path = Rails.logger.log_path
+    elsif Rails.logger.instance_variable_get('@logdev').try(:instance_variable_get, '@dev').try(:path)
+      log_path = Rails.logger.instance_variable_get('@logdev').instance_variable_get('@dev').path
+    else
+      return
+    end
+    Rails.logger.auto_flushing = true if Rails.logger.respond_to?(:auto_flushing=)
+    Thread.new do
+      f = File.open(log_path, 'r')
+      f.seek(0, IO::SEEK_END)
+      loop do
+        content = f.read
+        content.present? ? STDOUT.print(content) : sleep(0.5)
+      end
+    end
+  end
+end
+end

data/lib/delayed/pool.rb

@@ -1,103 +1,28 @@
-require 'erb'
-require 'optparse'
-require 'yaml'
-require 'fileutils'
-
 module Delayed
 class Pool
   mattr_accessor :on_fork
   self.on_fork = ->{ }
 
-  attr_reader :options, :workers
+  attr_reader :workers
 
-  def initialize(args = ARGV)
-    @args = args
-    @workers = {}
-    @config = { :workers => [] }
-    @options = {
-      :config_file => Settings.default_worker_config_name,
-      :pid_folder => Settings.expand_rails_path("tmp/pids"),
-      :tail_logs => true, # only in FG mode
-    }
-  end
-
-  def run
-    parse_cli_options!
-
-    read_config(options[:config_file])
-
-    command = @args.shift
-    case command
-    when 'start'
-      exit 1 if status(print: :alive) == :running
-      daemonize
-      start
-    when 'stop'
-      stop(kill: options[:kill])
-    when 'run'
-      start
-    when 'status'
-      if status
-        exit 0
-      else
-        exit 1
-      end
-    when 'restart'
-      pid = self.pid
-      alive = status(pid: pid, print: false)
-      if alive == :running || (options[:kill] && alive == :draining)
-        stop(pid: pid, kill: options[:kill])
-        if options[:kill]
-          sleep(0.5) while status(pid: pid, print: false)
-        else
-          sleep(0.5) while status(pid: pid, print: false) == :running
-        end
-      end
-      daemonize
-      start
-    when nil
-      puts op
+  def initialize(*args)
+    if args.size == 1 && args.first.is_a?(Array)
+      worker_configs = args.first
     else
-      raise("Unknown command: #{command.inspect}")
+      warn "Calling Delayed::Pool.new directly is deprecated. Use `Delayed::CLI.new.run()` instead."
     end
+    @workers = {}
+    @config = { workers: worker_configs }
   end
 
-  def parse_cli_options!
-    op = OptionParser.new do |opts|
-      opts.banner = "Usage #{$0} <command> <options>"
-      opts.separator %{\nWhere <command> is one of:
-  start      start the jobs daemon
-  stop       stop the jobs daemon
-  run        start and run in the foreground
-  restart    stop and then start the jobs daemon
-  status     show daemon status
-}
-
-      opts.separator "\n<options>"
-      opts.on("-c", "--config [CONFIG_PATH]", "Use alternate config file (default #{options[:config_file]})") { |c| options[:config_file] = c }
-      opts.on("-p", "--pid", "Use alternate folder for PID files (default #{options[:pid_folder]})") { |p| options[:pid_folder] = p }
-      opts.on("--no-tail", "Don't tail the logs (only affects non-daemon mode)") { options[:tail_logs] = false }
-      opts.on("--with-prejudice", "When stopping, interrupt jobs in progress, instead of letting them drain") { options[:kill] ||= true }
-      opts.on("--with-extreme-prejudice", "When stopping, immediately kill jobs in progress, instead of letting them drain") { options[:kill] = 9 }
-      opts.on_tail("-h", "--help", "Show this message") { puts opts; exit }
-    end
-    op.parse!(@args)
-  end
-
-  protected
-
-  def procname
-    "delayed_jobs_pool#{Settings.pool_procname_suffix}"
+  def run
+    warn "Delayed::Pool#run is deprecated and will be removed. Use `Delayed::CLI.new.run()` instead."
+    Delayed::CLI.new.run()
   end
 
   def start
-    load_rails
-    tail_rails_log unless @daemon
-
     say "Started job master", :info
     $0 = procname
-    apply_config
-
     # fork to handle unlocking (to prevent polluting the parent with worker objects)
     unlock_pid = fork_with_reconnects do
       unlock_orphaned_jobs
@@ -116,6 +41,12 @@ class Pool
     raise
   end
 
+  protected
+
+  def procname
+    "delayed_jobs_pool#{Settings.pool_procname_suffix}"
+  end
+
   def say(msg, level = :debug)
     if defined?(Rails.logger) && Rails.logger
       Rails.logger.send(level, "[#{Process.pid}]P #{msg}")
@@ -124,16 +55,8 @@ class Pool
     end
   end
 
-  def load_rails
-    require(Settings.expand_rails_path("config/environment.rb"))
-    Dir.chdir(Rails.root)
-  end
-
   def unlock_orphaned_jobs(worker = nil, pid = nil)
-    # don't bother trying to unlock jobs by process name if the name is overridden
-    return if @config.key?(:name)
-    return if @config[:disable_automatic_orphan_unlocking]
-    return if @config[:workers].any? { |worker_config| worker_config.key?(:name) || worker_config.key?('name') }
+    return if Settings.disable_automatic_orphan_unlocking
 
     unlocked_jobs = Delayed::Job.unlock_orphaned_jobs(pid)
     say "Unlocked #{unlocked_jobs} orphaned jobs" if unlocked_jobs > 0
@@ -144,8 +67,7 @@ class Pool
     ActiveRecord::Base.connection_handler.clear_all_connections!
 
     @config[:workers].each do |worker_config|
-      worker_config = worker_config.with_indifferent_access
-      (worker_config[:workers] || 1).times { spawn_worker(@config.merge(worker_config)) }
+      (worker_config[:workers] || 1).times { spawn_worker(worker_config) }
     end
   end
 
@@ -174,7 +96,7 @@ class Pool
   end
 
   def spawn_periodic_auditor
-    return if @config[:disable_periodic_jobs]
+    return if Settings.disable_periodic_jobs
 
     @periodic_thread = Thread.new do
       # schedule the initial audit immediately on startup
@@ -217,122 +139,5 @@ class Pool
       end
     end
   end
-
-  def tail_rails_log
-    return if !@options[:tail_logs]
-    if Rails.logger.respond_to?(:log_path)
-      log_path = Rails.logger.log_path
-    elsif Rails.logger.instance_variable_get('@logdev').try(:instance_variable_get, '@dev').try(:path)
-      log_path = Rails.logger.instance_variable_get('@logdev').instance_variable_get('@dev').path
-    else
-      return
-    end
-    Rails.logger.auto_flushing = true if Rails.logger.respond_to?(:auto_flushing=)
-    Thread.new do
-      f = File.open(log_path, 'r')
-      f.seek(0, IO::SEEK_END)
-      loop do
-        content = f.read
-        content.present? ? STDOUT.print(content) : sleep(0.5)
-      end
-    end
-  end
-
-  def daemonize
-    FileUtils.mkdir_p(pid_folder)
-    puts "Daemonizing..."
-
-    exit if fork
-    Process.setsid
-    exit if fork
-    Process.setpgrp
-
-    @daemon = true
-    lock_file = File.open(pid_file, 'wb')
-    # someone else is already running; just exit
-    unless lock_file.flock(File::LOCK_EX | File::LOCK_NB)
-      exit
-    end
-    at_exit { lock_file.flock(File::LOCK_UN) }
-    lock_file.puts(Process.pid.to_s)
-    lock_file.flush
-
-    # if we blow up so badly that we can't syslog the error, try to send
-    # it somewhere useful
-    last_ditch_logfile = self.last_ditch_logfile || "log/delayed_job.log"
-    if last_ditch_logfile[0] != '|'
-      last_ditch_logfile = Settings.expand_rails_path(last_ditch_logfile)
-    end
-    STDIN.reopen("/dev/null")
-    STDOUT.reopen(open(last_ditch_logfile, 'a'))
-    STDERR.reopen(STDOUT)
-    STDOUT.sync = STDERR.sync = true
-  end
-
-  def pid_folder
-    options[:pid_folder]
-  end
-
-  def pid_file
-    File.join(pid_folder, 'delayed_jobs_pool.pid')
-  end
-
-  def last_ditch_logfile
-    @config['last_ditch_logfile']
-  end
-
-  def stop(options = {})
-    kill = options[:kill]
-    pid = options[:pid] || self.pid
-    if pid && status(pid: pid, print: false)
-      puts "Stopping pool #{pid}..."
-      signal = 'INT'
-      if kill
-        pid = -pid # send to the whole group
-        if kill == 9
-          signal = 'KILL'
-        else
-          signal = 'TERM'
-        end
-      end
-      begin
-        Process.kill(signal, pid)
-      rescue Errno::ESRCH
-        # ignore if the pid no longer exists
-      end
-    else
-      status
-    end
-  end
-
-  def pid
-    if File.file?(pid_file)
-      pid = File.read(pid_file).to_i
-      pid = nil unless pid > 0
-    end
-    pid
-  end
-
-  def status(options = { print: true })
-    print = options[:print]
-    pid = options[:pid] || self.pid
-    alive = pid && (Process.kill(0, pid) rescue false) && :running
-    alive ||= :draining if pid && Process.kill(0, -pid) rescue false
-    if alive
-      puts "Delayed jobs #{alive}, pool PID: #{pid}" if print
-    else
-      puts "No delayed jobs pool running" if print && print != :alive
-    end
-    alive
-  end
-
-  def read_config(config_filename)
-    @config = Settings.worker_config(config_filename)
-  end
-
-  def apply_config
-    Settings.apply_worker_config!(@config)
-  end
-
 end
 end

data/lib/delayed/settings.rb

@@ -1,3 +1,6 @@
+require 'yaml'
+require 'erb'
+
 module Delayed
   module Settings
     SETTINGS = [
@@ -11,6 +14,9 @@ module Delayed
       :pool_procname_suffix,
       :default_job_options,
       :silence_periodic_log,
+      :disable_periodic_jobs,
+      :disable_automatic_orphan_unlocking,
+      :last_ditch_logfile,
     ]
     SETTINGS_WITH_ARGS = [ :num_strands ]
 
@@ -52,7 +58,11 @@ module Delayed
         raise ArgumentError,
           "Invalid config file #{config_filename}"
       end
-      config.with_indifferent_access
+      config = config.with_indifferent_access
+      config[:workers].map! do |worker_config|
+        config.except(:workers).merge(worker_config.with_indifferent_access)
+      end
+      config
     end
 
     def self.apply_worker_config!(config)

data/lib/delayed/version.rb

@@ -1,3 +1,3 @@
 module Delayed
-  VERSION = "0.10.5"
+  VERSION = "0.10.6"
 end

data/lib/delayed/worker.rb

@@ -55,10 +55,6 @@ class Worker
     plugins.each { |plugin| plugin.inject! }
   end
 
-  def name=(name)
-    @name = name
-  end
-
   def name
     @name ||= "#{Socket.gethostname rescue "X"}:#{self.id}"
   end
@@ -80,9 +76,11 @@ class Worker
 
     trap('INT') { say 'Exiting'; @exit = true }
 
-    loop do
-      run
-      break if exit?
+    self.class.lifecycle.run_callbacks(:execute, self) do
+      loop do
+        run
+        break if exit?
+      end
     end
 
     say "Stopping worker", :info
@@ -94,37 +92,39 @@ class Worker
   end
 
   def run
-    job =
-        self.class.lifecycle.run_callbacks(:pop, self) do
-          Delayed::Job.get_and_lock_next_available(
-            name,
-            queue,
-            min_priority,
-            max_priority)
-        end
-
-    if job
-      configure_for_job(job) do
-        @job_count += perform(job)
+    self.class.lifecycle.run_callbacks(:loop, self) do
+      job =
+          self.class.lifecycle.run_callbacks(:pop, self) do
+            Delayed::Job.get_and_lock_next_available(
+              name,
+              queue,
+              min_priority,
+              max_priority)
+          end
 
-        if @max_job_count > 0 && @job_count >= @max_job_count
-          say "Max job count of #{@max_job_count} exceeded, dying"
-          @exit = true
-        end
+      if job
+        configure_for_job(job) do
+          @job_count += perform(job)
 
-        if @max_memory_usage > 0
-          memory = sample_memory
-          if memory > @max_memory_usage
-            say "Memory usage of #{memory} exceeds max of #{@max_memory_usage}, dying"
+          if @max_job_count > 0 && @job_count >= @max_job_count
+            say "Max job count of #{@max_job_count} exceeded, dying"
             @exit = true
-          else
-            say "Memory usage: #{memory}"
+          end
+
+          if @max_memory_usage > 0
+            memory = sample_memory
+            if memory > @max_memory_usage
+              say "Memory usage of #{memory} exceeds max of #{@max_memory_usage}, dying"
+              @exit = true
+            else
+              say "Memory usage: #{memory}"
+            end
           end
         end
+      else
+        set_process_name("wait:#{Settings.worker_procname_prefix}#{@queue}:#{min_priority || 0}:#{max_priority || 'max'}")
+        sleep(Settings.sleep_delay + (rand * Settings.sleep_delay_stagger))
       end
-    else
-      set_process_name("wait:#{Settings.worker_procname_prefix}#{@queue}:#{min_priority || 0}:#{max_priority || 'max'}")
-      sleep(Settings.sleep_delay + (rand * Settings.sleep_delay_stagger))
     end
   end
 

data/lib/delayed_job.rb

@@ -24,8 +24,11 @@ require 'delayed/backend/base'
 require 'delayed/backend/active_record'
 require 'delayed/backend/redis/job'
 require 'delayed/batch'
+require 'delayed/cli'
+require 'delayed/daemon'
 require 'delayed/job_tracking'
 require 'delayed/lifecycle'
+require 'delayed/log_tailer'
 require 'delayed/message_sending'
 require 'delayed/performable_method'
 require 'delayed/periodic'

data/spec/delayed/cli_spec.rb

@@ -0,0 +1,23 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::CLI do
+  describe '#parse_cli_options!' do
+    it 'correctly parses the --config option' do
+      cli = described_class.new(%w{run --config /path/to/some/file.yml})
+      options = cli.parse_cli_options!
+      expect(options).to include config_file: '/path/to/some/file.yml'
+    end
+  end
+
+  describe '#run' do
+    before do
+      expect(Delayed::Settings).to receive(:worker_config).and_return({})
+    end
+
+    it 'prints help when no command is given' do
+      cli = described_class.new([])
+      expect(cli).to receive(:puts).with(/Usage/)
+      cli.run
+    end
+  end
+end

data/spec/delayed/daemon_spec.rb

@@ -0,0 +1,35 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::Daemon do
+  let(:pid_folder) { "/test/pid/folder" }
+  let(:pid) { 9999 }
+  let(:subject) { described_class.new(pid_folder) }
+
+  before do
+    allow(subject).to receive(:pid).and_return(pid)
+  end
+
+  describe '#stop' do
+    it 'prints status if not running' do
+      expect(subject).to receive(:status).with(print: false, pid: pid).and_return(false)
+      expect(subject).to receive(:status).with(no_args)
+      expect(Process).to receive(:kill).never
+      subject.stop
+    end
+
+    it 'prints status if draining' do
+      expect(subject).to receive(:status).with(print: false, pid: pid).and_return(:draining)
+      expect(subject).to receive(:status).with(no_args)
+      expect(Process).to receive(:kill).never
+      subject.stop
+    end
+
+    it 'sends INT by default' do
+      expect(subject).to receive(:status).with(print: false, pid: pid).and_return(:running)
+      expect(subject).to receive(:puts).with(/Stopping pool/)
+      expect(Process).to receive(:kill).with('INT', pid)
+      expect(subject).to receive(:wait).with(false)
+      subject.stop
+    end
+  end
+end

data/spec/delayed/settings_spec.rb

@@ -0,0 +1,32 @@
+require 'spec_helper'
+
+RSpec.describe Delayed::Settings do
+  let(:configfile) {<<-YAML
+default:
+  workers:
+  - queue: myqueue
+    workers: 2
+  - queue: secondqueue
+    max_priority: 7
+  max_attempts: 1
+    YAML
+  }
+
+  describe '.worker_config' do
+    it 'merges each worker config with the top-level config' do
+      expect(File).to receive(:read).with("fname").and_return(configfile)
+      config = described_class.worker_config("fname")
+      expect(config[:workers]).to eq([
+        {'queue' => 'myqueue', 'workers' => 2, 'max_attempts' => 1},
+        {'queue' => 'secondqueue', 'max_priority' => 7, 'max_attempts' => 1},
+      ])
+    end
+  end
+
+  describe '.apply_worker_config!' do
+    it 'applies global settings from the given config' do
+      expect(described_class).to receive(:last_ditch_logfile=).with(true)
+      described_class.apply_worker_config!('last_ditch_logfile' => true)
+    end
+  end
+end

data/spec/shared/worker.rb

@@ -46,7 +46,7 @@ shared_examples_for 'Delayed::Worker' do
         @worker.perform(batch_job).should == 3
         expect(@runs).to eql 4 # batch, plus all jobs
       end
-    
+
       it "should succeed regardless of the success/failure of its component jobs" do
         change_setting(Delayed::Settings, :max_attempts, 2) do
           batch = Delayed::Batch::PerformableBatch.new(:serial, [
@@ -66,7 +66,7 @@ shared_examples_for 'Delayed::Worker' do
           to_retry[0].attempts.should == 1
         end
       end
-  
+
       it "should retry a failed individual job" do
         batch = Delayed::Batch::PerformableBatch.new(:serial, [
           { :payload_object => Delayed::PerformableMethod.new(1, :/, [0]) },
@@ -108,21 +108,17 @@ shared_examples_for 'Delayed::Worker' do
   end
 
   context "while running with locked jobs" do
-    before(:each) do
-      @worker.name = 'worker1'
-    end
-    
     it "should not run jobs locked by another worker" do
       job_create(:locked_by => 'other_worker', :locked_at => (Delayed::Job.db_time_now - 1.minutes))
       lambda { @worker.run }.should_not change { SimpleJob.runs }
     end
-    
+
     it "should run open jobs" do
       job_create
       lambda { @worker.run }.should change { SimpleJob.runs }.from(0).to(1)
     end
   end
-  
+
   describe "failed jobs" do
     before do
       # reset defaults
@@ -150,7 +146,7 @@ shared_examples_for 'Delayed::Worker' do
 
       Delayed::Job.find_available(100, @job.queue).should == []
     end
-    
+
     it "should re-schedule jobs after failing" do
       @worker.perform(@job)
       @job = Delayed::Job.find(@job.id)
@@ -174,18 +170,18 @@ shared_examples_for 'Delayed::Worker' do
       ErrorJob.permanent_failure_runs.should == 1
     end
   end
-  
+
   context "reschedule" do
     before do
       @job = Delayed::Job.create :payload_object => SimpleJob.new
     end
-    
+
     context "and we want to destroy jobs" do
       it "should be destroyed if it failed more than Settings.max_attempts times" do
         expect(@job).to receive(:destroy)
         Delayed::Settings.max_attempts.times { @job.reschedule }
       end
-      
+
       it "should not be destroyed if failed fewer than Settings.max_attempts times" do
         expect(@job).to receive(:destroy).never
         (Delayed::Settings.max_attempts - 1).times { @job.reschedule }
@@ -205,7 +201,7 @@ shared_examples_for 'Delayed::Worker' do
         job.reschedule
       end
     end
-    
+
     context "and we don't want to destroy jobs" do
       before do
         Delayed::Worker.on_max_failures = proc { false }
@@ -268,7 +264,7 @@ shared_examples_for 'Delayed::Worker' do
       SimpleJob.runs.should == 0
       worker.run
       SimpleJob.runs.should == 1
-      
+
       SimpleJob.runs = 0
 
       worker = worker_create(:queue=>'queue2')
@@ -291,7 +287,7 @@ shared_examples_for 'Delayed::Worker' do
       worker = worker_create(:queue=>nil)
       worker.queue.should == queue_name
     end
-    
+
     it "should override default queue name if specified in initialize" do
       queue_name = "my_queue"
       Delayed::Settings.queue = "default_queue"
@@ -357,4 +353,26 @@ shared_examples_for 'Delayed::Worker' do
       expect(ErrorJob.last_error.to_s).to eq 'did not work'
     end
   end
+
+  describe "#start" do
+    it "fires off an execute callback on the processing jobs loop" do
+      fired = false
+      expect(@worker).to receive(:run)
+      expect(@worker).to receive(:exit?).and_return(true)
+      Delayed::Worker.lifecycle.before(:execute) { |w| w == @worker && fired = true }
+      @worker.start
+      expect(fired).to eq(true)
+    end
+  end
+
+  describe "#run" do
+    it "fires off a loop callback on each call to run" do
+      fired = 0
+      Delayed::Worker.lifecycle.before(:loop) { |w| w == @worker && fired += 1 }
+      expect(Delayed::Job).to receive(:get_and_lock_next_available).twice.and_return(nil)
+      @worker.run
+      @worker.run
+      expect(fired).to eq(2)
+    end
+  end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: canvas-jobs
 version: !ruby/object:Gem::Version
-  version: 0.10.5
+  version: 0.10.6
 platform: ruby
 authors:
 - Tobias Luetke
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-02-16 00:00:00.000000000 Z
+date: 2016-03-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: after_transaction_commit
@@ -293,9 +293,12 @@ files:
 - lib/delayed/backend/redis/set_running.lua
 - lib/delayed/backend/redis/tickle_strand.lua
 - lib/delayed/batch.rb
+- lib/delayed/cli.rb
+- lib/delayed/daemon.rb
 - lib/delayed/engine.rb
 - lib/delayed/job_tracking.rb
 - lib/delayed/lifecycle.rb
+- lib/delayed/log_tailer.rb
 - lib/delayed/message_sending.rb
 - lib/delayed/performable_method.rb
 - lib/delayed/periodic.rb
@@ -314,11 +317,12 @@ files:
 - lib/delayed/yaml_extensions.rb
 - lib/delayed_job.rb
 - spec/active_record_job_spec.rb
-- spec/delayed/pool_spec.rb
+- spec/delayed/cli_spec.rb
+- spec/delayed/daemon_spec.rb
 - spec/delayed/server_spec.rb
+- spec/delayed/settings_spec.rb
 - spec/delayed/worker_spec.rb
 - spec/gemfiles/32.gemfile
-- spec/gemfiles/32.gemfile.lock
 - spec/gemfiles/40.gemfile
 - spec/gemfiles/41.gemfile
 - spec/gemfiles/42.gemfile
@@ -352,17 +356,18 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.7
+rubygems_version: 2.5.1
 signing_key: 
 specification_version: 4
 summary: Instructure-maintained fork of delayed_job
 test_files:
 - spec/active_record_job_spec.rb
-- spec/delayed/pool_spec.rb
+- spec/delayed/cli_spec.rb
+- spec/delayed/daemon_spec.rb
 - spec/delayed/server_spec.rb
+- spec/delayed/settings_spec.rb
 - spec/delayed/worker_spec.rb
 - spec/gemfiles/32.gemfile
-- spec/gemfiles/32.gemfile.lock
 - spec/gemfiles/40.gemfile
 - spec/gemfiles/41.gemfile
 - spec/gemfiles/42.gemfile
@@ -377,4 +382,3 @@ test_files:
 - spec/shared/worker.rb
 - spec/shared_jobs_specs.rb
 - spec/spec_helper.rb
-has_rdoc: 

data/spec/delayed/pool_spec.rb

@@ -1,11 +0,0 @@
-require 'spec_helper'
-
-RSpec.describe Delayed::Pool do
-  describe '#parse_cli_options!' do
-    it 'must correctly parse the --config option' do
-      pool = Delayed::Pool.new(%w{run --config /path/to/some/file.yml})
-      pool.parse_cli_options!
-      expect(pool.options).to include config_file: '/path/to/some/file.yml'
-    end
-  end
-end

data/spec/gemfiles/32.gemfile.lock

@@ -1,155 +0,0 @@
-PATH
-  remote: ../../
-  specs:
-    canvas-jobs (0.9.14)
-      after_transaction_commit (= 1.0.1)
-      rails (>= 3.2)
-      redis (> 3.0)
-      redis-scripting (~> 1.0.1)
-      rufus-scheduler (~> 3.1.2)
-
-GEM
-  remote: https://rubygems.org/
-  specs:
-    actionmailer (3.2.22)
-      actionpack (= 3.2.22)
-      mail (~> 2.5.4)
-    actionpack (3.2.22)
-      activemodel (= 3.2.22)
-      activesupport (= 3.2.22)
-      builder (~> 3.0.0)
-      erubis (~> 2.7.0)
-      journey (~> 1.0.4)
-      rack (~> 1.4.5)
-      rack-cache (~> 1.2)
-      rack-test (~> 0.6.1)
-      sprockets (~> 2.2.1)
-    activemodel (3.2.22)
-      activesupport (= 3.2.22)
-      builder (~> 3.0.0)
-    activerecord (3.2.22)
-      activemodel (= 3.2.22)
-      activesupport (= 3.2.22)
-      arel (~> 3.0.2)
-      tzinfo (~> 0.3.29)
-    activeresource (3.2.22)
-      activemodel (= 3.2.22)
-      activesupport (= 3.2.22)
-    activesupport (3.2.22)
-      i18n (~> 0.6, >= 0.6.4)
-      multi_json (~> 1.0)
-    after_transaction_commit (1.0.1)
-      activerecord (>= 3.2)
-    arel (3.0.3)
-    backports (3.6.6)
-    builder (3.0.4)
-    bump (0.5.2)
-    coderay (1.1.0)
-    database_cleaner (1.3.0)
-    diff-lcs (1.2.5)
-    erubis (2.7.0)
-    hike (1.2.3)
-    i18n (0.7.0)
-    journey (1.0.4)
-    json (1.8.3)
-    mail (2.5.4)
-      mime-types (~> 1.16)
-      treetop (~> 1.4.8)
-    method_source (0.8.2)
-    mime-types (1.25.1)
-    multi_json (1.11.2)
-    pg (0.18.2)
-    polyglot (0.3.5)
-    pry (0.10.1)
-      coderay (~> 1.1.0)
-      method_source (~> 0.8.1)
-      slop (~> 3.4)
-    rack (1.4.7)
-    rack-cache (1.2)
-      rack (>= 0.4)
-    rack-protection (1.5.3)
-      rack
-    rack-ssl (1.3.4)
-      rack
-    rack-test (0.6.3)
-      rack (>= 1.0)
-    rails (3.2.22)
-      actionmailer (= 3.2.22)
-      actionpack (= 3.2.22)
-      activerecord (= 3.2.22)
-      activeresource (= 3.2.22)
-      activesupport (= 3.2.22)
-      bundler (~> 1.0)
-      railties (= 3.2.22)
-    railties (3.2.22)
-      actionpack (= 3.2.22)
-      activesupport (= 3.2.22)
-      rack-ssl (~> 1.3.2)
-      rake (>= 0.8.7)
-      rdoc (~> 3.4)
-      thor (>= 0.14.6, < 2.0)
-    rake (10.4.2)
-    rdoc (3.12.2)
-      json (~> 1.4)
-    redis (3.2.1)
-    redis-scripting (1.0.1)
-      redis (>= 3.0)
-    rspec (3.1.0)
-      rspec-core (~> 3.1.0)
-      rspec-expectations (~> 3.1.0)
-      rspec-mocks (~> 3.1.0)
-    rspec-core (3.1.7)
-      rspec-support (~> 3.1.0)
-    rspec-expectations (3.1.2)
-      diff-lcs (>= 1.2.0, < 2.0)
-      rspec-support (~> 3.1.0)
-    rspec-mocks (3.1.3)
-      rspec-support (~> 3.1.0)
-    rspec-support (3.1.2)
-    rufus-scheduler (3.1.3)
-    sinatra (1.4.6)
-      rack (~> 1.4)
-      rack-protection (~> 1.4)
-      tilt (>= 1.3, < 3)
-    sinatra-contrib (1.4.6)
-      backports (>= 2.0)
-      multi_json
-      rack-protection
-      rack-test
-      sinatra (~> 1.4.0)
-      tilt (>= 1.3, < 3)
-    slop (3.6.0)
-    sprockets (2.2.3)
-      hike (~> 1.2)
-      multi_json (~> 1.0)
-      rack (~> 1.0)
-      tilt (~> 1.1, != 1.3.0)
-    test_after_commit (0.4.1)
-      activerecord (>= 3.2)
-    thor (0.19.1)
-    tilt (1.4.1)
-    timecop (0.7.1)
-    treetop (1.4.15)
-      polyglot
-      polyglot (>= 0.3.1)
-    tzinfo (0.3.44)
-    wwtd (0.7.0)
-
-PLATFORMS
-  ruby
-
-DEPENDENCIES
-  bump
-  canvas-jobs!
-  database_cleaner (= 1.3.0)
-  pg
-  pry
-  rack-test
-  rails (~> 3.2.19)
-  rake
-  rspec (= 3.1.0)
-  sinatra
-  sinatra-contrib
-  test_after_commit (= 0.4.1)
-  timecop (= 0.7.1)
-  wwtd (= 0.7.0)