inst-jobs 0.15.0 → 0.16.0

data/lib/delayed/backend/redis/job.rb

@@ -223,7 +223,8 @@ class Job
       min_priority = Delayed::MIN_PRIORITY,
       max_priority = Delayed::MAX_PRIORITY,
       prefetch: nil,
-      prefetch_owner: nil)
+      prefetch_owner: nil,
+      forced_latency: nil)
 
     check_queue(queue)
     check_priorities(min_priority, max_priority)
@@ -234,7 +235,9 @@ class Job
 
     # as an optimization this lua function returns the hash of job attributes,
     # rather than just a job id, saving a round trip
-    job_attrs = functions.get_and_lock_next_available(worker_name, queue, min_priority, max_priority, db_time_now)
+    now = db_time_now
+    now -= forced_latency if forced_latency
+    job_attrs = functions.get_and_lock_next_available(worker_name, queue, min_priority, max_priority, now)
     job = instantiate_from_attrs(job_attrs) # will return nil if the attrs are blank
     if multiple_workers
       if job.nil?
@@ -450,7 +453,10 @@ class Job
             [singleton.run_at, run_at].min
           when :overwrite
             run_at
+          when :loose
+            singleton.run_at
           end
+        singleton.handler = self.handler if self.on_conflict == :overwrite
         singleton.save! if singleton.changed?
         COLUMNS.each { |c| send("#{c}=", singleton.send(c)) }
       end

data/lib/delayed/batch.rb

@@ -30,7 +30,7 @@ module Delayed
       private
       def prepare_batches(mode, opts)
         raise "nested batching is not supported" if Delayed::Job.batches
-        Delayed::Job.batches = Hash.new { |h,k| h[k] = [] }
+        Delayed::Job.batches = Hash.new { |h,k| h[k] = Set.new }
         batch_enqueue_args = [:queue]
         batch_enqueue_args << :priority unless opts[:priority]
         Delayed::Job.batch_enqueue_args = batch_enqueue_args

data/lib/delayed/lifecycle.rb

@@ -11,6 +11,7 @@ module Delayed
       :perform          => [:worker, :job],
       :pop              => [:worker],
       :work_queue_pop   => [:work_queue, :worker_config],
+      :check_for_work   => [:work_queue],
     }
 
     def initialize

data/lib/delayed/periodic.rb

@@ -1,4 +1,4 @@
-require 'rufus/scheduler'
+require 'fugit'
 
 module Delayed
 class Periodic
@@ -15,7 +15,7 @@ class Periodic
   def self.add_overrides(overrides)
     overrides.each do |name, cron_line|
       # throws error if the line is malformed
-      Rufus::Scheduler::CronLine.new(cron_line)
+      Fugit.do_parse_cron(cron_line)
     end
     self.overrides.merge!(overrides)
   end
@@ -41,13 +41,16 @@ class Periodic
 
   def initialize(name, cron_line, job_args, block)
     @name = name
-    @cron = Rufus::Scheduler::CronLine.new(cron_line)
+    @cron = Fugit.do_parse_cron(cron_line)
     @job_args = { :priority => Delayed::LOW_PRIORITY }.merge(job_args.symbolize_keys)
     @block = block
   end
 
   def enqueue
-    Delayed::Job.enqueue(self, @job_args.merge(:max_attempts => 1, :run_at => @cron.next_time(Delayed::Periodic.now).utc.to_time, :singleton => tag))
+    Delayed::Job.enqueue(self, @job_args.merge(:max_attempts => 1,
+                                               :run_at => @cron.next_time(Delayed::Periodic.now).utc.to_time,
+                                               :singleton => tag,
+                                               on_conflict: :patient))
   end
 
   def perform

data/lib/delayed/server.rb

@@ -16,12 +16,18 @@ module Delayed
       if using_active_record? && !ActiveRecord::Base.connected?
         ActiveRecord::Base.establish_connection(ENV['DATABASE_URL'])
       end
+
+      @allow_update = args.length > 0 && args[0][:update]
     end
 
     def using_active_record?
       Delayed::Job == Delayed::Backend::ActiveRecord::Job
     end
 
+    def allow_update
+      @allow_update
+    end
+
     # Ensure we're connected to the DB before processing the request
     before do
       if ActiveRecord::Base.respond_to?(:verify_active_connections!) && using_active_record?
@@ -100,6 +106,19 @@ module Delayed
       })
     end
 
+    post '/bulk_update' do
+      content_type :json
+
+      halt 403 unless @allow_update
+
+      payload = JSON.parse(request.body.read).symbolize_keys
+      Delayed::Job.bulk_update(payload[:action], { ids: payload[:ids] })
+
+      json({
+        success: true
+      })
+    end
+
     private
 
     def extract_page_size

data/lib/delayed/server/helpers.rb

@@ -20,6 +20,7 @@ module Delayed
             running: url_path('running'),
             tags: url_path('tags'),
             jobs: url_path('jobs'),
+            bulkUpdate: url_path('bulk_update'),
           }
         }.to_json
       end

data/lib/delayed/server/public/js/app.js

@@ -97,7 +97,8 @@
                 {"data": "priority"},
                 {"data": "strand"},
                 {"data": "run_at"}
-            ]
+            ],
+            select: true
         });
 
         $('body').on('click', '.refresh_jobs_link', function(event) {
@@ -106,6 +107,53 @@
             return true
         });
 
+        function bulkUpdate(action) {
+            var selectedRows = jobsTable.rows( { selected: true } );
+            var n = selectedRows.count();
+            if(action == 'destroy') {
+                if(!confirm("Are you sure you want to delete " + n + " jobs?")) {
+                    return;
+                }
+            }
+            var itemIds = [];
+            var i;
+            for (i = 0; i < n; ++i) {
+                itemIds.push(selectedRows.data()[i].id);
+            }
+            var data = {
+                action: action,
+                items: itemIds
+            }
+            $.ajax({
+                type: "POST",
+                contentType : 'application/json',
+                url: ENV.Routes.bulkUpdate,
+                data: JSON.stringify(data),
+                success: function( ) {
+                    jobsTable.rows().deselect();
+                    jobsTable.ajax.reload();
+                },
+              });
+        }
+
+        $('body').on('click', '.hold_selection_link', function(event) {
+            event.preventDefault();
+            bulkUpdate('hold');
+            return true
+        });
+
+        $('body').on('click', '.unhold_selection_link', function(event) {
+            event.preventDefault();
+            bulkUpdate('unhold');
+            return true
+        });
+
+        $('body').on('click', '.delete_selection_link', function(event) {
+            event.preventDefault();
+            bulkUpdate('destroy');
+            return true
+        });
+
         $('body').on('change', 'select#current_jobs_flavor', function (event) {
             event.preventDefault();
             $selectBox = $(this);

data/lib/delayed/server/views/index.erb

@@ -60,13 +60,28 @@
         </span>
       </div>
     </div>
-    <div class="col-md-6">
+    <div class="col-md-<%= allow_update ? 2 : 6 %>">
     </div>
     <div class="col-md-1">
       <a href="" class="btn btn-default pull-right refresh_jobs_link" aria-label="Refresh Jobs Table">
         <span class="glyphicon glyphicon-refresh"></span> Refresh
       </a>
     </div>
+    <% if allow_update %>
+    <div class="col-md-4">
+      <span>With selection: </span>
+      <br/>
+      <a href="" class="btn btn-default hold_selection_link" aria-label="Hold Selected Jobs">
+        <span class="glyphicon glyphicon-pause"></span> Hold
+      </a>
+      <a href="" class="btn btn-default unhold_selection_link" aria-label="Un-hold Selected Jobs">
+        <span class="glyphicon glyphicon-play"></span> Un-Hold
+      </a>
+      <a href="" class="btn btn-default delete_selection_link" aria-label="Delete Selected Jobs">
+        <span class="glyphicon glyphicon-trash"></span> Delete
+      </a>
+    </div>
+    <% end %>
   </div>
 </div>
 

data/lib/delayed/server/views/layout.erb

@@ -9,12 +9,14 @@ window.ENV = <%= render_javascript_env %>
 </script>
 <script type="text/javascript" charset="utf8 "src="//code.jquery.com/jquery-2.1.3.min.js" defer></script>
 <script type="text/javascript" charset="utf8" src="//maxcdn.bootstrapcdn.com/bootstrap/3.3.4/js/bootstrap.min.js" defer></script>
-<script type="text/javascript" charset="utf8" src="//cdn.datatables.net/1.10.6/js/jquery.dataTables.js" defer></script>
-<script type="text/javascript" charset="utf8" src="//cdn.datatables.net/plug-ins/1.10.6/integration/bootstrap/3/dataTables.bootstrap.js" defer></script>
+<script type="text/javascript" charset="utf8" src="//cdn.datatables.net/1.10.19/js/jquery.dataTables.js" defer></script>
+<script type="text/javascript" charset="utf8" src="//cdn.datatables.net/select/1.2.7/js/dataTables.select.min.js" defer></script>
+<script type="text/javascript" charset="utf8" src="//cdn.datatables.net/plug-ins/1.10.19/integration/bootstrap/3/dataTables.bootstrap.js" defer></script>
 <script type="text/javascript" charset="utf8" src="<%= url_path 'js/app.js' %>" defer></script>
 
 <link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.4/css/bootstrap.min.css">
-<link rel="stylesheet" href="//cdn.datatables.net/plug-ins/1.10.6/integration/bootstrap/3/dataTables.bootstrap.css">
+<link rel="stylesheet" href="//cdn.datatables.net/plug-ins/1.10.19/integration/bootstrap/3/dataTables.bootstrap.css">
+<link rel="stylesheet" href="//cdn.datatables.net/select/1.2.7/css/select.dataTables.min.css">
 <link rel="stylesheet" href="<%= url_path 'css/app.css' %>">
 
 <title>Delayed Jobs</title>

data/lib/delayed/settings.rb

@@ -23,7 +23,10 @@ module Delayed
       :worker_health_check_config,
       :worker_procname_prefix,
     ]
-    SETTINGS_WITH_ARGS = [ :num_strands ]
+    SETTINGS_WITH_ARGS = [
+      :job_detailed_log_format,
+      :num_strands
+    ]
 
     SETTINGS.each do |setting|
       mattr_writer(setting)
@@ -65,6 +68,7 @@ module Delayed
 
     self.num_strands = ->(strand_name){ nil }
     self.default_job_options = ->{ Hash.new }
+    self.job_detailed_log_format = ->(job){ job.to_json(include_root: false, only: %w(tag strand priority attempts created_at max_attempts source)) }
 
     # Send workers KILL after QUIT if they haven't exited within the
     # slow_exit_timeout

data/lib/delayed/version.rb

@@ -1,3 +1,3 @@
 module Delayed
-  VERSION = "0.15.0"
+  VERSION = "0.16.0"
 end

data/lib/delayed/work_queue/parent_process/server.rb

@@ -5,7 +5,7 @@ class ParentProcess
     attr_reader :clients, :listen_socket
 
     include Delayed::Logging
-    SIGNALS = %i{INT TERM QUIT CHLD}
+    SIGNALS = %i{INT TERM QUIT}
 
     def initialize(listen_socket, parent_pid: nil, config: Settings.parent_process)
       @listen_socket = listen_socket
@@ -59,12 +59,20 @@ class ParentProcess
 
     def run_once
       handles = @clients.keys + [@listen_socket, @self_pipe[0]]
+      # if we're currently idle, then force a "latency" to job fetching - don't
+      # fetch recently queued jobs, allowing busier workers to fetch them first.
+      # if they're not keeping up, the jobs will slip back in time, and suddenly we'll become
+      # active and quickly pick up all the jobs we can. The latency is calculated to ensure that
+      # an active worker is guaranteed to have attempted to fetch new jobs in the meantime
+      forced_latency = Settings.sleep_delay + Settings.sleep_delay_stagger * 2 if all_workers_idle?
       timeout = Settings.sleep_delay + (rand * Settings.sleep_delay_stagger)
       readable, _, _ = IO.select(handles, nil, nil, timeout)
       if readable
         readable.each { |s| handle_read(s) }
       end
-      check_for_work
+      Delayed::Worker.lifecycle.run_callbacks(:check_for_work, self) do
+        check_for_work(forced_latency: forced_latency)
+      end
       unlock_timed_out_prefetched_jobs
     end
 
@@ -111,7 +119,7 @@ class ParentProcess
       drop_socket(socket)
     end
 
-    def check_for_work
+    def check_for_work(forced_latency: nil)
       @waiting_clients.each do |(worker_config, workers)|
         prefetched_jobs = @prefetched_jobs[worker_config] ||= []
         logger.debug("I have #{prefetched_jobs.length} jobs for #{workers.length} waiting workers")
@@ -124,6 +132,7 @@ class ParentProcess
             workers.unshift(client)
             next
           end
+          client.working = true
           begin
             logger.debug("Sending prefetched job #{job.id} to #{client.name}")
             client_timeout { Marshal.dump(job, client.socket) }
@@ -148,7 +157,8 @@ class ParentProcess
               worker_config[:min_priority],
               worker_config[:max_priority],
               prefetch: Settings.fetch_batch_size * (worker_config[:workers] || 1) - recipients.length,
-              prefetch_owner: prefetch_owner)
+              prefetch_owner: prefetch_owner,
+              forced_latency: forced_latency)
           logger.debug("Fetched and locked #{response.values.flatten.size} new jobs for workers (#{response.keys.join(', ')}).")
           response.each do |(worker_name, job)|
             if worker_name == prefetch_owner

data/lib/delayed/worker.rb

@@ -63,11 +63,19 @@ class Worker
     if app && !app.config.cache_classes
       Delayed::Worker.lifecycle.around(:perform) do |worker, job, &block|
         reload = app.config.reload_classes_only_on_change != true || app.reloaders.map(&:updated?).any?
-        ActionDispatch::Reloader.prepare! if reload
+
+        if reload
+          if defined?(ActiveSupport::Reloader)
+            Rails.application.reloader.reload!
+          else
+            ActionDispatch::Reloader.prepare!
+          end
+        end
+
         begin
           block.call(worker, job)
         ensure
-          ActionDispatch::Reloader.cleanup! if reload
+          ActionDispatch::Reloader.cleanup! if reload && !defined?(ActiveSupport::Reloader)
         end
       end
     end
@@ -101,7 +109,7 @@ class Worker
       trap(sig) { @signal_queue << sig; wake_up }
     end
 
-    health_check.start
+    raise 'Could not register health_check' unless health_check.start
 
     signal_processor = Thread.new do
       loop do
@@ -110,6 +118,7 @@ class Worker
         when :INT, :TERM
           @exit = true # get the main thread to bail early if it's waiting for a job
           work_thread.raise(SystemExit) # Force the main thread to bail out of the current job
+          cleanup! # we're going to get SIGKILL'd in a moment, so clean up asap
           break
         when :QUIT
           @exit = true
@@ -130,13 +139,26 @@ class Worker
     Rails.logger.fatal("Child process died: #{e.inspect}") rescue nil
     self.class.lifecycle.run_callbacks(:exceptional_exit, self, e) { }
   ensure
-    health_check.stop
-    work_queue.close
+    cleanup!
+
     if signal_processor
       signal_processor.kill
       signal_processor.join
     end
+  end
+
+  def cleanup!
+    return if cleaned?
+
+    health_check.stop
+    work_queue.close
     Delayed::Job.clear_locks!(name)
+
+    @cleaned = true
+  end
+
+  def cleaned?
+    @cleaned
   end
 
   def run
@@ -231,7 +253,7 @@ class Worker
   def log_job(job, format = :short)
     case format
     when :long
-      "#{job.full_name} #{ job.to_json(:include_root => false, :only => %w(tag strand priority attempts created_at max_attempts source)) }"
+      "#{job.full_name} #{ Settings.job_detailed_log_format.call(job) }"
     else
       job.full_name
     end

data/lib/delayed/worker/consul_health_check.rb

@@ -59,7 +59,7 @@ module Delayed
 
       def check_attributes
         {
-          script: check_script,
+          args: ['bash', '-c', check_script],
           status: 'passing',
           interval: @config.fetch(:check_interval, '5m'),
           deregister_critical_service_after: @config.fetch(:deregister_service_delay, '10m'),

data/lib/delayed/worker/health_check.rb

@@ -32,11 +32,15 @@ module Delayed
             # prefetched jobs have their own way of automatically unlocking themselves
             next if job.locked_by.start_with?("prefetch:")
             unless live_workers.include?(job.locked_by)
-              Delayed::Job.transaction do
-                # double check that the job is still there. locked_by will immediately be reset
-                # to nil in this transaction by Job#reschedule
-                next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
-                job.reschedule
+              begin
+                Delayed::Job.transaction do
+                  # double check that the job is still there. locked_by will immediately be reset
+                  # to nil in this transaction by Job#reschedule
+                  next unless Delayed::Job.where(id: job, locked_by: job.locked_by).update_all(locked_by: "abandoned job cleanup") == 1
+                  job.reschedule
+                end
+              rescue
+                ::Rails.logger.error "Failure rescheduling abandoned job #{job.id} #{$!.inspect}"
               end
             end
           end

data/lib/delayed/worker/null_health_check.rb

@@ -3,7 +3,13 @@ module Delayed
     class NullHealthCheck < HealthCheck
       self.type_name = :none
 
-      attr_reader *%i{start stop}
+      def start
+        true
+      end
+
+      def stop
+        true
+      end
 
       def live_workers; []; end
     end