inst-jobs 3.0.8 → 3.1.14

data/lib/delayed/work_queue/parent_process/server.rb

@@ -163,7 +163,7 @@ module Delayed
                 forced_latency: forced_latency
               )
               logger.debug(
-                "Fetched and locked #{response.values.flatten.size} new jobs for workers (#{response.keys.join(', ')})."
+                "Fetched and locked #{response.values.flatten.size} new jobs for workers (#{response.keys.join(", ")})."
               )
               response.each do |(worker_name, locked_jobs)|
                 if worker_name == prefetch_owner
@@ -226,7 +226,11 @@ module Delayed
 
             # otherwise just reconnect and let it retry
             logger.warn("failed to unlock prefetched jobs - connection terminated; skipping for now")
-            Delayed::Job.clear_all_connections!
+            if Rails.version < "6.1"
+              ::Delayed::Job.clear_all_connections!
+            else
+              ::Delayed::Job.clear_all_connections!(nil)
+            end
           end
         end
 
@@ -263,7 +267,7 @@ module Delayed
         end
 
         def prefetch_owner
-          "prefetch:#{Socket.gethostname rescue 'X'}"
+          "prefetch:#{Socket.gethostname rescue "X"}"
         end
 
         def parent_exited?

data/lib/delayed/worker/health_check.rb

@@ -34,7 +34,7 @@ module Delayed
             # no other worker is trying to do this right now (and if we abandon the
             # operation, the transaction will end, releasing the advisory lock).
             result = Delayed::Job.attempt_advisory_lock("Delayed::Worker::HealthCheck#reschedule_abandoned_jobs")
-            return unless result
+            next unless result
 
             horizon = 5.minutes.ago
 

data/lib/delayed/worker/process_helper.rb

@@ -9,9 +9,9 @@ module Delayed
       ALIVE_CHECK_LINUX = '[ -d "/proc/$WORKER_PID" ]'
       ALIVE_CHECK_MAC = "ps -p $WORKER_PID > /dev/null"
       ALIVE_CHECK = RUBY_PLATFORM.include?("darwin") ? ALIVE_CHECK_MAC : ALIVE_CHECK_LINUX
-      SCRIPT_TEMPLATE = <<-BASH
-        WORKER_PID="%<pid>d" # an example, filled from ruby when the check is created
-        ORIGINAL_MTIME="%<mtime>s" # an example, filled from ruby when the check is created
+      SCRIPT_TEMPLATE = <<~SH
+        WORKER_PID="%{pid}" # an example, filled from ruby when the check is created
+        ORIGINAL_MTIME="%{mtime}" # an example, filled from ruby when the check is created
 
         if #{ALIVE_CHECK}; then
             CURRENT_MTIME=$(#{STAT})
@@ -26,7 +26,7 @@ module Delayed
         else
             exit 255 # The process is no more, trigger a "critical" state.
         fi
-      BASH
+      SH
 
       def self.mtime(pid)
         if RUBY_PLATFORM.include?("darwin")

data/lib/delayed/worker.rb

@@ -64,6 +64,7 @@ module Delayed
       @max_priority = options[:max_priority]
       @max_job_count = options[:worker_max_job_count].to_i
       @max_memory_usage = options[:worker_max_memory_usage].to_i
+      @memory_high_water = options[:worker_memory_high_water]&.to_i || (@max_memory_usage / 2)
       @work_queue = options.delete(:work_queue) || WorkQueue::InProcess.new
       @health_check_type = Settings.worker_health_check_type
       @health_check_config = Settings.worker_health_check_config
@@ -77,7 +78,7 @@ module Delayed
     end
 
     def name
-      @name ||= "#{Socket.gethostname rescue 'X'}:#{id}"
+      @name ||= "#{Socket.gethostname rescue "X"}:#{id}"
     end
 
     def process_name=(new_name)
@@ -99,8 +100,13 @@ module Delayed
 
     def start
       logger.info "Starting worker"
+      begin
+        Process.setrlimit(:DATA, @max_memory_usage, @max_memory_usage * 2) if @max_memory_usage.positive?
+      rescue Errno::EINVAL
+        # couldn't set an OS-level limit
+      end
       self.process_name =
-        "start:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}"
+        "start:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || "max"}"
       @self_pipe = IO.pipe
       work_queue.init
 
@@ -170,7 +176,7 @@ module Delayed
 
       self.class.lifecycle.run_callbacks(:loop, self) do
         self.process_name =
-          "pop:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}"
+          "pop:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || "max"}"
         job = self.class.lifecycle.run_callbacks(:pop, self) do
           work_queue.get_and_lock_next_available(name, config)
         end
@@ -184,10 +190,10 @@ module Delayed
               @exit = true
             end
 
-            if @max_memory_usage.positive?
+            if @memory_high_water.positive?
               memory = sample_memory
-              if memory > @max_memory_usage
-                logger.debug "Memory usage of #{memory} exceeds max of #{@max_memory_usage}, dying"
+              if memory > @memory_high_water
+                logger.debug "Memory usage of #{memory} exceeds high water of #{@memory_high_water}, dying"
                 @exit = true
               else
                 logger.debug "Memory usage: #{memory}"
@@ -196,7 +202,7 @@ module Delayed
           end
         else
           self.process_name =
-            "wait:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || 'max'}"
+            "wait:#{Settings.worker_procname_prefix}#{@queue_name}:#{min_priority || 0}:#{max_priority || "max"}"
           sleep(Settings.sleep_delay + (rand * Settings.sleep_delay_stagger)) unless exit?
         end
       end
@@ -220,7 +226,7 @@ module Delayed
             end
             job.destroy
           end
-          logger.info("Completed #{log_job(job)} #{format('%.0fms', (runtime * 1000))}")
+          logger.info("Completed #{log_job(job, :short)} #{format("%.0fms", (runtime * 1000))}")
         end
       rescue ::Delayed::RetriableError => e
         can_retry = job.attempts + 1 < job.inferred_max_attempts
@@ -233,6 +239,11 @@ module Delayed
         # still reschedule the job though.
         job.reschedule(e)
       rescue Exception => e # rubocop:disable Lint/RescueException
+        if e.is_a?(NoMemoryError)
+          GC.start # try and free up some memory before reporting the error
+          logger.debug "Could not allocate memory (max is #{@max_memory_usage}), dying"
+          @exit = true
+        end
         self.class.lifecycle.run_callbacks(:error, self, job, e) do
           handle_failed_job(job, e)
         end
@@ -269,17 +280,17 @@ module Delayed
       when :long
         "#{job.full_name} #{Settings.job_detailed_log_format.call(job)}"
       else
-        job.full_name
+        "#{job.full_name} #{Settings.job_short_log_format.call(job)}".strip
       end
     end
 
     # set up the session context information, so that it gets logged with the job log lines
     # also set up a unique tmpdir, which will get removed at the end of the job.
     def configure_for_job(job)
-      previous_tmpdir = ENV["TMPDIR"]
+      previous_tmpdir = ENV.fetch("TMPDIR", nil)
 
       self.class.running_job(job) do
-        dir = Dir.mktmpdir("job-#{job.id}-#{name.gsub(/[^\w.]/, '.')}-")
+        dir = Dir.mktmpdir("job-#{job.id}-#{name.gsub(/[^\w.]/, ".")}-")
         begin
           ENV["TMPDIR"] = dir
           yield

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: inst-jobs
 version: !ruby/object:Gem::Version
-  version: 3.0.8
+  version: 3.1.14
 platform: ruby
 authors:
 - Cody Cutrer
@@ -10,7 +10,7 @@ authors:
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2022-02-03 00:00:00.000000000 Z
+date: 2023-11-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activerecord
@@ -116,20 +116,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '6.0'
-- !ruby/object:Gem::Dependency
-  name: appraisal
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.4'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '2.4'
 - !ruby/object:Gem::Dependency
   name: bump
   requirement: !ruby/object:Gem::Requirement
@@ -192,14 +178,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.1
+        version: 2.6.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 2.5.1
+        version: 2.6.3
 - !ruby/object:Gem::Dependency
   name: pg
   requirement: !ruby/object:Gem::Requirement
@@ -242,20 +228,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: rails
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -285,33 +257,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '3.10'
 - !ruby/object:Gem::Dependency
-  name: rubocop
+  name: rubocop-inst
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
+        version: '1'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.19'
-- !ruby/object:Gem::Dependency
-  name: rubocop-performance
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.12.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.12.0
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: rubocop-rails
   requirement: !ruby/object:Gem::Requirement
@@ -386,16 +344,16 @@ dependencies:
   name: timecop
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.4
+        version: '0.9'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.4
+        version: '0.9'
 - !ruby/object:Gem::Dependency
   name: webmock
   requirement: !ruby/object:Gem::Requirement
@@ -410,20 +368,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: wwtd
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.0
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: 1.4.0
 description: 
 email:
 - cody@instructure.com
@@ -473,6 +417,8 @@ files:
 - db/migrate/20220128084800_update_insert_trigger_for_singleton_unique_constraint_change.rb
 - db/migrate/20220128084900_update_delete_trigger_for_singleton_unique_constraint_change.rb
 - db/migrate/20220203063200_remove_old_singleton_index.rb
+- db/migrate/20220328152900_add_failed_jobs_indicies.rb
+- db/migrate/20220519204546_add_requeued_job_id_to_failed_jobs.rb
 - exe/inst_jobs
 - lib/delayed/backend/active_record.rb
 - lib/delayed/backend/base.rb
@@ -512,33 +458,10 @@ files:
 - lib/delayed/yaml_extensions.rb
 - lib/delayed_job.rb
 - lib/inst-jobs.rb
-- spec/active_record_job_spec.rb
-- spec/delayed/cli_spec.rb
-- spec/delayed/daemon_spec.rb
-- spec/delayed/message_sending_spec.rb
-- spec/delayed/periodic_spec.rb
-- spec/delayed/server_spec.rb
-- spec/delayed/settings_spec.rb
-- spec/delayed/work_queue/in_process_spec.rb
-- spec/delayed/work_queue/parent_process/client_spec.rb
-- spec/delayed/work_queue/parent_process/server_spec.rb
-- spec/delayed/work_queue/parent_process_spec.rb
-- spec/delayed/worker/consul_health_check_spec.rb
-- spec/delayed/worker/health_check_spec.rb
-- spec/delayed/worker_spec.rb
-- spec/migrate/20140924140513_add_story_table.rb
-- spec/sample_jobs.rb
-- spec/shared/delayed_batch.rb
-- spec/shared/delayed_method.rb
-- spec/shared/performable_method.rb
-- spec/shared/shared_backend.rb
-- spec/shared/testing.rb
-- spec/shared/worker.rb
-- spec/shared_jobs_specs.rb
-- spec/spec_helper.rb
 homepage: https://github.com/instructure/inst-jobs
 licenses: []
-metadata: {}
+metadata:
+  rubygems_mfa_required: 'true'
 post_install_message: 
 rdoc_options: []
 require_paths:
@@ -547,39 +470,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.6'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.1.6
 signing_key: 
 specification_version: 4
 summary: Instructure-maintained fork of delayed_job
-test_files:
-- spec/active_record_job_spec.rb
-- spec/delayed/cli_spec.rb
-- spec/delayed/daemon_spec.rb
-- spec/delayed/message_sending_spec.rb
-- spec/delayed/periodic_spec.rb
-- spec/delayed/server_spec.rb
-- spec/delayed/settings_spec.rb
-- spec/delayed/work_queue/in_process_spec.rb
-- spec/delayed/work_queue/parent_process/client_spec.rb
-- spec/delayed/work_queue/parent_process/server_spec.rb
-- spec/delayed/work_queue/parent_process_spec.rb
-- spec/delayed/worker/consul_health_check_spec.rb
-- spec/delayed/worker/health_check_spec.rb
-- spec/delayed/worker_spec.rb
-- spec/migrate/20140924140513_add_story_table.rb
-- spec/sample_jobs.rb
-- spec/shared/delayed_batch.rb
-- spec/shared/delayed_method.rb
-- spec/shared/performable_method.rb
-- spec/shared/shared_backend.rb
-- spec/shared/testing.rb
-- spec/shared/worker.rb
-- spec/shared_jobs_specs.rb
-- spec/spec_helper.rb
+test_files: []