web47core 3.4.11.1 → 3.4.11.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ec4e1cf4e1bcf71a17721efcb838e63544ad5633bedd5864974dfdbd31668fb0
-  data.tar.gz: f2b92e7bd1de8cecbb9c1647110bc439fa0ee329901d00861ced2233299d2226
+  metadata.gz: 0c6cbc481a022be4632e12894dcbf3c8e91dc7b86a450e94c99627651bd4c827
+  data.tar.gz: 9cf223d9031627668cebcb06ceff3c2f50865a50504140dfc3c24f9e12208a4e
 SHA512:
-  metadata.gz: a43c7ec4df0ca64cdd9e5618436ff57b0d9585c5bf440107051ccdb08a6bec9f587bd73da6e82a555e411d0289f387df92b2607a4bcc8e28c3afa20ded453592
-  data.tar.gz: c136d963dd42d8364a0ae3e4e5924894a0d20ebfad418eac8af67cfea38dfd03c5e6748fc316f44ec606f18a0a357a36e55e4c9ea46f23a79592d480c8d50dc9
+  metadata.gz: 186a56e4ee962b9b4ee7b925f684738f383fd2837ffbd2c74431c41faa35894d950dc4ee442503dadb0bd0d9fc85c1e183bb20640b3ba6655d9ea1620835eba7
+  data.tar.gz: 2597dad347f6e7083883202160e081418173b7f27ded2e5c4ce8034880bf7e446a5d9e4b599a4bf63ef1eba85df83ae55bbd2f491a45bb6f504adf911663e6f7

data/README.md

@@ -48,6 +48,10 @@ The `web47core` project is a gem that will be deployed via [Ruby Gems](https://r
 2.  Post the gem to the DevOps channel in slack, ask for it to be added to the S3 repo.
 
 ## Usage
+### Worker autoscaling
+
+`AutoScaler.check` follows `Web47core::Config.worker_ladder` and keeps existing workers until the non-failed job queue is empty. If jobs are queued but both the ladder and ECS desired count are zero, it requests one worker after more than `SystemConfiguration.aws_worker_max_wait_minutes` without a worker check-in (default: 45 minutes). Set the wait with `SystemConfiguration.configuration.update!(aws_worker_max_wait_minutes: 30)`; it must be a positive integer. With no check-in history, it requests one immediately. The cron primary checks once a minute; ECS startup takes additional time.
+
 ### Importing the gem
 To use the `web47core` gem in a project, first add the gem to your Gemfile in one of two ways
 
@@ -256,13 +260,14 @@ bundle exec delayed_job stop
 ```
 
 ##### Delayed Job Management
-Since 2.1.0, there is now a built in Delayed Job monitor that will monitor delayed jobs and auto restart them if they 
-exceed the max allowed time for a job to run. To configure this, update the delayed job initializer to add the `TimeKeeper` plugin.
+Register `TimeKeeper` once in the host application’s delayed-job initializer. It records worker check-ins and job runs, which the autoscaler uses to detect idle workers.
 
 ```ruby
-Delayed::Worker.plugins += [Delayed::Plugins::TimeKeeper]
+Delayed::Worker.plugins |= [Delayed::Plugins::TimeKeeper]
 ```
 
+Automatic orphaned-job restarts remain controlled by `delayed_job_restart_orphaned`, which defaults to false.
+
 This will add a few now cron jobs as well as objects to the project, you can view these by adding the following routes near the delayed jobs area
 
 ```ruby

data/lib/app/models/concerns/aws_configuration.rb

@@ -17,9 +17,12 @@ module AwsConfiguration
       field :aws_desired_worker_count, type: Integer, default: 0
       # ECS service desired count from the last DescribeServices. Not the running task count.
       field :aws_current_worker_count, type: Integer, default: 0
+      # Maximum time without a worker check-in before overriding a zero-worker rung.
+      field :aws_worker_max_wait_minutes, type: Integer, default: 45
 
       validates :aws_desired_worker_count, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
       validates :aws_current_worker_count, numericality: { only_integer: true, greater_than_or_equal_to: 0 }
+      validates :aws_worker_max_wait_minutes, numericality: { only_integer: true, greater_than: 0 }
     end
   end
 

data/lib/app/services/auto_scaler.rb

@@ -9,6 +9,10 @@ require 'singleton'
 # the worker ladder. It does not shrink the fleet while any job is still
 # queued, including when the ladder maps that depth to zero workers. It
 # scales to zero only when the queue is empty.
+# If the ladder and live desired count are zero with jobs queued, check
+# starts one worker after the configured wait without a worker check-in
+# (or immediately when there is no worker history). SystemConfiguration's
+# aws_worker_max_wait_minutes defaults to 45.
 # warm_up raises the fleet to at least the requested floor. bump_worker adds
 # one worker. Neither call consults the ladder or the queue, and neither
 # lowers a fleet that is already larger.
@@ -102,6 +106,8 @@ class AutoScaler
       return
     end
 
+    target = 1 if job_count.positive? && target.zero? && current.zero? && worker_check_in_overdue?
+
     # Keep the live fleet while any job remains. The ladder still chooses a
     # larger fleet. Scale down only once the queue is empty.
     if job_count.positive? && target < current
@@ -123,6 +129,14 @@ class AutoScaler
     update_service(target)
   end
 
+  def worker_check_in_overdue?
+    cutoff = configuration.aws_worker_max_wait_minutes.minutes.ago
+    !Delayed::Jobs::Worker.where(:last_check_in_at.gte => cutoff).exists?
+  rescue StandardError => error
+    App47Logger.log_error 'Unable to determine worker max wait minutes for worker', error
+    false
+  end
+
   def desired_for(job_count)
     worker_ladder.each do |range, workers|
       return workers if range.cover?(job_count)

data/lib/web47core/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Web47core
-  VERSION = '3.4.11.1'
+  VERSION = '3.4.11.2'
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: web47core
 version: !ruby/object:Gem::Version
-  version: 3.4.11.1
+  version: 3.4.11.2
 platform: ruby
 authors:
 - Chris Schroeder