thecore_backend_commons 3.9.1 → 3.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3aa559045828d8dc8ca4981fad05e6ff9ef2f80a7191f2e16db4f45ddcaf1c24
-  data.tar.gz: cd8fe33d14e2373a1b01ccd7290ceba1cb75048df57b6591a35c9cb135e077d5
+  metadata.gz: ae67ed15f7e3e684726abb005f023d769f0ee953ebce5d3f019baecb6d2a5d46
+  data.tar.gz: 76bfaf830d5bf571797f5fdd42794f087506eca3dd15c43ca0fc7ee6733871eb
 SHA512:
-  metadata.gz: 2a94d02e27478addb9660b69f693d4c8760e92dd11c5698130a7dc38601e1a745d3ca041ee6103204e4aeb070d138ef64877537debba042153ad5d2125650414
-  data.tar.gz: 15fc51c68e34e67042f8a02699639d5fb953e8710fce8e5923c225fa414e6c7c52c6ca5e22d3d618b817b88df09597dff65d31b35935bcbbc884b3678029ae2c
+  metadata.gz: 1b1554f540e0b404c3544ffee0cbc2ca531c861fc7d910a00892ca7867fbcc866f93e29860626afc4d41c4da31cc783cc73d5396235d8d3eb0c9fad78d9914a7
+  data.tar.gz: a25a66bf6cefa142a62fc176576efebb259424323563726a3256f132fea54c398b777b7782f501b59ea20de37ffca37201682aafb2eaa46a4c61db31d3d39b64

data/README.md

@@ -159,6 +159,67 @@ overrides it. Full mechanics (idempotency, abstract/STI exclusion, `to_prepare`
 
 ---
 
+## Background jobs: no retries by default
+
+Every app using this gem runs its background jobs on Sidekiq in production, and by default:
+
+- **a failed job is never retried** (`retry: false`): it runs once, and Sidekiq keeps it in
+  neither the retry set nor the Dead Set;
+- **only the latest failed job is kept in Redis**: `sidekiq-failures` (from
+  `thecore_background_jobs`) stores each failed job's full payload, arguments included, in the
+  Redis `failed` set; it is capped at 1, still visible in the Sidekiq web UI's Failures tab;
+- **every failure is written to the Rails log** by `ThecoreBackendCommons::DiscardedJobLogger`:
+  job class (or `Mailer#action`), queue, jid and error, never the arguments.
+
+Why zero: in a Thecore app in production, mail jobs carrying a whole PDF in their arguments
+failed with a permanent SMTP error and were retried for weeks, then kept in the Dead Set. Redis
+grew to 23 GB until the host's OOM killer put it in a restart loop and no job ran for three days
+(`docs/adr/0002-no-job-retries-failures-logged.md`).
+
+### Changing the defaults on an installation
+
+Set these in the environment of both the `backend` and the `worker` containers. On an
+installation deployed by `thecore_devcontainer` that is the target's
+`vendor/deploytargets/<provider>/<customer>.env`, which reaches both containers through their
+`env_file` (`thecore_devcontainer` ADR 0002); on a hand-written compose, its `environment:`.
+
+| Variable | Unset or blank | Accepted values | Effect |
+| --- | --- | --- | --- |
+| `THECORE_JOB_RETRIES` | `0` (no retries) | a whole number >= 0 | how many times Sidekiq retries a failed job, app-wide |
+| `THECORE_FAILED_JOBS_KEPT` | `1` | a whole number >= 1 | how many failed jobs stay in the Redis `failed` set |
+
+Any other value (`yes`, `-1`, `1.5`, ...) stops the app at boot with an error naming the
+variable: a typo never silently turns retries back on. With retries above 0, a job that uses
+them all ends in Sidekiq's Dead Set, which is capped by count and age but not by size.
+
+### Changing it for one job
+
+A job class sets its own retry count with Sidekiq's option, which wins over the app-wide value
+in both directions (it works on ActiveJob classes and mailers' jobs alike):
+
+```ruby
+class SyncInventoryJob < ApplicationJob
+  sidekiq_options retry: 3   # or retry: false when THECORE_JOB_RETRIES is above 0
+end
+```
+
+**Don't use ActiveJob's `retry_on` for this.** It retries independently of Sidekiq: on failure
+ActiveJob enqueues a new copy of the job, arguments included, whatever `THECORE_JOB_RETRIES`
+says, and the copies never show up as Sidekiq retries.
+
+### Logging failures somewhere else too
+
+Add your own Sidekiq death handler next to the default one; both run:
+
+```ruby
+# config/initializers/sidekiq.rb in the app
+Sidekiq.configure_server do |config|
+  config.death_handlers << ->(job, exception) { MyErrorLog.record(job["jid"], exception) }
+end
+```
+
+---
+
 ## Invio email e configurazione SMTP
 
 ### Configurazione

data/config/initializers/sidekiq.rb

@@ -0,0 +1,11 @@
+# What a failed background job leaves behind, app-wide, from the container's environment:
+# THECORE_JOB_RETRIES (default 0, no retries) and THECORE_FAILED_JOBS_KEPT (default 1, the latest
+# failure in the sidekiq-failures set); an invalid value refuses to boot (JobFailurePolicy).
+# Why: docs/adr/0002-no-job-retries-failures-logged.md.
+ThecoreBackendCommons::JobFailurePolicy.apply!
+
+# Every failed job leaves one line in the Rails log (DiscardedJobLogger); apps append their own
+# handlers next to it. Required, not autoloaded: see docs/adr/0002-no-job-retries-failures-logged.md.
+Sidekiq.configure_server do |config|
+  config.death_handlers << ThecoreBackendCommons::DiscardedJobLogger unless config.death_handlers.include?(ThecoreBackendCommons::DiscardedJobLogger)
+end

data/lib/thecore_backend_commons/discarded_job_logger.rb

@@ -0,0 +1,31 @@
+module ThecoreBackendCommons
+  # Generic Sidekiq death handler, registered by config/initializers/sidekiq.rb: with retries off
+  # Sidekiq drops a failed job without keeping it anywhere, so this writes one line to the Rails log
+  # per failure. Only the job's identity goes in: class (or Mailer#action for a mail job), queue, jid
+  # and the error, never the arguments, which can be huge or sensitive (docs/adr/0002). Apps add their own handlers next to this one (`config.death_handlers << MyHandler`
+  # inside `Sidekiq.configure_server`), e.g. to record failures in their own integration log.
+  module DiscardedJobLogger
+    MAX_MESSAGE_LENGTH = 500
+
+    def self.call(job, exception)
+      error = "#{exception.class}: #{exception.message}".truncate(MAX_MESSAGE_LENGTH)
+      Rails.logger.error(
+        "Background job #{name_of(job)} failed and was discarded (no retry left): " \
+        "queue=#{job["queue"]} jid=#{job["jid"]} error=#{error}"
+      )
+    rescue StandardError => e
+      Rails.logger.error("DiscardedJobLogger could not record a failed job: #{e.class}: #{e.message}")
+    end
+
+    def self.name_of(job)
+      wrapped = (job["wrapped"] || job["class"]).to_s
+      first = Array(job["args"]).first
+      arguments = first.is_a?(Hash) ? Array(first["arguments"]) : []
+      return wrapped unless wrapped == "ActionMailer::MailDeliveryJob" && arguments.size >= 2
+
+      "#{arguments[0]}##{arguments[1]}"
+    end
+
+    private_class_method :name_of
+  end
+end

data/lib/thecore_backend_commons/job_failure_policy.rb

@@ -0,0 +1,54 @@
+module ThecoreBackendCommons
+  # What happens to a failed background job, app-wide, read from the container's environment by
+  # config/initializers/sidekiq.rb (docs/adr/0002-no-job-retries-failures-logged.md):
+  #
+  # - THECORE_JOB_RETRIES: how many times Sidekiq retries a failed job. Unset or blank = 0, i.e.
+  #   `retry: false` (not `retry: 0`, which would still send the job to the Dead Set).
+  # - THECORE_FAILED_JOBS_KEPT: how many failed jobs sidekiq-failures keeps in the Redis `failed`
+  #   set, payload and arguments included (Sidekiq.failures_max_count). Unset or blank = 1, the
+  #   latest, still shown in the Failures tab.
+  #
+  # Any other value refuses to boot: a typo must never silently re-enable retries or keep every
+  # failed payload. A job class still overrides the retry count with `sidekiq_options retry: N`.
+  # Required, not autoloaded: see the ADR.
+  module JobFailurePolicy
+    RETRIES_VARIABLE = "THECORE_JOB_RETRIES".freeze
+    KEPT_VARIABLE = "THECORE_FAILED_JOBS_KEPT".freeze
+    DEFAULT_RETRIES = 0
+    DEFAULT_KEPT = 1
+
+    class InvalidValue < StandardError; end
+
+    # Sets Sidekiq's app-wide default retry option and the sidekiq-failures cap.
+    def self.apply!
+      Sidekiq.default_job_options = Sidekiq.default_job_options.merge("retry" => sidekiq_retry_option)
+      Sidekiq.failures_max_count = failed_jobs_kept
+    end
+
+    # The value for Sidekiq's `retry` option: false for no retries, else the count.
+    def self.sidekiq_retry_option
+      retries.zero? ? false : retries
+    end
+
+    def self.retries
+      whole_number_from_env(RETRIES_VARIABLE, default: DEFAULT_RETRIES, minimum: 0, expected: "a whole number >= 0")
+    end
+
+    def self.failed_jobs_kept
+      whole_number_from_env(KEPT_VARIABLE, default: DEFAULT_KEPT, minimum: 1, expected: "a whole number >= 1")
+    end
+
+    # The whole number in the environment variable, default when unset or blank; raises InvalidValue
+    # on anything else or below minimum.
+    def self.whole_number_from_env(variable, default:, minimum:, expected:)
+      raw = ENV.fetch(variable, nil)
+      return default if raw.nil? || raw.strip.empty?
+
+      value = raw.strip
+      return value.to_i if value.match?(/\A\d+\z/) && value.to_i >= minimum
+
+      raise InvalidValue, "#{variable}=#{raw.inspect} is not valid: use #{expected}, or leave it unset (default #{default})"
+    end
+    private_class_method :whole_number_from_env
+  end
+end

data/lib/thecore_backend_commons/version.rb

@@ -1,3 +1,3 @@
 module ThecoreBackendCommons
-  VERSION = "3.9.1".freeze
+  VERSION = "3.10.0".freeze
 end

data/lib/thecore_backend_commons.rb

@@ -19,6 +19,8 @@ require "thecore_backend_commons/smtp_config"
 require "thecore_backend_commons/smtp_tester"
 require "thecore_backend_commons/push_notification_service"
 require "thecore_backend_commons/default_module_registry"
+require "thecore_backend_commons/discarded_job_logger"
+require "thecore_backend_commons/job_failure_policy"
 
 module ThecoreBackendCommons
   # Deep-merges +dest+ into +src+ (mutating and returning +src+), extending existing arrays

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: thecore_backend_commons
 version: !ruby/object:Gem::Version
-  version: 3.9.1
+  version: 3.10.0
 platform: ruby
 authors:
 - Gabriele Tassoni
@@ -234,6 +234,7 @@ files:
 - config/initializers/default_module_registry.rb
 - config/initializers/extension_nil.rb
 - config/initializers/extension_string.rb
+- config/initializers/sidekiq.rb
 - config/locales/en.devise.custom.yml
 - config/locales/en.yml
 - config/locales/it.devise.custom.yml
@@ -247,7 +248,9 @@ files:
 - lib/tasks/thecore_backend_commons_tasks.rake
 - lib/thecore_backend_commons.rb
 - lib/thecore_backend_commons/default_module_registry.rb
+- lib/thecore_backend_commons/discarded_job_logger.rb
 - lib/thecore_backend_commons/engine.rb
+- lib/thecore_backend_commons/job_failure_policy.rb
 - lib/thecore_backend_commons/push_notification_service.rb
 - lib/thecore_backend_commons/smtp_config.rb
 - lib/thecore_backend_commons/smtp_tester.rb