rails_pod_kit 0.2.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 85fea66a44c5b2198b5c3c7bfd05f5f1f4c3655559ed701748661d0e148650a5
-  data.tar.gz: 9c921528f839ea1d5b962035600316ea37affe17cf8fdc910453c30108a248dd
+  metadata.gz: 0dda5e5202f1bb5e509fad032de8449c5d8e9eec78c0cedcd0a59d20cc4f0d32
+  data.tar.gz: ebccb551ba5dcfd23c1dc2e668d417f59fb1d76f06522e02c352cea1f5e0dca5
 SHA512:
-  metadata.gz: d7fbfee31ce338a6dd7c11d65214dc442cd2cb261848ae2403a415dc4a6e1333fa55b39e2d06441b4e05fcacff2ec3971ee009613a532f384fcc19fbe84f070d
-  data.tar.gz: 894f99f9fba4596aa7d69e143f5fa23d2ebc64ef85f6b7cac310096c902005dd003103cb12c3933e49dfe26263c68f9656cfc3e68c0d9551a8f118e79496fd48
+  metadata.gz: f6ab1b5432bf79341efabb50d0d3d1862dafa49523bbbc8f70ea718eb520d258b7c216da274f0d184dbeb1737c887ebfb19f04fbcc656e8fd39c6a0e057d6346
+  data.tar.gz: c2e8aa9a3b35a1ea4d557a47be04810c69e79de50a0297593388f90c1be48f8a144a281ad3d26a6c42c4564ba608d6ded064ba53b12f5f49279ddefa940e049c

data/README.md

@@ -159,6 +159,8 @@ endpoint every few seconds); pass `silence_controller_log: false` to keep it.
   `sidekiq_queue_latency`, `sidekiq_active_processes`,
   `sidekiq_active_workers_count`, `sidekiq_jobs_retry_count`,
   `sidekiq_jobs_dead_count`, `sidekiq_jobs_scheduled_count`.
+- **Sidekiq (cron scheduler):** `sidekiq_cron_poll_age_seconds`, on the process
+  hosting the poller (see GlobalScheduler).
 - **SolidQueue (DB-wide):** `solid_queue_backlog`,
   `solid_queue_latency_seconds`.
 
@@ -276,6 +278,12 @@ Sidekiq job class.
 | `sidekiq.active_processes` | — |
 | `sidekiq.active_workers_count` | — |
 
+**Sidekiq — cron scheduler** (only the process hosting the poller, `namespace: sidekiq`):
+
+| canonical Datadog metric | functional tags |
+|---|---|
+| `sidekiq.cron_poll_age_seconds` | — |
+
 **Sidekiq — per-process / job** (worker pod, `namespace: sidekiq`; emitted on job activity):
 
 | canonical Datadog metric | type | functional tags |
@@ -403,6 +411,32 @@ silently. The cron poller's own loop swallows StandardError, so a Redis blip
 costs one skipped tick; the supervisor makes anything it does *not* catch a
 skipped tick too.
 
+### The heartbeat
+
+The supervisor covers a poller thread that *dies*. It cannot see one that is
+running and no longer enqueueing — which from the outside is indistinguishable
+from an idle one: every gauge stays fresh, `/metrics` answers 200, the process
+looks healthy. On the only process carrying the schedule that is a silently
+stopped schedule, i.e. the very failure hosting the poller here was meant to
+eliminate, back through another door.
+
+So `start!` also publishes **`sidekiq_cron_poll_age_seconds`**: seconds since the
+poller last completed a tick. It measures the loop turning, so it stays flat on a
+healthy but idle schedule and climbs the moment ticks stop — the one shape an
+alert can be written against:
+
+```
+max:sidekiq.cron_poll_age_seconds{…} > 10 * <poll interval>
+```
+
+Before the first tick it measures from `start!`, so a poller that never manages
+one reads as climbing rather than as no-data; it is `nil` (and the series absent)
+on any process that hosts no poller, and again once `stop!` runs.
+
+Deliberately **not** "time since last enqueue", which would climb on any quiet
+schedule and so alert on nothing happening. Answering *should this job have run
+by now?* needs a per-job check against the cron expression, not a gauge.
+
 > **Every schedule entry must declare `active_job: true`.** This process has no
 > Rails, so it cannot resolve the job classes; sidekiq-cron then falls back to
 > pushing a raw message, and only that flag makes the message an ActiveJob

data/VERSION

@@ -1 +1 @@
-0.2.0
+0.3.0

data/lib/rails_pod_kit/global_scheduler/heartbeat.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module RailsPodKit
+  module GlobalScheduler
+    # The one failure the rest of the machinery cannot see.
+    #
+    # A poller thread that dies is restarted by the Supervisor, and a process
+    # that stops serving is caught by the exporter's own probe. But a poller
+    # that is *running and no longer enqueueing* is indistinguishable from an
+    # idle one from the outside: every gauge stays fresh, /metrics answers 200,
+    # the pod is Running and Ready. On the only process carrying the schedule
+    # that is a silently stopped schedule — exactly the failure hosting the
+    # poller here was meant to eliminate, coming back through another door.
+    #
+    # So publish the age of the last completed tick, as
+    # `sidekiq_cron_poll_age_seconds`. It measures the loop turning, which means
+    # it stays flat on a healthy but idle schedule and climbs the moment ticks
+    # stop — the one shape an alert can be written against.
+    #
+    # Deliberately *not* "time since last enqueue": that climbs on any quiet
+    # schedule, so it would alert on nothing happening. Answering "should this
+    # job have run by now?" needs a per-job check against the cron expression,
+    # not a gauge.
+    module Heartbeat
+      module_function
+
+      # Declares the gauge. One-shot, and safe either side of
+      # `Yabeda.configure!` — a metric declared after it is registered with the
+      # adapters immediately.
+      def install!
+        return false if @installed
+
+        require 'yabeda'
+        declare!
+        @installed = true
+      end
+
+      def declare!
+        Yabeda.configure do
+          group :sidekiq do
+            gauge :cron_poll_age,
+                  unit: :seconds,
+                  tags: [],
+                  aggregation: :most_recent,
+                  comment: 'Seconds since the sidekiq-cron poller last completed a tick'
+
+            collect do
+              age = RailsPodKit::GlobalScheduler::Heartbeat.age
+              Yabeda.sidekiq.cron_poll_age.set({}, age) if age
+            end
+          end
+        end
+      end
+
+      # Begins measuring, from before the first tick — so a poller that never
+      # manages one reads as climbing rather than as no-data.
+      def start!
+        @started_at = monotonic_now
+        @last_poll_at = nil
+      end
+
+      # Drops the series with the poller: a stopped scheduler should read as
+      # no-data, not as an age climbing forever.
+      def stop!
+        @started_at = nil
+        @last_poll_at = nil
+      end
+
+      def record!
+        @last_poll_at = monotonic_now
+      end
+
+      # nil until started, which is what keeps the series off any process that
+      # hosts no poller.
+      def age
+        reference = @last_poll_at || @started_at
+        return nil unless reference
+
+        monotonic_now - reference
+      end
+
+      # Monotonic: this is a duration, and a wall-clock step (NTP, a node coming
+      # back from suspend) must not read as the schedule having stalled.
+      def monotonic_now
+        ::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/rails_pod_kit/global_scheduler.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'rails_pod_kit/config'
+require 'rails_pod_kit/global_scheduler/heartbeat'
 require 'rails_pod_kit/supervisor'
 
 module RailsPodKit
@@ -66,6 +67,8 @@ module RailsPodKit
                  reschedule_grace_period: reschedule_grace_period)
       load_schedule!
 
+      Heartbeat.install!
+      Heartbeat.start!
       @supervisor = build_supervisor(supervision_interval).start
     end
 
@@ -74,6 +77,7 @@ module RailsPodKit
     def stop!
       @supervisor&.stop
       @supervisor = nil
+      Heartbeat.stop!
     end
 
     def poller
@@ -132,7 +136,25 @@ module RailsPodKit
       config[:cron_poll_interval] = ::Sidekiq::Cron.configuration.cron_poll_interval.to_i
       config[:cron_poll_process_count] = ::Sidekiq::Cron.configuration.cron_poll_process_count || 1
 
-      ::Sidekiq::Cron::Poller.new(config)
+      poller_class.new(config)
+    end
+
+    # A subclass rather than a prepended module, so the heartbeat hook is
+    # confined to the poller this module builds and never touches sidekiq-cron's
+    # own for a host that also runs a Sidekiq server. Built lazily because the
+    # superclass does not exist until `sidekiq-cron` is required.
+    #
+    # Records only on a normal return: `Poller#enqueue` rescues internally, so
+    # anything that still escapes it means the tick did not complete, and
+    # counting it as a heartbeat would be exactly the lie the gauge exists to
+    # prevent.
+    def poller_class
+      @poller_class ||= Class.new(::Sidekiq::Cron::Poller) do
+        def enqueue
+          super
+          Heartbeat.record!
+        end
+      end
     end
 
     # An entry whose class this process cannot load and which does not declare

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: rails_pod_kit
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Fabio Napoleoni
@@ -159,6 +159,7 @@ files:
 - lib/rails_pod_kit/exporter.rb
 - lib/rails_pod_kit/global_exporter.rb
 - lib/rails_pod_kit/global_scheduler.rb
+- lib/rails_pod_kit/global_scheduler/heartbeat.rb
 - lib/rails_pod_kit/health.rb
 - lib/rails_pod_kit/puma.rb
 - lib/rails_pod_kit/railtie.rb