cronwatch 0.3.1

data/lib/cronwatch/rails/active_job.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require "active_support/concern"
+require "active_support/inflector"
+require "active_job"
+require_relative "../scheduler"
+
+module Cronwatch
+  # Monitors an ActiveJob class. Each perform is a recorded run with the
+  # trigger "active_job"; `cronwatch` in the job is the run's context, for
+  # log and metric. A perform that raises is recorded as failed and then
+  # raises as before, so retry_on, discard_on and error reporters see it
+  # unchanged.
+  #
+  #   class NightlyReportJob < ApplicationJob
+  #     include Cronwatch::ActiveJob
+  #     cronwatch schedule: "0 2 * * *", grace: "15m", expect: "Report written" # name: "nightly-report"
+  #
+  #     def perform
+  #       cronwatch.log("Report written")
+  #       cronwatch.metric(:cost, 1.2)
+  #     end
+  #   end
+  #
+  # `cronwatch schedule: :from_scheduler` takes the schedule from the class's
+  # entry in config/recurring.yml (Solid Queue) or sidekiq-cron's schedule,
+  # so the cron expression is written once.
+  #
+  # The job is declared on Cronwatch.client once the app has booted (after
+  # config/initializers/cronwatch.rb ran), and again if the client is
+  # replaced. Only classes that call `cronwatch` are monitored; subclasses
+  # declare their own.
+  module ActiveJob
+    extend ActiveSupport::Concern
+    include Cronwatch::Monitored
+
+    TRIGGER = "active_job"
+
+    included do
+      around_perform :cronwatch_perform
+    end
+
+    class_methods do
+      include Cronwatch::Monitored::ClassMethods
+    end
+
+    private
+
+    # Runs the perform as a recorded run. A job whose declaration is broken
+    # still performs, unrecorded, with the error sent to on_error.
+    def cronwatch_perform(&block)
+      Cronwatch::Monitored.record(self.class.cronwatch_declaration, TRIGGER, self, &block)
+    end
+  end
+end

data/lib/cronwatch/rails/check_job.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+require "active_job"
+
+module Cronwatch
+  # Looks for missed and stuck runs across every job, sends alerts, retries
+  # undelivered ones and prunes old runs: Cronwatch.client.check, as a job.
+  # Schedule it every few minutes; nothing else notices a job that never ran.
+  #
+  #   # config/recurring.yml (Solid Queue)
+  #   production:
+  #     cronwatch_check:
+  #       class: Cronwatch::CheckJob
+  #       schedule: every 5 minutes
+  #
+  #   # config/schedule.yml (sidekiq-cron)
+  #   cronwatch_check:
+  #     cron: "*/5 * * * *"
+  #     class: "Cronwatch::CheckJob"
+  #
+  # With Sidekiq but no ActiveJob adapter for it, Cronwatch::Sidekiq::CheckWorker
+  # does the same. Returns the CheckResult.
+  class CheckJob < ::ActiveJob::Base
+    queue_as :default
+
+    def perform
+      Cronwatch::Monitored.load_app_jobs
+      Cronwatch::Monitored.register_all(strict: false)
+      Cronwatch.client.check
+    end
+  end
+end

data/lib/cronwatch/rails/railtie.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require "rails/railtie"
+
+module Cronwatch
+  # Hooks the gem into a Rails app. There is nothing to start: runs are
+  # recorded as monitored jobs perform, and checks come from
+  # Cronwatch::CheckJob (or Cronwatch::Sidekiq::CheckWorker) on your
+  # scheduler, so no interval thread runs inside web or worker processes.
+  # Errors outside jobs (the store, a channel) go to Rails.logger unless the
+  # initializer sets on_error.
+  class Railtie < ::Rails::Railtie
+    # With Sidekiq in the bundle, its server records the runs of
+    # Cronwatch::Sidekiq jobs. Checked here rather than when the gem loads,
+    # so the order of the Gemfile does not matter.
+    initializer "cronwatch.sidekiq" do
+      if defined?(::Sidekiq) && ::Sidekiq.respond_to?(:configure_server)
+        require "cronwatch/sidekiq"
+        Cronwatch::Sidekiq.install
+      end
+    end
+
+    # Once config/initializers/cronwatch.rb has run and, in production, app/jobs
+    # is loaded: declare every monitored job, and what
+    # Cronwatch.declare_from_scheduler! asked for, so a check knows each
+    # schedule before the job first runs. A bad declaration (or a schedule
+    # that cannot be read from the scheduler's config) stops the boot, as it
+    # would in plain Ruby.
+    config.after_initialize do
+      Cronwatch::Monitored.boot!
+    end
+
+    rake_tasks do
+      require_relative "tasks"
+    end
+  end
+end

data/lib/cronwatch/rails/tasks.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+# `bin/rails cronwatch:check`: one check, for a plain crontab instead of a job scheduler.
+namespace :cronwatch do
+  desc "Check for missed and stuck runs and send alerts (what Cronwatch::CheckJob does)"
+  task check: :environment do
+    result = Cronwatch::CheckJob.perform_now
+    jobs = result.jobs.length
+    alerts = result.alerts.length
+    puts "cronwatch: checked #{jobs} job#{jobs == 1 ? "" : "s"}, sent #{alerts} alert#{alerts == 1 ? "" : "s"}"
+  end
+end

data/lib/cronwatch/rails.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+# The Rails integration: a Railtie, the Cronwatch::ActiveJob concern,
+# Cronwatch::CheckJob, Cronwatch::Web and the `cronwatch:install` generator.
+# Needs railties and activejob; the ActiveRecord store loads on first use,
+# and so does Cronwatch::Sidekiq when the app has Sidekiq (the Railtie loads
+# it at boot).
+#
+# `gem "cronwatch"` loads this file on its own when Rails is already loaded,
+# as it is under Bundler.require; require it by hand only otherwise.
+require "cronwatch" unless defined?(Cronwatch::Client) # loaded from cronwatch.rb when Rails is already up
+begin
+  require "rails"
+  require "active_job"
+rescue LoadError => e
+  raise LoadError, "cronwatch/rails needs the railties and activejob gems, which a Rails app has (#{e.message})"
+end
+
+# Rails always brings Rack, so the dashboard loads here and the mount line in
+# config/routes.rb needs no require of its own.
+require_relative "web"
+
+module Cronwatch
+  autoload :Sidekiq, File.expand_path("sidekiq", __dir__) unless const_defined?(:Sidekiq, false)
+
+  module Stores
+    autoload :ActiveRecord, File.expand_path("stores/active_record", __dir__) unless const_defined?(:ActiveRecord, false)
+  end
+end
+
+require_relative "sidekiq" if defined?(::Sidekiq::Job) || defined?(::Sidekiq::Worker)
+
+require_relative "rails/active_job"
+require_relative "rails/check_job"
+require_relative "rails/railtie"

data/lib/cronwatch/schedule.rb

@@ -0,0 +1,191 @@
+# frozen_string_literal: true
+
+require "fugit"
+require_relative "zone"
+require_relative "cron_pattern"
+
+module Cronwatch
+  # Schedules: "0 2 * * *" (cron, five or six fields), "@hourly", or
+  # "every 5m". The SDK computes fire times with croner; this module gives
+  # the same times. Fugit parses the cron fields. Croner's rules decide what
+  # is accepted and how a fire lands on a clock that jumps (daylight saving),
+  # so both are ported here and a Node and a Ruby process sharing one store
+  # agree on every due time.
+  module Schedule
+    # How early a run may start and still count for the fire it was meant for.
+    EARLY_SLACK_MS = 60_000
+
+    Parsed = Struct.new(:kind, :source, :timezone, :every_ms, keyword_init: true) do
+      include Serializable
+
+      # The compiled cron fields, kept out of the public shape.
+      attr_accessor :pattern
+
+      def cron? = kind == :cron
+      def interval? = kind == :interval
+
+      def to_h
+        out = { "kind" => kind.to_s, "source" => source }
+        out["timezone"] = timezone if timezone
+        out["everyMs"] = every_ms if every_ms
+        out
+      end
+    end
+
+    Expectation = Struct.new(:due_at, :deadline, keyword_init: true) do
+      include Serializable
+
+      def to_h = { "dueAt" => due_at, "deadline" => deadline }
+    end
+
+    @cache = {}
+    @lock = Mutex.new
+
+    module_function
+
+    # Parsed once per (schedule, timezone) pair and cached. Without a timezone
+    # the expression is read in the process timezone, like crontab. Vercel and
+    # GitHub Actions run their crons in UTC, so pass timezone: "UTC" for those.
+    def parse(schedule, timezone = nil)
+      key = "#{timezone}|#{schedule}"
+      hit = @lock.synchronize { @cache[key] }
+      return hit if hit
+
+      text = JS.trim(schedule.to_s)
+      every = Regexp.new("\\Aevery[#{JS::WHITESPACE}]+(.+)\\z", Regexp::IGNORECASE).match(text)
+      if every
+        every_ms = Duration.parse(every[1], "schedule interval")
+        raise ArgumentError, "schedule \"#{schedule}\" is shorter than one second" if every_ms < 1000
+
+        parsed = Parsed.new(kind: :interval, source: text, every_ms: every_ms)
+      else
+        begin
+          pattern = CronPattern.compile(text)
+        rescue ArgumentError => e
+          raise ArgumentError, "schedule \"#{schedule}\" is not a cron expression or \"every <duration>\": #{e.message}"
+        end
+        parsed = Parsed.new(kind: :cron, source: text, timezone: timezone.nil? || timezone == "" ? nil : timezone)
+        parsed.pattern = pattern
+      end
+      parsed.freeze
+      @lock.synchronize { @cache[key] = parsed }
+    end
+
+    # The first fire strictly after `from`, or nil when the cron never fires
+    # again. The walker works on the wall clock and Zone.to_utc takes the
+    # earlier of a repeated time, so, like croner, asked from inside the hour
+    # that repeats when clocks go back it can answer with times in the past.
+    # Its answers are filtered, and a stretch of nothing but past times is
+    # stepped over an hour at a time. Ported from fireAfter in schedule.ts.
+    def fire_after(parsed, from)
+      raise ArgumentError, "schedule \"#{parsed.source}\" was not made by Schedule.parse" unless parsed.pattern
+
+      probe = from
+      4.times do
+        runs = next_runs(parsed, 8, probe)
+        return nil if runs.empty?
+
+        found = runs.find { |t| t > from }
+        return found if found
+
+        probe += 3_600_000
+      end
+      nil
+    end
+
+    # Up to `count` fires, each found from the one before, as croner's nextRuns.
+    def next_runs(parsed, count, from)
+      runs = []
+      at = from
+      count.times do
+        at = parsed.pattern.next_after(at, parsed.timezone)
+        break if at.nil?
+
+        runs << at
+      end
+      runs
+    end
+
+    # The next time the schedule fires strictly after `from`. For an interval, counted from the last run when there is one.
+    def next_fire(parsed, from, last_run_at)
+      return (last_run_at || from) + parsed.every_ms if parsed.interval?
+
+      fire_after(parsed, from)
+    end
+
+    # When the schedule next wants a run, given the last one. For a cron that is
+    # the first fire the last run does not already cover; with no run yet, the
+    # first fire at or after registration. For an interval it is the last run's
+    # start (or registration) plus the interval. Nil for a cron that never
+    # fires again.
+    #
+    # Counting forward from the last run, rather than back from now, is what
+    # lets a job whose period is shorter than its grace be missed at all, and it
+    # works for a cron that fires once a year or less.
+    def expectation(parsed, last_run_at, registered_at, grace_ms)
+      due_at =
+        if parsed.interval? then (last_run_at || registered_at) + parsed.every_ms
+        elsif last_run_at.nil? then fire_after(parsed, registered_at - 1)
+        else due_after_run(parsed, last_run_at)
+        end
+      due_at.nil? ? nil : Expectation.new(due_at: due_at, deadline: due_at + grace_ms)
+    end
+
+    # The first fire that a run starting at `started_at` does not cover.
+    def due_after_run(parsed, started_at)
+      # A fire at or before the start is covered by the run itself.
+      following_fire = fire_after(parsed, started_at)
+      return nil if following_fire.nil?
+
+      following = fire_after(parsed, following_fire)
+      covers = run_covers?(started_at, following_fire, following) || in_spring_forward_gap?(parsed, started_at, following_fire)
+      covers ? following : following_fire
+    end
+
+    # Whether a run starting at `started_at` covers the fire at `due_at`. A minute
+    # of slack before the tick absorbs schedulers that fire a touch early. When
+    # the fire after `due_at` is known, the slack is at most half the gap between
+    # the two, so one run of an every-minute cron never covers two fires.
+    def run_covers?(started_at, due_at, following_at = nil)
+      slack = following_at.nil? ? EARLY_SLACK_MS : [EARLY_SLACK_MS, (following_at - due_at).div(2)].min
+      started_at >= due_at - slack
+    end
+
+    # On the night clocks spring forward, a fire whose local time does not exist
+    # (02:30 when 02:00 jumps to 03:00) is moved by croner to the same distance
+    # past the jump (03:30), while vixie cron runs it at the jump itself (03:00).
+    # A run that starts at or after the jump, and before the first fire after
+    # it when that fire lies within one gap of it, is taken to cover that fire,
+    # so neither scheduler's run is reported as missed. A cron that really fires
+    # at 03:30 that night is treated the same way, which only matters if it also
+    # ran early by up to an hour.
+    def in_spring_forward_gap?(parsed, started_at, fire_at)
+      lookback = 3 * 3_600_000
+      after = utc_offset(fire_at, parsed.timezone)
+      before = utc_offset(fire_at - lookback, parsed.timezone)
+      gap = after - before
+      return false if gap <= 0
+
+      # Find the jump: the first minute in the window with the later offset.
+      lo = fire_at - lookback
+      hi = fire_at
+      while hi - lo > 60_000
+        mid = lo + (hi - lo).div(2)
+        if utc_offset(mid, parsed.timezone) == after then hi = mid
+        else lo = mid
+        end
+      end
+      jump_at = hi.div(60_000) * 60_000
+      return false if fire_at - jump_at >= gap || started_at < jump_at - EARLY_SLACK_MS || started_at >= fire_at
+
+      # Only the first fire after the jump can be a moved one; a cron that also
+      # fires at the jump (every 10 minutes, say) was not moved at all.
+      fire_after(parsed, jump_at - 1) == fire_at
+    end
+
+    # Milliseconds the zone's wall clock is ahead of UTC at `at`.
+    def utc_offset(at, timezone)
+      Zone.offset(at.div(1000), timezone) * 1000
+    end
+  end
+end