railwatch 0.1.3 → 0.2.0.pre1

data/app/controllers/railwatch/spans_controller.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Railwatch
+    class SpansController < DashboardController
+    def index
+      spans = grouped("span", limit: 200, order: params[:sort], dir: params[:dir])
+      spans = apply_filters(spans, FilterQuery.parse(params[:q]).fetch(:fields))
+      render inertia: { spans: spans, series: series("span"), summary: summary_with_delta("span"),
+                        sort: params[:sort] || "count", dir: params[:dir] || "desc", q: params[:q].to_s }
+    end
+
+    def show
+      group_hash = params[:id]
+      rows = telemetry { Telemetry::Span.where(group_hash: group_hash).between(*window_range).recent.limit(200).to_a }
+      render inertia: {
+        group_hash: group_hash,
+        name: rows.first&.name || telemetry { Telemetry::Rollup.for_type("span").where(group_hash: group_hash).pick(:name) },
+        summary: summary_with_delta("span", group_hash: group_hash), series: series("span", group_hash: group_hash),
+        facets: attribute_facets(rows), samples: rows.first(50).map { |s| span_row(s) }
+      }
+    end
+
+    private
+
+    # Rollup rows for "span" carry the span name and a failure count, so
+    # status:failed is "this group failed at least once in the window" rather
+    # than a per-span lookup.
+    def apply_filters(spans, fields)
+      spans = spans.select { |s| s[:name].to_s.include?(fields["name"]) } if fields["name"].present?
+      spans = spans.select { |s| s[:errors].positive? } if fields["status"] == "failed"
+      spans
+    end
+
+    # Which attribute keys these spans carry, and the most common values for
+    # each -- the span equivalent of the query page's "called from" breakdown.
+    def attribute_facets(rows)
+      rows.flat_map { |r| r.payload.to_a }.group_by(&:first).map { |key, pairs|
+        { key: key, count: pairs.size, values: pairs.map { |_k, value| value.to_s }.tally.sort_by { |_v, c| -c }.first(5) }
+      }.sort_by { |facet| -facet[:count] }.first(10)
+    end
+
+    def span_row(s)
+      { id: s.id, name: s.name, duration: s.duration_ms.round(3), status: s.status, attributes: s.payload,
+        occurred_at: s.occurred_at, execution_id: s.execution_id, execution_preview: s.execution_preview }
+    end
+    end
+end

data/app/controllers/railwatch/storage_ops_controller.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Railwatch
+    class StorageOpsController < DashboardController
+    def index
+      from, to = window_range
+      ops = telemetry do
+        Telemetry::StorageOp.between(from, to).group(:service, :op).pluck(:service, :op, Arel.sql("COUNT(*)"), Arel.sql("AVG(duration)"), Arel.sql("MAX(duration)"))
+          .map { |s, o, c, avg, mx| { service: s, op: o, count: c, avg: (avg.to_f / 1000.0).round(1), max: (mx.to_f / 1000.0).round(1) } }
+      end
+      recent = telemetry { Telemetry::StorageOp.between(from, to).recent.limit(100).map { |r| { id: r.id, service: r.service, op: r.op, key: r.key, duration: r.duration_ms.round(1), occurred_at: r.occurred_at, execution_id: r.execution_id, execution_preview: r.execution_preview } } }
+      render inertia: { ops: ops, recent: recent }
+    end
+    end
+end

data/app/controllers/railwatch/tenants_controller.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Railwatch
+    class TenantsController < DashboardController
+    def index
+      from, to = window_range
+      tenants, summary = telemetry do
+        rows = Telemetry::Tenant.index(from, to, q: params[:q], sort: params[:sort], dir: params[:dir])
+        [ rows, Telemetry::Tenant.overview(rows, from, to) ]
+      end
+      render inertia: { tenants: tenants, summary: summary,
+                        sort: params[:sort] || "requests", dir: params[:dir] || "desc", q: params[:q].to_s }
+    end
+
+    # :id is the tenant string. Rails reads a dot in a path segment as a format
+    # separator, so tenants like "acme.co" arrive percent-encoded (see the
+    # tenants index page's tenantPath).
+    def show
+      tenant = params[:id]
+      from, to = window_range
+      data = telemetry do
+        { tenant: tenant, summary: { current: Telemetry::Tenant.summary(tenant, from, to),
+                                    previous: Telemetry::Tenant.summary(tenant, *previous_window_range) },
+          series: Telemetry::Tenant.series(tenant, from, to), routes: Telemetry::Tenant.routes(tenant, from, to),
+          jobs: Telemetry::Tenant.job_classes(tenant, from, to), exceptions: Telemetry::Tenant.exceptions(tenant, from, to),
+          people: Telemetry::Tenant.people(tenant), recent_requests: Telemetry::Tenant.recent_requests(tenant, from, to) }
+      end
+      issues = environment.issues.where(group_hash: data[:exceptions].map { |e| e[:group_hash] }.uniq).index_by(&:group_hash)
+      render inertia: data.merge(exceptions: data[:exceptions].map { |e| with_issue(e, issues) }, links: links(tenant))
+    end
+
+    private
+
+    def with_issue(exception, issues)
+      issue = issues[exception[:group_hash]]
+      exception.merge(issue_id: issue&.id, issue_key: issue&.key)
+    end
+
+    def links(tenant)
+      { logs: application_environment_logs_path(application, environment, window: window_key, q: "tenant:#{tenant}"),
+        people: application_environment_people_path(application, environment, window: window_key) }
+    end
+    end
+end

data/app/controllers/railwatch/thresholds_controller.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module Railwatch
+    class ThresholdsController < DashboardController
+    def index
+      render inertia: { thresholds: environment.thresholds.order(:target_kind, :target).map { |t| t.slice(:id, :target_kind, :target, :metric, :limit, :window_minutes).merge(description: t.description) },
+                        routes: telemetry { Telemetry::Rollup.for_type("request").where("bucket > ?", 7.days.ago).distinct.pluck(:name).sort },
+                        jobs: telemetry { Telemetry::Rollup.for_type("job_attempt").where("bucket > ?", 7.days.ago).distinct.pluck(:name).sort },
+                        kinds: Threshold::TARGET_KINDS, metrics: Threshold::METRICS,
+                        anomaly_rules: environment.anomaly_rules.order(:target_kind, :target).map { |r| r.slice(:id, :target_kind, :target, :metric, :deviation, :window_minutes, :baseline_days, :enabled, :last_fired_at).merge(description: r.description) },
+                        anomaly_target_kinds: AnomalyRule::TARGET_KINDS, anomaly_metrics: AnomalyRule::METRICS }
+    end
+
+    def create
+      threshold = environment.thresholds.new(threshold_params)
+      if threshold.save
+        redirect_to application_environment_thresholds_path(application, environment), notice: "Threshold added"
+      else
+        redirect_to application_environment_thresholds_path(application, environment), inertia: { errors: threshold.errors }
+      end
+    end
+
+    def update
+      threshold = environment.thresholds.find(params[:id])
+      threshold.update(threshold_params)
+      redirect_to application_environment_thresholds_path(application, environment), notice: "Threshold updated"
+    end
+
+    def destroy
+      environment.thresholds.find(params[:id]).destroy
+      redirect_to application_environment_thresholds_path(application, environment), notice: "Threshold removed"
+    end
+
+    private
+
+    def threshold_params
+      params.permit(:target_kind, :target, :metric, :limit, :window_minutes)
+    end
+    end
+end

data/app/controllers/railwatch/traces_controller.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+# One distributed trace: every execution that shares a trace_id, nested by
+# the parent_id the gem propagates (the enqueuing execution's id for jobs,
+# the W3C traceparent parent-id for HTTP hops).
+module Railwatch
+    class TracesController < DashboardController
+    LIMIT = 200
+
+    def show
+      trace_id = params[:trace_id]
+      executions, outgoing = telemetry do
+        rows = Telemetry::Execution.where(trace_id: trace_id).order(:occurred_at).limit(LIMIT).to_a
+        [ rows, Telemetry::OutgoingRequest.where(execution_id: rows.map(&:execution_id)).order(:occurred_at).group_by(&:execution_id) ]
+      end
+      raise ActiveRecord::RecordNotFound if executions.empty?
+
+      @start = executions.first.occurred_at
+      @outgoing = outgoing
+      render inertia: { trace_id: trace_id, roots: tree(executions),
+                        span_count: executions.size + outgoing.values.sum(&:size),
+                        services: executions.filter_map(&:server).uniq, duration: total_duration(executions) }
+    end
+
+    private
+
+    # An execution's parent is the one whose execution_id matches its
+    # parent_id outright (a job enqueued by that execution) or whose id
+    # matches once dashes are dropped and it is cut to the 16 hex chars a
+    # W3C traceparent carries (an incoming HTTP hop). Anything whose parent
+    # is missing -- the trace's entry point, or a hop whose caller was not
+    # reported -- is a root.
+    def tree(executions)
+      by_id = executions.index_by(&:execution_id)
+      by_short_id = executions.index_by { |e| e.execution_id.to_s.delete("-").first(16) }
+      children = Hash.new { |hash, key| hash[key] = [] }
+      roots = []
+      executions.each do |exe|
+        parent = exe.parent_id.presence && (by_id[exe.parent_id] || by_short_id[exe.parent_id])
+        parent = nil if parent && parent.execution_id == exe.execution_id
+        parent ? children[parent.execution_id] << exe : roots << exe
+      end
+      roots.map { |exe| node(exe, children) }
+    end
+
+    def node(exe, children)
+      { execution: execution_row(exe), children: children[exe.execution_id].map { |child| node(child, children) } }
+    end
+
+    def execution_row(exe)
+      { execution_id: exe.execution_id, kind: exe.kind, name: exe.name, status: exe.status, outcome: exe.outcome,
+        parent_id: exe.parent_id, server: exe.server, occurred_at: exe.occurred_at,
+        duration: exe.duration_ms.round(2), offset: offset(exe.occurred_at),
+        outgoing: @outgoing.fetch(exe.execution_id, []).map { |o| outgoing_row(o) } }
+    end
+
+    def outgoing_row(o)
+      { id: o.id, method: o.method, url: o.url.to_s.first(200), status_code: o.status_code,
+        duration: o.duration_ms&.round(2), offset: offset(o.occurred_at) }
+    end
+
+    def offset(time)
+      ((time - @start) * 1000.0).round(2)
+    end
+
+    # Wall-clock span of the whole trace: the last thing to finish, minus the
+    # first thing to start.
+    def total_duration(executions)
+      executions.map { |e| offset(e.occurred_at) + e.duration_ms }.max.round(2)
+    end
+    end
+end

data/app/controllers/railwatch/transactions_controller.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Railwatch
+    class TransactionsController < DashboardController
+    def index
+      from, to = window_range
+      rows = grouped("transaction", limit: 200, order: params[:sort], dir: params[:dir])
+      recent = telemetry { Telemetry::Transaction.between(from, to).recent.limit(50).map { |t| transaction_row(t) } }
+      render inertia: { transactions: rows, series: series("transaction"), recent: recent,
+                        sort: params[:sort] || "count", dir: params[:dir] || "desc" }
+    end
+
+    private
+
+    def transaction_row(t)
+      { id: t.id, outcome: t.outcome, connection: t.connection, duration: t.duration_ms.round(3), statement_count: t.statement_count, occurred_at: t.occurred_at,
+        execution_id: t.execution_id, execution_source: t.execution_source, execution_preview: t.execution_preview,
+        group_hash: t.group_hash }
+    end
+    end
+end

data/app/controllers/railwatch/view_renders_controller.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Railwatch
+    class ViewRendersController < DashboardController
+    def index
+      from, to = window_range
+      fields = FilterQuery.parse(params[:q]).fetch(:fields)
+      rows = grouped("view_render", limit: 200, order: params[:sort], dir: params[:dir])
+      rows = rows.select { |r| r[:name].include?(fields["template"]) } if fields["template"].present?
+      kinds = telemetry do
+        Telemetry::Rollup.for_type("view_render").between(from, to).to_a.group_by(&:group_hash)
+          .transform_values { |group_rows| group_rows.max_by(&:bucket).extra["kind"] }
+      end
+      rows.each { |r| r[:kind] = kinds[r[:group_hash]] }
+      slowest = telemetry { Telemetry::ViewRender.between(from, to).order(duration: :desc).limit(30).map { |v| render_row(v) } }
+      render inertia: { renders: rows, series: series("view_render"), slowest: slowest,
+                        sort: params[:sort] || "count", dir: params[:dir] || "desc", q: params[:q].to_s }
+    end
+
+    private
+
+    def render_row(v)
+      { id: v.id, identifier: v.identifier, kind: v.kind, layout: v.layout, duration: v.duration_ms.round(3),
+        occurred_at: v.occurred_at, execution_id: v.execution_id, execution_source: v.execution_source,
+        execution_preview: v.execution_preview, group_hash: v.group_hash }
+    end
+    end
+end

data/app/controllers/railwatch/visits_controller.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Railwatch
+    class VisitsController < DashboardController
+    # Vitals come from raw visit rows (rollups only carry the visit duration),
+    # so cap how many we read for a p75.
+    MAX_VITAL_SAMPLES = 50_000
+
+    def index
+      fields = FilterQuery.parse(params[:q]).fetch(:fields)
+      rows, vitals, component_vitals = telemetry do
+        scope = Telemetry::Visit.between(*window_range)
+        recent = fields["component"].present? ? scope.where(component: fields["component"]) : scope
+        [ recent.recent.limit(200).to_a, web_vitals(scope), component_web_vitals(scope) ]
+      end
+      components = grouped("visit", limit: 100, order: params[:sort], dir: params[:dir])
+        .map { |row| row.merge(component_vitals.transform_values { |p75s| p75s[row[:name]] }) }
+      render inertia: { components: components, series: series("visit"), vitals: vitals,
+                        sort: params[:sort] || "count", dir: params[:dir] || "desc", q: params[:q].to_s,
+                        recent: rows.map { |v| { id: v.id, component: v.component, url: v.url, method: v.method, duration: v.duration_ms.round(1), status: v.status, partial: v.partial, only: v.only, props_bytes: v.props_bytes, occurred_at: v.occurred_at, user_ref: v.user_ref, lcp: v.lcp, cls: v.cls, inp: v.inp, ttfb: v.ttfb } } }
+    end
+
+    private
+
+    # p75 of each web vital in the window, with its rating and sample count.
+    # Metrics nobody reported are left out entirely.
+    def web_vitals(scope)
+      Telemetry::Visit::VITALS.keys.filter_map { |metric|
+        values = samples(scope, metric).pluck(metric)
+        next if values.empty?
+        value = p75(values)
+        [ metric, { value: value, rating: Telemetry::Visit.rating(metric, value), samples: values.size } ]
+      }.to_h
+    end
+
+    # { lcp: { "widgets/index" => 2100 }, inp: { ... } } for the components table.
+    def component_web_vitals(scope)
+      %i[lcp inp].index_with do |metric|
+        samples(scope, metric).pluck(:component, metric).group_by(&:first)
+          .transform_values { |pairs| p75(pairs.map(&:last)) }
+      end
+    end
+
+    def samples(scope, metric)
+      scope.where.not(metric => nil).recent.limit(MAX_VITAL_SAMPLES)
+    end
+
+    def p75(values)
+      sorted = values.sort
+      sorted[[ (sorted.size * 0.75).ceil - 1, 0 ].max].round(3)
+    end
+    end
+end

data/app/helpers/railwatch/assets_helper.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Emits the dashboard's script and stylesheet tags from the manifest Vite
+  # wrote at gem build time. The gem ships the built bundle under
+  # public/railwatch, so a host app needs no Node toolchain and no asset
+  # pipeline integration: the engine serves those files itself and this
+  # helper does the little that rails_vite's vite_tags would have done.
+  module AssetsHelper
+    MANIFEST = Railwatch::Engine.root.join("public/railwatch/manifest.json")
+    ASSET_PREFIX = "/railwatch/assets"
+
+    def railwatch_asset_tags(nonce: nil)
+      manifest = railwatch_manifest
+      tags = []
+      preloaded = Set.new
+      %w[app/frontend/entrypoints/application.css app/frontend/entrypoints/inertia.tsx].each do |entry|
+        info = manifest.fetch(entry)
+        Array(info["imports"]).each do |key|
+          file = manifest.fetch(key)["file"]
+          next unless preloaded.add?(file)
+          tags << tag.link(rel: "modulepreload", href: "#{ASSET_PREFIX}/#{file}", nonce: nonce)
+        end
+        tags << if info["file"].end_with?(".css")
+          tag.link(rel: "stylesheet", href: "#{ASSET_PREFIX}/#{info['file']}", nonce: nonce)
+        else
+          tag.script(src: "#{ASSET_PREFIX}/#{info['file']}", type: "module", nonce: nonce)
+        end
+        Array(info["css"]).each { |css| tags << tag.link(rel: "stylesheet", href: "#{ASSET_PREFIX}/#{css}", nonce: nonce) }
+      end
+      safe_join(tags, "\n")
+    end
+
+    def railwatch_font_preloads
+      safe_join(railwatch_manifest.select { |k, _| k.end_with?(".woff2") && k.include?("Roboto") }.map { |_, v|
+        tag.link(rel: "preload", href: "#{ASSET_PREFIX}/#{v['file']}", as: "font", type: "font/woff2", crossorigin: "anonymous")
+      }, "\n")
+    end
+
+    # The bundle digest doubles as the Inertia asset version: a gem upgrade
+    # changes it and every open tab does a full reload on its next visit.
+    def self.digest
+      @digest ||= Digest::SHA256.file(MANIFEST).hexdigest[0, 16]
+    end
+
+    private
+
+    def railwatch_manifest
+      @railwatch_manifest ||= JSON.parse(File.read(MANIFEST))
+    end
+  end
+end

data/app/jobs/railwatch/anomaly_scan_job.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Railwatch
+  class AnomalyScanJob < ApplicationJob
+    queue_as :default
+
+    def perform
+      DetectAnomaliesJob.perform_later(Environment.current) if AnomalyRule.where(enabled: true).exists?
+    end
+  end
+end

data/app/jobs/railwatch/application_job.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # The engine's jobs run on whatever Active Job adapter the host configured.
+  class ApplicationJob < ActiveJob::Base
+  end
+end

data/app/jobs/railwatch/auto_resolve_issues_job.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Auto-resolves open issues that have gone quiet for a configured number of
+  # days (nil = off). Resolved with no deploy attached, so any recurrence is
+  # treated as a regression rather than a duplicate. Runs daily (recurring.yml).
+  class AutoResolveIssuesJob < ApplicationJob
+    queue_as :default
+
+    def perform
+      days = Railwatch::Embedded::Account.auto_resolve_after_days
+      return unless days
+
+      Issue.open.where("last_seen_at < ?", days.days.ago).find_each do |issue|
+        issue.resolve!(deploy: nil)
+      end
+    end
+  end
+end

data/app/jobs/railwatch/check_scheduled_tasks_job.rb

@@ -0,0 +1,113 @@
+# frozen_string_literal: true
+
+module Railwatch
+  # Flags scheduled tasks that have not run when expected. Uses the schedule
+  # string the gem sends (Solid Queue's cron / natural language) parsed with
+  # fugit, which Solid Queue already depends on.
+  class CheckScheduledTasksJob < ApplicationJob
+    include DetectionSnapshotting
+
+    queue_as :default
+
+    GRACE = 5.minutes
+    # A deploy restarts the worker, and Solid Queue skips (does not replay) the
+    # recurring ticks that fell inside the restart window. A tick missed within
+    # this long after a deploy is the deploy, not the scheduler.
+    DEPLOY_GRACE = 15.minutes
+
+    def perform(environment)
+      require "fugit"
+      tasks = environment.with_telemetry do
+        last_runs = Telemetry::Execution.scheduled.where("occurred_at > ?", 30.days.ago).group(:task_key).maximum(:occurred_at)
+        schedules = Telemetry::Execution.latest_schedules(last_runs.keys)
+        last_runs.map { |key, last| [ key, last, schedules[key] ] }
+      end
+      last_deploy_at = environment.deploys.maximum(:deployed_at)
+      configured = environment.with_telemetry { Telemetry::HealthSample.recurring_task_keys }
+      open_missed = environment.issues.open.where("group_hash LIKE 'missed:%'").index_by(&:group_hash)
+      tasks.each do |key, last_run, schedule|
+        next if schedule.blank?
+        # A task taken out of config/recurring.yml stops running on purpose.
+        # Its history would otherwise read as a miss every window for 30 days
+        # (reconcile_ingest_batches was live for four minutes and flagged 62
+        # times). The gem ships the schedule Solid Queue is actually running
+        # on every health sample; a key absent from the newest manifest is
+        # removed, not missed. No manifest yet means an older gem: judge from
+        # history alone, as before.
+        if configured && !configured.include?(key)
+          resolve_missed!(open_missed["missed:#{key}"], "Auto-resolved: #{key} is no longer a configured recurring task.")
+          next
+        end
+        cron = Fugit.parse(schedule) or next
+        expected = cron.next_time(last_run).to_t
+        if expected > Time.current - GRACE
+          # The task is back on schedule; a missed-run issue for it heals on its own.
+          resolve_missed!(open_missed["missed:#{key}"], "Auto-resolved: #{key} ran again on schedule.")
+          next
+        end
+        next if last_deploy_at && expected.between?(last_deploy_at - DEPLOY_GRACE, last_deploy_at + DEPLOY_GRACE)
+        diagnosis = diagnose(key, expected)
+        issue, outcome = Issue.record_occurrence!(
+          environment: environment, group_hash: "missed:#{key}", kind: "performance",
+          title: "Scheduled task #{key} missed its run (expected #{expected.utc.iso8601})#{diagnosis[:title_suffix]}",
+          culprit: key, occurred_at: Time.current, deploy: nil, user_ref: nil,
+          sample: { task_key: key, schedule: schedule, last_run: last_run.iso8601, expected: expected.iso8601 }.merge(diagnosis[:sample]))
+        next unless outcome == :new || outcome == :regressed
+
+        # Only on a new or regressed issue: the detectors re-run every few minutes
+        # and capturing a snapshot on every breached window would query telemetry
+        # for an issue nobody is newly looking at.
+        latest = environment.with_telemetry { Telemetry::Execution.scheduled.where(task_key: key).recent.first }
+        issue.update!(sample: issue.sample.merge("detection" => IssueDetectionSnapshot.capture(
+          environment: environment, kind: "performance", telemetry_type: "scheduled_task",
+          telemetry_group_hash: latest.group_hash, target: key, metric: "missed", from: expected, to: Time.current,
+          representative_from: last_run,
+          rule: { type: "schedule", task_key: key, schedule: schedule,
+                 last_run: last_run.iso8601, expected: expected.iso8601 }
+        )))
+        payload = { issue_key: issue.key, title: issue.title, environment: environment.name }
+        environment.application.alert_rules.where(event: "threshold").find_each do |rule|
+          next unless rule.matches?(issue: issue, payload: payload)
+          rule.fire!(event: "threshold", issue: issue, payload: payload)
+        end
+      end
+    end
+
+    private
+
+    # Where the run went, from Solid Queue's own ledger when it is in this
+    # process. A RecurringExecution row is written by the scheduler the moment
+    # it enqueues a run, so its presence without a scheduled_task execution
+    # means "enqueued, never performed" (no worker, or a wedged one); its
+    # absence means the scheduler itself never fired. The hosted platform, and
+    # any adapter other than Solid Queue, only ever sees the absence of the
+    # run and reports it as before.
+    def diagnose(key, expected)
+      return NO_DIAGNOSIS unless defined?(::SolidQueue::RecurringExecution)
+
+      enqueued_at = Railwatch.ignore { ::SolidQueue::RecurringExecution.where(task_key: key).where("run_at >= ?", expected - 1.minute).maximum(:run_at) }
+      workers = Railwatch.ignore { ::SolidQueue::Process.where(kind: "Worker").count }
+      if enqueued_at
+        { title_suffix: workers.zero? ? ": enqueued, but no Solid Queue worker is running" : ": enqueued, not yet performed",
+          sample: { enqueued_at: enqueued_at.iso8601, workers: workers, cause: workers.zero? ? "no_worker" : "backlog" } }
+      else
+        { title_suffix: ": the scheduler never enqueued it",
+          sample: { enqueued_at: nil, workers: workers, cause: "scheduler" } }
+      end
+    rescue StandardError
+      NO_DIAGNOSIS
+    end
+
+    NO_DIAGNOSIS = { title_suffix: "", sample: {} }.freeze
+
+    # Sentry Crons marked a monitor OK again on the next successful check-in;
+    # the same here: once the task has run inside its window again, the open
+    # missed-run issue resolves with an activity row, so it does not sit open
+    # until someone notices it is stale.
+    def resolve_missed!(issue, body)
+      return unless issue
+      issue.resolve!
+      issue.activities.create!(kind: "comment", data: { body: body })
+    end
+  end
+end