@tokenoftrust/storefront-runner 1.4.0 → 1.4.2-rc.0

package/apps/storefront/src/lib/activity/alerts.ts

@@ -0,0 +1,426 @@
+/**
+ * Operational-activity ALERT conditions — the detection layer on top of
+ * the activity query surface.
+ *
+ * The query surface makes operational activity queryable; this turns four specific,
+ * high-value conditions in that stream into ALERTS. It follows the same shape as
+ * this repo's other alert contract, `analytics/rumAlert.ts`: a set of
+ * PURE, transport-agnostic detectors that take a window of activity and return a
+ * structured `ActivityAlert[]`. WHERE the alerts go (a log line, a
+ * PagerDuty/Slack sink, a polled endpoint) is the sink's job — there is NO outbound
+ * notification transport in this repo (rumAlert is likewise detection-only),
+ * so this layer does not invent one. It also does NOT add a cron: the checker is on-demand
+ * (computed when `/admin/ops-timeline` loads, or when the read-only alerts endpoint
+ * is polled).
+ *
+ * THE FOUR CONDITIONS (and how each avoids false-positive noise):
+ *
+ *   1. silent-terminal — a `reconcile.terminal` with status `failed` (errorClass
+ *      `reconcile_fault` = an unexpected crash → critical, or `reconcile_blocked` =
+ *      a safety gate refused the candidate → warning). A single failure is worth an
+ *      alert (a reviewer's PR silently didn't materialize), so there is no count
+ *      threshold; noise is bounded by (a) status = `failed` only, never `succeeded`,
+ *      and (b) collapsing repeat failures for one changeId into one alert.
+ *
+ *   2. recurring-preview-miss — a REPEATED `preview.serve`/`failed`/`miss_no_env`
+ *      for the SAME changeId. One miss is an expected race (a reviewer opened the
+ *      preview before its ReviewEnvironment finished materializing), so a single
+ *      miss NEVER alerts; only `threshold`+ misses (default 3) for one changeId
+ *      inside a short `window` (default 15m) do — that pattern means a LIVE PR whose
+ *      preview never came up, not a transient.
+ *
+ *   3. reconcile-hang — a `webhook.reconcile.received` for a changeId with NO
+ *      `reconcile.terminal` (any status) at-or-after it, once a `grace` period
+ *      (default 10m) has elapsed. This is an ABSENCE detector (architecturally
+ *      unlike 1/2, which fire on a bad event's PRESENCE): it correlates two action
+ *      types by changeId + time. Noise is bounded by the grace window — a reconcile
+ *      still legitimately in flight (received < grace ago) never fires; a terminal
+ *      after the received (any status) clears it.
+ *
+ *   4. deploy-skew — promotes the visibility-only `deriveDeploySkew()` into an
+ *      alert. `skewDetected` is true on EVERY normal rollout (two isolates coexist
+ *      briefly), so alerting on it directly would page on every deploy. Instead this
+ *      fires ONLY when the build serving this request differs from the newest
+ *      observed build AND that newest build has been latest for longer than a `grace`
+ *      period (default 15m) — i.e. THIS isolate failed to roll to a deploy that
+ *      finished long ago (a persistent stale isolate), not a rollout in progress.
+ *
+ * DATA DEPENDENCY. The detectors read the D1 `activity_events` timeline through the
+ * `queryActivity` seam, so they alert on whatever is in that timeline. Getting
+ * server-emitted lifecycle events into the timeline is the fan-out wiring's job, not
+ * this layer's. `preview.serve` stays `analytics`-sink by design (too
+ * high-volume to timeline wholesale); detector 2 is source-agnostic (the rumAlert
+ * precedent) and fires against an Analytics-Engine-sourced or seeded feed. See
+ * `ALERT_CONDITIONS` for the per-condition source + wiring note.
+ */
+import {
+  queryActivity,
+  deriveDeploySkew,
+  type ActivityTimelineRow,
+  type DeploySkewSummary,
+} from "./query.js";
+import type { Queryable } from "../d1/catalog.js";
+
+/** Severity, mirroring `rumAlert.AlertSeverity` / `manifest.Severity`. */
+export type AlertSeverity = "page" | "critical" | "warning" | "info";
+
+/** The four conditions. Stable ids — a downstream sink/dashboard keys on them. */
+export type ActivityAlertType =
+  | "silent-terminal"
+  | "recurring-preview-miss"
+  | "reconcile-hang"
+  | "deploy-skew";
+
+/** One active alert. `details` is low-cardinality + already-redacted-safe (it is
+ *  built only from CORE fields — action/status/errorClass/changeId/at/version —
+ *  never from the payload zone), so it is safe to log or serialize verbatim. */
+export interface ActivityAlert {
+  type: ActivityAlertType;
+  severity: AlertSeverity;
+  /** Stable human title for the condition. */
+  title: string;
+  /** One-line human summary for a banner / log line / notification body. */
+  summary: string;
+  /** The change this alert concerns, when it is change-scoped. */
+  changeId?: string;
+  /** Structured, low-cardinality specifics (safe to serialize). */
+  details: Record<string, string | number>;
+  /** ISO-8601 instant the condition has been active SINCE (earliest evidence). */
+  since: string;
+}
+
+// ─── Thresholds (the launch defaults; every one is overridable per call) ──────
+export const DEFAULTS = {
+  /** How far back the on-demand checker scans the timeline. */
+  lookbackMs: 24 * 60 * 60 * 1000,
+  /** recurring-preview-miss: misses of the SAME changeId before it alerts. */
+  previewMissThreshold: 3,
+  /** recurring-preview-miss: the window those misses must fall inside. */
+  previewMissWindowMs: 15 * 60 * 1000,
+  /** reconcile-hang: how long a `received` may sit un-terminated before alerting. */
+  reconcileHangGraceMs: 10 * 60 * 1000,
+  /** deploy-skew: how long the newest build must have been latest before a
+   *  still-stale serving isolate is a persistent (not rollout-transient) skew. */
+  deploySkewGraceMs: 15 * 60 * 1000,
+} as const;
+
+const SEVERITY_RANK: Record<AlertSeverity, number> = { page: 0, critical: 1, warning: 2, info: 3 };
+
+/** Deterministic ordering: most-severe first, then most-recently-active first. */
+export function sortAlerts(alerts: ActivityAlert[]): ActivityAlert[] {
+  return [...alerts].sort(
+    (a, b) => SEVERITY_RANK[a.severity] - SEVERITY_RANK[b.severity] || (a.since < b.since ? 1 : a.since > b.since ? -1 : 0),
+  );
+}
+
+// ─── Detector 1: silent-terminal ─────────────────────────────────────────────
+
+/**
+ * Alert on `reconcile.terminal`/`failed` rows. Pass ONLY failed terminals (the
+ * orchestrator queries `status: "failed"`). Repeat failures for one changeId
+ * collapse into a single alert carrying the count, so a flapping PR does not spam.
+ */
+export function detectSilentTerminals(failedTerminals: readonly ActivityTimelineRow[]): ActivityAlert[] {
+  const byChange = new Map<string, ActivityTimelineRow[]>();
+  for (const r of failedTerminals) {
+    if (r.status !== "failed") continue; // defensive: only failures
+    const key = r.changeId ?? r.id;
+    (byChange.get(key) ?? byChange.set(key, []).get(key)!).push(r);
+  }
+  const out: ActivityAlert[] = [];
+  for (const [key, group] of byChange) {
+    const sorted = [...group].sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
+    const latest = sorted[sorted.length - 1]!;
+    const errorClass = latest.errorClass ?? "unknown";
+    // reconcile_fault = an UNEXPECTED crash of the reconcile → critical; a
+    // reconcile_blocked = a safety gate correctly refused the candidate, an
+    // operator should look but nothing crashed → warning.
+    const severity: AlertSeverity = errorClass === "reconcile_fault" ? "critical" : "warning";
+    out.push({
+      type: "silent-terminal",
+      severity,
+      title: "Reconcile terminated in failure",
+      summary:
+        `Candidate ${latest.changeId ?? key} reconcile ${errorClass}` +
+        (group.length > 1 ? ` (${group.length}× in window)` : "") +
+        ` — its preview did not materialize.`,
+      ...(latest.changeId ? { changeId: latest.changeId } : {}),
+      details: { errorClass, failures: group.length, lastAt: latest.at },
+      since: sorted[0]!.at,
+    });
+  }
+  return out;
+}
+
+// ─── Detector 2: recurring preview.serve miss_no_env ─────────────────────────
+
+export interface PreviewMissOptions {
+  /** "now" as epoch ms (injected for testing). */
+  now: number;
+  threshold?: number;
+  windowMs?: number;
+}
+
+/**
+ * Alert when the SAME changeId has `threshold`+ `preview.serve`/`failed`/
+ * `miss_no_env` events inside `windowMs`. Pass `preview.serve`/`failed` rows (the
+ * orchestrator queries them); this filters to `miss_no_env` and groups by changeId.
+ * A single miss (an expected pre-materialization race) is below threshold → silent.
+ */
+export function detectRecurringPreviewMisses(
+  serveFailures: readonly ActivityTimelineRow[],
+  opts: PreviewMissOptions,
+): ActivityAlert[] {
+  const threshold = opts.threshold ?? DEFAULTS.previewMissThreshold;
+  const windowMs = opts.windowMs ?? DEFAULTS.previewMissWindowMs;
+  const floor = opts.now - windowMs;
+
+  const byChange = new Map<string, ActivityTimelineRow[]>();
+  for (const r of serveFailures) {
+    if (r.errorClass !== "miss_no_env" || !r.changeId) continue;
+    const t = Date.parse(r.at);
+    if (Number.isNaN(t) || t < floor || t > opts.now) continue; // only misses inside the window
+    (byChange.get(r.changeId) ?? byChange.set(r.changeId, []).get(r.changeId)!).push(r);
+  }
+
+  const out: ActivityAlert[] = [];
+  for (const [changeId, misses] of byChange) {
+    if (misses.length < threshold) continue; // below threshold ⇒ transient, not an alert
+    const sorted = [...misses].sort((a, b) => (a.at < b.at ? -1 : a.at > b.at ? 1 : 0));
+    out.push({
+      type: "recurring-preview-miss",
+      severity: "warning",
+      title: "Preview repeatedly missing for a live change",
+      summary:
+        `Change ${changeId} has ${misses.length} preview.serve misses (miss_no_env) in ` +
+        `${Math.round(windowMs / 60000)}m — reviewers are hitting a preview that never came up.`,
+      changeId,
+      details: { misses: misses.length, windowMinutes: Math.round(windowMs / 60000), lastAt: sorted[sorted.length - 1]!.at },
+      since: sorted[0]!.at,
+    });
+  }
+  return out;
+}
+
+// ─── Detector 3: reconcile-hang watchdog (ABSENCE correlation) ───────────────
+
+export interface ReconcileHangOptions {
+  /** "now" as epoch ms (injected for testing). */
+  now: number;
+  graceMs?: number;
+}
+
+/**
+ * Alert on a `webhook.reconcile.received` for a changeId that has NO
+ * `reconcile.terminal` at-or-after it, once `graceMs` has elapsed. Correlates the
+ * two action streams by changeId + time and reports the ABSENCE of a terminal.
+ *
+ * Per changeId we take the LATEST received (a PR pushed N times → only the newest
+ * cycle matters) and look for ANY terminal (succeeded/failed/blocked) with
+ * `at >= received.at`. None + past-grace ⇒ hang. A terminal after the received, OR
+ * a received still inside the grace window, clears it (the true-negative cases).
+ */
+export function detectReconcileHangs(
+  received: readonly ActivityTimelineRow[],
+  terminals: readonly ActivityTimelineRow[],
+  opts: ReconcileHangOptions,
+): ActivityAlert[] {
+  const graceMs = opts.graceMs ?? DEFAULTS.reconcileHangGraceMs;
+
+  // Latest `received` per changeId.
+  const latestReceived = new Map<string, ActivityTimelineRow>();
+  for (const r of received) {
+    if (!r.changeId) continue;
+    const prev = latestReceived.get(r.changeId);
+    if (!prev || r.at > prev.at) latestReceived.set(r.changeId, r);
+  }
+  // Latest terminal `at` per changeId (any status).
+  const latestTerminalAt = new Map<string, string>();
+  for (const t of terminals) {
+    if (!t.changeId) continue;
+    const prev = latestTerminalAt.get(t.changeId);
+    if (!prev || t.at > prev) latestTerminalAt.set(t.changeId, t.at);
+  }
+
+  const out: ActivityAlert[] = [];
+  for (const [changeId, r] of latestReceived) {
+    const terminalAt = latestTerminalAt.get(changeId);
+    if (terminalAt && terminalAt >= r.at) continue; // terminated after this receive ⇒ not hung
+    const receivedMs = Date.parse(r.at);
+    if (Number.isNaN(receivedMs)) continue;
+    const elapsedMs = opts.now - receivedMs;
+    if (elapsedMs <= graceMs) continue; // still legitimately in flight ⇒ no alert
+    out.push({
+      type: "reconcile-hang",
+      severity: "critical",
+      title: "Reconcile received but never terminated",
+      summary:
+        `Change ${changeId} got a reconcile webhook ${Math.round(elapsedMs / 60000)}m ago with no ` +
+        `terminal state — the review loop is stuck.`,
+      changeId,
+      details: { receivedAt: r.at, minutesElapsed: Math.round(elapsedMs / 60000), graceMinutes: Math.round(graceMs / 60000) },
+      since: r.at,
+    });
+  }
+  return out;
+}
+
+// ─── Detector 4: deploy-skew (persistent, not rollout-transient) ─────────────
+
+export interface DeploySkewAlertOptions {
+  /** "now" as epoch ms (injected for testing). */
+  now: number;
+  graceMs?: number;
+}
+
+/**
+ * Promote the visibility-only `DeploySkewSummary` into an alert, WITHOUT firing on
+ * every normal rollout. Fires only when the build serving this request differs from
+ * the newest observed build AND that newest build has been latest for longer than
+ * `graceMs` — i.e. THIS isolate never rolled to a deploy that finished long ago.
+ * A fresh rollout (newest observed < grace old) is the expected transient → silent.
+ */
+export function detectDeploySkew(skew: DeploySkewSummary, opts: DeploySkewAlertOptions): ActivityAlert[] {
+  const graceMs = opts.graceMs ?? DEFAULTS.deploySkewGraceMs;
+  const { runningVersion, latestObservedVersion } = skew;
+  // Need to know BOTH the serving build and the newest observed build to judge skew.
+  if (!runningVersion || !latestObservedVersion) return [];
+  if (runningVersion === latestObservedVersion) return []; // this isolate is current ⇒ fine
+  const latest = skew.observedVersions[0];
+  if (!latest) return [];
+  const latestAgeMs = opts.now - Date.parse(latest.lastObservedAt);
+  if (Number.isNaN(latestAgeMs) || latestAgeMs <= graceMs) return []; // rollout still in its grace window
+  return [
+    {
+      type: "deploy-skew",
+      severity: "warning",
+      title: "Stale build still serving after rollout",
+      summary:
+        `This isolate serves build ${runningVersion} but ${latestObservedVersion} has been the ` +
+        `latest for ${Math.round(latestAgeMs / 60000)}m — a stale isolate never rolled forward.`,
+      details: {
+        runningVersion,
+        latestObservedVersion,
+        minutesStale: Math.round(latestAgeMs / 60000),
+        graceMinutes: Math.round(graceMs / 60000),
+      },
+      since: latest.lastObservedAt,
+    },
+  ];
+}
+
+// ─── The on-demand orchestrator ──────────────────────────────────────────────
+
+export interface CheckAlertsOptions {
+  /** Injected clock (tests). Defaults to `new Date()`. */
+  now?: Date;
+  lookbackMs?: number;
+  previewMissThreshold?: number;
+  previewMissWindowMs?: number;
+  reconcileHangGraceMs?: number;
+  deploySkewGraceMs?: number;
+  /** Pin the serving build for deploy-skew (tests); real callers omit it. */
+  runningOverride?: string;
+}
+
+/**
+ * Run all four detectors for a tenant against the D1 timeline and return the active
+ * alerts, most-severe-first. Tenant-scoped through `queryActivity`/`deriveDeploySkew`
+ * (the same isolation invariant covers the whole check). On-demand: called when the
+ * ops-timeline page loads and by the read-only alerts endpoint — no cron.
+ */
+export async function checkActivityAlerts(
+  db: Queryable,
+  tenantId: string,
+  opts: CheckAlertsOptions = {},
+): Promise<ActivityAlert[]> {
+  const now = opts.now ?? new Date();
+  const nowMs = now.getTime();
+  const lookbackMs = opts.lookbackMs ?? DEFAULTS.lookbackMs;
+  const since = new Date(nowMs - lookbackMs).toISOString();
+
+  const [failedTerminals, allTerminals, received, serveFailures, skew] = await Promise.all([
+    queryActivity(db, { tenantId, action: "reconcile.terminal", status: "failed", since, order: "desc" }),
+    queryActivity(db, { tenantId, action: "reconcile.terminal", since, order: "desc" }),
+    queryActivity(db, { tenantId, action: "webhook.reconcile.received", since, order: "desc" }),
+    queryActivity(db, { tenantId, action: "preview.serve", status: "failed", since, order: "desc" }),
+    deriveDeploySkew(db, tenantId, { since, ...(opts.runningOverride ? { runningOverride: opts.runningOverride } : {}) }),
+  ]);
+
+  return sortAlerts([
+    ...detectSilentTerminals(failedTerminals),
+    ...detectRecurringPreviewMisses(serveFailures, {
+      now: nowMs,
+      ...(opts.previewMissThreshold != null ? { threshold: opts.previewMissThreshold } : {}),
+      ...(opts.previewMissWindowMs != null ? { windowMs: opts.previewMissWindowMs } : {}),
+    }),
+    ...detectReconcileHangs(received, allTerminals, {
+      now: nowMs,
+      ...(opts.reconcileHangGraceMs != null ? { graceMs: opts.reconcileHangGraceMs } : {}),
+    }),
+    ...detectDeploySkew(skew, {
+      now: nowMs,
+      ...(opts.deploySkewGraceMs != null ? { graceMs: opts.deploySkewGraceMs } : {}),
+    }),
+  ]);
+}
+
+/** One-line alert summary for a log line / notification body (rumAlert parity). */
+export function formatActivityAlert(a: ActivityAlert): string {
+  return `[${a.severity}] ${a.type}: ${a.summary} (since ${a.since})`;
+}
+
+// ─── Coherent surface descriptor (F16 `MONITORS` parity) ─────────────────────
+
+/** Honest per-condition source + wiring note — the audit trail for "is this
+ *  actually alerting?", mirroring F16's `WiringStatus`. */
+export interface AlertCondition {
+  type: ActivityAlertType;
+  title: string;
+  defaultSeverity: AlertSeverity;
+  threshold: string;
+  /** Which timeline signal(s) it reads. */
+  source: string;
+  /** Honest wiring status. */
+  wiring: "in-code" | "needs-timeline-fanout" | "needs-external";
+  notes?: string;
+}
+
+export const ALERT_CONDITIONS: readonly AlertCondition[] = [
+  {
+    type: "silent-terminal",
+    title: "Reconcile terminated in failure",
+    defaultSeverity: "critical",
+    threshold: "any reconcile.terminal/failed (reconcile_fault ⇒ critical, reconcile_blocked ⇒ warning)",
+    source: "reconcile.terminal (timeline sink)",
+    wiring: "needs-timeline-fanout",
+    notes: "Emitted by candidate-reconcile.ts via recordActivity() (logs-only today); reaches D1 once the D1/D4 server→timeline fan-out lands.",
+  },
+  {
+    type: "recurring-preview-miss",
+    title: "Preview repeatedly missing for a live change",
+    defaultSeverity: "warning",
+    threshold: "≥3 preview.serve/failed/miss_no_env for one changeId within 15m",
+    source: "preview.serve (analytics sink — high volume)",
+    wiring: "needs-external",
+    notes: "preview.serve stays analytics-sink by design; detector is source-agnostic (rumAlert precedent) and fires against an Analytics-Engine-sourced or seeded feed.",
+  },
+  {
+    type: "reconcile-hang",
+    title: "Reconcile received but never terminated",
+    defaultSeverity: "critical",
+    threshold: "webhook.reconcile.received with no reconcile.terminal at-or-after it, past a 10m grace",
+    source: "webhook.reconcile.received + reconcile.terminal (both timeline sink)",
+    wiring: "needs-timeline-fanout",
+    notes: "D7 promoted webhook.reconcile.received to the timeline sink and emits it at reconcile start; reaches D1 once server→timeline fan-out lands.",
+  },
+  {
+    type: "deploy-skew",
+    title: "Stale build still serving after rollout",
+    defaultSeverity: "warning",
+    threshold: "serving build ≠ newest observed AND newest has been latest > 15m",
+    source: "deploy.version_observed (timeline sink) + this build's __GIT_SHA__",
+    wiring: "needs-timeline-fanout",
+    notes: "Promotes D5's deriveDeploySkew from visibility to an alert; grace period suppresses normal-rollout noise.",
+  },
+] as const;

package/apps/storefront/src/lib/activity/changeActorAttribution.ts

@@ -0,0 +1,85 @@
+/**
+ * Owner-facing "who did this" attribution for a candidate change or aggregate —
+ * DELIBERATELY separate from the anonymized activity spine. `ActivityActor.id`
+ * is a salted, non-reversible hash by design (see `recordActivity.ts`'s header:
+ * a raw PII id in the core zone would defeat the redaction contract), so it can never
+ * answer "who submitted/accepted/rejected/shipped this" for an owner reviewing
+ * `/admin#publish`. This persists the real email (never joined into the general
+ * `queryActivity()` surface, never cross-tenant) alongside the existing hashed
+ * event, so the two concerns stay separate: analytics stays anonymized, and
+ * this stays a narrow, owner-only-readable accountability record.
+ *
+ * `sub` (the tot20-issued, stable user id) would be a better key than email —
+ * email is a mutable account attribute, not the identity itself — but `sub`
+ * is dropped before `SessionRecord` today (see `auth/session.ts`). Threading
+ * it through is a real follow-up, not done here; email is what every call site
+ * already has.
+ */
+import { displayNameFromEmail } from "./recordActivity";
+
+export type ChangeActorAction = "submit" | "reject" | "resolve" | "accept" | "ship" | "build";
+
+export interface ChangeActorRecord {
+  email: string;
+  /** Best-effort display name — no real name is captured anywhere today, so
+   *  this is always {@link displayNameFromEmail}'s local-part guess. Swap the
+   *  source here (not the callers) if a real name ever becomes available. */
+  name: string;
+  at: string;
+}
+
+export interface ChangeActorKv {
+  get(key: string): Promise<string | null>;
+  put(key: string, value: string): Promise<void>;
+}
+
+function attributionKey(tenantId: string, subjectId: string, action: ChangeActorAction): string {
+  return `cust:change-actor:${tenantId}:${subjectId}:${action}`;
+}
+
+/**
+ * Persist who took `action` on `subjectId` (a `changeId` for reject/accept/build,
+ * or the aggregate id for ship). Best-effort — never blocks or fails the
+ * mutation it describes; a write fault is swallowed, mirroring `recordActivity`.
+ */
+export async function recordChangeActor(
+  kv: ChangeActorKv,
+  tenantId: string,
+  subjectId: string,
+  action: ChangeActorAction,
+  email: string,
+): Promise<void> {
+  try {
+    const record: ChangeActorRecord = {
+      email,
+      name: displayNameFromEmail(email),
+      at: new Date().toISOString(),
+    };
+    await kv.put(attributionKey(tenantId, subjectId, action), JSON.stringify(record));
+  } catch {
+    /* best-effort — the mutation it describes already succeeded */
+  }
+}
+
+/** Read every recorded action's attribution for one subject. Best-effort per
+ *  action: a missing/corrupt record is simply absent from the result, never
+ *  a thrown error. */
+export async function readChangeActors(
+  kv: ChangeActorKv,
+  tenantId: string,
+  subjectId: string,
+): Promise<Partial<Record<ChangeActorAction, ChangeActorRecord>>> {
+  const actions: ChangeActorAction[] = ["submit", "reject", "resolve", "accept", "ship", "build"];
+  const out: Partial<Record<ChangeActorAction, ChangeActorRecord>> = {};
+  await Promise.all(
+    actions.map(async (action) => {
+      try {
+        const raw = await kv.get(attributionKey(tenantId, subjectId, action));
+        if (raw) out[action] = JSON.parse(raw) as ChangeActorRecord;
+      } catch {
+        /* a corrupt/missing record just isn't shown */
+      }
+    }),
+  );
+  return out;
+}

package/apps/storefront/src/lib/activity/deployVersion.ts

@@ -0,0 +1,120 @@
+/**
+ * deploy.version_observed — the deploy-provenance / deploy-skew signal.
+ *
+ * "A deployed worker version/commit was observed." This module makes
+ * `worker_commit` reliably present on server events and EMITs this observation at
+ * least once per running build. It intentionally does NO comparison — it has no
+ * notion of what the umbrella tip is, only what THIS build is; the "== umbrella
+ * tip?" deploy-skew comparison and the timeline query live elsewhere.
+ *
+ * WHAT IT PRODUCES. A catalog `deploy.version_observed` event (ACTION_CATALOG:
+ * actors [system|dev], source server, sinks logs+analytics+timeline). The FULL
+ * 40-char commit rides `scope.workerCommit` — the CORE zone, which createActivityEvent
+ * never redacts — because that is where the authoritative, unscrubbed sha belongs.
+ *
+ * DELIBERATELY the full sha does NOT go in `payload`. The action's argsAllow names
+ * `commit`, but a 40-char hex sha trips the `high_entropy` VALUE canary (redaction.ts:
+ * ≥28 chars of base64url) — it would be scrubbed to «redacted» AND set
+ * `report.canaryTripped`, which alerting reads, firing a FALSE PII alarm on every cold
+ * start. So payload carries only the SHORT `version` (7 chars, safe) plus `env` +
+ * `observedFrom`; a reader wanting the full sha reads scope.workerCommit (a git sha is
+ * public, so the core zone is its correct, alert-free home). Treat scope.workerCommit
+ * as the authoritative provenance, payload.version as a
+ * human-eyeball convenience.
+ *
+ * WHERE IT'S EMITTED. observeDeployVersion() fires ONCE PER WORKER ISOLATE (guarded
+ * by a module-level flag) from the middleware's front door — a cold start emits one
+ * observation, not one per request, so the stream is low-volume and honest. The
+ * emission goes to the universal `logs` sink (structured `[activity]` JSON to Workers
+ * Logs, already provisioned via `[observability]`).
+ */
+import { createActivityEvent } from "@tot/public-runtime";
+import type { ActivityEvent, RedactionReport } from "@tot/public-runtime";
+import { workerCommit, workerCommitShort, UNKNOWN_COMMIT } from "./workerCommit.js";
+import { isActivityDisabled } from "./killSwitch.js";
+
+export interface DeployVersionObservedInput {
+  /** Deploy env/target name (e.g. "preview", "production"), when known. Optional. */
+  env?: string;
+  /** What surfaced the observation (e.g. "startup", "health"). Defaults "startup". */
+  observedFrom?: string;
+  /** Opaque actor id. Defaults "system" (the platform observed its own build). */
+  actorId?: string;
+  /** Override the build commit — tests only. Real callers omit (reads __GIT_SHA__). */
+  commit?: string;
+}
+
+/**
+ * Build (do not emit) the `deploy.version_observed` ActivityEvent for the running
+ * build. Pure + side-effect-free — the piece downstream consumers ultimately consume
+ * and the piece tests assert against. Returns the safe-constructed event plus its redaction report.
+ */
+export function buildDeployVersionObservedEvent(
+  input: DeployVersionObservedInput = {},
+): { event: ActivityEvent; report: RedactionReport } {
+  const commit = workerCommit(input.commit);
+  const version = workerCommitShort(input.commit);
+  return createActivityEvent({
+    action: "deploy.version_observed",
+    actor: { kind: "system", id: input.actorId ?? "system" },
+    source: "server",
+    outcome: { status: "succeeded" },
+    // worker_commit (full sha) in the CORE zone — never redacted; the authoritative
+    // provenance every server event carries and deploy skew is derived from.
+    scope: commit !== UNKNOWN_COMMIT ? { workerCommit: commit } : {},
+    // Payload carries the SHORT version (safe from the high_entropy canary that would
+    // scrub a full sha) + env + observedFrom. Unknown builds omit `version` rather than
+    // store the sentinel, but still emit the observation.
+    payload: {
+      args: {
+        ...(commit !== UNKNOWN_COMMIT ? { version } : {}),
+        ...(input.env ? { env: input.env } : {}),
+        observedFrom: input.observedFrom ?? "startup",
+      },
+    },
+  });
+}
+
+/**
+ * Serialize an ActivityEvent to the universal `logs` sink (Workers Logs) —
+ * structured `[activity]` JSON so one grep finds all activity lines.
+ * Best-effort, never throws.
+ */
+function emitToLogs(event: ActivityEvent): void {
+  try {
+    console.log(`[activity] ${JSON.stringify(event)}`);
+  } catch {
+    // best-effort only — telemetry must never break a request
+  }
+}
+
+/** Per-isolate guard: emit the observation once per cold start, not per request. */
+let observedThisIsolate = false;
+
+/**
+ * Emit `deploy.version_observed` ONCE for this worker isolate. Idempotent within an
+ * isolate (subsequent calls are no-ops), best-effort, never throws — safe to call on
+ * every request from the middleware front door. Returns the event it emitted (or
+ * `undefined` if it already fired this isolate) so callers/tests can inspect it.
+ */
+export function observeDeployVersion(
+  input: DeployVersionObservedInput = {},
+): ActivityEvent | undefined {
+  if (observedThisIsolate) return undefined;
+  // Kill switch (D8): telemetry off ⇒ skip the observation entirely (fail-open). Do
+  // NOT set the per-isolate guard, so re-enabling emits the observation on the next call.
+  if (isActivityDisabled()) return undefined;
+  observedThisIsolate = true;
+  try {
+    const { event } = buildDeployVersionObservedEvent(input);
+    emitToLogs(event);
+    return event;
+  } catch {
+    return undefined;
+  }
+}
+
+/** Test-only: reset the per-isolate guard so a test can re-exercise the first-fire path. */
+export function __resetDeployVersionObservedForTest(): void {
+  observedThisIsolate = false;
+}