@obsunified/collector 2.2.0 → 2.2.3

package/src/__tests__/autosre-closed-loop.test.ts

@@ -1,6 +1,9 @@
-import { buildCanonicalNodeId, buildFindingFingerprint } from "@obsunified/types";
+import {
+	buildCanonicalNodeId,
+	buildFindingFingerprint,
+} from "@obsunified/types";
 import { Hono } from "hono";
-import { describe, expect, it } from "vitest";
+import { afterEach, describe, expect, it, vi } from "vitest";
 import { CollectorRuntime } from "../framework/collector";
 import type { CollectorEnv } from "../framework/env";
 import {
@@ -8,13 +11,25 @@ import {
 	findCrossTraceContention,
 	recordChangeEvent,
 } from "../lib/autosre-change-correlation";
+import { recordInvestigationFinding } from "../lib/autosre-findings";
 import {
 	computeTarjanScc,
 	extractSpanLineageAndTopology,
 	localizeTraceFaultLocus,
 	upsertOntologyFromSpans,
 } from "../lib/autosre-ontology";
-import { evaluateRemediationWatchdogs } from "../lib/autosre-remediation-gate";
+import {
+	acquireIncidentLease,
+	claimRemediationGrant,
+	completeRemediationProposal,
+	dispatchIncidentWebhook,
+	evaluateRemediationWatchdogs,
+	failRemediationProposal,
+	openOrDeduplicateIncident,
+	proposeRemediationAction,
+	RemediationGateError,
+	rejectRemediationProposal,
+} from "../lib/autosre-remediation-gate";
 import {
 	createMigratedSqliteDb,
 	type NodeSqliteDb,
@@ -207,7 +222,8 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 
 		// Verify unscoped DB contention caps relationshipCertainty at "inferred"
 		const baseMs = Date.parse("2026-10-15T01:10:00.000Z");
-		const unscopedNodeId = unscopedCtx.targetNode!.id;
+		const unscopedNodeId = unscopedCtx.targetNode?.id;
+		if (!unscopedNodeId) throw new Error("expected an unscoped target node");
 		for (let i = 0; i < 6; i++) {
 			insertSpan(db, {
 				traceId: "tr_aggressor_unscoped",
@@ -629,8 +645,7 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 
 		// 1. Seed concurrent v2.4.0 traffic (60 spans, 1 error = 1.67% error rate)
 		// and canary v2.4.1 traffic (60 spans, 15 errors = 25% error rate) with code stackframe in src/checkout/inventory.ts
-		const dbNodeId =
-			"database:prod:postgresql:pg-primary.internal:5432/orders";
+		const dbNodeId = "database:prod:postgresql:pg-primary.internal:5432/orders";
 		const serviceNodeId = "service:prod:default:checkout-api";
 
 		for (let i = 0; i < 60; i++) {
@@ -704,10 +719,7 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 					commitSha: "8f3a1b2",
 					previousCommitSha: "1111111",
 					summary: "Deploy checkout-api v2.4.1 (batch inventory lock)",
-					touchedPaths: [
-						"src/checkout/inventory.ts",
-						"src/checkout/routes.ts",
-					],
+					touchedPaths: ["src/checkout/inventory.ts", "src/checkout/routes.ts"],
 					occurredAt: new Date(deployTime).toISOString(),
 				}),
 			},
@@ -835,7 +847,8 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 					faultLocusEntityId: "sp_new_0",
 					rootCauseEntityKind: "change",
 					rootCauseEntityId: "chg_deploy_241",
-					contributingChangeId: "chg_deploy_241",
+					// No contributingChangeId: the server cannot recompute a cohort
+					// verdict, so the proof tier stays corroborated_candidate.
 					cohortVerdict: "shared_degradation_new_version_aggressor",
 					causalChain: [
 						{
@@ -852,7 +865,7 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 						{
 							subjectKind: "span",
 							subjectId: "sp_new_0",
-							predicate: "stackframe_overlaps_changed_file",
+							predicate: "touched_same_code_path",
 							objectKind: "change",
 							objectId: "chg_deploy_241",
 							relationshipCertainty: "exact",
@@ -926,9 +939,13 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 				headers: {
 					"Content-Type": "application/json",
 					Cookie: "obs_session=sess_human_operator_123",
+					// Client-supplied identities must be ignored (B15).
 					"X-Obs-User-Actor": "sawan@obsunified.dev",
 				},
-				body: JSON.stringify({ environment: "prod" }),
+				body: JSON.stringify({
+					environment: "prod",
+					approvedByUserId: "ceo@example.com",
+				}),
 			},
 			env,
 		);
@@ -938,8 +955,11 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 			grantToken: string;
 		};
 		expect(humanApproveBody.proposal.status).toBe("grant_issued");
-		expect(humanApproveBody.proposal.approvedByUserId).toBe(
-			"sawan@obsunified.dev",
+		expect(humanApproveBody.proposal.approvedByUserId).toMatch(
+			/^dashboard_session:[0-9a-f]{16}$/,
+		);
+		expect(humanApproveBody.proposal.approvedByUserId).not.toContain(
+			"sess_human_operator_123",
 		);
 		expect(humanApproveBody.grantToken).toMatch(/^rgt_/);
 
@@ -954,7 +974,8 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 					sourceEntityType: "trace",
 					sourceEntityId: "tr_new_0",
 					name: "Reproduce checkout lock timeout",
-					expectedOutcome: "Checkout completes under 200ms without lock timeout",
+					expectedOutcome:
+						"Checkout completes under 200ms without lock timeout",
 				}),
 			},
 			env,
@@ -1136,6 +1157,7 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 				headers: { "Content-Type": "application/json" },
 				body: JSON.stringify({
 					environment: "prod",
+					runnerId: "runner-alpha",
 					resultingChangeId: rollbackChange.id,
 					remediationAgentRunId: "arun_autosre_1",
 				}),
@@ -1185,9 +1207,20 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 			)
 			.run(new Date(postRollbackBase + 40_000).toISOString(), finding.id);
 
-		const watchdogRecovered = await evaluateRemediationWatchdogs(db, {
+		// A single healthy checkpoint is not enough (B11): two consecutive
+		// sufficient-traffic ok checkpoints are required before `recovered`.
+		const watchdogFirstOk = await evaluateRemediationWatchdogs(db, {
 			nowIso: new Date(postRollbackBase + 50_000).toISOString(),
 		});
+		expect(watchdogFirstOk.recovered).toBe(0);
+		db.raw
+			.prepare(
+				`UPDATE investigation_findings SET next_watchdog_check_at = ? WHERE id = ?`,
+			)
+			.run(new Date(postRollbackBase + 60_000).toISOString(), finding.id);
+		const watchdogRecovered = await evaluateRemediationWatchdogs(db, {
+			nowIso: new Date(postRollbackBase + 70_000).toISOString(),
+		});
 		expect(watchdogRecovered.recovered).toBe(1);
 
 		// Verify incident is now resolved and finding recoveryStatus === 'recovered'
@@ -1270,6 +1303,1510 @@ describe("RFC 0013 — AutoSRE Causal Ontology, Change Correlation & Closed-Loop
 			),
 		).toBe(true);
 
+		const changeBundleRes = await app.request(
+			`/internal/evidence/bundle`,
+			{
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({
+					intent: "triage_incident",
+					anchor: {
+						entityKind: "change",
+						entityId: "chg_deploy_241",
+					},
+				}),
+			},
+			env,
+		);
+		expect(changeBundleRes.status).toBe(200);
+		const changeBundleBody = (await changeBundleRes.json()) as {
+			anchor: { entityKind: string; entityId: string };
+			summary: string;
+		};
+		expect(changeBundleBody.anchor).toEqual({
+			entityKind: "change",
+			entityId: "chg_deploy_241",
+		});
+		expect(changeBundleBody.summary).toContain("chg_deploy_241");
+
+		const findingBundleRes = await app.request(
+			`/internal/evidence/bundle`,
+			{
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({
+					intent: "triage_incident",
+					anchor: {
+						entityKind: "finding",
+						entityId: finding.id,
+					},
+				}),
+			},
+			env,
+		);
+		expect(findingBundleRes.status).toBe(200);
+		const findingBundleBody = (await findingBundleRes.json()) as {
+			anchor: { entityKind: string; entityId: string };
+			findings: Array<{ title: string }>;
+		};
+		expect(findingBundleBody.anchor).toEqual({
+			entityKind: "finding",
+			entityId: finding.id,
+		});
+		expect(
+			findingBundleBody.findings.some((f) =>
+				f.title.includes("Remediation proposal"),
+			),
+		).toBe(true);
+
+		db.close();
+	});
+});
+
+describe("AutoSRE governance hardening", () => {
+	const DEPLOY = Date.parse("2026-10-15T03:00:00.000Z");
+	const SERVICE_NODE = "service:prod:default:checkout-api";
+
+	afterEach(() => {
+		vi.unstubAllGlobals();
+	});
+
+	/** v2.4.0 healthy, v2.4.1 at 25% errors, plus the deploy change event. */
+	const seedIsolatedCohort = async (db: NodeSqliteDb) => {
+		for (let i = 0; i < 60; i++) {
+			const t = new Date(DEPLOY + (i + 1) * 1000).toISOString();
+			const end = new Date(DEPLOY + (i + 1) * 1000 + 40).toISOString();
+			insertSpan(db, {
+				traceId: `g_base_${i}`,
+				spanId: `g_base_sp_${i}`,
+				serviceName: "checkout-api",
+				spanName: "POST /checkout",
+				statusCode: i === 0 ? 2 : 1,
+				startTime: t,
+				endTime: end,
+				durationMs: 40,
+				serviceVersion: "v2.4.0",
+			});
+			insertSpan(db, {
+				traceId: `g_new_${i}`,
+				spanId: `g_new_sp_${i}`,
+				serviceName: "checkout-api",
+				spanName: "POST /checkout",
+				statusCode: i < 15 ? 2 : 1,
+				startTime: t,
+				endTime: end,
+				durationMs: 40,
+				serviceVersion: "v2.4.1",
+				errorSignature: "checkout 500",
+			});
+		}
+		await recordChangeEvent(db, "default", {
+			id: "chg_gov",
+			environment: "prod",
+			kind: "deploy",
+			serviceName: "checkout-api",
+			version: "v2.4.1",
+			previousVersion: "v2.4.0",
+			summary: "Deploy v2.4.1",
+			occurredAt: new Date(DEPLOY).toISOString(),
+		});
+	};
+
+	const newIncident = async (db: NodeSqliteDb, fingerprint = "fp_gov") =>
+		(
+			await openOrDeduplicateIncident(db, "default", {
+				environment: "prod",
+				targetNodeId: SERVICE_NODE,
+				symptomAnchorKind: "trace",
+				symptomAnchorId: "t1",
+				symptomFingerprint: fingerprint,
+				symptomCategory: fingerprint,
+			})
+		).incident;
+
+	const newFinding = (
+		db: NodeSqliteDb,
+		incidentId: string,
+		extra: Partial<Parameters<typeof recordInvestigationFinding>[2]> = {},
+	) =>
+		recordInvestigationFinding(db, "default", {
+			environment: "prod",
+			incidentId,
+			title: "Checkout regression",
+			summary: "Errors after deploy",
+			causalProofTier: "corroborated_candidate",
+			symptomEntityKind: "trace",
+			symptomEntityId: "t1",
+			faultLocusEntityKind: "service",
+			faultLocusEntityId: SERVICE_NODE,
+			causalChain: [],
+			citedEvidenceIds: [],
+			...extra,
+		});
+
+	const propose = (
+		db: NodeSqliteDb,
+		incidentId: string,
+		findingId: string,
+		key: string,
+		extra: Record<string, unknown> = {},
+	) =>
+		proposeRemediationAction(db, "default", {
+			environment: "prod",
+			incidentId,
+			findingId,
+			idempotencyKey: key,
+			actionKind: "rollback_deploy",
+			targetNodeId: SERVICE_NODE,
+			expectedCurrentState: {},
+			desiredTargetState: { version: "v2.4.0" },
+			requestedAutonomy: "autonomous_write",
+			...extra,
+		} as Parameters<typeof proposeRemediationAction>[2]);
+
+	const grantedProposal = async (db: NodeSqliteDb, key = "idem_granted") => {
+		await seedIsolatedCohort(db);
+		const incident = await newIncident(db);
+		const finding = await newFinding(db, incident.id, {
+			contributingChangeId: "chg_gov",
+		});
+		const proposal = await propose(db, incident.id, finding.id, key);
+		return { incident, finding, proposal };
+	};
+
+	it("B1: a client-asserted isolated_to_new_version verdict cannot buy an autonomous rollback grant", async () => {
+		const db = createMigratedSqliteDb();
+		const incident = await newIncident(db);
+		const finding = await newFinding(db, incident.id, {
+			cohortVerdict: "isolated_to_new_version",
+			causalProofTier: "cohort_rate_isolated",
+			causalChain: [
+				{
+					subjectKind: "service",
+					subjectId: SERVICE_NODE,
+					predicate: "isolated_to_version_cohort",
+					objectKind: "change",
+					objectId: "chg_missing",
+					relationshipCertainty: "exact",
+					causalProofTier: "cohort_rate_isolated",
+					confidence: 0.99,
+					reason: "asserted by client",
+				},
+			],
+		});
+		expect(finding.causalProofTier).toBe("corroborated_candidate");
+		expect(finding.cohortVerdict).toBeNull();
+
+		const proposal = await propose(db, incident.id, finding.id, "idem_b1");
+		expect(proposal.status).toBe("pending_human_approval");
+		expect(proposal.grantToken).toBeUndefined();
+
+		// Pointing at a real change whose telemetry does not show isolation
+		// (no spans at all) does not help either.
+		await recordChangeEvent(db, "default", {
+			id: "chg_empty",
+			environment: "prod",
+			kind: "deploy",
+			serviceName: "checkout-api",
+			version: "v9",
+			summary: "no telemetry",
+		});
+		const f2 = await newFinding(db, incident.id, {
+			cohortVerdict: "isolated_to_new_version",
+			contributingChangeId: "chg_empty",
+		});
+		expect(f2.causalProofTier).toBe("corroborated_candidate");
+		db.close();
+	});
+
+	it("B1: the server-recomputed isolated verdict is what yields cohort_rate_isolated and an autonomous grant", async () => {
+		const db = createMigratedSqliteDb();
+		const { finding, proposal } = await grantedProposal(db);
+		expect(finding.causalProofTier).toBe("cohort_rate_isolated");
+		expect(finding.cohortVerdict).toBe("isolated_to_new_version");
+		expect(proposal.status).toBe("grant_issued");
+		expect(proposal.grantToken).toMatch(/^rgt_/);
+		db.close();
+	});
+
+	it("B2: a grant token is single-use, sequentially and under concurrent claims", async () => {
+		const db = createMigratedSqliteDb();
+		const { proposal } = await grantedProposal(db);
+		const token = proposal.grantToken as string;
+
+		await claimRemediationGrant(db, "default", proposal.id, {
+			environment: "prod",
+			grantToken: token,
+			runnerId: "r1",
+		});
+		await expect(
+			claimRemediationGrant(db, "default", proposal.id, {
+				environment: "prod",
+				grantToken: token,
+				runnerId: "r2",
+			}),
+		).rejects.toMatchObject({ statusCode: 409 });
+
+		// Concurrent claims on a fresh grant: exactly one wins.
+		const second = await propose(
+			db,
+			proposal.incidentId,
+			proposal.findingId,
+			"idem_b2_conc",
+			{ actionKind: "toggle_feature_flag" },
+		);
+		const results = await Promise.allSettled(
+			["a", "b", "c", "d"].map((runnerId) =>
+				claimRemediationGrant(db, "default", second.id, {
+					environment: "prod",
+					grantToken: second.grantToken as string,
+					runnerId,
+				}),
+			),
+		);
+		expect(results.filter((r) => r.status === "fulfilled")).toHaveLength(1);
+		for (const r of results) {
+			if (r.status === "rejected") {
+				expect(r.reason).toBeInstanceOf(RemediationGateError);
+				expect((r.reason as RemediationGateError).statusCode).toBe(409);
+			}
+		}
+		db.close();
+	});
+
+	it("B2: a wrong token is rejected with 403 and does not consume the grant", async () => {
+		const db = createMigratedSqliteDb();
+		const { proposal } = await grantedProposal(db);
+		await expect(
+			claimRemediationGrant(db, "default", proposal.id, {
+				environment: "prod",
+				grantToken: "rgt_wrong",
+			}),
+		).rejects.toMatchObject({ statusCode: 403 });
+		const ok = await claimRemediationGrant(db, "default", proposal.id, {
+			environment: "prod",
+			grantToken: proposal.grantToken as string,
+		});
+		expect(ok.status).toBe("claimed_executing");
+		db.close();
+	});
+
+	it("B3: concurrent lease acquisition has exactly one winner; expired leases can be taken over", async () => {
+		const db = createMigratedSqliteDb();
+		const incident = await newIncident(db);
+		const results = await Promise.all(
+			["a", "b", "c"].map((runnerId) =>
+				acquireIncidentLease(db, "default", incident.id, {
+					environment: "prod",
+					runnerId,
+				}),
+			),
+		);
+		const winners = results.filter((r) => r.acquired);
+		expect(winners).toHaveLength(1);
+		const owner = (winners[0] as { incident: { leaseOwnerId: string } })
+			.incident.leaseOwnerId;
+		// The owner can renew; a rival cannot until expiry.
+		expect(
+			(
+				await acquireIncidentLease(db, "default", incident.id, {
+					environment: "prod",
+					runnerId: owner,
+				})
+			).acquired,
+		).toBe(true);
+		const rival = owner === "a" ? "b" : "a";
+		expect(
+			(
+				await acquireIncidentLease(db, "default", incident.id, {
+					environment: "prod",
+					runnerId: rival,
+				})
+			).acquired,
+		).toBe(false);
+		const later = new Date(Date.now() + 3600_000).toISOString();
+		const takeover = await acquireIncidentLease(db, "default", incident.id, {
+			environment: "prod",
+			runnerId: rival,
+			nowIso: later,
+		});
+		expect(takeover.acquired).toBe(true);
+		db.close();
+	});
+
+	it("B4: concurrent alerts for one symptom open exactly one active incident", async () => {
+		const db = createMigratedSqliteDb();
+		const results = await Promise.all(
+			Array.from({ length: 6 }, () =>
+				openOrDeduplicateIncident(db, "default", {
+					environment: "prod",
+					targetNodeId: SERVICE_NODE,
+					symptomAnchorKind: "alert",
+					symptomAnchorId: "a1",
+					symptomCategory: "race",
+				}),
+			),
+		);
+		expect(results.filter((r) => !r.deduplicated)).toHaveLength(1);
+		expect(new Set(results.map((r) => r.incident.id)).size).toBe(1);
+		const count = db.raw
+			.prepare(`SELECT COUNT(*) AS n FROM incident_investigations`)
+			.get() as { n: number };
+		expect(count.n).toBe(1);
+		expect(results[0].incident.alertCount).toBeGreaterThanOrEqual(1);
+
+		// The database itself refuses a second active row for the same key.
+		expect(() =>
+			db.raw
+				.prepare(
+					`INSERT INTO incident_investigations (project_id, environment, id, dedup_key, status, target_node_id, symptom_anchor_kind, symptom_anchor_id, opened_at, updated_at)
+					 SELECT project_id, environment, 'dupe', dedup_key, 'open', target_node_id, symptom_anchor_kind, symptom_anchor_id, opened_at, updated_at FROM incident_investigations`,
+				)
+				.run(),
+		).toThrow(/UNIQUE/i);
+		// Once resolved, a new incident for the same symptom is allowed.
+		db.raw
+			.prepare(`UPDATE incident_investigations SET status = 'resolved'`)
+			.run();
+		const reopened = await openOrDeduplicateIncident(db, "default", {
+			environment: "prod",
+			targetNodeId: SERVICE_NODE,
+			symptomAnchorKind: "alert",
+			symptomAnchorId: "a1",
+			symptomCategory: "race",
+		});
+		expect(reopened.deduplicated).toBe(false);
+		db.close();
+	});
+
+	it("B5: concurrent proposals with one idempotency key return the same row instead of a DB error", async () => {
+		const db = createMigratedSqliteDb();
+		const incident = await newIncident(db);
+		const finding = await newFinding(db, incident.id);
+		const results = await Promise.all(
+			Array.from({ length: 4 }, () =>
+				propose(db, incident.id, finding.id, "idem_same"),
+			),
+		);
+		expect(new Set(results.map((r) => r.id)).size).toBe(1);
+		await expect(
+			propose(db, incident.id, finding.id, "idem_same", {
+				actionKind: "apply_config",
+			}),
+		).rejects.toMatchObject({ statusCode: 409 });
+		db.close();
+	});
+
+	it("B6: the watchdog route ignores a client clock and only evaluates the caller's project", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		db.raw
+			.prepare(
+				`INSERT INTO investigation_findings (id, project_id, environment, fingerprint, fingerprint_input_json, title, summary, causal_proof_tier, causal_verification_status, confidence, confidence_breakdown_json, symptom_entity_kind, symptom_entity_id, fault_locus_entity_kind, fault_locus_entity_id, causal_chain_json, cited_evidence_ids_json, actor_type, actor_id, created_at, updated_at, recovery_status, watchdog_step, next_watchdog_check_at)
+				 VALUES ('fnd_other', 'other-project', 'prod', 'fp', '{}', 't', 's', 'corroborated_candidate', 'unverified', 0.5, '{}', 'trace', 't', 'service', 'service:prod:default:x', '[]', '[]', 'agent', 'a', '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z', 'monitoring', 1, '2026-01-01T00:02:00.000Z')`,
+			)
+			.run();
+		const res = await app.request(
+			"/internal/remediations/watchdog",
+			{
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({
+					nowIso: "2099-01-01T00:00:00.000Z",
+					minComparableRequests: 0,
+				}),
+			},
+			env,
+		);
+		expect(res.status).toBe(200);
+		expect(((await res.json()) as { evaluated: number }).evaluated).toBe(0);
+		const row = db.raw
+			.prepare(
+				`SELECT recovery_status FROM investigation_findings WHERE id = 'fnd_other'`,
+			)
+			.get() as { recovery_status: string };
+		expect(row.recovery_status).toBe("monitoring");
+		db.close();
+	});
+
+	it("B7: only the claiming runner can complete or fail, and only while claimed (and, for complete, unexpired)", async () => {
+		const db = createMigratedSqliteDb();
+		const { proposal } = await grantedProposal(db);
+		// Not yet claimed.
+		await expect(
+			completeRemediationProposal(db, "default", proposal.id, {
+				environment: "prod",
+				runnerId: "r1",
+			}),
+		).rejects.toMatchObject({ statusCode: 409 });
+		await expect(
+			failRemediationProposal(db, "default", proposal.id, {
+				environment: "prod",
+				runnerId: "r1",
+				failureReason: "x",
+			}),
+		).rejects.toMatchObject({ statusCode: 409 });
+
+		const t0 = new Date().toISOString();
+		await claimRemediationGrant(db, "default", proposal.id, {
+			environment: "prod",
+			grantToken: proposal.grantToken as string,
+			runnerId: "r1",
+			nowIso: t0,
+		});
+		await expect(
+			completeRemediationProposal(db, "default", proposal.id, {
+				environment: "prod",
+				runnerId: "someone-else",
+			}),
+		).rejects.toMatchObject({ statusCode: 409 });
+		await expect(
+			completeRemediationProposal(db, "default", proposal.id, {
+				environment: "prod",
+			}),
+		).rejects.toMatchObject({ statusCode: 409 });
+		await expect(
+			failRemediationProposal(db, "default", proposal.id, {
+				environment: "prod",
+				runnerId: "someone-else",
+				failureReason: "x",
+			}),
+		).rejects.toMatchObject({ statusCode: 409 });
+		await expect(
+			completeRemediationProposal(db, "default", proposal.id, {
+				environment: "prod",
+				runnerId: "r1",
+				nowIso: new Date(Date.parse(t0) + 6 * 60_000).toISOString(),
+			}),
+		).rejects.toMatchObject({ statusCode: 409 });
+
+		const failed = await failRemediationProposal(db, "default", proposal.id, {
+			environment: "prod",
+			runnerId: "r1",
+			failureReason: "boom",
+		});
+		expect(failed.status).toBe("failed_execution");
+		// Terminal: cannot be failed or completed again.
+		await expect(
+			failRemediationProposal(db, "default", proposal.id, {
+				environment: "prod",
+				runnerId: "r1",
+				failureReason: "again",
+			}),
+		).rejects.toMatchObject({ statusCode: 409 });
+		db.close();
+	});
+
+	it("B8: incident webhooks refuse private targets and redirects, never follow redirects, and time out", async () => {
+		const db = createMigratedSqliteDb();
+		const incident = await newIncident(db);
+		const calls: Array<{ url: string; init: RequestInit }> = [];
+		const okFetch = (async (url: string, init: RequestInit) => {
+			calls.push({ url, init });
+			return new Response("ok", { status: 200 });
+		}) as unknown as typeof fetch;
+
+		const privateLookup = vi.fn(async () => ["10.0.0.5"]);
+		const refused = await dispatchIncidentWebhook(db, "default", incident.id, {
+			environment: "prod",
+			url: "https://internal.example.com/hook",
+			hostLookup: privateLookup,
+			fetchImpl: okFetch,
+		});
+		expect(refused.delivered).toBe(false);
+		expect(refused.error).toMatch(/private/);
+		const literal = await dispatchIncidentWebhook(db, "default", incident.id, {
+			environment: "prod",
+			url: "http://127.0.0.1:8080/hook",
+			fetchImpl: okFetch,
+		});
+		expect(literal.delivered).toBe(false);
+		const unresolvable = await dispatchIncidentWebhook(
+			db,
+			"default",
+			incident.id,
+			{
+				environment: "prod",
+				url: "https://nowhere.example.com/hook",
+				hostLookup: async () => {
+					throw new Error("ENOTFOUND");
+				},
+				fetchImpl: okFetch,
+			},
+		);
+		expect(unresolvable.delivered).toBe(false);
+		expect(calls).toHaveLength(0);
+
+		const delivered = await dispatchIncidentWebhook(
+			db,
+			"default",
+			incident.id,
+			{
+				environment: "prod",
+				url: "https://hooks.example.com/hook",
+				secret: "s3cret",
+				hostLookup: async () => ["93.184.216.34"],
+				fetchImpl: okFetch,
+			},
+		);
+		expect(delivered.delivered).toBe(true);
+		expect(calls).toHaveLength(1);
+		expect(calls[0].init.redirect).toBe("manual");
+		expect(calls[0].init.signal).toBeInstanceOf(AbortSignal);
+		expect(
+			(calls[0].init.headers as Record<string, string>)["X-Obs-Signature-256"],
+		).toMatch(/^sha256=[0-9a-f]{64}$/);
+
+		const redirected = await dispatchIncidentWebhook(
+			db,
+			"default",
+			incident.id,
+			{
+				environment: "prod",
+				url: "https://hooks.example.com/hook",
+				hostLookup: async () => ["93.184.216.34"],
+				fetchImpl: (async () =>
+					new Response(null, {
+						status: 302,
+						headers: { Location: "http://169.254.169.254/" },
+					})) as unknown as typeof fetch,
+			},
+		);
+		expect(redirected.delivered).toBe(false);
+		expect(redirected.error).toMatch(/Redirect/);
+		db.close();
+	});
+
+	it("B8: the dispatch route cannot disable the SSRF guard from the request body", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		const incident = await newIncident(db);
+		const fetchSpy = vi.fn(async () => new Response("ok"));
+		vi.stubGlobal("fetch", fetchSpy);
+		const res = await app.request(
+			`/internal/incidents/${incident.id}/dispatch`,
+			{
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({
+					environment: "prod",
+					webhookUrl: "http://127.0.0.1:9/hook",
+					allowPrivateNetworks: true,
+					allowPrivateWebhookTargets: true,
+				}),
+			},
+			env,
+		);
+		expect(res.status).toBe(502);
+		expect(fetchSpy).not.toHaveBeenCalled();
+
+		// The operator-level env flag is the only way to allow private targets.
+		const allowed = await app.request(
+			`/internal/incidents/${incident.id}/dispatch`,
+			{
+				method: "POST",
+				headers: { "Content-Type": "application/json" },
+				body: JSON.stringify({
+					environment: "prod",
+					webhookUrl: "http://127.0.0.1:9/hook",
+				}),
+			},
+			{ ...env, ALERT_WEBHOOK_ALLOW_PRIVATE: "true" },
+		);
+		expect(allowed.status).toBe(200);
+		expect(fetchSpy).toHaveBeenCalledTimes(1);
+		db.close();
+	});
+
+	it("B9: approve -> grant token pushed over the webhook -> claim succeeds once -> second claim 409", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		const incident = await newIncident(db);
+		const finding = await newFinding(db, incident.id); // corroborated: needs a human
+		const proposal = await propose(db, incident.id, finding.id, "idem_b9");
+		expect(proposal.status).toBe("pending_human_approval");
+
+		const bodies: Array<Record<string, unknown>> = [];
+		vi.stubGlobal(
+			"fetch",
+			vi.fn(async (_url: string, init: RequestInit) => {
+				bodies.push(JSON.parse(String(init.body)));
+				return new Response("ok", { status: 200 });
+			}),
+		);
+		const approveEnv = { ...env, ALERT_WEBHOOK_ALLOW_PRIVATE: "true" };
+		const approve = await app.request(
+			`/internal/remediations/${proposal.id}/approve`,
+			{
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+					Cookie: "obs_session=sess_b9",
+				},
+				body: JSON.stringify({
+					environment: "prod",
+					webhookUrl: "https://runner.example.com/grants",
+					secret: "hmac-secret",
+				}),
+			},
+			approveEnv,
+		);
+		expect(approve.status).toBe(200);
+		const approved = (await approve.json()) as {
+			grantToken: string;
+			webhookDelivery?: { delivered: boolean };
+		};
+		expect(approved.grantToken).toMatch(/^rgt_/);
+		expect(approved.webhookDelivery?.delivered).toBe(true);
+		expect(bodies).toHaveLength(1);
+		expect(bodies[0]).toMatchObject({
+			event: "autosre.remediation.grant_issued",
+			proposalId: proposal.id,
+			status: "grant_issued",
+			grantToken: approved.grantToken,
+			incidentId: incident.id,
+		});
+
+		const claim = (runnerId: string) =>
+			app.request(
+				`/internal/remediations/${proposal.id}/claim`,
+				{
+					method: "POST",
+					headers: { "Content-Type": "application/json" },
+					body: JSON.stringify({
+						environment: "prod",
+						grantToken: approved.grantToken,
+						runnerId,
+					}),
+				},
+				env,
+			);
+		expect((await claim("runner-x")).status).toBe(200);
+		expect((await claim("runner-y")).status).toBe(409);
+		db.close();
+	});
+
+	it("B12: the circuit breaker is scoped to fingerprint + action kind and ignores rejected proposals", async () => {
+		const db = createMigratedSqliteDb();
+		const incident = await newIncident(db);
+		const finding = await newFinding(db, incident.id);
+		const mk = (key: string, extra: Record<string, unknown> = {}) =>
+			propose(db, incident.id, finding.id, key, extra);
+
+		const a = await mk("k1");
+		await mk("k2");
+		await mk("k3");
+		// A different action kind has its own budget.
+		expect(
+			(await mk("k_other", { actionKind: "apply_config" })).status,
+		).not.toBe("rejected");
+		// Rejecting one frees a slot, so the next proposal is not blocked.
+		await rejectRemediationProposal(db, "default", a.id, {
+			environment: "prod",
+			rejectedByUserId: "tester",
+		});
+		expect((await mk("k4")).status).toBe("pending_human_approval");
+		// Three live ones now exist: the next is blocked by policy.
+		const blocked = await mk("k5");
+		expect(blocked.status).toBe("rejected");
+		expect(blocked.policyDecisionReason).toMatch(/Circuit breaker/);
+		db.close();
+	});
+
+	it("B18: unknown incidents 404 and read_only/suggested_action requests never become grants", async () => {
+		const db = createMigratedSqliteDb();
+		const { incident, finding } = await (async () => {
+			await seedIsolatedCohort(db);
+			const incident = await newIncident(db);
+			const finding = await newFinding(db, incident.id, {
+				contributingChangeId: "chg_gov",
+			});
+			return { incident, finding };
+		})();
+		await expect(
+			propose(db, "inc_missing", finding.id, "idem_404"),
+		).rejects.toMatchObject({ statusCode: 404 });
+		for (const level of ["read_only", "suggested_action"]) {
+			const p = await propose(db, incident.id, finding.id, `idem_${level}`, {
+				requestedAutonomy: level,
+			});
+			expect(p.status).toBe("pending_human_approval");
+			expect(p.effectiveAutonomy).toBe(level);
+			expect(p.grantToken).toBeUndefined();
+		}
+		db.close();
+	});
+
+	it("B11: watchdog checkpoints are anchored to completion time and recovery needs two consecutive healthy checkpoints", async () => {
+		const db = createMigratedSqliteDb();
+		const { proposal, incident, finding } = await grantedProposal(db);
+		const t0 = Date.now();
+		const at = (offsetMs: number) => new Date(t0 + offsetMs).toISOString();
+		await claimRemediationGrant(db, "default", proposal.id, {
+			environment: "prod",
+			grantToken: proposal.grantToken as string,
+			runnerId: "r1",
+			nowIso: at(0),
+		});
+		await completeRemediationProposal(db, "default", proposal.id, {
+			environment: "prod",
+			runnerId: "r1",
+			nowIso: at(60_000),
+		});
+		const completedAt = t0 + 60_000;
+		const state = () =>
+			db.raw
+				.prepare(
+					`SELECT recovery_status, watchdog_step, consecutive_ok_checks, next_watchdog_check_at FROM investigation_findings WHERE id = ?`,
+				)
+				.get(finding.id) as {
+				recovery_status: string;
+				watchdog_step: number;
+				consecutive_ok_checks: number;
+				next_watchdog_check_at: string | null;
+			};
+		expect(state().next_watchdog_check_at).toBe(
+			new Date(completedAt + 2 * 60_000).toISOString(),
+		);
+
+		for (let i = 0; i < 60; i++) {
+			const t = new Date(completedAt + 30_000 + i * 100).toISOString();
+			insertSpan(db, {
+				traceId: `g_ok_${i}`,
+				spanId: `g_ok_sp_${i}`,
+				serviceName: "checkout-api",
+				spanName: "POST /checkout",
+				statusCode: 1,
+				startTime: t,
+				endTime: t,
+				durationMs: 20,
+				serviceVersion: "v2.4.0",
+			});
+		}
+
+		// Checkpoint 1 runs late (+10m): next checkpoint is still anchored on
+		// completion (+5m), not on "now + offset".
+		const first = await evaluateRemediationWatchdogs(db, {
+			projectId: "default",
+			nowIso: at(60_000 + 10 * 60_000),
+		});
+		expect(first.recovered).toBe(0);
+		expect(state()).toMatchObject({
+			recovery_status: "monitoring",
+			watchdog_step: 2,
+			consecutive_ok_checks: 1,
+			next_watchdog_check_at: new Date(completedAt + 5 * 60_000).toISOString(),
+		});
+
+		const second = await evaluateRemediationWatchdogs(db, {
+			projectId: "default",
+			nowIso: at(60_000 + 11 * 60_000),
+		});
+		expect(second.recovered).toBe(1);
+		expect(state().recovery_status).toBe("recovered");
+		const inc = db.raw
+			.prepare(`SELECT status FROM incident_investigations WHERE id = ?`)
+			.get(incident.id) as { status: string };
+		expect(inc.status).toBe("resolved");
+		db.close();
+	});
+
+	it("B14/B17: routes reject unknown predicates and invalid timestamps", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		const post = (path: string, body: unknown) =>
+			app.request(
+				path,
+				{
+					method: "POST",
+					headers: { "Content-Type": "application/json" },
+					body: JSON.stringify(body),
+				},
+				env,
+			);
+
+		const badPredicate = await post("/internal/findings", {
+			title: "t",
+			summary: "s",
+			symptomEntityKind: "trace",
+			symptomEntityId: "t1",
+			faultLocusEntityKind: "service",
+			faultLocusEntityId: SERVICE_NODE,
+			causalChain: [
+				{
+					subjectKind: "trace",
+					subjectId: "t1",
+					predicate: "made_up_predicate",
+					objectKind: "span",
+					objectId: "s1",
+				},
+			],
+		});
+		expect(badPredicate.status).toBe(400);
+
+		const badDate = await post("/v1/changes", {
+			kind: "deploy",
+			summary: "x",
+			serviceName: "checkout-api",
+			occurredAt: "yesterday-ish",
+		});
+		expect(badDate.status).toBe(400);
+		const badKind = await post("/v1/changes", { kind: "nope", summary: "x" });
+		expect(badKind.status).toBe(400);
+
+		const offset = await post("/v1/changes", {
+			id: "chg_tz",
+			kind: "deploy",
+			summary: "x",
+			serviceName: "checkout-api",
+			environment: "prod",
+			source: "ci_webhook",
+			occurredAt: "2026-10-15T05:00:00+02:00",
+		});
+		expect(offset.status).toBe(201);
+		const created = (await offset.json()) as {
+			changeEvent: { occurredAt: string; source: string };
+		};
+		expect(created.changeEvent.occurredAt).toBe("2026-10-15T03:00:00.000Z");
+		expect(created.changeEvent.source).toBe("ci_webhook");
+
+		const badSince = await app.request(
+			"/internal/changes?since=garbage",
+			{},
+			env,
+		);
+		expect(badSince.status).toBe(400);
+		const okSince = await app.request(
+			"/internal/changes?since=2026-10-15T04:30:00%2B02:00",
+			{},
+			env,
+		);
+		expect(okSince.status).toBe(200);
+		expect(
+			((await okSince.json()) as { changes: unknown[] }).changes,
+		).toHaveLength(1);
+		db.close();
+	});
+
+	it("B15: the approver is derived from the session, never from client-supplied identity", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		const incident = await newIncident(db);
+		const finding = await newFinding(db, incident.id);
+		const proposal = await propose(db, incident.id, finding.id, "idem_b15");
+		const res = await app.request(
+			`/internal/remediations/${proposal.id}/approve`,
+			{
+				method: "POST",
+				headers: {
+					"Content-Type": "application/json",
+					Cookie: "obs_session=sess_secret_cookie_value",
+					"X-Obs-User-Actor": "vp-of-engineering",
+				},
+				body: JSON.stringify({
+					environment: "prod",
+					approvedByUserId: "cto",
+					approvedBySession: "cfo",
+				}),
+			},
+			env,
+		);
+		expect(res.status).toBe(200);
+		const body = (await res.json()) as {
+			proposal: { approvedByUserId: string };
+		};
+		expect(body.proposal.approvedByUserId).toMatch(
+			/^dashboard_session:[0-9a-f]{16}$/,
+		);
+		db.close();
+	});
+
+	const json = (body: unknown) => ({
+		method: "POST",
+		headers: { "Content-Type": "application/json" },
+		body: JSON.stringify(body),
+	});
+	const approveInit = (body: Record<string, unknown>) => ({
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+			Cookie: "obs_session=sess_lifecycle",
+		},
+		body: JSON.stringify(body),
+	});
+
+	it("J10: span-id fault loci on one service/route share a fingerprint, so the breaker is not fragmented", async () => {
+		const db = createMigratedSqliteDb();
+		await seedIsolatedCohort(db);
+		const incident = await newIncident(db);
+		const spanFinding = (n: number) =>
+			newFinding(db, incident.id, {
+				title: `Checkout regression attempt ${n}`,
+				summary: `Investigation ${n} found errors after the deploy`,
+				contributingChangeId: "chg_gov",
+				faultLocusEntityKind: "span",
+				faultLocusEntityId: `g_new_sp_${n}`,
+			});
+
+		const findings = [];
+		for (let n = 0; n < 4; n++) findings.push(await spanFinding(n));
+		expect(new Set(findings.map((f) => f.fingerprint)).size).toBe(1);
+		expect(findings[0].fingerprintInput.originatingNodeId).toBe(SERVICE_NODE);
+		expect(findings[0].fingerprintInput.normalizedOperation).toContain(
+			"/checkout",
+		);
+
+		for (let n = 0; n < 3; n++) {
+			const granted = await propose(
+				db,
+				incident.id,
+				findings[n].id,
+				`j10_${n}`,
+			);
+			expect(granted.status).toBe("grant_issued");
+		}
+		const blocked = await propose(db, incident.id, findings[3].id, "j10_3");
+		expect(blocked.status).toBe("rejected");
+		expect(blocked.policyDecisionReason).toMatch(/Circuit breaker/);
+
+		// A span that is not in telemetry keeps the legacy behaviour.
+		const unknown = await spanFinding(99);
+		expect(unknown.fingerprint).not.toBe(findings[0].fingerprint);
+		db.close();
+	});
+
+	it("J10: traceId:spanId span loci resolve the service, and empty error signatures fall back to the summary", async () => {
+		const db = createMigratedSqliteDb();
+		const t0 = Date.parse("2026-10-15T06:00:00.000Z");
+		const start = new Date(t0).toISOString();
+		const end = new Date(t0 + 50).toISOString();
+		const traceId = "a".repeat(32);
+		insertSpan(db, {
+			traceId,
+			spanId: "b".repeat(16),
+			serviceName: "pay-api",
+			spanName: "POST /pay",
+			statusCode: 2,
+			statusMessage: "card declined",
+			startTime: start,
+			endTime: end,
+			durationMs: 50,
+		});
+		insertSpan(db, {
+			traceId,
+			spanId: "c".repeat(16),
+			serviceName: "pay-api",
+			spanName: "POST /refund",
+			statusCode: 2,
+			statusMessage: "",
+			errorSignature: "",
+			startTime: start,
+			endTime: end,
+			durationMs: 50,
+		});
+		const incident = await newIncident(db);
+		const mk = (id: string, summary = "Declines after deploy") =>
+			newFinding(db, incident.id, {
+				title: "Pay declines",
+				summary,
+				faultLocusEntityKind: "span",
+				faultLocusEntityId: id,
+			});
+		const bare = await mk("b".repeat(16));
+		const qualified = await mk(`${traceId}:${"b".repeat(16)}`);
+		expect(qualified.fingerprintInput.originatingNodeId).toBe(
+			"service:prod:default:pay-api",
+		);
+		expect(qualified.fingerprint).toBe(bare.fingerprint);
+
+		// No error signature or status message: the summary is the signature.
+		const noSig = await mk("c".repeat(16), "Refunds time out");
+		expect(noSig.fingerprintInput.errorSignature).toBe("Refunds time out");
+		expect(noSig.fingerprintInput.normalizedOperation).toBe("POST /refund");
+		db.close();
+	});
+
+	it("J10: a recurring failure on a different trace surfaces the prior finding by fingerprint", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		const t0 = Date.parse("2026-10-15T05:00:00.000Z");
+		for (const [trace, offset] of [
+			["tr_prior", 0],
+			["tr_recur", 60_000],
+		] as const) {
+			const start = new Date(t0 + offset).toISOString();
+			const end = new Date(t0 + offset + 80).toISOString();
+			insertSpan(db, {
+				traceId: trace,
+				spanId: `${trace}_root`,
+				serviceName: "pay-api",
+				spanName: "POST /pay",
+				statusCode: 2,
+				startTime: start,
+				endTime: end,
+				durationMs: 80,
+			});
+			insertSpan(db, {
+				traceId: trace,
+				spanId: `${trace}_err`,
+				parentSpanId: `${trace}_root`,
+				serviceName: "pay-api",
+				spanName: "POST /pay",
+				statusCode: 2,
+				statusMessage: "card declined",
+				startTime: start,
+				endTime: end,
+				durationMs: 70,
+			});
+		}
+		const incident = await newIncident(db);
+		const prior = await newFinding(db, incident.id, {
+			title: "Pay declines",
+			summary: "Provider returns declines",
+			symptomEntityId: "tr_prior",
+			faultLocusEntityKind: "span",
+			faultLocusEntityId: "tr_prior_err",
+		});
+		const bundleFor = async (trace: string) => {
+			const res = await app.request(
+				"/internal/evidence/bundle",
+				json({
+					intent: "triage_incident",
+					anchor: { entityKind: "trace", entityId: trace },
+				}),
+				env,
+			);
+			expect(res.status).toBe(200);
+			return (await res.json()) as {
+				autosre?: { priorFindings?: Array<{ findingId: string }> };
+			};
+		};
+		const recur = await bundleFor("tr_recur");
+		expect(
+			recur.autosre?.priorFindings?.some((f) => f.findingId === prior.id),
+		).toBe(true);
+		db.close();
+	});
+
+	it("J9: a finding idempotencyKey returns the stored finding on retry and survives route round-trips", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		const body = {
+			environment: "prod",
+			idempotencyKey: "retry-me",
+			title: "Idempotent finding",
+			summary: "Same call twice",
+			symptomEntityKind: "trace",
+			symptomEntityId: "t1",
+			faultLocusEntityKind: "service",
+			faultLocusEntityId: SERVICE_NODE,
+			causalChain: [],
+			citedEvidenceIds: [],
+		};
+		const first = await app.request("/internal/findings", json(body), env);
+		const second = await app.request("/internal/findings", json(body), env);
+		expect(first.status).toBe(201);
+		const a = (await first.json()) as { id: string };
+		const b = (await second.json()) as { id: string };
+		expect(b.id).toBe(a.id);
+		const n = db.raw
+			.prepare(`SELECT COUNT(*) AS n FROM investigation_findings`)
+			.get() as { n: number };
+		expect(n.n).toBe(1);
+		// A different key is a different finding.
+		const third = await app.request(
+			"/internal/findings",
+			json({ ...body, idempotencyKey: "other" }),
+			env,
+		);
+		expect(((await third.json()) as { id: string }).id).not.toBe(a.id);
 		db.close();
 	});
+
+	it("J9: GET /internal/evals/runs/:id reports existence so counterfactual prerequisites can be checked first", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		const missing = await app.request("/internal/evals/runs/nope", {}, env);
+		expect(missing.status).toBe(404);
+		db.raw
+			.prepare(
+				`INSERT INTO eval_runs (project_id, id, eval_case_id, status, pass_count, fail_count, created_at)
+				 VALUES ('default', 'run_exists', 'case_1', 'completed', 0, 1, ?)`,
+			)
+			.run(new Date().toISOString());
+		const found = await app.request("/internal/evals/runs/run_exists", {}, env);
+		expect(found.status).toBe(200);
+		expect(((await found.json()) as { run: { id: string } }).run.id).toBe(
+			"run_exists",
+		);
+		db.close();
+	});
+
+	it("J17: callback stored at propose time -> approve without callback -> webhook receives the token -> claim succeeds", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		const incident = await newIncident(db);
+		const finding = await newFinding(db, incident.id); // corroborated: needs a human
+		const callbackEnv = { ...env, ALERT_WEBHOOK_ALLOW_PRIVATE: "true" };
+
+		const bodies: Array<Record<string, unknown>> = [];
+		const headers: Array<Record<string, string>> = [];
+		vi.stubGlobal(
+			"fetch",
+			vi.fn(async (_url: string, init: RequestInit) => {
+				bodies.push(JSON.parse(String(init.body)));
+				headers.push(init.headers as Record<string, string>);
+				return new Response("ok", { status: 200 });
+			}),
+		);
+
+		const proposeRes = await app.request(
+			"/internal/remediations/propose",
+			json({
+				environment: "prod",
+				incidentId: incident.id,
+				findingId: finding.id,
+				idempotencyKey: "j17",
+				actionKind: "rollback_deploy",
+				targetNodeId: SERVICE_NODE,
+				expectedCurrentState: {},
+				desiredTargetState: { version: "v2.4.0" },
+				webhookUrl: "https://runner.example.com/grants?token=urlsecret",
+				secret: "hmac-secret-j17",
+			}),
+			callbackEnv,
+		);
+		expect(proposeRes.status).toBe(201);
+		const proposed = (await proposeRes.json()) as {
+			proposal: { id: string; status: string; callbackHost: string };
+		};
+		expect(proposed.proposal.status).toBe("pending_human_approval");
+		expect(proposed.proposal.callbackHost).toBe("runner.example.com");
+		expect(bodies).toHaveLength(0);
+
+		// Neither the URL path/query nor the secret is ever returned by a GET.
+		const getRes = await app.request(
+			`/internal/remediations/${proposed.proposal.id}?environment=prod`,
+			{},
+			callbackEnv,
+		);
+		const getText = await getRes.text();
+		expect(getText).toContain("runner.example.com");
+		expect(getText).not.toContain("urlsecret");
+		expect(getText).not.toContain("hmac-secret-j17");
+		expect(getText).not.toContain("/grants");
+		const list = await (
+			await app.request(
+				"/internal/remediations?environment=prod",
+				{},
+				callbackEnv,
+			)
+		).text();
+		expect(list).not.toContain("hmac-secret-j17");
+
+		const approve = await app.request(
+			`/internal/remediations/${proposed.proposal.id}/approve`,
+			approveInit({ environment: "prod" }),
+			callbackEnv,
+		);
+		expect(approve.status).toBe(200);
+		const approved = (await approve.json()) as {
+			grantToken: string;
+			webhookDelivery?: { delivered: boolean };
+			proposal: { callbackHost: string | null };
+		};
+		expect(approved.webhookDelivery?.delivered).toBe(true);
+		expect(approved.proposal.callbackHost).toBeNull();
+		expect(bodies).toHaveLength(1);
+		expect(bodies[0]).toMatchObject({
+			event: "autosre.remediation.grant_issued",
+			proposalId: proposed.proposal.id,
+			grantToken: approved.grantToken,
+		});
+		expect(headers[0]["X-Obs-Signature-256"]).toMatch(/^sha256=[0-9a-f]{64}$/);
+
+		const claim = await app.request(
+			`/internal/remediations/${proposed.proposal.id}/claim`,
+			json({
+				environment: "prod",
+				grantToken: approved.grantToken,
+				runnerId: "runner-j17",
+			}),
+			callbackEnv,
+		);
+		expect(claim.status).toBe(200);
+
+		// The stored callback is gone from the row once consumed.
+		const row = db.raw
+			.prepare(
+				`SELECT callback_url, callback_secret FROM remediation_proposals WHERE id = ?`,
+			)
+			.get(proposed.proposal.id) as {
+			callback_url: string | null;
+			callback_secret: string | null;
+		};
+		expect(row.callback_url).toBeNull();
+		expect(row.callback_secret).toBeNull();
+		db.close();
+	});
+
+	it("J17: a callback on the approve request wins, a bad propose callback is a 400, and reject clears the stored one", async () => {
+		const db = createMigratedSqliteDb();
+		const { app, env } = createTestApp(db);
+		const incident = await newIncident(db);
+		const finding = await newFinding(db, incident.id);
+		const callbackEnv = { ...env, ALERT_WEBHOOK_ALLOW_PRIVATE: "true" };
+		const urls: string[] = [];
+		vi.stubGlobal(
+			"fetch",
+			vi.fn(async (url: string) => {
+				urls.push(url);
+				return new Response("ok", { status: 200 });
+			}),
+		);
+		const mk = (key: string, webhookUrl: string) =>
+			app.request(
+				"/internal/remediations/propose",
+				json({
+					environment: "prod",
+					incidentId: incident.id,
+					findingId: finding.id,
+					idempotencyKey: key,
+					actionKind: "rollback_deploy",
+					targetNodeId: SERVICE_NODE,
+					expectedCurrentState: {},
+					desiredTargetState: {},
+					webhookUrl,
+				}),
+				callbackEnv,
+			);
+		expect((await mk("j17_bad", "ftp://example.com/x")).status).toBe(400);
+
+		const one = (await (
+			await mk("j17_a", "https://stored.example.com/a")
+		).json()) as {
+			proposal: { id: string };
+		};
+		const approve = await app.request(
+			`/internal/remediations/${one.proposal.id}/approve`,
+			approveInit({
+				environment: "prod",
+				webhookUrl: "https://request.example.com/b",
+			}),
+			callbackEnv,
+		);
+		expect(approve.status).toBe(200);
+		expect(urls).toEqual(["https://request.example.com/b"]);
+
+		const two = (await (
+			await mk("j17_b", "https://stored.example.com/c")
+		).json()) as {
+			proposal: { id: string };
+		};
+		await rejectRemediationProposal(db, "default", two.proposal.id, {
+			environment: "prod",
+			rejectedByUserId: "tester",
+		});
+		const row = db.raw
+			.prepare(`SELECT callback_url FROM remediation_proposals WHERE id = ?`)
+			.get(two.proposal.id) as { callback_url: string | null };
+		expect(row.callback_url).toBeNull();
+		db.close();
+	});
+
+	describe("grant delivery retries", () => {
+		type Delivery = {
+			attempted: boolean;
+			delivered: boolean;
+			attempts: number;
+			host: string | null;
+			error?: string;
+		};
+		const setup = async (key: string, webhookUrl: string, secret?: string) => {
+			const db = createMigratedSqliteDb();
+			const { app, env } = createTestApp(db);
+			const incident = await newIncident(db);
+			const finding = await newFinding(db, incident.id);
+			const callbackEnv = { ...env, ALERT_WEBHOOK_ALLOW_PRIVATE: "true" };
+			const proposed = (await (
+				await app.request(
+					"/internal/remediations/propose",
+					json({
+						environment: "prod",
+						incidentId: incident.id,
+						findingId: finding.id,
+						idempotencyKey: key,
+						actionKind: "rollback_deploy",
+						targetNodeId: SERVICE_NODE,
+						expectedCurrentState: {},
+						desiredTargetState: { version: "v2.4.0" },
+						webhookUrl,
+						...(secret ? { secret } : {}),
+					}),
+					callbackEnv,
+				)
+			).json()) as { proposal: { id: string } };
+			const approve = async () =>
+				(await (
+					await app.request(
+						`/internal/remediations/${proposed.proposal.id}/approve`,
+						approveInit({ environment: "prod" }),
+						callbackEnv,
+					)
+				).json()) as {
+					grantToken: string;
+					grantDelivery: Delivery;
+					proposal: { callbackHost: string | null };
+				};
+			const get = async () =>
+				(await (
+					await app.request(
+						`/internal/remediations/${proposed.proposal.id}?environment=prod`,
+						{},
+						callbackEnv,
+					)
+				).json()) as {
+					proposal: {
+						callbackDelivery: {
+							status: string;
+							attempts: number;
+							lastError: string | null;
+							at: string;
+						} | null;
+					};
+				};
+			return { db, proposed, approve, get };
+		};
+
+		it("delivers on the 2nd attempt after a 503", async () => {
+			const statuses = [503, 200];
+			const fetchMock = vi.fn(
+				async () => new Response("x", { status: statuses.shift() ?? 200 }),
+			);
+			vi.stubGlobal("fetch", fetchMock);
+			const { db, approve, get } = await setup(
+				"retry-503",
+				"https://runner.example.com/g?token=urlsecret",
+				"hmac-retry",
+			);
+			const approved = await approve();
+			expect(fetchMock).toHaveBeenCalledTimes(2);
+			expect(approved.grantDelivery).toEqual({
+				attempted: true,
+				delivered: true,
+				attempts: 2,
+				host: "runner.example.com",
+			});
+			expect(JSON.stringify(approved.grantDelivery)).not.toContain("urlsecret");
+			const got = await get();
+			expect(got.proposal.callbackDelivery).toMatchObject({
+				status: "delivered",
+				attempts: 2,
+				lastError: null,
+			});
+			db.close();
+		}, 15_000);
+
+		it("fails after 3 attempts on a closed port but still returns the token and clears the callback", async () => {
+			vi.unstubAllGlobals();
+			const net = await import("node:net");
+			const port = await new Promise<number>((resolve) => {
+				const srv = net.createServer();
+				srv.listen(0, "127.0.0.1", () => {
+					const { port: p } = srv.address() as { port: number };
+					srv.close(() => resolve(p));
+				});
+			});
+			const { db, proposed, approve, get } = await setup(
+				"retry-closed",
+				`http://127.0.0.1:${port}/grant?token=urlsecret`,
+				"hmac-closed",
+			);
+			const approved = await approve();
+			expect(approved.grantToken).toMatch(/^rgt_/);
+			expect(approved.grantDelivery).toMatchObject({
+				attempted: true,
+				delivered: false,
+				attempts: 3,
+				host: `127.0.0.1:${port}`,
+			});
+			expect(approved.grantDelivery.error).toBeTruthy();
+			expect(JSON.stringify(approved.grantDelivery)).not.toContain("urlsecret");
+			expect(approved.proposal.callbackHost).toBeNull();
+			const row = db.raw
+				.prepare(
+					`SELECT callback_url, callback_secret FROM remediation_proposals WHERE id = ?`,
+				)
+				.get(proposed.proposal.id) as {
+				callback_url: string | null;
+				callback_secret: string | null;
+			};
+			expect(row.callback_url).toBeNull();
+			expect(row.callback_secret).toBeNull();
+			const got = await get();
+			expect(got.proposal.callbackDelivery).toMatchObject({
+				status: "failed",
+				attempts: 3,
+			});
+			expect(got.proposal.callbackDelivery?.lastError).toBeTruthy();
+			db.close();
+		}, 20_000);
+
+		it("does not retry a 4xx", async () => {
+			const fetchMock = vi.fn(async () => new Response("no", { status: 403 }));
+			vi.stubGlobal("fetch", fetchMock);
+			const { db, approve, get } = await setup(
+				"retry-4xx",
+				"https://runner.example.com/g",
+			);
+			const approved = await approve();
+			expect(fetchMock).toHaveBeenCalledTimes(1);
+			expect(approved.grantToken).toMatch(/^rgt_/);
+			expect(approved.grantDelivery).toMatchObject({
+				attempted: true,
+				delivered: false,
+				attempts: 1,
+				error: "HTTP 403",
+			});
+			expect((await get()).proposal.callbackDelivery?.status).toBe("failed");
+			db.close();
+		});
+
+		it("reports attempted=false when no callback exists", async () => {
+			const db = createMigratedSqliteDb();
+			const { app, env } = createTestApp(db);
+			const incident = await newIncident(db);
+			const finding = await newFinding(db, incident.id);
+			const proposed = (await (
+				await app.request(
+					"/internal/remediations/propose",
+					json({
+						environment: "prod",
+						incidentId: incident.id,
+						findingId: finding.id,
+						idempotencyKey: "no-cb",
+						actionKind: "rollback_deploy",
+						targetNodeId: SERVICE_NODE,
+						expectedCurrentState: {},
+						desiredTargetState: {},
+					}),
+					env,
+				)
+			).json()) as { proposal: { id: string } };
+			const approved = (await (
+				await app.request(
+					`/internal/remediations/${proposed.proposal.id}/approve`,
+					approveInit({ environment: "prod" }),
+					env,
+				)
+			).json()) as { grantDelivery: Delivery };
+			expect(approved.grantDelivery).toEqual({
+				attempted: false,
+				delivered: false,
+				attempts: 0,
+				host: null,
+			});
+			db.close();
+		});
+	});
 });