@obsunified/collector 2.2.1 → 2.3.0

package/src/plugins/autosre-routes.ts

@@ -1,9 +1,11 @@
 import type {
-	CausalProofTier,
 	CausalVerificationStatus,
 	ChangeEventKind,
-	CohortAttributionVerdict,
 	EvidenceEntityKind,
+	GrantCallbackDelivery,
+	GrantDeliveryResult,
+	InternalEndpointKey,
+	InternalEndpoints,
 	InvestigationFindingInput,
 	JsonValue,
 	RemediationActionKind,
@@ -12,11 +14,16 @@ import type {
 	RemediationStateSpec,
 } from "@obsunified/types";
 import {
+	ChangeEventInputSchema,
 	ChangeEventKindSchema,
 	EvidencePredicateSchema,
+	InvestigationFindingHttpInputSchema,
 	RemediationActionKindSchema,
+	RemediationProposalInputSchema,
+	validateContractResponse,
 } from "@obsunified/types";
 import { AgentAutonomyLevel } from "@obsunified/types/constants";
+import type { Context } from "hono";
 import type { CollectorPlugin } from "../framework/collector";
 import type { CollectorEnv } from "../framework/env";
 import {
@@ -46,19 +53,37 @@ import {
 	evaluateRemediationWatchdogs,
 	failRemediationProposal,
 	getIncidentInvestigation,
+	getRemediationCallback,
 	getRemediationProposal,
 	listIncidentInvestigations,
 	listRemediationProposals,
 	openOrDeduplicateIncident,
 	proposeRemediationAction,
 	RemediationGateError,
+	recordCallbackDelivery,
 	rejectRemediationProposal,
 } from "../lib/autosre-remediation-gate";
 import { sha256Hex } from "../lib/hash";
 import { sqlDbFor } from "../lib/sql-db";
+import { assertSafeWebhookUrl } from "../lib/webhook-url";
 import { getProjectId } from "./_context";
 import { readJsonObject } from "./_json";
 
+export function respondWithContract<
+	K extends InternalEndpointKey,
+	C extends {
+		json: (data: unknown, status?: 200 | 201 | 202) => Response;
+	},
+>(
+	c: C,
+	endpoint: K,
+	payload: InternalEndpoints[K]["response"],
+	status: 200 | 201 | 202 = 200,
+): Response {
+	validateContractResponse(endpoint, payload, status);
+	return c.json(payload, status);
+}
+
 const CHANGE_KINDS = new Set<string>(ChangeEventKindSchema.options);
 
 const REMEDIATION_ACTION_KINDS = new Set<string>(
@@ -126,6 +151,12 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 		 * `webhookUrl` (+ optional `secret`) in that same request and the token is
 		 * pushed to the runner over the SSRF-guarded, HMAC-signed incident webhook.
 		 * Without a `webhookUrl` the token is simply returned in the response.
+		 *
+		 * A `webhookUrl` (+ `secret`) given to `propose` for a proposal that is
+		 * parked for human approval is persisted on the proposal and used when
+		 * the approve request carries none. A callback on the approve request
+		 * wins over the stored one. The stored callback is cleared on approve
+		 * or reject, and only its host is ever returned by GET.
 		 */
 		const deliverGrant = async (
 			env: CollectorEnv | undefined,
@@ -139,23 +170,87 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 				status: string;
 				grantToken?: string;
 			},
+			/** Callback persisted at propose time; used when the request has none. */
+			stored?: { url: string; secret?: string } | null,
 		) => {
-			const webhookUrl =
+			const bodyUrl =
 				typeof body.webhookUrl === "string" ? body.webhookUrl.trim() : "";
-			if (!webhookUrl || !proposal.grantToken) return undefined;
-			if (proposal.status !== "grant_issued") return undefined;
-			return dispatchIncidentWebhook(db, projectId, proposal.incidentId, {
-				url: webhookUrl,
-				secret: typeof body.secret === "string" ? body.secret : undefined,
-				environment: proposal.environment,
-				allowPrivateNetworks: env?.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
-				logger: runtime.logger,
-				grant: {
-					proposalId: proposal.id,
-					status: proposal.status,
-					grantToken: proposal.grantToken,
+			const target = bodyUrl
+				? {
+						url: bodyUrl,
+						secret: typeof body.secret === "string" ? body.secret : undefined,
+					}
+				: (stored ?? null);
+			const notAttempted: GrantDeliveryResult = {
+				attempted: false,
+				delivered: false,
+				attempts: 0,
+				host: null,
+			};
+			if (!target || !proposal.grantToken) {
+				return { webhookDelivery: undefined, grantDelivery: notAttempted };
+			}
+			if (proposal.status !== "grant_issued") {
+				return { webhookDelivery: undefined, grantDelivery: notAttempted };
+			}
+			let host: string | null = null;
+			try {
+				host = new URL(target.url).host || null;
+			} catch {
+				host = null;
+			}
+			const webhookDelivery = await dispatchIncidentWebhook(
+				db,
+				projectId,
+				proposal.incidentId,
+				{
+					url: target.url,
+					secret: target.secret,
+					environment: proposal.environment,
+					allowPrivateNetworks: env?.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
+					logger: runtime.logger,
+					grant: {
+						proposalId: proposal.id,
+						status: proposal.status,
+						grantToken: proposal.grantToken,
+					},
 				},
-			});
+			);
+			// Error text is surfaced to the human: strip anything URL-shaped and
+			// the secret so neither the query string nor the HMAC key can leak.
+			let error = webhookDelivery.error
+				? webhookDelivery.error.replace(
+						/\b[a-z][a-z0-9+.-]*:\/\/\S+/gi,
+						"<url>",
+					)
+				: undefined;
+			if (error && target.secret) {
+				error = error.split(target.secret).join("<redacted>");
+			}
+			const record = await recordCallbackDelivery(
+				db,
+				projectId,
+				proposal.environment,
+				proposal.id,
+				{
+					delivered: webhookDelivery.delivered,
+					attempts: webhookDelivery.attempts,
+					error,
+				},
+			);
+			(
+				proposal as { callbackDelivery?: GrantCallbackDelivery | null }
+			).callbackDelivery = record;
+			const grantDelivery: GrantDeliveryResult = {
+				attempted: true,
+				delivered: webhookDelivery.delivered,
+				attempts: webhookDelivery.attempts,
+				host,
+				...(webhookDelivery.delivered
+					? {}
+					: { error: error ?? "unknown error" }),
+			};
+			return { webhookDelivery, grantDelivery };
 		};
 
 		// 1. Change Ingestion (Ingest Auth /v1/changes)
@@ -225,52 +320,63 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 						? body.actor
 						: null;
 
+			const rawInput = {
+				id: typeof body.id === "string" ? body.id : undefined,
+				environment:
+					typeof body.environment === "string" ? body.environment : undefined,
+				kind: normalizedKind,
+				source:
+					body.source === "inferred_telemetry" ||
+					body.source === "autosre_remediation" ||
+					body.source === "ci_webhook"
+						? body.source
+						: ("explicit_api" as const),
+				targetNodeId:
+					typeof body.targetNodeId === "string" ? body.targetNodeId : undefined,
+				serviceName,
+				namespace,
+				version: typeof body.version === "string" ? body.version : null,
+				previousVersion:
+					typeof body.previousVersion === "string"
+						? body.previousVersion
+						: null,
+				commitSha: typeof body.commitSha === "string" ? body.commitSha : null,
+				previousCommitSha:
+					typeof body.previousCommitSha === "string"
+						? body.previousCommitSha
+						: null,
+				configDigest:
+					typeof body.configDigest === "string" ? body.configDigest : null,
+				summary,
+				actorId,
+				repositoryUrl,
+				pullRequestUrl,
+				causedByActionId:
+					typeof body.causedByActionId === "string"
+						? body.causedByActionId
+						: null,
+				agentRunId:
+					typeof body.agentRunId === "string" ? body.agentRunId : null,
+				touchedPaths,
+				attributes,
+				occurredAt:
+					typeof body.occurredAt === "string" ? body.occurredAt : undefined,
+			};
+			const parsedInput = ChangeEventInputSchema.safeParse(rawInput);
+			if (!parsedInput.success) {
+				return c.json(
+					{
+						error: parsedInput.error.issues
+							.map((issue) => `${issue.path.join(".")}: ${issue.message}`)
+							.join("; "),
+					},
+					400,
+				);
+			}
+
 			let changeEvent: Awaited<ReturnType<typeof recordChangeEvent>>;
 			try {
-				changeEvent = await recordChangeEvent(db, projectId, {
-					id: typeof body.id === "string" ? body.id : undefined,
-					environment:
-						typeof body.environment === "string" ? body.environment : undefined,
-					kind: normalizedKind,
-					source:
-						body.source === "inferred_telemetry" ||
-						body.source === "autosre_remediation" ||
-						body.source === "ci_webhook"
-							? body.source
-							: "explicit_api",
-					targetNodeId:
-						typeof body.targetNodeId === "string"
-							? body.targetNodeId
-							: undefined,
-					serviceName,
-					namespace,
-					version: typeof body.version === "string" ? body.version : null,
-					previousVersion:
-						typeof body.previousVersion === "string"
-							? body.previousVersion
-							: null,
-					commitSha: typeof body.commitSha === "string" ? body.commitSha : null,
-					previousCommitSha:
-						typeof body.previousCommitSha === "string"
-							? body.previousCommitSha
-							: null,
-					configDigest:
-						typeof body.configDigest === "string" ? body.configDigest : null,
-					summary,
-					actorId,
-					repositoryUrl,
-					pullRequestUrl,
-					causedByActionId:
-						typeof body.causedByActionId === "string"
-							? body.causedByActionId
-							: null,
-					agentRunId:
-						typeof body.agentRunId === "string" ? body.agentRunId : null,
-					touchedPaths,
-					attributes,
-					occurredAt:
-						typeof body.occurredAt === "string" ? body.occurredAt : undefined,
-				});
+				changeEvent = await recordChangeEvent(db, projectId, parsedInput.data);
 			} catch (err) {
 				if (err instanceof InvalidTimestampError) {
 					return c.json({ error: err.message }, 400);
@@ -278,7 +384,12 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 				throw err;
 			}
 
-			return c.json({ changeEvent, change: changeEvent }, 201);
+			return respondWithContract(
+				c,
+				"POST /v1/changes",
+				{ ...changeEvent, changeEvent, change: changeEvent },
+				201,
+			);
 		});
 
 		// 2. Ontology Graph & Fault Localization
@@ -296,7 +407,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 				? await loadTraceFaultLocusFromDb(db, projectId, traceId)
 				: null;
 
-			return c.json({
+			return respondWithContract(c, "GET /internal/ontology", {
 				...graph,
 				faultLocus,
 			});
@@ -323,7 +434,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 					since: c.req.query("since") || undefined,
 					limit: parseLimit(c.req.query("limit"), 50),
 				});
-				return c.json({ changes });
+				return respondWithContract(c, "GET /internal/changes", { changes });
 			} catch (err) {
 				if (err instanceof InvalidTimestampError) {
 					return c.json({ error: err.message }, 400);
@@ -359,7 +470,10 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 			if (!blastRadius) {
 				return c.json({ error: "Change event not found" }, 404);
 			}
-			return c.json({ blastRadius, ...blastRadius });
+			return respondWithContract(c, "GET /internal/changes/:id/blast-radius", {
+				blastRadius,
+				...blastRadius,
+			});
 		});
 
 		// 4. Incident Investigations & Lease Coordination
@@ -372,10 +486,12 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 				status: c.req.query("status") || undefined,
 				limit: parseLimit(c.req.query("limit"), 50),
 			});
-			return c.json({ incidents });
+			return respondWithContract(c, "GET /internal/incidents", { incidents });
 		});
 
-		app.post("/internal/incidents", async (c) => {
+		const openIncidentHandler = async (
+			c: Context<{ Bindings: CollectorEnv }>,
+		) => {
 			const projectId = getProjectId(c);
 			const body = await readJsonObject<Record<string, unknown>>(c);
 			if (!body) {
@@ -429,8 +545,15 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 						? (body.latestBundle as Record<string, unknown>)
 						: null,
 			});
-			return c.json(result, result.deduplicated ? 200 : 201);
-		});
+			return respondWithContract(
+				c,
+				"POST /internal/incidents",
+				result,
+				result.deduplicated ? 200 : 201,
+			);
+		};
+		app.post("/internal/incidents", openIncidentHandler);
+		app.post("/internal/incidents/open", openIncidentHandler);
 
 		app.post("/internal/incidents/:id/lease", async (c) => {
 			const projectId = getProjectId(c);
@@ -444,7 +567,9 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 					? body.runnerId.trim()
 					: typeof body.leaseOwner === "string"
 						? body.leaseOwner.trim()
-						: "";
+						: typeof body.leaseOwnerId === "string"
+							? body.leaseOwnerId.trim()
+							: "";
 			if (!runnerId) {
 				return c.json({ error: "runnerId is required" }, 400);
 			}
@@ -456,17 +581,28 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 					: typeof body.ttlSeconds === "number" &&
 							Number.isFinite(body.ttlSeconds)
 						? body.ttlSeconds
-						: typeof body.leaseDurationMs === "number" &&
-								Number.isFinite(body.leaseDurationMs)
-							? Math.ceil(body.leaseDurationMs / 1000)
-							: undefined;
+						: typeof body.leaseSeconds === "number" &&
+								Number.isFinite(body.leaseSeconds)
+							? body.leaseSeconds
+							: typeof body.leaseDurationMs === "number" &&
+									Number.isFinite(body.leaseDurationMs)
+								? Math.ceil(body.leaseDurationMs / 1000)
+								: undefined;
 			const result = await acquireIncidentLease(db, projectId, incidentId, {
 				environment:
 					typeof body.environment === "string" ? body.environment : undefined,
 				runnerId,
 				leaseDurationSeconds,
 			});
-			return c.json(result, result.acquired ? 200 : 409);
+			if (!result.acquired) {
+				return c.json(result, 409);
+			}
+			return respondWithContract(
+				c,
+				"POST /internal/incidents/:id/lease",
+				result,
+				200,
+			);
 		});
 
 		app.post("/internal/incidents/:id/dispatch", async (c) => {
@@ -493,7 +629,15 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 				allowPrivateNetworks: c.env?.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
 				logger: runtime.logger,
 			});
-			return c.json(result, result.delivered ? 200 : 502);
+			if (!result.delivered) {
+				return c.json({ ...result, dispatched: result.delivered }, 502);
+			}
+			return respondWithContract(
+				c,
+				"POST /internal/incidents/:id/dispatch",
+				{ ...result, dispatched: result.delivered },
+				200,
+			);
 		});
 
 		// 5. Investigation Findings & Counterfactual Verification
@@ -516,7 +660,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 				incidentId: c.req.query("incidentId") || undefined,
 				limit: parseLimit(c.req.query("limit"), 20),
 			});
-			return c.json({ findings });
+			return respondWithContract(c, "GET /internal/findings", { findings });
 		});
 
 		app.post("/internal/findings", async (c) => {
@@ -582,65 +726,67 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 				}
 			}
 
-			const db = sqlDbFor(c.env);
-			const input: InvestigationFindingInput = {
-				id: typeof body.id === "string" ? body.id : undefined,
-				environment:
-					typeof body.environment === "string" ? body.environment : undefined,
-				incidentId:
-					typeof body.incidentId === "string" ? body.incidentId : null,
-				fingerprintInput:
-					body.fingerprintInput && typeof body.fingerprintInput === "object"
-						? (body.fingerprintInput as InvestigationFindingInput["fingerprintInput"])
-						: undefined,
+			const parsed = InvestigationFindingHttpInputSchema.safeParse({
+				...body,
 				title,
 				summary,
-				causalProofTier:
-					typeof body.causalProofTier === "string"
-						? (body.causalProofTier as CausalProofTier)
-						: undefined,
-				confidence:
-					typeof body.confidence === "number" ? body.confidence : undefined,
-				symptomEntityKind: symptomEntityKind as EvidenceEntityKind,
+				symptomEntityKind,
 				symptomEntityId,
-				faultLocusEntityKind: faultLocusEntityKind as EvidenceEntityKind,
+				faultLocusEntityKind,
 				faultLocusEntityId,
-				rootCauseEntityKind:
-					typeof body.rootCauseEntityKind === "string"
-						? (body.rootCauseEntityKind as EvidenceEntityKind)
-						: null,
-				rootCauseEntityId:
-					typeof body.rootCauseEntityId === "string"
-						? body.rootCauseEntityId
-						: null,
-				contributingChangeId:
-					typeof body.contributingChangeId === "string"
-						? body.contributingChangeId
-						: null,
-				cohortVerdict:
-					typeof body.cohortVerdict === "string"
-						? (body.cohortVerdict as CohortAttributionVerdict)
-						: null,
-				contendingTraceId:
-					typeof body.contendingTraceId === "string"
-						? body.contendingTraceId
-						: null,
-				causalChain: Array.isArray(body.causalChain)
-					? (body.causalChain as InvestigationFindingInput["causalChain"])
-					: [],
-				citedEvidenceIds: Array.isArray(body.citedEvidenceIds)
-					? body.citedEvidenceIds.filter(
-							(id): id is string => typeof id === "string",
-						)
-					: [],
-				agentRunId:
-					typeof body.agentRunId === "string" ? body.agentRunId : null,
+				idempotencyKey:
+					typeof body.idempotencyKey === "string"
+						? body.idempotencyKey.trim() || undefined
+						: body.idempotencyKey,
+			});
+			if (!parsed.success) {
+				return c.json(
+					{
+						error: parsed.error.issues
+							.map((issue) => `${issue.path.join(".")}: ${issue.message}`)
+							.join("; "),
+					},
+					400,
+				);
+			}
+			const data = parsed.data;
+
+			const db = sqlDbFor(c.env);
+			const input: InvestigationFindingInput = {
+				id: data.id,
+				idempotencyKey: data.idempotencyKey,
+				environment: data.environment,
+				incidentId: data.incidentId ?? null,
+				fingerprintInput: data.fingerprintInput,
+				title: data.title,
+				summary: data.summary,
+				causalProofTier: data.causalProofTier,
+				confidence: data.confidence,
+				symptomEntityKind: data.symptomEntityKind,
+				symptomEntityId: data.symptomEntityId,
+				faultLocusEntityKind: data.faultLocusEntityKind,
+				faultLocusEntityId: data.faultLocusEntityId,
+				rootCauseEntityKind: data.rootCauseEntityKind ?? null,
+				rootCauseEntityId: data.rootCauseEntityId ?? null,
+				contributingChangeId: data.contributingChangeId ?? null,
+				cohortVerdict: data.cohortVerdict ?? null,
+				contendingTraceId: data.contendingTraceId ?? null,
+				causalChain: (data.causalChain ??
+					[]) as InvestigationFindingInput["causalChain"],
+				citedEvidenceIds: data.citedEvidenceIds ?? [],
+				codeReferences: data.codeReferences,
+				agentRunId: data.agentRunId ?? null,
 			};
 
 			const finding = await recordInvestigationFinding(db, projectId, input, {
 				logger: runtime.logger,
 			});
-			return c.json({ finding, id: finding.id }, 201);
+			return respondWithContract(
+				c,
+				"POST /internal/findings",
+				{ ...finding, finding, id: finding.id },
+				201,
+			);
 		});
 
 		app.post("/internal/findings/:id/counterfactual", async (c) => {
@@ -688,7 +834,15 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 								: undefined,
 					},
 				);
-				return c.json({ verified: true, finding });
+				return respondWithContract(
+					c,
+					"POST /internal/findings/:id/counterfactual",
+					{
+						verified: true,
+						finding,
+						proof: finding.counterfactualProof,
+					},
+				);
 			} catch (err) {
 				const message = err instanceof Error ? err.message : String(err);
 				if (err instanceof CounterfactualVerificationError) {
@@ -702,6 +856,40 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 			}
 		});
 
+		// Lightweight existence check so callers can validate counterfactual
+		// prerequisites before persisting a finding.
+		app.get("/internal/evals/runs/:id", async (c) => {
+			const projectId = getProjectId(c);
+			const runId = c.req.param("id");
+			const db = sqlDbFor(c.env);
+			const row = await db
+				.prepare(
+					`SELECT id, eval_case_id, service_version, commit_sha,
+					        pass_count, fail_count
+					 FROM eval_runs WHERE project_id = ? AND id = ? LIMIT 1`,
+				)
+				.bind(projectId, runId)
+				.first<{
+					id: string;
+					eval_case_id: string | null;
+					service_version: string | null;
+					commit_sha: string | null;
+					pass_count: number | null;
+					fail_count: number | null;
+				}>();
+			if (!row) return c.json({ error: "Eval run not found" }, 404);
+			return respondWithContract(c, "GET /internal/evals/runs/:id", {
+				run: {
+					id: row.id,
+					evalCaseId: row.eval_case_id,
+					serviceVersion: row.service_version,
+					commitSha: row.commit_sha,
+					passCount: Number(row.pass_count ?? 0),
+					failCount: Number(row.fail_count ?? 0),
+				},
+			});
+		});
+
 		// 6. Remediation Proposals, Human Approval, Claim Grant, Watchdog
 		app.get("/internal/remediations", async (c) => {
 			const projectId = getProjectId(c);
@@ -716,7 +904,9 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 					undefined,
 				limit: parseLimit(c.req.query("limit"), 50),
 			});
-			return c.json({ proposals });
+			return respondWithContract(c, "GET /internal/remediations", {
+				proposals,
+			});
 		});
 
 		app.post("/internal/remediations/propose", async (c) => {
@@ -774,7 +964,23 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 				: "autonomous_write";
 
 			const db = sqlDbFor(c.env);
-			const input: RemediationProposalInput = {
+			const callbackUrl =
+				typeof body.webhookUrl === "string" ? body.webhookUrl.trim() : "";
+			if (callbackUrl) {
+				try {
+					assertSafeWebhookUrl(callbackUrl, {
+						allowPrivateNetworks: c.env?.ALERT_WEBHOOK_ALLOW_PRIVATE === "true",
+					});
+				} catch (err) {
+					return c.json(
+						{
+							error: `webhookUrl rejected: ${err instanceof Error ? err.message : String(err)}`,
+						},
+						400,
+					);
+				}
+			}
+			const rawInput: RemediationProposalInput = {
 				environment:
 					typeof body.environment === "string" ? body.environment : undefined,
 				incidentId,
@@ -786,23 +992,39 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 				desiredTargetState,
 				requestedAutonomy,
 			};
+			const parsedInput = RemediationProposalInputSchema.safeParse(rawInput);
+			const input: RemediationProposalInput = parsedInput.success
+				? parsedInput.data
+				: rawInput;
 
 			try {
 				const proposal = await proposeRemediationAction(db, projectId, input, {
 					logger: runtime.logger,
+					callback: callbackUrl
+						? {
+								url: callbackUrl,
+								secret:
+									typeof body.secret === "string" && body.secret
+										? body.secret
+										: undefined,
+							}
+						: undefined,
 				});
-				const webhookDelivery = await deliverGrant(
+				const { webhookDelivery, grantDelivery } = await deliverGrant(
 					c.env,
 					db,
 					projectId,
 					body,
 					proposal,
 				);
-				return c.json(
+				return respondWithContract(
+					c,
+					"POST /internal/remediations/propose",
 					{
 						proposal,
 						...proposal,
 						grantToken: proposal.grantToken ?? null,
+						grantDelivery,
 						...(webhookDelivery ? { webhookDelivery } : {}),
 					},
 					201,
@@ -836,6 +1058,13 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 
 			const db = sqlDbFor(c.env);
 			try {
+				// Read before approving: the approve UPDATE clears the stored callback.
+				const storedCallback = await getRemediationCallback(
+					db,
+					projectId,
+					proposalId,
+					environment,
+				);
 				const proposal = await approveRemediationProposal(
 					db,
 					projectId,
@@ -845,19 +1074,25 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 						approvedByUserId,
 					},
 				);
-				const webhookDelivery = await deliverGrant(
+				const { webhookDelivery, grantDelivery } = await deliverGrant(
 					c.env,
 					db,
 					projectId,
 					body,
 					proposal,
+					storedCallback,
+				);
+				return respondWithContract(
+					c,
+					"POST /internal/remediations/:id/approve",
+					{
+						proposal,
+						...proposal,
+						grantToken: proposal.grantToken ?? null,
+						grantDelivery,
+						...(webhookDelivery ? { webhookDelivery } : {}),
+					},
 				);
-				return c.json({
-					proposal,
-					...proposal,
-					grantToken: proposal.grantToken ?? null,
-					...(webhookDelivery ? { webhookDelivery } : {}),
-				});
 			} catch (err) {
 				const message = err instanceof Error ? err.message : String(err);
 				const status =
@@ -897,7 +1132,11 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 						reason,
 					},
 				);
-				return c.json({ proposal, ...proposal });
+				return respondWithContract(
+					c,
+					"POST /internal/remediations/:id/reject",
+					{ proposal, ...proposal },
+				);
 			} catch (err) {
 				const message = err instanceof Error ? err.message : String(err);
 				const status =
@@ -937,7 +1176,11 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 						logger: runtime.logger,
 					},
 				);
-				return c.json({ claimed: true, proposal, ...proposal });
+				return respondWithContract(c, "POST /internal/remediations/:id/claim", {
+					claimed: true,
+					proposal,
+					...proposal,
+				});
 			} catch (err) {
 				const message = err instanceof Error ? err.message : String(err);
 				const status =
@@ -983,7 +1226,11 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 								: null,
 					},
 				);
-				return c.json({ proposal, ...proposal });
+				return respondWithContract(
+					c,
+					"POST /internal/remediations/:id/complete",
+					{ proposal, ...proposal },
+				);
 			} catch (err) {
 				const message = err instanceof Error ? err.message : String(err);
 				const status =
@@ -1022,7 +1269,10 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 								: "Execution failed",
 					},
 				);
-				return c.json({ proposal, ...proposal });
+				return respondWithContract(c, "POST /internal/remediations/:id/fail", {
+					proposal,
+					...proposal,
+				});
 			} catch (err) {
 				const message = err instanceof Error ? err.message : String(err);
 				const status =
@@ -1049,7 +1299,10 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 						: undefined,
 				logger: runtime.logger,
 			});
-			return c.json({ summary, ...summary });
+			return respondWithContract(c, "POST /internal/remediations/watchdog", {
+				summary,
+				...summary,
+			});
 		});
 
 		app.get("/internal/remediations/:id", async (c) => {
@@ -1065,7 +1318,9 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 			if (!proposal) {
 				return c.json({ error: "Proposal not found" }, 404);
 			}
-			return c.json({ proposal });
+			return respondWithContract(c, "GET /internal/remediations/:id", {
+				proposal,
+			});
 		});
 
 		app.get("/internal/incidents/:id", async (c) => {
@@ -1096,7 +1351,11 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 					limit: 20,
 				}),
 			]);
-			return c.json({ incident, findings, proposals });
+			return respondWithContract(c, "GET /internal/incidents/:id", {
+				incident,
+				findings,
+				proposals,
+			});
 		});
 
 		app.get("/internal/findings/:id", async (c) => {
@@ -1112,7 +1371,7 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 			if (!finding) {
 				return c.json({ error: "Finding not found" }, 404);
 			}
-			return c.json({ finding });
+			return respondWithContract(c, "GET /internal/findings/:id", { finding });
 		});
 
 		app.get("/internal/changes/:id", async (c) => {
@@ -1128,7 +1387,10 @@ export const autosreRoutesPlugin: CollectorPlugin = {
 			if (!changeEvent) {
 				return c.json({ error: "Change event not found" }, 404);
 			}
-			return c.json({ changeEvent, change: changeEvent });
+			return respondWithContract(c, "GET /internal/changes/:id", {
+				changeEvent,
+				change: changeEvent,
+			});
 		});
 	},
 };