create-safest-tools 0.2.0

package/template/src/report-operations.ts

@@ -0,0 +1,685 @@
+import { canonicalJson } from "./audit";
+import { ApiError } from "./report-http";
+import type { ActionProposalJob, AiJob, DeliveryJob, EvidenceFetchJob, OperatorSession, ReportJob, WorkflowDispatchJob } from "./report-types";
+
+type QueueJob = DeliveryJob | ReportJob | AiJob | EvidenceFetchJob | WorkflowDispatchJob | ActionProposalJob;
+
+interface CountRow {
+  total: number;
+  pending: number;
+  delivering: number;
+  failed: number;
+  terminal_failed: number;
+  oldest_due_at: string | null;
+}
+
+interface RoutingCountRow {
+  total: number;
+  pending: number;
+  enqueued: number;
+  processing: number;
+  failed: number;
+  dead_letter: number;
+  oldest_available_at: string | null;
+}
+
+function now(): string {
+  return new Date().toISOString();
+}
+
+function auditEvent(
+  db: D1Database,
+  input: { idempotencyKey: string; targetType: string; targetId: string; details: unknown; createdAt: string },
+): D1PreparedStatement {
+  return db.prepare(`
+    INSERT OR IGNORE INTO audit_events (
+      id, idempotency_key, action, actor_type, actor_id, target_type, target_id, details_json, created_at
+    ) VALUES (?1, ?2, 'queue.dead_lettered', 'system', 'queue-consumer', ?3, ?4, ?5, ?6)
+  `).bind(crypto.randomUUID(), input.idempotencyKey, input.targetType, input.targetId, canonicalJson(input.details), input.createdAt);
+}
+
+function validJobType(value: QueueJob): "route_report" | "process_ai_report" | "fetch_evidence" | "dispatch_workflow" | "deliver_action" | "deliver_notification" | "deliver_email" | "apply_action_proposal" {
+  return value.type;
+}
+
+export async function recordDeadLetter(
+  db: D1Database,
+  sourceQueue: string,
+  sourceMessageId: string,
+  job: QueueJob | null,
+): Promise<void> {
+  const createdAt = now();
+  const jobType = job ? validJobType(job) : "invalid";
+  const jobId = job?.jobId ?? null;
+  const reportId = job?.type === "route_report" || job?.type === "process_ai_report" || job?.type === "fetch_evidence"
+    ? job.reportId
+    : job?.type === "apply_action_proposal"
+      ? (await db.prepare(`SELECT report_id AS reportId FROM action_proposals WHERE id = ?1 LIMIT 1`).bind(job.proposalId).first<{ reportId: string }>())?.reportId ?? null
+      : null;
+  const outboxId = job && (job.type === "deliver_action" || job.type === "deliver_notification" || job.type === "deliver_email" || job.type === "dispatch_workflow") ? job.outboxId : null;
+  const fingerprint = `${sourceQueue}:${jobId ?? `message:${sourceMessageId}`}`.slice(0, 500);
+  const deadLetterId = crypto.randomUUID();
+  const incidentId = crypto.randomUUID();
+  const statements: D1PreparedStatement[] = [
+    db.prepare(`
+      INSERT INTO operations_incidents (
+        id, installation_id, incident_key, severity, category, state,
+        title, summary, entity_type, entity_id, occurrence_count,
+        first_seen_at, last_seen_at, metadata_json
+      ) VALUES (?1, 'default', ?2, 'high', 'queue_dead_letter', 'open',
+        'Queue delivery exhausted retries', ?3, ?4, ?5, 1, ?6, ?6, ?7)
+      ON CONFLICT(installation_id, incident_key) DO UPDATE SET
+        state = 'open', occurrence_count = operations_incidents.occurrence_count + 1,
+        last_seen_at = excluded.last_seen_at, metadata_json = excluded.metadata_json,
+        resolved_by = NULL, resolved_at = NULL, resolution_summary = NULL
+    `).bind(
+      incidentId, fingerprint, `The ${jobType} job exhausted Queue retries and requires repair.`,
+      job ? "queue_job" : "queue_message", jobId ?? sourceMessageId, createdAt,
+      canonicalJson({ sourceQueue, jobType, reportId, outboxId }),
+    ),
+    db.prepare(`
+      INSERT INTO dead_letter_items (
+        id, fingerprint, source_queue, source_message_id, job_type, job_id, report_id, outbox_id,
+        incident_id, state, error_code, details_json, first_seen_at, last_seen_at
+      ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8,
+        (SELECT id FROM operations_incidents WHERE installation_id = 'default' AND incident_key = ?2),
+        'open', 'queue_retries_exhausted', ?9, ?10, ?10)
+      ON CONFLICT(fingerprint) DO UPDATE SET
+        source_message_id = excluded.source_message_id,
+        incident_id = excluded.incident_id,
+        state = 'open', error_code = excluded.error_code,
+        occurrence_count = dead_letter_items.occurrence_count + 1,
+        details_json = excluded.details_json, last_seen_at = excluded.last_seen_at,
+        resolved_at = NULL
+    `).bind(
+      deadLetterId, fingerprint, sourceQueue.slice(0, 200), sourceMessageId.slice(0, 200), jobType,
+      jobId?.slice(0, 200) ?? null, reportId, outboxId,
+      canonicalJson({ version: 1, jobType, hasReportReference: Boolean(reportId), hasOutboxReference: Boolean(outboxId) }),
+      createdAt,
+    ),
+  ];
+
+  if (job?.type === "route_report") {
+    statements.push(
+      db.prepare(`
+        UPDATE background_jobs
+        SET state = 'dead_letter', last_error_code = 'queue_retries_exhausted', updated_at = ?2
+        WHERE id = ?1 AND state != 'complete'
+      `).bind(job.jobId, createdAt),
+      db.prepare(`
+        UPDATE reports
+        SET routing_status = 'failed', routing_error_code = 'queue_retries_exhausted', updated_at = ?2
+        WHERE id = ?1 AND routing_status != 'complete'
+      `).bind(job.reportId, createdAt),
+      auditEvent(db, {
+        idempotencyKey: `dead-letter:${fingerprint}`,
+        targetType: "report",
+        targetId: job.reportId,
+        details: { jobId: job.jobId, sourceQueue, errorCode: "queue_retries_exhausted" },
+        createdAt,
+      }),
+    );
+  } else if (job?.type === "process_ai_report") {
+    const fallback = await db.prepare(`
+      SELECT q.id, q.active_version_id
+      FROM background_jobs j
+      JOIN ai_queue_config_versions c ON c.id = j.ai_config_version_id
+      JOIN queue_definitions q ON q.id = c.fallback_queue_id
+      JOIN queue_versions qv ON qv.id = q.active_version_id
+      WHERE j.id = ?1 AND q.status = 'active' AND qv.management_mode = 'human'
+      LIMIT 1
+    `).bind(job.jobId).first<{ id: string; active_version_id: string }>()
+      ?? await db.prepare(`
+        SELECT q.id, q.active_version_id
+        FROM queue_definitions q JOIN queue_versions qv ON qv.id = q.active_version_id
+        WHERE q.purpose = 'reports' AND q.status = 'active' AND qv.management_mode = 'human'
+        ORDER BY q.position, q.id LIMIT 1
+      `).first<{ id: string; active_version_id: string }>();
+    statements.push(
+      db.prepare(`
+        UPDATE background_jobs
+        SET state = 'dead_letter', last_error_code = 'queue_retries_exhausted', updated_at = ?2
+        WHERE id = ?1 AND state != 'complete'
+      `).bind(job.jobId, createdAt),
+      db.prepare(`
+        INSERT INTO report_state_history (
+          id, report_id, background_job_id, from_state, to_state, reason_code, actor_type, actor_id, created_at
+        )
+        SELECT ?1, ?2, ?3, 'ai_review', 'human_review', 'ai_queue_retries_exhausted',
+          'system', 'bounded-ai-v2', ?4
+        FROM reports WHERE id = ?2 AND state = 'ai_review'
+      `).bind(crypto.randomUUID(), job.reportId, job.jobId, createdAt),
+      db.prepare(`
+        UPDATE reports SET state = 'human_review', queue_id = ?2, queue_version_id = ?3, priority = MAX(priority, 60),
+          routing_error_code = 'ai_queue_retries_exhausted', updated_at = ?4
+        WHERE id = ?1 AND state = 'ai_review'
+      `).bind(job.reportId, fallback?.id ?? null, fallback?.active_version_id ?? null, createdAt),
+      auditEvent(db, {
+        idempotencyKey: `dead-letter:${fingerprint}`,
+        targetType: "report",
+        targetId: job.reportId,
+        details: { jobId: job.jobId, sourceQueue, errorCode: "ai_queue_retries_exhausted", fallbackQueue: fallback?.id ?? null },
+        createdAt,
+      }),
+    );
+  } else if (job?.type === "fetch_evidence") {
+    statements.push(
+      db.prepare(`
+        UPDATE evidence_fetch_requests SET state = 'failed',
+          last_error_code = 'queue_retries_exhausted', updated_at = ?2
+        WHERE id = ?1 AND state != 'available'
+      `).bind(job.requestId, createdAt),
+      auditEvent(db, {
+        idempotencyKey: `dead-letter:${fingerprint}`,
+        targetType: "evidence_fetch_request",
+        targetId: job.requestId,
+        details: { jobId: job.jobId, reportId: job.reportId, sourceQueue, errorCode: "queue_retries_exhausted" },
+        createdAt,
+      }),
+    );
+  } else if (job?.type === "dispatch_workflow") {
+    statements.push(
+      db.prepare(`
+        UPDATE workflow_dispatch_outbox
+        SET state = 'dead_letter', lease_expires_at = NULL,
+          last_error_code = 'queue_retries_exhausted', updated_at = ?2
+        WHERE id = ?1 AND state != 'delivered'
+      `).bind(job.outboxId, createdAt),
+      db.prepare(`
+        UPDATE workflow_runs
+        SET state = 'repair_required', error_code = 'workflow_dispatch_queue_retries_exhausted',
+          error_summary = 'Workflow dispatch exhausted infrastructure Queue retries.',
+          updated_at = ?2, revision = revision + 1
+        WHERE id = ?1 AND state IN ('created','dispatch_pending','queued')
+      `).bind(job.runId, createdAt),
+      auditEvent(db, {
+        idempotencyKey: `dead-letter:${fingerprint}`,
+        targetType: "workflow_run",
+        targetId: job.runId,
+        details: { sourceQueue, outboxId: job.outboxId, errorCode: "queue_retries_exhausted" },
+        createdAt,
+      }),
+    );
+  } else if (job?.type === "deliver_action") {
+    statements.push(
+      db.prepare(`
+        UPDATE decision_action_outbox
+        SET status = 'failed', terminal = 1, lease_expires_at = NULL,
+            last_error_code = 'queue_retries_exhausted', updated_at = ?2
+        WHERE id = ?1 AND terminal = 0
+      `).bind(job.outboxId, createdAt),
+      auditEvent(db, {
+        idempotencyKey: `dead-letter:${fingerprint}`,
+        targetType: "application_action",
+        targetId: job.outboxId,
+        details: { sourceQueue, errorCode: "queue_retries_exhausted" },
+        createdAt,
+      }),
+    );
+  } else if (job?.type === "deliver_notification") {
+    statements.push(
+      db.prepare(`
+        UPDATE notification_outbox
+        SET status = 'failed', terminal = 1, lease_expires_at = NULL,
+            last_error_code = 'queue_retries_exhausted', updated_at = ?2
+        WHERE id = ?1 AND terminal = 0
+      `).bind(job.outboxId, createdAt),
+      db.prepare(`
+        UPDATE notices SET delivery_state = 'failed', updated_at = ?2
+        WHERE id = (SELECT notice_id FROM notification_outbox WHERE id = ?1)
+      `).bind(job.outboxId, createdAt),
+      db.prepare(`
+        UPDATE case_messages SET delivery_state = 'failed'
+        WHERE id = (
+          SELECT n.message_id FROM notices n
+          JOIN notification_outbox o ON o.notice_id = n.id
+          WHERE o.id = ?1
+        )
+      `).bind(job.outboxId),
+      auditEvent(db, {
+        idempotencyKey: `dead-letter:${fingerprint}`,
+        targetType: "notification",
+        targetId: job.outboxId,
+        details: { sourceQueue, errorCode: "queue_retries_exhausted" },
+        createdAt,
+      }),
+    );
+  } else if (job?.type === "deliver_email") {
+    statements.push(
+      db.prepare(`
+        UPDATE email_deliveries SET state = 'failed',
+          last_error_code = 'queue_retries_exhausted', updated_at = ?2
+        WHERE id = ?1 AND state != 'delivered'
+      `).bind(job.outboxId, createdAt),
+      db.prepare(`
+        UPDATE case_messages SET delivery_state = 'failed'
+        WHERE id = (SELECT message_id FROM email_deliveries WHERE id = ?1)
+          AND delivery_state != 'delivered'
+      `).bind(job.outboxId),
+      auditEvent(db, {
+        idempotencyKey: `dead-letter:${fingerprint}`,
+        targetType: "email_delivery",
+        targetId: job.outboxId,
+        details: { sourceQueue, errorCode: "queue_retries_exhausted" },
+        createdAt,
+      }),
+    );
+  } else if (job?.type === "apply_action_proposal") {
+    statements.push(
+      db.prepare(`
+        UPDATE action_proposals SET state = 'failed', delivery_terminal = 1,
+          lease_expires_at = NULL, last_error_code = 'queue_retries_exhausted', updated_at = ?2
+        WHERE id = ?1 AND state != 'applied'
+      `).bind(job.proposalId, createdAt),
+      auditEvent(db, {
+        idempotencyKey: `dead-letter:${fingerprint}`,
+        targetType: "action_proposal",
+        targetId: job.proposalId,
+        details: { runId: job.runId, sourceQueue, errorCode: "queue_retries_exhausted" },
+        createdAt,
+      }),
+    );
+  }
+  await db.batch(statements);
+}
+
+function numeric(value: number | null | undefined): number {
+  return Number.isFinite(value) ? Number(value) : 0;
+}
+
+export async function loadOperationsSnapshot(db: D1Database): Promise<Record<string, unknown>> {
+  const [routing, actions, notifications, emails, workflows, evidence, budgets, incidents, surge, deadLetters, failures] = await Promise.all([
+    db.prepare(`
+      SELECT COUNT(*) AS total,
+        SUM(CASE WHEN state = 'pending' THEN 1 ELSE 0 END) AS pending,
+        SUM(CASE WHEN state = 'enqueued' THEN 1 ELSE 0 END) AS enqueued,
+        SUM(CASE WHEN state = 'processing' THEN 1 ELSE 0 END) AS processing,
+        SUM(CASE WHEN state = 'failed' THEN 1 ELSE 0 END) AS failed,
+        SUM(CASE WHEN state = 'dead_letter' THEN 1 ELSE 0 END) AS dead_letter,
+        MIN(CASE WHEN state IN ('pending', 'failed') THEN available_at END) AS oldest_available_at
+      FROM background_jobs WHERE job_type = 'route_report'
+    `).first<RoutingCountRow>(),
+    db.prepare(`
+      SELECT COUNT(*) AS total,
+        SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending,
+        SUM(CASE WHEN status = 'delivering' THEN 1 ELSE 0 END) AS delivering,
+        SUM(CASE WHEN status = 'failed' AND terminal = 0 THEN 1 ELSE 0 END) AS failed,
+        SUM(CASE WHEN status = 'failed' AND terminal = 1 THEN 1 ELSE 0 END) AS terminal_failed,
+        MIN(CASE WHEN terminal = 0 AND status IN ('pending', 'failed') THEN next_attempt_at END) AS oldest_due_at
+      FROM decision_action_outbox
+    `).first<CountRow>(),
+    db.prepare(`
+      SELECT COUNT(*) AS total,
+        SUM(CASE WHEN status = 'pending' THEN 1 ELSE 0 END) AS pending,
+        SUM(CASE WHEN status = 'delivering' THEN 1 ELSE 0 END) AS delivering,
+        SUM(CASE WHEN status = 'failed' AND terminal = 0 THEN 1 ELSE 0 END) AS failed,
+        SUM(CASE WHEN status = 'failed' AND terminal = 1 THEN 1 ELSE 0 END) AS terminal_failed,
+        MIN(CASE WHEN terminal = 0 AND status IN ('pending', 'failed') THEN next_attempt_at END) AS oldest_due_at
+      FROM notification_outbox
+    `).first<CountRow>(),
+    db.prepare(`
+      SELECT COUNT(*) AS total,
+        SUM(CASE WHEN state = 'pending' THEN 1 ELSE 0 END) AS pending,
+        SUM(CASE WHEN state = 'sending' THEN 1 ELSE 0 END) AS delivering,
+        SUM(CASE WHEN state = 'failed' AND attempt_count < 8 THEN 1 ELSE 0 END) AS failed,
+        SUM(CASE WHEN state = 'failed' AND attempt_count >= 8 THEN 1 ELSE 0 END) AS terminal_failed,
+        MIN(CASE WHEN state IN ('pending','failed') THEN COALESCE(next_attempt_at, created_at) END) AS oldest_due_at
+      FROM email_deliveries
+    `).first<CountRow>(),
+    db.prepare(`
+      SELECT COUNT(*) AS total,
+        SUM(CASE WHEN state IN ('created','dispatch_pending','queued','running','waiting','paused') THEN 1 ELSE 0 END) AS active,
+        SUM(CASE WHEN state = 'repair_required' THEN 1 ELSE 0 END) AS repairRequired,
+        SUM(CASE WHEN state = 'failed' THEN 1 ELSE 0 END) AS failed,
+        MIN(CASE WHEN state IN ('created','dispatch_pending','queued','running','waiting','paused','repair_required') THEN updated_at END) AS oldestActiveAt
+      FROM workflow_runs
+    `).first<Record<string, number | string | null>>(),
+    db.prepare(`
+      SELECT COUNT(*) AS total,
+        SUM(CASE WHEN state = 'pending' THEN 1 ELSE 0 END) AS pending,
+        SUM(CASE WHEN state = 'fetching' THEN 1 ELSE 0 END) AS fetching,
+        SUM(CASE WHEN state IN ('failed','unavailable') THEN 1 ELSE 0 END) AS unavailable,
+        MIN(CASE WHEN state IN ('pending','fetching') THEN updated_at END) AS oldestActiveAt
+      FROM evidence_fetch_requests
+    `).first<Record<string, number | string | null>>(),
+    db.prepare(`
+      SELECT COUNT(*) AS policyCount,
+        (SELECT COUNT(*) FROM budget_reservations WHERE status = 'reserved') AS activeReservations,
+        (SELECT COALESCE(SUM(consumed_value), 0) FROM budget_usage) AS consumedUnits,
+        (SELECT MIN(expires_at) FROM budget_reservations WHERE status = 'reserved') AS nextReservationExpiry
+      FROM budget_policies WHERE retired_at IS NULL
+    `).first<Record<string, number | string | null>>(),
+    db.prepare(`
+      SELECT COUNT(*) AS total,
+        SUM(CASE WHEN state = 'open' THEN 1 ELSE 0 END) AS open,
+        SUM(CASE WHEN state = 'acknowledged' THEN 1 ELSE 0 END) AS acknowledged,
+        SUM(CASE WHEN severity IN ('high','critical') AND state != 'resolved' THEN 1 ELSE 0 END) AS highPriority
+      FROM operations_incidents
+    `).first<Record<string, number | string | null>>(),
+    db.prepare(`
+      SELECT state AS mode, activated_at AS enteredAt, recover_after AS recoverAfter,
+        reason_codes_json AS reasonCodesJson, revision, updated_at AS updatedAt
+      FROM abuse_surge_states WHERE installation_id = 'default' LIMIT 1
+    `).first<Record<string, unknown>>(),
+    db.prepare(`
+      SELECT id, source_queue AS sourceQueue, job_type AS jobType, job_id AS jobId,
+        report_id AS reportId, outbox_id AS outboxId, state, error_code AS errorCode,
+        occurrence_count AS occurrenceCount, first_seen_at AS firstSeenAt, last_seen_at AS lastSeenAt
+      FROM dead_letter_items WHERE state != 'resolved'
+      ORDER BY last_seen_at DESC, id DESC LIMIT 100
+    `).all<Record<string, unknown>>(),
+    db.prepare(`
+      SELECT 'action' AS deliveryType, o.id, r.public_reference AS reportReference,
+        o.status, o.attempt_count AS attemptCount, o.last_error_code AS errorCode, o.updated_at AS updatedAt
+      FROM decision_action_outbox o JOIN reports r ON r.id = o.report_id
+      WHERE o.status = 'failed'
+      UNION ALL
+      SELECT 'notification' AS deliveryType, o.id, r.public_reference AS reportReference,
+        o.status, o.attempt_count AS attemptCount, o.last_error_code AS errorCode, o.updated_at AS updatedAt
+      FROM notification_outbox o JOIN notices n ON n.id = o.notice_id JOIN reports r ON r.id = n.report_id
+      WHERE o.status = 'failed'
+      ORDER BY updatedAt DESC LIMIT 100
+    `).all<Record<string, unknown>>(),
+  ]);
+  return {
+    generatedAt: now(),
+    routing: {
+      total: numeric(routing?.total), pending: numeric(routing?.pending), enqueued: numeric(routing?.enqueued),
+      processing: numeric(routing?.processing), failed: numeric(routing?.failed), deadLetter: numeric(routing?.dead_letter),
+      oldestAvailableAt: routing?.oldest_available_at ?? null,
+    },
+    actions: {
+      total: numeric(actions?.total), pending: numeric(actions?.pending), delivering: numeric(actions?.delivering),
+      retrying: numeric(actions?.failed), terminalFailed: numeric(actions?.terminal_failed), oldestDueAt: actions?.oldest_due_at ?? null,
+    },
+    notifications: {
+      total: numeric(notifications?.total), pending: numeric(notifications?.pending), delivering: numeric(notifications?.delivering),
+      retrying: numeric(notifications?.failed), terminalFailed: numeric(notifications?.terminal_failed), oldestDueAt: notifications?.oldest_due_at ?? null,
+    },
+    emails: {
+      total: numeric(emails?.total), pending: numeric(emails?.pending), sending: numeric(emails?.delivering),
+      retrying: numeric(emails?.failed), terminalFailed: numeric(emails?.terminal_failed), oldestDueAt: emails?.oldest_due_at ?? null,
+    },
+    workflows: workflows ?? {},
+    evidence: evidence ?? {},
+    budgets: budgets ?? {},
+    incidents: incidents ?? {},
+    intakeSurge: surge ? {
+      ...surge,
+      reasonCodes: typeof surge.reasonCodesJson === "string" ? JSON.parse(surge.reasonCodesJson) : [],
+      reasonCodesJson: undefined,
+    } : { mode: "normal" },
+    deadLetters: deadLetters.results,
+    deliveryFailures: failures.results,
+  };
+}
+
+export async function listOperationsIncidents(
+  db: D1Database,
+  input: { state?: string | null; severity?: string | null; limit?: number },
+): Promise<Record<string, unknown>[]> {
+  const rows = await db.prepare(`
+    SELECT id, incident_key AS incidentKey, severity, category, state, title, summary,
+      entity_type AS entityType, entity_id AS entityId, occurrence_count AS occurrenceCount,
+      first_seen_at AS firstSeenAt, last_seen_at AS lastSeenAt,
+      acknowledged_by AS acknowledgedBy, acknowledged_at AS acknowledgedAt,
+      resolved_by AS resolvedBy, resolved_at AS resolvedAt,
+      resolution_summary AS resolutionSummary, metadata_json AS metadataJson
+    FROM operations_incidents
+    WHERE installation_id = 'default' AND (?1 IS NULL OR state = ?1)
+      AND (?2 IS NULL OR severity = ?2)
+    ORDER BY CASE severity WHEN 'critical' THEN 0 WHEN 'high' THEN 1 WHEN 'medium' THEN 2 WHEN 'low' THEN 3 ELSE 4 END,
+      last_seen_at DESC, id DESC LIMIT ?3
+  `).bind(input.state ?? null, input.severity ?? null, Math.max(1, Math.min(100, input.limit ?? 50)))
+    .all<Record<string, unknown>>();
+  return rows.results.map((row) => ({
+    ...row,
+    metadata: typeof row.metadataJson === "string" ? JSON.parse(row.metadataJson) : {},
+    metadataJson: undefined,
+  }));
+}
+
+export async function loadOperationsIncident(db: D1Database, incidentId: string): Promise<Record<string, unknown> | null> {
+  const incident = await db.prepare(`SELECT * FROM operations_incidents WHERE id = ?1 AND installation_id = 'default' LIMIT 1`)
+    .bind(incidentId).first<Record<string, unknown>>();
+  if (!incident) return null;
+  const [deadLetters, repairs] = await Promise.all([
+    db.prepare(`SELECT * FROM dead_letter_items WHERE incident_id = ?1 ORDER BY last_seen_at DESC, id DESC`).bind(incidentId).all(),
+    db.prepare(`SELECT * FROM repair_actions WHERE incident_id = ?1 ORDER BY planned_at DESC, id DESC`).bind(incidentId).all(),
+  ]);
+  return {
+    incident: {
+      ...incident,
+      metadata: typeof incident.metadata_json === "string" ? JSON.parse(incident.metadata_json) : {},
+      metadata_json: undefined,
+    },
+    deadLetters: deadLetters.results,
+    repairs: repairs.results,
+  };
+}
+
+async function assertIncidentMutationKey(
+  db: D1Database,
+  idempotencyKey: string,
+  action: string,
+  incidentId: string,
+): Promise<boolean> {
+  const prior = await db.prepare(`SELECT action, target_id AS targetId FROM audit_events WHERE idempotency_key = ?1 LIMIT 1`)
+    .bind(idempotencyKey).first<{ action: string; targetId: string }>();
+  if (!prior) return false;
+  if (prior.action !== action || prior.targetId !== incidentId) {
+    throw new ApiError(409, "idempotency_key_reused", "This Idempotency-Key was already used for another operations mutation.");
+  }
+  return true;
+}
+
+function operationsAudit(
+  db: D1Database,
+  input: { key: string; action: string; incidentId: string; session: OperatorSession; details: unknown; createdAt: string },
+): D1PreparedStatement {
+  return db.prepare(`
+    INSERT INTO audit_events (
+      id, idempotency_key, action, actor_type, actor_id,
+      target_type, target_id, details_json, created_at
+    ) VALUES (?1, ?2, ?3, ?4, ?5, 'operations_incident', ?6, ?7, ?8)
+  `).bind(
+    crypto.randomUUID(), input.key, input.action, input.session.actor.type,
+    input.session.actor.id, input.incidentId, canonicalJson(input.details), input.createdAt,
+  );
+}
+
+export async function acknowledgeOperationsIncident(
+  db: D1Database,
+  incidentId: string,
+  session: OperatorSession,
+  idempotencyKey: string,
+): Promise<{ incidentId: string; state: string; idempotentReplay: boolean }> {
+  const key = `incident-ack:${idempotencyKey}`;
+  if (await assertIncidentMutationKey(db, key, "operations.incident_acknowledged", incidentId)) {
+    return { incidentId, state: "acknowledged", idempotentReplay: true };
+  }
+  const incident = await db.prepare(`SELECT state FROM operations_incidents WHERE id = ?1 LIMIT 1`).bind(incidentId).first<{ state: string }>();
+  if (!incident) throw new ApiError(404, "operations_incident_not_found", "The operations incident does not exist.");
+  if (incident.state === "resolved") throw new ApiError(409, "operations_incident_resolved", "A resolved incident cannot be acknowledged.");
+  const timestamp = now();
+  await db.batch([
+    db.prepare(`UPDATE operations_incidents SET state = 'acknowledged', acknowledged_by = ?2, acknowledged_at = COALESCE(acknowledged_at, ?3) WHERE id = ?1 AND state != 'resolved'`)
+      .bind(incidentId, session.actor.id, timestamp),
+    db.prepare(`UPDATE dead_letter_items SET state = 'acknowledged' WHERE incident_id = ?1 AND state = 'open'`).bind(incidentId),
+    operationsAudit(db, { key, action: "operations.incident_acknowledged", incidentId, session, details: {}, createdAt: timestamp }),
+  ]);
+  return { incidentId, state: "acknowledged", idempotentReplay: false };
+}
+
+export async function resolveOperationsIncident(
+  db: D1Database,
+  incidentId: string,
+  resolution: { code: string; summary: string },
+  session: OperatorSession,
+  idempotencyKey: string,
+): Promise<{ incidentId: string; state: string; idempotentReplay: boolean }> {
+  const key = `incident-resolve:${idempotencyKey}`;
+  if (await assertIncidentMutationKey(db, key, "operations.incident_resolved", incidentId)) {
+    return { incidentId, state: "resolved", idempotentReplay: true };
+  }
+  if (!/^[a-z][a-z0-9_]{1,63}$/u.test(resolution.code) || resolution.summary.trim().length < 3 || resolution.summary.length > 2_000) {
+    throw new ApiError(400, "incident_resolution_invalid", "A valid resolution code and summary are required.");
+  }
+  const incident = await db.prepare(`SELECT state FROM operations_incidents WHERE id = ?1 LIMIT 1`).bind(incidentId).first<{ state: string }>();
+  if (!incident) throw new ApiError(404, "operations_incident_not_found", "The operations incident does not exist.");
+  const timestamp = now();
+  const summary = `[${resolution.code}] ${resolution.summary.trim()}`;
+  await db.batch([
+    db.prepare(`UPDATE operations_incidents SET state = 'resolved', resolved_by = ?2, resolved_at = ?3, resolution_summary = ?4 WHERE id = ?1`)
+      .bind(incidentId, session.actor.id, timestamp, summary),
+    db.prepare(`UPDATE dead_letter_items SET state = 'resolved', resolved_at = ?2 WHERE incident_id = ?1 AND state != 'resolved'`)
+      .bind(incidentId, timestamp),
+    operationsAudit(db, { key, action: "operations.incident_resolved", incidentId, session, details: { resolutionCode: resolution.code, summary: resolution.summary.trim() }, createdAt: timestamp }),
+  ]);
+  return { incidentId, state: "resolved", idempotentReplay: false };
+}
+
+function requestIdFromJob(jobType: string, jobId: string | null): string | null {
+  if (!jobId) return null;
+  const prefix = jobType === "fetch_evidence" ? "evidence:" : null;
+  return prefix && jobId.startsWith(prefix) ? jobId.slice(prefix.length) : null;
+}
+
+export async function retryOperationsIncident(
+  db: D1Database,
+  incidentId: string,
+  session: OperatorSession,
+  idempotencyKey: string,
+): Promise<{ incidentId: string; repairActionId: string; state: string; idempotentReplay: boolean }> {
+  const key = `incident-retry:${idempotencyKey}`;
+  const existing = await db.prepare(`SELECT id FROM repair_actions WHERE operation_key = ?1 LIMIT 1`).bind(key).first<{ id: string }>();
+  if (existing) return { incidentId, repairActionId: existing.id, state: "applied", idempotentReplay: true };
+  const item = await db.prepare(`
+    SELECT d.id, d.job_type AS jobType, d.job_id AS jobId, d.outbox_id AS outboxId,
+      d.report_id AS reportId, i.state AS incidentState
+    FROM operations_incidents i JOIN dead_letter_items d ON d.incident_id = i.id
+    WHERE i.id = ?1 ORDER BY d.last_seen_at DESC, d.id DESC LIMIT 1
+  `).bind(incidentId).first<{ id: string; jobType: string; jobId: string | null; outboxId: string | null; reportId: string | null; incidentState: string }>();
+  if (!item) throw new ApiError(404, "operations_incident_not_repairable", "The incident has no typed dead-letter record to repair.");
+  if (item.incidentState === "resolved") throw new ApiError(409, "operations_incident_resolved", "A resolved incident cannot be retried.");
+  const timestamp = now();
+  let entityType: string;
+  let entityId: string;
+  let repair: D1PreparedStatement;
+  if (item.jobType === "route_report" || item.jobType === "process_ai_report") {
+    if (!item.jobId) throw new ApiError(409, "operations_repair_reference_missing", "The background job reference is missing.");
+    entityType = "background_job"; entityId = item.jobId;
+    repair = db.prepare(`UPDATE background_jobs SET state = 'pending', attempt_count = 0, available_at = ?2, last_error_code = NULL, updated_at = ?2 WHERE id = ?1`)
+      .bind(entityId, timestamp);
+  } else if (item.jobType === "fetch_evidence") {
+    const requestId = requestIdFromJob(item.jobType, item.jobId);
+    if (!requestId) throw new ApiError(409, "operations_repair_reference_missing", "The evidence request reference is missing.");
+    entityType = "evidence_fetch_request"; entityId = requestId;
+    repair = db.prepare(`UPDATE evidence_fetch_requests SET state = 'pending', attempt_count = 0, next_attempt_at = ?2, last_error_code = NULL, updated_at = ?2 WHERE id = ?1`)
+      .bind(entityId, timestamp);
+  } else if (item.jobType === "dispatch_workflow") {
+    if (!item.outboxId) throw new ApiError(409, "operations_repair_reference_missing", "The workflow outbox reference is missing.");
+    entityType = "workflow_dispatch_outbox"; entityId = item.outboxId;
+    repair = db.prepare(`UPDATE workflow_dispatch_outbox SET state = 'pending', attempt_count = 0, available_at = ?2, lease_expires_at = NULL, last_error_code = NULL, updated_at = ?2 WHERE id = ?1`)
+      .bind(entityId, timestamp);
+  } else if (item.jobType === "deliver_action" || item.jobType === "deliver_notification") {
+    if (!item.outboxId) throw new ApiError(409, "operations_repair_reference_missing", "The delivery outbox reference is missing.");
+    entityType = item.jobType === "deliver_action" ? "decision_action_outbox" : "notification_outbox";
+    entityId = item.outboxId;
+    const table = item.jobType === "deliver_action" ? "decision_action_outbox" : "notification_outbox";
+    repair = db.prepare(`UPDATE ${table} SET status = 'pending', terminal = 0, attempt_count = 0, next_attempt_at = ?2, lease_expires_at = NULL, last_error_code = NULL, updated_at = ?2 WHERE id = ?1`)
+      .bind(entityId, timestamp);
+  } else if (item.jobType === "deliver_email") {
+    if (!item.outboxId) throw new ApiError(409, "operations_repair_reference_missing", "The email delivery reference is missing.");
+    entityType = "email_delivery"; entityId = item.outboxId;
+    repair = db.prepare(`UPDATE email_deliveries SET state = 'pending', attempt_count = 0, next_attempt_at = ?2, last_error_code = NULL, updated_at = ?2 WHERE id = ?1`)
+      .bind(entityId, timestamp);
+  } else if (item.jobType === "apply_action_proposal") {
+    if (!item.jobId?.startsWith("action-proposal:")) throw new ApiError(409, "operations_repair_reference_missing", "The action proposal reference is missing.");
+    entityType = "action_proposal"; entityId = item.jobId.slice("action-proposal:".length);
+    repair = db.prepare(`UPDATE action_proposals SET state = 'queued', delivery_terminal = 0, delivery_attempt_count = 0, next_attempt_at = ?2, lease_expires_at = NULL, last_error_code = NULL, updated_at = ?2 WHERE id = ?1 AND state != 'applied'`)
+      .bind(entityId, timestamp);
+  } else {
+    throw new ApiError(409, "operations_incident_not_repairable", "This incident cannot be repaired automatically.");
+  }
+  const repairActionId = crypto.randomUUID();
+  await db.batch([
+    repair,
+    db.prepare(`INSERT INTO repair_actions (id, incident_id, action_type, entity_type, entity_id, operation_key, state, before_json, after_json, planned_by, planned_at, applied_at) VALUES (?1, ?2, 'retry_authoritative_record', ?3, ?4, ?5, 'applied', ?6, ?7, ?8, ?9, ?9)`)
+      .bind(repairActionId, incidentId, entityType, entityId, key, canonicalJson({ deadLetterId: item.id, state: "dead_letter" }), canonicalJson({ state: "pending", automaticAttemptsReset: true }), session.actor.id, timestamp),
+    db.prepare(`UPDATE operations_incidents SET state = 'acknowledged', acknowledged_by = ?2, acknowledged_at = COALESCE(acknowledged_at, ?3) WHERE id = ?1`).bind(incidentId, session.actor.id, timestamp),
+    db.prepare(`UPDATE dead_letter_items SET state = 'acknowledged' WHERE id = ?1`).bind(item.id),
+    operationsAudit(db, { key: `audit:${key}`, action: "operations.incident_retry_applied", incidentId, session, details: { repairActionId, entityType, entityId, jobType: item.jobType }, createdAt: timestamp }),
+  ]);
+  return { incidentId, repairActionId, state: "applied", idempotentReplay: false };
+}
+
+export async function beginReconciliationRun(db: D1Database, kind: string, actorId: string): Promise<string> {
+  if (!/^[a-z][a-z0-9_]{1,63}$/u.test(kind)) throw new ApiError(400, "reconciliation_kind_invalid", "The reconciliation kind is invalid.");
+  const id = crypto.randomUUID();
+  await db.prepare(`INSERT INTO reconciliation_runs (id, installation_id, reconciliation_kind, state, cursor_json, started_at) VALUES (?1, 'default', ?2, 'running', ?3, ?4)`)
+    .bind(id, kind, canonicalJson({ actorId, limit: 100 }), now()).run();
+  return id;
+}
+
+export async function finishReconciliationRun(
+  db: D1Database,
+  runId: string,
+  input: { scanned: number; repaired: number; incidents?: number; errorCode?: string },
+): Promise<void> {
+  const state = input.errorCode ? (input.repaired > 0 ? "partial" : "failed") : "succeeded";
+  await db.prepare(`UPDATE reconciliation_runs SET state = ?2, scanned_count = ?3, repaired_count = ?4, incident_count = ?5, error_code = ?6, finished_at = ?7 WHERE id = ?1 AND state = 'running'`)
+    .bind(runId, state, Math.max(0, input.scanned), Math.max(0, input.repaired), Math.max(0, input.incidents ?? 0), input.errorCode ?? null, now()).run();
+}
+
+export async function repairStaleOperationalLeases(
+  db: D1Database,
+  reconciliationRunId: string,
+  actorId: string,
+  limit = 100,
+): Promise<{ scanned: number; repaired: number }> {
+  const timestamp = now();
+  const staleAt = new Date(Date.now() - 5 * 60_000).toISOString();
+  const bounded = Math.max(1, Math.min(200, limit));
+  const [evidence, workflows, connections] = await Promise.all([
+    db.prepare(`SELECT id, updated_at AS marker FROM evidence_fetch_requests WHERE state = 'fetching' AND updated_at <= ?1 ORDER BY updated_at, id LIMIT ?2`)
+      .bind(staleAt, bounded).all<{ id: string; marker: string }>(),
+    db.prepare(`SELECT id, lease_expires_at AS marker FROM workflow_dispatch_outbox WHERE state = 'leased' AND lease_expires_at <= ?1 ORDER BY lease_expires_at, id LIMIT ?2`)
+      .bind(timestamp, bounded).all<{ id: string; marker: string }>(),
+    db.prepare(`SELECT id, expires_at AS marker FROM connection_execution_leases WHERE expires_at <= ?1 ORDER BY expires_at, id LIMIT ?2`)
+      .bind(timestamp, bounded).all<{ id: string; marker: string }>(),
+  ]);
+  const statements: D1PreparedStatement[] = [];
+  for (const row of evidence.results) {
+    statements.push(
+      db.prepare(`UPDATE evidence_fetch_requests SET state = 'pending', next_attempt_at = ?2, last_error_code = 'stale_fetch_lease_recovered', updated_at = ?2 WHERE id = ?1 AND state = 'fetching' AND updated_at <= ?3`)
+        .bind(row.id, timestamp, staleAt),
+      db.prepare(`INSERT OR IGNORE INTO repair_actions (id, reconciliation_run_id, action_type, entity_type, entity_id, operation_key, state, before_json, after_json, planned_by, planned_at, applied_at) VALUES (?1, ?2, 'release_stale_lease', 'evidence_fetch_request', ?3, ?4, 'applied', ?5, ?6, ?7, ?8, ?8)`)
+        .bind(crypto.randomUUID(), reconciliationRunId, row.id, `stale-evidence:${row.id}:${row.marker}`, canonicalJson({ state: "fetching", updatedAt: row.marker }), canonicalJson({ state: "pending" }), actorId, timestamp),
+    );
+  }
+  for (const row of workflows.results) {
+    statements.push(
+      db.prepare(`UPDATE workflow_dispatch_outbox SET state = 'failed', available_at = ?2, lease_expires_at = NULL, last_error_code = 'stale_dispatch_lease_recovered', updated_at = ?2 WHERE id = ?1 AND state = 'leased' AND lease_expires_at <= ?2`)
+        .bind(row.id, timestamp),
+      db.prepare(`INSERT OR IGNORE INTO repair_actions (id, reconciliation_run_id, action_type, entity_type, entity_id, operation_key, state, before_json, after_json, planned_by, planned_at, applied_at) VALUES (?1, ?2, 'release_stale_lease', 'workflow_dispatch_outbox', ?3, ?4, 'applied', ?5, ?6, ?7, ?8, ?8)`)
+        .bind(crypto.randomUUID(), reconciliationRunId, row.id, `stale-workflow:${row.id}:${row.marker}`, canonicalJson({ state: "leased", leaseExpiresAt: row.marker }), canonicalJson({ state: "failed" }), actorId, timestamp),
+    );
+  }
+  for (const row of connections.results) {
+    statements.push(
+      db.prepare(`DELETE FROM connection_execution_leases WHERE id = ?1 AND expires_at <= ?2`).bind(row.id, timestamp),
+      db.prepare(`INSERT OR IGNORE INTO repair_actions (id, reconciliation_run_id, action_type, entity_type, entity_id, operation_key, state, before_json, after_json, planned_by, planned_at, applied_at) VALUES (?1, ?2, 'release_stale_lease', 'connection_execution_lease', ?3, ?4, 'applied', ?5, ?6, ?7, ?8, ?8)`)
+        .bind(crypto.randomUUID(), reconciliationRunId, row.id, `stale-connection:${row.id}:${row.marker}`, canonicalJson({ expiresAt: row.marker }), canonicalJson({ deleted: true }), actorId, timestamp),
+    );
+  }
+  if (statements.length) await db.batch(statements);
+  const repaired = evidence.results.length + workflows.results.length + connections.results.length;
+  return { scanned: repaired, repaired };
+}
+
+export async function listDeadLetters(db: D1Database, limit = 100): Promise<Record<string, unknown>[]> {
+  const rows = await db.prepare(`SELECT * FROM dead_letter_items ORDER BY CASE state WHEN 'open' THEN 0 WHEN 'acknowledged' THEN 1 ELSE 2 END, last_seen_at DESC, id DESC LIMIT ?1`)
+    .bind(Math.max(1, Math.min(200, limit))).all<Record<string, unknown>>();
+  return rows.results;
+}