create-safest-tools 0.2.0

package/template/src/report-delivery.ts

@@ -0,0 +1,579 @@
+import { canonicalJson } from "./audit";
+import type { DeliveryJob } from "./report-types";
+import { decryptReportContact } from "./report-crypto";
+import { BudgetControlError, reserveBudgets, settleBudgets } from "./budget-control";
+
+const MAX_ATTEMPTS = 8;
+const RESPONSE_LIMIT = 16 * 1024;
+const REQUEST_TIMEOUT_MS = 7_500;
+const RETRY_DELAYS = [10, 30, 120, 600, 1_800, 3_600, 7_200, 21_600] as const;
+
+interface ActionOutboxRow {
+  id: string;
+  report_id: string;
+  idempotency_key: string;
+  payload_json: string;
+  status: string;
+  attempt_count: number;
+}
+
+interface NotificationOutboxRow {
+  id: string;
+  report_id: string;
+  notice_id: string;
+  idempotency_key: string;
+  payload_json: string;
+  status: string;
+  attempt_count: number;
+  audience: "reporter" | "affected_user";
+  recipient_reference: string | null;
+  contact_ciphertext: string | null;
+}
+
+interface EmailDeliveryRow {
+  id: string;
+  reportId: string;
+  messageId: string;
+  audience: "reporter" | "affected_user";
+  state: string;
+  attemptCount: number;
+  recipientReference: string | null;
+  contactCiphertext: string | null;
+  body: string;
+  reportReference: string;
+  installationId: string;
+  queueId: string | null;
+}
+
+export class RetryableDeliveryError extends Error {
+  constructor(readonly delaySeconds: number) {
+    super("Delivery failed temporarily and was scheduled for retry.");
+    this.name = "RetryableDeliveryError";
+  }
+}
+
+export class DeliveryConfigurationError extends Error {
+  constructor(message: string) {
+    super(message);
+    this.name = "DeliveryConfigurationError";
+  }
+}
+
+function now(): string {
+  return new Date().toISOString();
+}
+
+function plusSeconds(timestamp: string, seconds: number): string {
+  return new Date(Date.parse(timestamp) + seconds * 1_000).toISOString();
+}
+
+function delay(attempt: number): number {
+  return RETRY_DELAYS[Math.min(RETRY_DELAYS.length - 1, Math.max(0, attempt - 1))] ?? 21_600;
+}
+
+function webhookUrl(raw: string | undefined, variable: string): string {
+  try {
+    const url = new URL(raw ?? "");
+    if (url.protocol !== "https:" || url.username || url.password || url.hash) throw new Error("unsafe webhook URL");
+    return url.toString();
+  } catch {
+    throw new DeliveryConfigurationError(`${variable} is not configured as a safe HTTPS URL.`);
+  }
+}
+
+async function signature(secret: string, timestamp: string, payload: string): Promise<string> {
+  if (new TextEncoder().encode(secret).byteLength < 32) {
+    throw new DeliveryConfigurationError("Webhook signing secrets must contain at least 32 bytes.");
+  }
+  const key = await crypto.subtle.importKey(
+    "raw",
+    new TextEncoder().encode(secret),
+    { name: "HMAC", hash: "SHA-256" },
+    false,
+    ["sign"],
+  );
+  const digest = new Uint8Array(await crypto.subtle.sign("HMAC", key, new TextEncoder().encode(`${timestamp}.${payload}`)));
+  return `v1=${[...digest].map((byte) => byte.toString(16).padStart(2, "0")).join("")}`;
+}
+
+export async function boundedResponse(response: Response): Promise<Record<string, unknown> | null> {
+  const declared = Number.parseInt(response.headers.get("content-length") ?? "0", 10);
+  if (declared > RESPONSE_LIMIT) {
+    await response.body?.cancel().catch(() => {});
+    return null;
+  }
+  if (!response.body) return {};
+  const reader = response.body.getReader();
+  const chunks: Uint8Array[] = [];
+  let length = 0;
+  while (true) {
+    const result = await reader.read();
+    if (result.done) break;
+    length += result.value.byteLength;
+    if (length > RESPONSE_LIMIT) {
+      await reader.cancel().catch(() => {});
+      return null;
+    }
+    chunks.push(result.value);
+  }
+  const bytes = new Uint8Array(length);
+  let offset = 0;
+  for (const chunk of chunks) {
+    bytes.set(chunk, offset);
+    offset += chunk.byteLength;
+  }
+  if (!bytes.length) return {};
+  try {
+    const value: unknown = JSON.parse(new TextDecoder().decode(bytes));
+    return value && typeof value === "object" && !Array.isArray(value) ? value as Record<string, unknown> : null;
+  } catch {
+    return null;
+  }
+}
+
+export async function sendWebhook(input: {
+  url: string;
+  urlVariable?: string;
+  secret: string;
+  event: string;
+  idempotencyKey: string;
+  payload: string;
+}, fetchImpl: typeof fetch = fetch): Promise<{ status: number; body: Record<string, unknown> | null }> {
+  const url = webhookUrl(input.url, input.urlVariable ?? "webhook URL");
+  const timestamp = Math.floor(Date.now() / 1_000).toString();
+  const controller = new AbortController();
+  const timeout = setTimeout(() => controller.abort("webhook_timeout"), REQUEST_TIMEOUT_MS);
+  try {
+    const response = await fetchImpl(url, {
+      method: "POST",
+      headers: {
+        "content-type": "application/json",
+        "user-agent": "Safest-Resolve/1",
+        "x-safest-event": input.event,
+        "x-safest-idempotency-key": input.idempotencyKey,
+        "x-safest-signature": await signature(input.secret, timestamp, input.payload),
+        "x-safest-timestamp": timestamp,
+      },
+      body: input.payload,
+      signal: controller.signal,
+    });
+    return { status: response.status, body: await boundedResponse(response) };
+  } finally {
+    clearTimeout(timeout);
+  }
+}
+
+export function responseClass(status: number): "success" | "temporary_failure" | "permanent_failure" {
+  if (status >= 200 && status < 300) return "success";
+  if (status === 408 || status === 409 || status === 425 || status === 429 || status >= 500) return "temporary_failure";
+  return "permanent_failure";
+}
+
+async function claimAction(db: D1Database, id: string): Promise<ActionOutboxRow | null> {
+  const claimedAt = now();
+  const result = await db.prepare(`
+    UPDATE decision_action_outbox
+    SET status = 'delivering', attempt_count = attempt_count + 1,
+        lease_expires_at = ?2, updated_at = ?1
+    WHERE id = ?3 AND terminal = 0 AND attempt_count < ?4
+      AND (
+        status = 'pending'
+        OR (status = 'failed' AND next_attempt_at <= ?1)
+        OR (status = 'delivering' AND lease_expires_at <= ?1)
+      )
+  `).bind(claimedAt, plusSeconds(claimedAt, 60), id, MAX_ATTEMPTS).run();
+  if ((result.meta.changes ?? 0) < 1) return null;
+  return db.prepare(`
+    SELECT id, report_id, idempotency_key, payload_json, status, attempt_count
+    FROM decision_action_outbox WHERE id = ?1 LIMIT 1
+  `).bind(id).first<ActionOutboxRow>();
+}
+
+async function claimNotification(db: D1Database, id: string): Promise<NotificationOutboxRow | null> {
+  const claimedAt = now();
+  const result = await db.prepare(`
+    UPDATE notification_outbox
+    SET status = 'delivering', attempt_count = attempt_count + 1,
+        lease_expires_at = ?2, updated_at = ?1
+    WHERE id = ?3 AND terminal = 0 AND attempt_count < ?4
+      AND (
+        status = 'pending'
+        OR (status = 'failed' AND next_attempt_at <= ?1)
+        OR (status = 'delivering' AND lease_expires_at <= ?1)
+      )
+  `).bind(claimedAt, plusSeconds(claimedAt, 60), id, MAX_ATTEMPTS).run();
+  if ((result.meta.changes ?? 0) < 1) return null;
+  return db.prepare(`
+    SELECT o.id, n.report_id, o.notice_id, o.idempotency_key, o.payload_json, o.status, o.attempt_count,
+      n.audience, p.reference AS recipient_reference, p.contact_ciphertext
+    FROM notification_outbox o JOIN notices n ON n.id = o.notice_id
+    LEFT JOIN report_participants p ON p.report_id = n.report_id AND p.audience = n.audience
+    WHERE o.id = ?1 LIMIT 1
+  `).bind(id).first<NotificationOutboxRow>();
+}
+
+async function claimEmail(db: D1Database, id: string): Promise<EmailDeliveryRow | null> {
+  const claimedAt = now();
+  const staleAt = plusSeconds(claimedAt, -60);
+  const result = await db.prepare(`
+    UPDATE email_deliveries SET state = 'sending', attempt_count = attempt_count + 1,
+      updated_at = ?2 WHERE id = ?1 AND attempt_count < ?3
+      AND (state = 'pending' OR (state = 'failed' AND next_attempt_at <= ?2)
+        OR (state = 'sending' AND updated_at <= ?4))
+  `).bind(id, claimedAt, MAX_ATTEMPTS, staleAt).run();
+  if ((result.meta.changes ?? 0) < 1) return null;
+  return db.prepare(`
+    SELECT e.id, m.report_id AS reportId, e.message_id AS messageId,
+      e.audience, e.state, e.attempt_count AS attemptCount,
+      p.reference AS recipientReference, p.contact_ciphertext AS contactCiphertext,
+      m.body, r.public_reference AS reportReference,
+      r.installation_id AS installationId, r.queue_id AS queueId
+    FROM email_deliveries e
+    JOIN case_messages m ON m.id = e.message_id
+    JOIN reports r ON r.id = m.report_id
+    LEFT JOIN report_participants p ON p.report_id = m.report_id AND p.audience = e.audience
+    WHERE e.id = ?1 LIMIT 1
+  `).bind(id).first<EmailDeliveryRow>();
+}
+
+function deliveryAudit(
+  db: D1Database,
+  input: { key: string; action: string; targetType: string; targetId: string; reportId: string; details: unknown; createdAt: string },
+): D1PreparedStatement {
+  return db.prepare(`
+    INSERT OR IGNORE INTO audit_events (
+      id, idempotency_key, action, actor_type, actor_id, target_type, target_id, details_json, created_at
+    ) VALUES (?1, ?2, ?3, 'system', 'delivery-worker', ?4, ?5, ?6, ?7)
+  `).bind(
+    crypto.randomUUID(), input.key, input.action, input.targetType, input.targetId,
+    canonicalJson({ reportId: input.reportId, ...input.details as Record<string, unknown> }), input.createdAt,
+  );
+}
+
+async function processAction(db: D1Database, outboxId: string, env: Env): Promise<void> {
+  const row = await claimAction(db, outboxId);
+  if (!row) return;
+  const startedAt = now();
+  let status: number | null = null;
+  let body: Record<string, unknown> | null = null;
+  let errorCode: string | null = null;
+  try {
+    const response = await sendWebhook({
+      url: env.ACTION_WEBHOOK_URL,
+      urlVariable: "ACTION_WEBHOOK_URL",
+      secret: env.ACTION_WEBHOOK_SECRET,
+      event: "application.action.requested",
+      idempotencyKey: row.idempotency_key,
+      payload: row.payload_json,
+    });
+    status = response.status;
+    body = response.body;
+  } catch (error) {
+    errorCode = error instanceof DeliveryConfigurationError
+      ? "webhook_configuration_error"
+      : error instanceof DOMException && error.name === "AbortError" ? "webhook_timeout" : "webhook_network_error";
+  }
+  const finishedAt = now();
+  const attemptId = crypto.randomUUID();
+  if (status !== null && responseClass(status) === "success") {
+    const result = body?.status;
+    if (result === "accepted" || result === "already_applied" || result === "rejected") {
+      await db.batch([
+        db.prepare(`
+          UPDATE decision_action_outbox SET status = ?2, terminal = 1, lease_expires_at = NULL,
+            last_error_code = NULL, updated_at = ?3 WHERE id = ?1
+        `).bind(row.id, result, finishedAt),
+        db.prepare(`
+          INSERT INTO decision_action_attempts (
+            id, action_id, attempt_number, response_class, response_status, started_at, finished_at
+          ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)
+        `).bind(attemptId, row.id, row.attempt_count, result, status, startedAt, finishedAt),
+        deliveryAudit(db, {
+          key: `action-delivery:${row.id}:${result}`,
+          action: "application_action.delivery_completed",
+          targetType: "application_action",
+          targetId: row.id,
+          reportId: row.report_id,
+          details: { result, responseStatus: status, attemptCount: row.attempt_count },
+          createdAt: finishedAt,
+        }),
+      ]);
+      return;
+    }
+    errorCode = "invalid_action_response";
+  } else if (status !== null) {
+    errorCode = `webhook_http_${status}`;
+  }
+  const configurationFailure = errorCode === "webhook_configuration_error";
+  const temporary = !configurationFailure
+    && (status === null || responseClass(status) === "temporary_failure" || errorCode === "invalid_action_response");
+  const terminal = !temporary || row.attempt_count >= MAX_ATTEMPTS;
+  const retryDelay = delay(row.attempt_count);
+  const statements = [
+    db.prepare(`
+      UPDATE decision_action_outbox
+      SET status = ?2, terminal = ?3, next_attempt_at = ?4, lease_expires_at = NULL,
+          last_error_code = ?5, updated_at = ?6
+      WHERE id = ?1
+    `).bind(
+      row.id,
+      terminal && status !== null && responseClass(status) === "permanent_failure" ? "rejected" : "failed",
+      terminal ? 1 : 0,
+      plusSeconds(finishedAt, retryDelay), errorCode, finishedAt,
+    ),
+    db.prepare(`
+      INSERT INTO decision_action_attempts (
+        id, action_id, attempt_number, response_class, response_status, error_code, started_at, finished_at
+      ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
+    `).bind(attemptId, row.id, row.attempt_count, terminal ? "permanent_failure" : "temporary_failure", status, errorCode, startedAt, finishedAt),
+  ];
+  if (terminal) statements.push(deliveryAudit(db, {
+    key: `action-delivery:${row.id}:failed`,
+    action: "application_action.delivery_failed",
+    targetType: "application_action",
+    targetId: row.id,
+    reportId: row.report_id,
+    details: { errorCode, responseStatus: status, attemptCount: row.attempt_count },
+    createdAt: finishedAt,
+  }));
+  await db.batch(statements);
+  if (!terminal) throw new RetryableDeliveryError(retryDelay);
+}
+
+async function updateNoticeDelivery(db: D1Database, noticeId: string, state: "delivered" | "failed", timestamp: string): Promise<void> {
+  await db.batch([
+    db.prepare(`UPDATE notices SET delivery_state = ?2, updated_at = ?3 WHERE id = ?1`).bind(noticeId, state, timestamp),
+    db.prepare(`
+      UPDATE case_messages SET delivery_state = ?2
+      WHERE id = (SELECT message_id FROM notices WHERE id = ?1)
+    `).bind(noticeId, state),
+  ]);
+}
+
+async function processNotification(db: D1Database, outboxId: string, env: Env): Promise<void> {
+  const row = await claimNotification(db, outboxId);
+  if (!row) return;
+  const startedAt = now();
+  let status: number | null = null;
+  let errorCode: string | null = null;
+  try {
+    let payload: Record<string, unknown>;
+    try {
+      const parsed: unknown = JSON.parse(row.payload_json);
+      payload = parsed && typeof parsed === "object" && !Array.isArray(parsed) ? parsed as Record<string, unknown> : {};
+    } catch {
+      payload = {};
+    }
+    const recipientEmail = row.contact_ciphertext
+      ? (await decryptReportContact(env.CREDENTIAL_ENCRYPTION_KEY, row.contact_ciphertext, row.report_id, row.audience)).trim().toLowerCase()
+      : null;
+    const deliveryPayload = canonicalJson({
+      ...payload,
+      ...(row.recipient_reference ? { recipient_reference: row.recipient_reference } : {}),
+      ...(recipientEmail ? { recipient_email: recipientEmail } : {}),
+    });
+    const response = await sendWebhook({
+      url: env.NOTIFICATION_WEBHOOK_URL,
+      urlVariable: "NOTIFICATION_WEBHOOK_URL",
+      secret: env.NOTIFICATION_WEBHOOK_SECRET,
+      event: "participant.notification.requested",
+      idempotencyKey: row.idempotency_key,
+      payload: deliveryPayload,
+    });
+    status = response.status;
+  } catch (error) {
+    errorCode = error instanceof DeliveryConfigurationError
+      ? "webhook_configuration_error"
+      : error instanceof DOMException && error.name === "AbortError" ? "webhook_timeout" : "webhook_network_error";
+  }
+  const finishedAt = now();
+  const attemptId = crypto.randomUUID();
+  if (status !== null && responseClass(status) === "success") {
+    await db.batch([
+      db.prepare(`
+        UPDATE notification_outbox SET status = 'delivered', terminal = 1,
+          lease_expires_at = NULL, last_error_code = NULL, updated_at = ?2 WHERE id = ?1
+      `).bind(row.id, finishedAt),
+      db.prepare(`
+        INSERT INTO notification_attempts (
+          id, notification_id, attempt_number, response_class, response_status, started_at, finished_at
+        ) VALUES (?1, ?2, ?3, 'delivered', ?4, ?5, ?6)
+      `).bind(attemptId, row.id, row.attempt_count, status, startedAt, finishedAt),
+      deliveryAudit(db, {
+        key: `notification-delivery:${row.id}:delivered`,
+        action: "notification.delivered",
+        targetType: "notification",
+        targetId: row.id,
+        reportId: row.report_id,
+        details: { responseStatus: status, attemptCount: row.attempt_count },
+        createdAt: finishedAt,
+      }),
+    ]);
+    await updateNoticeDelivery(db, row.notice_id, "delivered", finishedAt);
+    return;
+  }
+  if (status !== null) errorCode = `webhook_http_${status}`;
+  const temporary = errorCode !== "webhook_configuration_error"
+    && (status === null || responseClass(status) === "temporary_failure");
+  const terminal = !temporary || row.attempt_count >= MAX_ATTEMPTS;
+  const retryDelay = delay(row.attempt_count);
+  const statements = [
+    db.prepare(`
+      UPDATE notification_outbox SET status = 'failed', terminal = ?2,
+        next_attempt_at = ?3, lease_expires_at = NULL, last_error_code = ?4, updated_at = ?5
+      WHERE id = ?1
+    `).bind(row.id, terminal ? 1 : 0, plusSeconds(finishedAt, retryDelay), errorCode, finishedAt),
+    db.prepare(`
+      INSERT INTO notification_attempts (
+        id, notification_id, attempt_number, response_class, response_status, error_code, started_at, finished_at
+      ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)
+    `).bind(attemptId, row.id, row.attempt_count, terminal ? "permanent_failure" : "temporary_failure", status, errorCode, startedAt, finishedAt),
+  ];
+  if (terminal) statements.push(deliveryAudit(db, {
+    key: `notification-delivery:${row.id}:failed`,
+    action: "notification.delivery_failed",
+    targetType: "notification",
+    targetId: row.id,
+    reportId: row.report_id,
+    details: { errorCode, responseStatus: status, attemptCount: row.attempt_count },
+    createdAt: finishedAt,
+  }));
+  await db.batch(statements);
+  if (terminal) await updateNoticeDelivery(db, row.notice_id, "failed", finishedAt);
+  if (!terminal) throw new RetryableDeliveryError(retryDelay);
+}
+
+async function processEmail(db: D1Database, deliveryId: string, env: Env): Promise<void> {
+  const row = await claimEmail(db, deliveryId);
+  if (!row) return;
+  const email = row.contactCiphertext
+    ? (await decryptReportContact(env.CREDENTIAL_ENCRYPTION_KEY, row.contactCiphertext, row.reportId, row.audience)).trim().toLowerCase()
+    : row.recipientReference?.trim().toLowerCase() ?? "";
+  const from = env.EMAIL_FROM_ADDRESS?.trim().toLowerCase() ?? "";
+  const binding = (env as unknown as { EMAIL?: { send(message: Record<string, unknown>): Promise<{ messageId?: string }> } }).EMAIL;
+  let reservations;
+  try {
+    reservations = await reserveBudgets(db, {
+      installationId: row.installationId,
+      scopes: [...(row.queueId ? [{ type: "queue", id: row.queueId }] : [])],
+      metric: "emails", amount: 1, operationType: "email_delivery",
+      operationId: row.id, attemptId: String(row.attemptCount),
+    });
+  } catch (error) {
+    if (!(error instanceof BudgetControlError)) throw error;
+    const timestamp = now();
+    if (error.retryable) {
+      await db.prepare(`UPDATE email_deliveries SET state = 'failed', last_error_code = 'email_budget_deferred', next_attempt_at = ?2, updated_at = ?3 WHERE id = ?1`)
+        .bind(row.id, plusSeconds(timestamp, error.delaySeconds), timestamp).run();
+      throw new RetryableDeliveryError(error.delaySeconds);
+    }
+    await db.batch([
+      db.prepare(`UPDATE email_deliveries SET state = 'suppressed', last_error_code = 'email_budget_exhausted', next_attempt_at = NULL, updated_at = ?2 WHERE id = ?1`).bind(row.id, timestamp),
+      deliveryAudit(db, {
+        key: `email-delivery:${row.id}:budget-suppressed`, action: "email.budget_exhausted",
+        targetType: "email_delivery", targetId: row.id, reportId: row.reportId,
+        details: { fallback: "customer_webhook", fallbackReference: error.fallbackReference }, createdAt: timestamp,
+      }),
+    ]);
+    return;
+  }
+  let providerMessageId: string | null = null;
+  let errorCode: string | null = null;
+  try {
+    if (!binding) throw new DeliveryConfigurationError("The Cloudflare Email Service binding is unavailable.");
+    if (!/^[^\s@]+@[^\s@]+\.[^\s@]+$/u.test(email) || !/^[^\s@]+@[^\s@]+\.[^\s@]+$/u.test(from)) {
+      throw new DeliveryConfigurationError("Email sender or recipient configuration is invalid.");
+    }
+    const prefix = env.EMAIL_SUBJECT_PREFIX?.trim().slice(0, 120) || "Safest report update";
+    const result = await binding.send({
+      to: email,
+      from,
+      subject: `${prefix} — ${row.reportReference}`,
+      text: `${row.body}\n\nReport reference: ${row.reportReference}`,
+    });
+    providerMessageId = typeof result.messageId === "string" ? result.messageId.slice(0, 300) : null;
+  } catch (error) {
+    errorCode = error instanceof DeliveryConfigurationError
+      ? "email_configuration_error"
+      : typeof error === "object" && error !== null && "code" in error && typeof error.code === "string"
+        ? `email_${error.code}`.slice(0, 100) : "email_provider_error";
+  }
+  const finishedAt = now();
+  if (!errorCode) {
+    await settleBudgets(db, reservations, true);
+    await db.batch([
+      db.prepare(`UPDATE email_deliveries SET state = 'delivered', provider_message_id = ?2, last_error_code = NULL, next_attempt_at = NULL, delivered_at = ?3, updated_at = ?3 WHERE id = ?1`)
+        .bind(row.id, providerMessageId, finishedAt),
+      db.prepare(`UPDATE case_messages SET delivery_state = 'delivered' WHERE id = ?1`).bind(row.messageId),
+      db.prepare(`UPDATE notices SET delivery_state = 'delivered', updated_at = ?2 WHERE message_id = ?1`).bind(row.messageId, finishedAt),
+      deliveryAudit(db, {
+        key: `email-delivery:${row.id}:delivered`, action: "email.delivered",
+        targetType: "email_delivery", targetId: row.id, reportId: row.reportId,
+        details: { provider: "cloudflare_email_service", attemptCount: row.attemptCount }, createdAt: finishedAt,
+      }),
+    ]);
+    return;
+  }
+  const terminal = errorCode === "email_configuration_error" || row.attemptCount >= MAX_ATTEMPTS;
+  await settleBudgets(db, reservations, errorCode !== "email_configuration_error");
+  const retryDelay = delay(row.attemptCount);
+  const statements: D1PreparedStatement[] = [
+    db.prepare(`UPDATE email_deliveries SET state = 'failed', last_error_code = ?2, next_attempt_at = ?3, updated_at = ?4 WHERE id = ?1`)
+      .bind(row.id, errorCode, plusSeconds(finishedAt, retryDelay), finishedAt),
+  ];
+  if (terminal) {
+    statements.push(
+      db.prepare(`UPDATE case_messages SET delivery_state = 'failed' WHERE id = ?1 AND delivery_state != 'delivered'`).bind(row.messageId),
+      db.prepare(`UPDATE notices SET delivery_state = 'failed', updated_at = ?2 WHERE message_id = ?1 AND delivery_state != 'delivered'`).bind(row.messageId, finishedAt),
+      deliveryAudit(db, {
+        key: `email-delivery:${row.id}:failed`, action: "email.delivery_failed",
+        targetType: "email_delivery", targetId: row.id, reportId: row.reportId,
+        details: { errorCode, attemptCount: row.attemptCount }, createdAt: finishedAt,
+      }),
+    );
+  }
+  await db.batch(statements);
+  if (!terminal) throw new RetryableDeliveryError(retryDelay);
+}
+
+export async function processDeliveryJob(db: D1Database, job: DeliveryJob, env: Env): Promise<void> {
+  if (job.type === "deliver_action") return processAction(db, job.outboxId, env);
+  if (job.type === "deliver_notification") return processNotification(db, job.outboxId, env);
+  return processEmail(db, job.outboxId, env);
+}
+
+export async function pendingDeliveryJobs(db: D1Database, limit = 50): Promise<DeliveryJob[]> {
+  const at = now();
+  const boundedLimit = Math.max(1, Math.min(100, limit));
+  const [actions, notifications, emails] = await Promise.all([
+    db.prepare(`
+      SELECT id FROM decision_action_outbox
+      WHERE terminal = 0 AND attempt_count < ?2 AND (
+        (status IN ('pending', 'failed') AND next_attempt_at <= ?1)
+        OR (status = 'delivering' AND lease_expires_at <= ?1)
+      )
+      ORDER BY next_attempt_at, id LIMIT ?3
+    `).bind(at, MAX_ATTEMPTS, boundedLimit).all<{ id: string }>(),
+    db.prepare(`
+      SELECT id FROM notification_outbox
+      WHERE terminal = 0 AND attempt_count < ?2 AND (
+        (status IN ('pending', 'failed') AND next_attempt_at <= ?1)
+        OR (status = 'delivering' AND lease_expires_at <= ?1)
+      )
+      ORDER BY next_attempt_at, id LIMIT ?3
+    `).bind(at, MAX_ATTEMPTS, boundedLimit).all<{ id: string }>(),
+    db.prepare(`
+      SELECT id FROM email_deliveries
+      WHERE attempt_count < ?2 AND (
+        (state IN ('pending', 'failed') AND COALESCE(next_attempt_at, created_at) <= ?1)
+        OR (state = 'sending' AND updated_at <= ?3)
+      )
+      ORDER BY COALESCE(next_attempt_at, created_at), id LIMIT ?4
+    `).bind(at, MAX_ATTEMPTS, plusSeconds(at, -60), boundedLimit).all<{ id: string }>(),
+  ]);
+  return [
+    ...actions.results.map((row): DeliveryJob => ({ version: 1, jobId: `action:${row.id}`, type: "deliver_action", outboxId: row.id })),
+    ...notifications.results.map((row): DeliveryJob => ({ version: 1, jobId: `notification:${row.id}`, type: "deliver_notification", outboxId: row.id })),
+    ...emails.results.map((row): DeliveryJob => ({ version: 1, jobId: `email:${row.id}`, type: "deliver_email", outboxId: row.id })),
+  ].slice(0, boundedLimit);
+}

package/template/src/report-form-validation.ts

@@ -0,0 +1,51 @@
+import { ApiError } from "./report-http";
+
+export function validateAnswersAgainstFormFields(
+  answers: Record<string, string | boolean | number | string[]>,
+  definitionsValue: unknown[],
+): void {
+  const definitions = definitionsValue
+    .filter((value): value is Record<string, unknown> => Boolean(value && typeof value === "object" && !Array.isArray(value)))
+    .filter((value) => typeof value.key === "string");
+  const byKey = new Map(definitions.map((field) => [String(field.key), field]));
+  const unsupported = Object.keys(answers).filter((key) => !byKey.has(key));
+  if (unsupported.length) {
+    throw new ApiError(400, "answer_field_not_allowed", "The report contains answers that are not part of this form.", { fields: unsupported.slice(0, 10) });
+  }
+  for (const field of definitions) {
+    const key = String(field.key);
+    const value = answers[key];
+    const missing = value === undefined || value === null || value === "" || Array.isArray(value) && value.length === 0;
+    if (field.required === true && missing) {
+      throw new ApiError(400, "required_answer_missing", `The ${key} answer is required.`, { field: `answers.${key}` });
+    }
+    if (missing) continue;
+    const type = String(field.type);
+    const answerField = `answers.${key}`;
+    if (type === "short_text" || type === "long_text" || type === "url") {
+      if (typeof value !== "string") throw new ApiError(400, "answer_type_invalid", `${answerField} must be text.`, { field: answerField });
+      const defaultMaximum = type === "short_text" ? 512 : type === "url" ? 2_048 : 4_000;
+      const maximum = typeof field.max_length === "number" ? Math.min(defaultMaximum, field.max_length) : defaultMaximum;
+      if (value.length > maximum) throw new ApiError(400, "answer_length_invalid", `${answerField} exceeds its ${maximum}-character limit.`, { field: answerField, maximum });
+      if (type === "url") {
+        try {
+          const url = new URL(value);
+          const local = ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname);
+          if ((url.protocol !== "https:" && !(local && url.protocol === "http:")) || url.username || url.password) throw new Error("unsafe URL");
+        } catch { throw new ApiError(400, "answer_url_invalid", `${answerField} must be a valid HTTPS URL.`, { field: answerField }); }
+      }
+    } else if (type === "number") {
+      if (typeof value !== "number" || !Number.isFinite(value)) throw new ApiError(400, "answer_type_invalid", `${answerField} must be a number.`, { field: answerField });
+      if (typeof field.min_value === "number" && value < field.min_value) throw new ApiError(400, "answer_range_invalid", `${answerField} is below its minimum.`, { field: answerField, minimum: field.min_value });
+      if (typeof field.max_value === "number" && value > field.max_value) throw new ApiError(400, "answer_range_invalid", `${answerField} is above its maximum.`, { field: answerField, maximum: field.max_value });
+    } else if (type === "boolean") {
+      if (typeof value !== "boolean") throw new ApiError(400, "answer_type_invalid", `${answerField} must be yes or no.`, { field: answerField });
+    } else if (type === "select" || type === "multi_select") {
+      const allowed = new Set(Array.isArray(field.options) ? field.options.map((option) => option && typeof option === "object" && !Array.isArray(option) ? (option as Record<string, unknown>).value : null).filter((option): option is string => typeof option === "string") : []);
+      if (type === "select" && (typeof value !== "string" || !allowed.has(value))) throw new ApiError(400, "answer_choice_invalid", `${answerField} contains an unavailable choice.`, { field: answerField });
+      if (type === "multi_select" && (!Array.isArray(value) || value.length > 20 || value.some((choice) => !allowed.has(choice)))) throw new ApiError(400, "answer_choice_invalid", `${answerField} contains unavailable choices.`, { field: answerField });
+    } else {
+      throw new ApiError(400, "form_field_type_invalid", "The published form contains an unsupported field type.", { field: answerField });
+    }
+  }
+}