create-safest-tools 0.2.0

package/template/src/workflow-runs.ts

@@ -0,0 +1,686 @@
+import { canonicalJson } from "./audit";
+import { sha256 } from "./report-crypto";
+import { ApiError } from "./report-http";
+import type { OperatorSession, WorkflowDispatchJob } from "./report-types";
+import { startDynamicWorkflow } from "./workflow-dynamic-runtime";
+import type { JsonObject } from "./workflow-platform-types";
+
+interface MutationRow {
+  action: string;
+  actor_id: string;
+  response_json: string;
+}
+
+interface DispatchRow {
+  outboxId: string;
+  runId: string;
+  operationKey: string;
+  attemptCount: number;
+  workflowVersionId: string;
+  artifactId: string;
+  bundleDigest: string;
+  instanceId: string | null;
+  runState: string;
+  operation: "start" | "event";
+  eventType: string | null;
+  eventPayloadJson: string | null;
+}
+
+export class RetryableWorkflowDispatchError extends Error {
+  constructor(readonly delaySeconds: number) {
+    super("Workflow dispatch should be retried.");
+    this.name = "RetryableWorkflowDispatchError";
+  }
+}
+
+export async function preparePinnedWorkflowRunsForRouting(
+  db: D1Database,
+  input: {
+    reportId: string;
+    installationId: string;
+    queueId: string;
+    queueVersionId: string | null;
+    createdAt: string;
+  },
+): Promise<{ statements: D1PreparedStatement[]; jobs: WorkflowDispatchJob[] }> {
+  const pins = await db.prepare(`
+    SELECT p.id AS pinId, p.mode, p.workflow_version_id AS workflowVersionId,
+      v.authority_mode AS authorityMode, v.graph_digest AS graphDigest,
+      a.bundle_digest AS bundleDigest
+    FROM queue_workflow_pins p JOIN workflow_versions v ON v.id = p.workflow_version_id
+    JOIN workflow_artifacts a ON a.id = v.artifact_id
+    WHERE p.queue_id = ?1 AND p.deactivated_at IS NULL
+      AND v.retired_at IS NULL AND a.status = 'ready'
+    ORDER BY CASE p.mode WHEN 'primary' THEN 0 ELSE 1 END, p.activated_at, p.id
+  `).bind(input.queueId).all<{
+    pinId: string; mode: "primary" | "shadow"; workflowVersionId: string;
+    authorityMode: "off" | "shadow" | "assist" | "bounded_auto";
+    graphDigest: string; bundleDigest: string;
+  }>();
+  const statements: D1PreparedStatement[] = [];
+  const jobs: WorkflowDispatchJob[] = [];
+  for (const pin of pins.results) {
+    if (pin.mode === "primary" && pin.authorityMode === "shadow") continue;
+    if (pin.mode === "shadow" && pin.authorityMode !== "shadow") continue;
+    const existing = await db.prepare(`SELECT id FROM workflow_runs WHERE report_id = ?1 AND pin_id = ?2 LIMIT 1`)
+      .bind(input.reportId, pin.pinId).first();
+    if (existing) continue;
+    const runId = crypto.randomUUID();
+    const outboxId = crypto.randomUUID();
+    const instanceId = `run-${runId}`;
+    statements.push(
+      db.prepare(`
+        INSERT INTO workflow_runs (
+          id, installation_id, report_id, workflow_version_id, queue_id, queue_version_id,
+          pin_id, run_kind, authority_mode, state, cloudflare_instance_id,
+          graph_digest, artifact_digest, input_json, created_at, updated_at
+        ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'dispatch_pending', ?10, ?11, ?12, ?13, ?14, ?14)
+      `).bind(
+        runId, input.installationId, input.reportId, pin.workflowVersionId,
+        input.queueId, input.queueVersionId, pin.pinId,
+        pin.mode === "primary" ? "primary" : "shadow", pin.authorityMode,
+        instanceId, pin.graphDigest, pin.bundleDigest,
+        canonicalJson({ schemaVersion: 1, reportId: input.reportId, queueId: input.queueId, routingTriggered: true }),
+        input.createdAt,
+      ),
+      db.prepare(`
+        INSERT INTO workflow_dispatch_outbox (
+          id, run_id, operation, operation_key, state, available_at, created_at, updated_at
+        ) VALUES (?1, ?2, 'start', ?3, 'pending', ?4, ?4, ?4)
+      `).bind(outboxId, runId, `workflow-start:${runId}`, input.createdAt),
+    );
+    jobs.push({ version: 1, type: "dispatch_workflow", jobId: `workflow-dispatch:${outboxId}`, runId, outboxId });
+  }
+  return { statements, jobs };
+}
+
+function now(): string {
+  return new Date().toISOString();
+}
+
+function parsedJson<T>(value: string | null, fallback: T): T {
+  if (!value) return fallback;
+  try {
+    return JSON.parse(value) as T;
+  } catch {
+    return fallback;
+  }
+}
+
+async function mutationReplay(
+  db: D1Database,
+  key: string,
+  action: string,
+  actorId: string,
+): Promise<Record<string, unknown> | null> {
+  const row = await db.prepare(`
+    SELECT action, actor_id, response_json FROM configuration_mutations WHERE idempotency_key = ?1 LIMIT 1
+  `).bind(key).first<MutationRow>();
+  if (!row) return null;
+  if (row.action !== action || row.actor_id !== actorId) throw new ApiError(409, "idempotency_key_reused", "This Idempotency-Key was already used for another workflow operation.");
+  return parsedJson(row.response_json, {});
+}
+
+function auditStatement(
+  db: D1Database,
+  input: {
+    key: string;
+    action: string;
+    session: OperatorSession;
+    targetId: string;
+    details: unknown;
+    createdAt: string;
+  },
+): D1PreparedStatement {
+  return db.prepare(`
+    INSERT OR IGNORE INTO audit_events (
+      id, idempotency_key, action, actor_type, actor_id, target_type, target_id, details_json, created_at
+    ) VALUES (?1, ?2, ?3, ?4, ?5, 'workflow_run', ?6, ?7, ?8)
+  `).bind(
+    crypto.randomUUID(), input.key, input.action, input.session.actor.type,
+    input.session.actor.id, input.targetId, canonicalJson(input.details), input.createdAt,
+  );
+}
+
+export async function createPinnedWorkflowRun(
+  db: D1Database,
+  reportId: string,
+  mode: "shadow" | "primary",
+  session: OperatorSession,
+  idempotencyKey: string,
+): Promise<{
+  runId: string;
+  outboxId: string;
+  job: WorkflowDispatchJob;
+  workflowVersionId: string;
+  idempotentReplay: boolean;
+}> {
+  const replay = await mutationReplay(db, idempotencyKey, "workflow.run.start", session.actor.id);
+  if (replay) return {
+    runId: String(replay.runId), outboxId: String(replay.outboxId),
+    workflowVersionId: String(replay.workflowVersionId),
+    job: {
+      version: 1,
+      type: "dispatch_workflow",
+      jobId: `workflow-dispatch:${String(replay.outboxId)}`,
+      runId: String(replay.runId),
+      outboxId: String(replay.outboxId),
+    },
+    idempotentReplay: true,
+  };
+  const selected = await db.prepare(`
+    SELECT r.id AS report_id, r.installation_id, r.queue_id, r.queue_version_id,
+      p.id AS pin_id, p.workflow_version_id, v.authority_mode, v.graph_digest,
+      a.bundle_digest, a.status AS artifact_status
+    FROM reports r
+    JOIN queue_workflow_pins p ON p.queue_id = r.queue_id AND p.mode = ?2 AND p.deactivated_at IS NULL
+    JOIN workflow_versions v ON v.id = p.workflow_version_id AND v.retired_at IS NULL
+    JOIN workflow_artifacts a ON a.id = v.artifact_id
+    WHERE r.id = ?1 LIMIT 1
+  `).bind(reportId, mode).first<{
+    report_id: string;
+    installation_id: string;
+    queue_id: string;
+    queue_version_id: string | null;
+    pin_id: string;
+    workflow_version_id: string;
+    authority_mode: "off" | "shadow" | "assist" | "bounded_auto";
+    graph_digest: string;
+    bundle_digest: string;
+    artifact_status: string;
+  }>();
+  if (!selected) throw new ApiError(409, "workflow_pin_unavailable", `The report's queue has no active ${mode} workflow pin.`);
+  if (selected.artifact_status !== "ready") throw new ApiError(409, "workflow_artifact_not_ready", "The pinned workflow artifact is not verified and ready.");
+  if (mode === "primary" && selected.authority_mode === "shadow") throw new ApiError(409, "shadow_workflow_not_primary", "A shadow-authority workflow cannot create a primary run.");
+  if (mode === "shadow" && selected.authority_mode !== "shadow") throw new ApiError(409, "shadow_run_authority_required", "A shadow run requires a workflow version compiled with shadow authority.");
+  const runId = crypto.randomUUID();
+  const outboxId = crypto.randomUUID();
+  const createdAt = now();
+  const instanceId = `run-${runId}`;
+  const inputJson = canonicalJson({ schemaVersion: 1, reportId, queueId: selected.queue_id });
+  const response = { runId, outboxId, workflowVersionId: selected.workflow_version_id };
+  await db.batch([
+    db.prepare(`
+      INSERT INTO workflow_runs (
+        id, installation_id, report_id, workflow_version_id, queue_id, queue_version_id,
+        pin_id, run_kind, authority_mode, state, cloudflare_instance_id,
+        graph_digest, artifact_digest, input_json, created_at, updated_at
+      ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'dispatch_pending', ?10, ?11, ?12, ?13, ?14, ?14)
+    `).bind(
+      runId, selected.installation_id, reportId, selected.workflow_version_id,
+      selected.queue_id, selected.queue_version_id, selected.pin_id,
+      mode === "primary" ? "primary" : "shadow", selected.authority_mode,
+      instanceId, selected.graph_digest, selected.bundle_digest, inputJson, createdAt,
+    ),
+    db.prepare(`
+      INSERT INTO workflow_dispatch_outbox (
+        id, run_id, operation, operation_key, state, available_at, created_at, updated_at
+      ) VALUES (?1, ?2, 'start', ?3, 'pending', ?4, ?4, ?4)
+    `).bind(outboxId, runId, `workflow-start:${runId}`, createdAt),
+    db.prepare(`
+      INSERT INTO configuration_mutations (idempotency_key, action, actor_id, target_id, response_json, created_at)
+      VALUES (?1, 'workflow.run.start', ?2, ?3, ?4, ?5)
+    `).bind(idempotencyKey, session.actor.id, runId, canonicalJson(response), createdAt),
+    auditStatement(db, {
+      key: `workflow-run-start:${idempotencyKey}`, action: "workflow.run_requested", session,
+      targetId: runId,
+      details: { reportId, mode, workflowVersionId: selected.workflow_version_id, pinId: selected.pin_id },
+      createdAt,
+    }),
+  ]);
+  return {
+    ...response,
+    job: { version: 1, type: "dispatch_workflow", jobId: `workflow-dispatch:${outboxId}`, runId, outboxId },
+    idempotentReplay: false,
+  };
+}
+
+export async function pendingWorkflowDispatchJobs(db: D1Database, limit: number): Promise<WorkflowDispatchJob[]> {
+  const rows = await db.prepare(`
+    SELECT id, run_id FROM workflow_dispatch_outbox
+    WHERE operation IN ('start','event') AND state IN ('pending','failed') AND available_at <= ?1
+      AND (lease_expires_at IS NULL OR lease_expires_at <= ?1)
+    ORDER BY available_at, id LIMIT ?2
+  `).bind(now(), Math.max(1, Math.min(limit, 100))).all<{ id: string; run_id: string }>();
+  return rows.results.map((row) => ({
+    version: 1,
+    type: "dispatch_workflow",
+    jobId: `workflow-dispatch:${row.id}`,
+    runId: row.run_id,
+    outboxId: row.id,
+  }));
+}
+
+export async function createLinkedWorkflowRun(
+  env: Env,
+  input: {
+    parentRunId: string;
+    nodeId: string;
+    workflowVersionId: string;
+    payload: JsonObject;
+  },
+): Promise<{ runId: string; workflowVersionId: string; idempotentReplay: boolean }> {
+  const existing = await env.DB.prepare(`
+    SELECT l.child_run_id AS childRunId FROM workflow_run_links l
+    WHERE l.parent_run_id = ?1 AND l.node_id = ?2 LIMIT 1
+  `).bind(input.parentRunId, input.nodeId).first<{ childRunId: string }>();
+  if (existing) return { runId: existing.childRunId, workflowVersionId: input.workflowVersionId, idempotentReplay: true };
+  const depth = await env.DB.prepare(`
+    WITH RECURSIVE ancestors(id, depth) AS (
+      SELECT id, 1 FROM workflow_runs WHERE id = ?1
+      UNION ALL
+      SELECT r.parent_run_id, ancestors.depth + 1 FROM workflow_runs r
+      JOIN ancestors ON r.id = ancestors.id WHERE r.parent_run_id IS NOT NULL AND ancestors.depth < 10
+    ) SELECT MAX(depth) AS depth FROM ancestors
+  `).bind(input.parentRunId).first<{ depth: number }>();
+  if (Number(depth?.depth ?? 0) >= 5) throw new ApiError(409, "linked_workflow_depth_exceeded", "Linked workflow depth is limited to five runs.");
+  const target = await env.DB.prepare(`
+    SELECT p.installation_id AS installationId, p.report_id AS reportId,
+      p.queue_id AS queueId, p.queue_version_id AS queueVersionId,
+      v.id AS workflowVersionId, v.authority_mode AS authorityMode,
+      v.graph_digest AS graphDigest, w.purpose, a.bundle_digest AS bundleDigest,
+      a.status AS artifactStatus
+    FROM workflow_runs p CROSS JOIN workflow_versions v
+    JOIN workflow_definitions w ON w.id = v.workflow_id
+    JOIN workflow_artifacts a ON a.id = v.artifact_id
+    WHERE p.id = ?1 AND v.id = ?2 AND v.retired_at IS NULL AND w.status = 'active'
+    LIMIT 1
+  `).bind(input.parentRunId, input.workflowVersionId).first<{
+    installationId: string; reportId: string | null; queueId: string | null;
+    queueVersionId: string | null; workflowVersionId: string; authorityMode: "off" | "shadow" | "assist" | "bounded_auto";
+    graphDigest: string; purpose: "report" | "appeal" | "quality" | "operations";
+    bundleDigest: string; artifactStatus: string;
+  }>();
+  if (!target) throw new ApiError(409, "linked_workflow_unavailable", "The linked workflow version is unavailable.");
+  if (target.purpose === "report") throw new ApiError(409, "linked_primary_workflow_prohibited", "A linked workflow cannot create a second primary report run.");
+  if (target.artifactStatus !== "ready") throw new ApiError(409, "linked_workflow_artifact_not_ready", "The linked workflow artifact is not ready.");
+  const runId = crypto.randomUUID();
+  const outboxId = crypto.randomUUID();
+  const timestamp = now();
+  const instanceId = `run-${runId}`;
+  const runKind = target.purpose === "appeal" ? "appeal" : target.purpose === "quality" ? "quality" : "linked";
+  const runInput = canonicalJson({
+    schemaVersion: 1,
+    parentRunId: input.parentRunId,
+    parentNodeId: input.nodeId,
+    reportId: target.reportId,
+    payload: input.payload,
+  });
+  await env.DB.batch([
+    env.DB.prepare(`
+      INSERT INTO workflow_runs (
+        id, installation_id, report_id, parent_run_id, workflow_version_id,
+        queue_id, queue_version_id, run_kind, authority_mode, state,
+        cloudflare_instance_id, graph_digest, artifact_digest, input_json,
+        created_at, updated_at
+      ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 'dispatch_pending', ?10, ?11, ?12, ?13, ?14, ?14)
+    `).bind(
+      runId, target.installationId, target.reportId, input.parentRunId,
+      target.workflowVersionId, target.queueId, target.queueVersionId,
+      runKind, target.authorityMode, instanceId, target.graphDigest,
+      target.bundleDigest, runInput, timestamp,
+    ),
+    env.DB.prepare(`
+      INSERT INTO workflow_run_links (parent_run_id, child_run_id, link_type, node_id, created_at)
+      VALUES (?1, ?2, 'subworkflow', ?3, ?4)
+    `).bind(input.parentRunId, runId, input.nodeId, timestamp),
+    env.DB.prepare(`
+      INSERT INTO workflow_dispatch_outbox (
+        id, run_id, operation, operation_key, state, available_at, created_at, updated_at
+      ) VALUES (?1, ?2, 'start', ?3, 'pending', ?4, ?4, ?4)
+    `).bind(outboxId, runId, `workflow-start:${runId}`, timestamp),
+  ]);
+  const job: WorkflowDispatchJob = {
+    version: 1,
+    type: "dispatch_workflow",
+    jobId: `workflow-dispatch:${outboxId}`,
+    runId,
+    outboxId,
+  };
+  await env.CONTROL_JOBS.send(job, { contentType: "json" }).catch(() => {});
+  return { runId, workflowVersionId: target.workflowVersionId, idempotentReplay: false };
+}
+
+async function leaseDispatch(db: D1Database, job: WorkflowDispatchJob): Promise<DispatchRow | null> {
+  const timestamp = now();
+  const leaseExpiresAt = new Date(Date.now() + 60_000).toISOString();
+  const leased = await db.prepare(`
+    UPDATE workflow_dispatch_outbox
+    SET state = 'leased', attempt_count = attempt_count + 1,
+      lease_expires_at = ?3, updated_at = ?2
+    WHERE id = ?1 AND run_id = ?4
+      AND state IN ('pending','failed') AND available_at <= ?2
+      AND (lease_expires_at IS NULL OR lease_expires_at <= ?2)
+  `).bind(job.outboxId, timestamp, leaseExpiresAt, job.runId).run();
+  if (leased.meta.changes !== 1) {
+    const delivered = await db.prepare(`SELECT state FROM workflow_dispatch_outbox WHERE id = ?1 AND run_id = ?2`)
+      .bind(job.outboxId, job.runId).first<{ state: string }>();
+    if (delivered?.state === "delivered") return null;
+    throw new RetryableWorkflowDispatchError(5);
+  }
+  return db.prepare(`
+    SELECT o.id AS outboxId, o.run_id AS runId, o.operation_key AS operationKey,
+      o.attempt_count AS attemptCount, o.operation, r.workflow_version_id AS workflowVersionId,
+      v.artifact_id AS artifactId, a.bundle_digest AS bundleDigest,
+      r.cloudflare_instance_id AS instanceId, r.state AS runState,
+      e.event_type AS eventType, e.payload_json AS eventPayloadJson
+    FROM workflow_dispatch_outbox o
+    JOIN workflow_runs r ON r.id = o.run_id
+    JOIN workflow_versions v ON v.id = r.workflow_version_id
+    JOIN workflow_artifacts a ON a.id = v.artifact_id
+    LEFT JOIN workflow_event_inbox e ON e.id = o.event_inbox_id
+    WHERE o.id = ?1 AND o.run_id = ?2 LIMIT 1
+  `).bind(job.outboxId, job.runId).first<DispatchRow>();
+}
+
+export async function processWorkflowDispatchJob(
+  env: Env,
+  ctx: ExecutionContext,
+  job: WorkflowDispatchJob,
+): Promise<void> {
+  const dispatch = await leaseDispatch(env.DB, job);
+  if (!dispatch) return;
+  if (["succeeded", "failed", "terminated", "superseded"].includes(dispatch.runState)) {
+    await env.DB.prepare(`
+      UPDATE workflow_dispatch_outbox SET state = 'dead_letter', lease_expires_at = NULL,
+        last_error_code = 'run_terminal_before_dispatch', updated_at = ?2 WHERE id = ?1
+    `).bind(dispatch.outboxId, now()).run();
+    return;
+  }
+  const instanceId = dispatch.instanceId ?? `run-${dispatch.runId}`;
+  try {
+    if (dispatch.operation === "event") {
+      if (!dispatch.instanceId || !dispatch.eventType || !dispatch.eventPayloadJson) throw new Error("workflow_event_dispatch_not_ready");
+      const instance = await env.WORKFLOWS.get(dispatch.instanceId);
+      await instance.sendEvent({ type: dispatch.eventType, payload: parsedJson(dispatch.eventPayloadJson, {}) });
+      const deliveredAt = now();
+      await env.DB.batch([
+        env.DB.prepare(`
+          UPDATE workflow_event_inbox SET delivered_at = ?2,
+            delivery_attempts = delivery_attempts + 1, last_error_code = NULL
+          WHERE id = (SELECT event_inbox_id FROM workflow_dispatch_outbox WHERE id = ?1)
+        `).bind(dispatch.outboxId, deliveredAt),
+        env.DB.prepare(`
+          UPDATE workflow_dispatch_outbox SET state = 'delivered', lease_expires_at = NULL,
+            last_error_code = NULL, updated_at = ?2 WHERE id = ?1 AND state = 'leased'
+        `).bind(dispatch.outboxId, deliveredAt),
+      ]);
+      return;
+    }
+    const platformInstanceId = await startDynamicWorkflow(env, ctx, {
+      artifactId: dispatch.artifactId,
+      workflowVersionId: dispatch.workflowVersionId,
+      instanceId,
+      runId: dispatch.runId,
+    });
+    const timestamp = now();
+    await env.DB.batch([
+      env.DB.prepare(`
+        UPDATE workflow_runs SET state = 'queued', cloudflare_instance_id = ?2,
+          updated_at = ?3, revision = revision + 1
+        WHERE id = ?1 AND state = 'dispatch_pending'
+      `).bind(dispatch.runId, platformInstanceId, timestamp),
+      env.DB.prepare(`
+        UPDATE workflow_dispatch_outbox SET state = 'delivered', lease_expires_at = NULL,
+          last_error_code = NULL, updated_at = ?2 WHERE id = ?1 AND state = 'leased'
+      `).bind(dispatch.outboxId, timestamp),
+    ]);
+  } catch (error) {
+    const delaySeconds = Math.min(900, 5 * (2 ** Math.min(dispatch.attemptCount, 7)));
+    const timestamp = now();
+    await env.DB.prepare(`
+      UPDATE workflow_dispatch_outbox SET state = CASE WHEN attempt_count >= 8 THEN 'dead_letter' ELSE 'failed' END,
+        lease_expires_at = NULL, available_at = ?2, last_error_code = ?3, updated_at = ?4
+      WHERE id = ?1 AND state = 'leased'
+    `).bind(
+      dispatch.outboxId, new Date(Date.now() + delaySeconds * 1_000).toISOString(),
+      error instanceof Error ? error.message.slice(0, 200) : "workflow_dispatch_failed", timestamp,
+    ).run();
+    if (dispatch.attemptCount >= 8) {
+      await env.DB.prepare(`
+        UPDATE workflow_runs SET state = 'repair_required', error_code = 'workflow_dispatch_exhausted',
+          error_summary = 'Workflow dispatch exhausted automatic attempts.', updated_at = ?2,
+          revision = revision + 1 WHERE id = ?1 AND state = 'dispatch_pending'
+      `).bind(dispatch.runId, timestamp).run();
+      return;
+    }
+    throw new RetryableWorkflowDispatchError(delaySeconds);
+  }
+}
+
+export async function listWorkflowRuns(
+  db: D1Database,
+  input: { reportId?: string; state?: string; allowedQueueIds?: string[] | null; limit: number },
+): Promise<Record<string, unknown>[]> {
+  const rows = await db.prepare(`
+    SELECT r.id, r.report_id AS reportId, r.appeal_id AS appealId,
+      r.parent_run_id AS parentRunId, r.workflow_version_id AS workflowVersionId,
+      r.queue_id AS queueId, r.run_kind AS runKind, r.authority_mode AS authorityMode,
+      r.state, r.cloudflare_instance_id AS cloudflareInstanceId,
+      r.current_node_id AS currentNodeId, r.error_code AS errorCode,
+      r.started_at AS startedAt, r.waiting_at AS waitingAt,
+      r.finished_at AS finishedAt, r.created_at AS createdAt, r.updated_at AS updatedAt,
+      w.workflow_key AS workflowKey, w.name AS workflowName, v.version,
+      (SELECT COUNT(*) FROM workflow_step_runs s WHERE s.run_id = r.id) AS stepCount,
+      (SELECT COUNT(*) FROM human_tasks t WHERE t.run_id = r.id AND t.state IN ('open','claimed')) AS openTaskCount
+    FROM workflow_runs r
+    JOIN workflow_versions v ON v.id = r.workflow_version_id
+    JOIN workflow_definitions w ON w.id = v.workflow_id
+    WHERE (?1 IS NULL OR r.report_id = ?1) AND (?2 IS NULL OR r.state = ?2)
+      AND (?3 = 1 OR r.queue_id IN (SELECT value FROM json_each(?4)))
+    ORDER BY r.created_at DESC, r.id DESC LIMIT ?5
+  `).bind(
+    input.reportId ?? null, input.state ?? null,
+    input.allowedQueueIds === null || input.allowedQueueIds === undefined ? 1 : 0,
+    canonicalJson(input.allowedQueueIds ?? []), Math.max(1, Math.min(input.limit, 100)),
+  ).all<Record<string, unknown>>();
+  return rows.results;
+}
+
+export async function loadWorkflowRun(db: D1Database, runId: string): Promise<Record<string, unknown> | null> {
+  const run = await db.prepare(`
+    SELECT r.*, w.workflow_key, w.name AS workflow_name, v.version,
+      v.graph_digest AS version_graph_digest, a.bundle_digest, a.status AS artifact_status
+    FROM workflow_runs r JOIN workflow_versions v ON v.id = r.workflow_version_id
+    JOIN workflow_definitions w ON w.id = v.workflow_id
+    JOIN workflow_artifacts a ON a.id = v.artifact_id
+    WHERE r.id = ?1 LIMIT 1
+  `).bind(runId).first<Record<string, unknown>>();
+  if (!run) return null;
+  const [steps, waits, events, tasks] = await Promise.all([
+    db.prepare(`SELECT * FROM workflow_step_runs WHERE run_id = ?1 ORDER BY started_at, node_id, occurrence`).bind(runId).all(),
+    db.prepare(`SELECT * FROM workflow_waits WHERE run_id = ?1 ORDER BY created_at, id`).bind(runId).all(),
+    db.prepare(`SELECT * FROM workflow_event_inbox WHERE run_id = ?1 ORDER BY received_at, id`).bind(runId).all(),
+    db.prepare(`SELECT * FROM human_tasks WHERE run_id = ?1 ORDER BY created_at, id`).bind(runId).all(),
+  ]);
+  return { run, steps: steps.results, waits: waits.results, events: events.results, tasks: tasks.results };
+}
+
+export async function sendWorkflowEvent(
+  env: Env,
+  runId: string,
+  input: { eventType: string; eventKey: string; sourceType: string; sourceId?: string; payload: unknown },
+): Promise<{ eventId: string; idempotentReplay: boolean; delivered: boolean }> {
+  if (!/^[A-Za-z0-9][A-Za-z0-9._:-]{0,99}$/u.test(input.eventType)) throw new ApiError(400, "workflow_event_type_invalid", "event_type is invalid.");
+  if (input.eventKey.length < 1 || input.eventKey.length > 200) throw new ApiError(400, "workflow_event_key_invalid", "event_key is invalid.");
+  const run = await env.DB.prepare(`
+    SELECT id, cloudflare_instance_id, state FROM workflow_runs WHERE id = ?1 LIMIT 1
+  `).bind(runId).first<{ id: string; cloudflare_instance_id: string | null; state: string }>();
+  if (!run) throw new ApiError(404, "workflow_run_not_found", "The workflow run does not exist.");
+  const existing = await env.DB.prepare(`
+    SELECT id, delivered_at FROM workflow_event_inbox
+    WHERE run_id = ?1 AND event_type = ?2 AND event_key = ?3 LIMIT 1
+  `).bind(runId, input.eventType, input.eventKey).first<{ id: string; delivered_at: string | null }>();
+  if (existing) return { eventId: existing.id, idempotentReplay: true, delivered: Boolean(existing.delivered_at) };
+  const eventId = crypto.randomUUID();
+  const payloadJson = canonicalJson(input.payload);
+  const timestamp = now();
+  await env.DB.batch([
+    env.DB.prepare(`
+      INSERT INTO workflow_event_inbox (
+        id, run_id, event_type, event_key, source_type, source_id,
+        payload_json, payload_digest, received_at
+      ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
+    `).bind(
+      eventId, runId, input.eventType, input.eventKey, input.sourceType,
+      input.sourceId ?? null, payloadJson, `sha256:${await sha256(payloadJson)}`, timestamp,
+    ),
+    env.DB.prepare(`
+      INSERT INTO workflow_dispatch_outbox (
+        id, run_id, operation, event_inbox_id, operation_key,
+        state, available_at, created_at, updated_at
+      ) VALUES (?1, ?2, 'event', ?1, ?3, 'pending', ?4, ?4, ?4)
+    `).bind(eventId, runId, `workflow-event:${runId}:${input.eventType}:${input.eventKey}`, timestamp),
+  ]);
+  if (!run.cloudflare_instance_id) return { eventId, idempotentReplay: false, delivered: false };
+  try {
+    const instance = await env.WORKFLOWS.get(run.cloudflare_instance_id);
+    await instance.sendEvent({ type: input.eventType, payload: input.payload });
+    const deliveredAt = now();
+    await env.DB.batch([
+      env.DB.prepare(`UPDATE workflow_event_inbox SET delivered_at = ?2, delivery_attempts = delivery_attempts + 1 WHERE id = ?1`).bind(eventId, deliveredAt),
+      env.DB.prepare(`UPDATE workflow_dispatch_outbox SET state = 'delivered', updated_at = ?2 WHERE id = ?1`).bind(eventId, deliveredAt),
+    ]);
+    return { eventId, idempotentReplay: false, delivered: true };
+  } catch {
+    await env.DB.batch([
+      env.DB.prepare(`UPDATE workflow_event_inbox SET delivery_attempts = delivery_attempts + 1, last_error_code = 'platform_event_send_failed' WHERE id = ?1`).bind(eventId),
+      env.DB.prepare(`UPDATE workflow_dispatch_outbox SET state = 'failed', last_error_code = 'platform_event_send_failed', updated_at = ?2 WHERE id = ?1`).bind(eventId, now()),
+    ]);
+    return { eventId, idempotentReplay: false, delivered: false };
+  }
+}
+
+export async function controlWorkflowRun(
+  env: Env,
+  runId: string,
+  operation: "pause" | "resume" | "terminate",
+): Promise<void> {
+  const run = await env.DB.prepare(`SELECT cloudflare_instance_id, state FROM workflow_runs WHERE id = ?1 LIMIT 1`)
+    .bind(runId).first<{ cloudflare_instance_id: string | null; state: string }>();
+  if (!run) throw new ApiError(404, "workflow_run_not_found", "The workflow run does not exist.");
+  if (!run.cloudflare_instance_id) throw new ApiError(409, "workflow_instance_unavailable", "The workflow run has not been dispatched.");
+  const instance = await env.WORKFLOWS.get(run.cloudflare_instance_id);
+  if (operation === "pause") await instance.pause();
+  else if (operation === "resume") await instance.resume();
+  else await instance.terminate();
+  const timestamp = now();
+  await env.DB.prepare(`
+    UPDATE workflow_runs SET state = ?2, finished_at = CASE WHEN ?2 = 'terminated' THEN ?3 ELSE finished_at END,
+      waiting_at = CASE WHEN ?2 = 'paused' THEN waiting_at ELSE NULL END,
+      updated_at = ?3, revision = revision + 1 WHERE id = ?1
+  `).bind(runId, operation === "pause" ? "paused" : operation === "resume" ? "running" : "terminated", timestamp).run();
+}
+
+export async function reconcileWorkflowRun(
+  env: Env,
+  runId: string,
+  session: OperatorSession,
+  idempotencyKey: string,
+): Promise<{ runId: string; state: string; platformState: string | null; repairApplied: boolean; idempotentReplay: boolean }> {
+  const auditKey = `workflow-run-reconcile:${idempotencyKey}`;
+  const replay = await env.DB.prepare(`SELECT details_json AS detailsJson, target_id AS targetId FROM audit_events WHERE idempotency_key = ?1 AND action = 'workflow.run_reconciled' LIMIT 1`)
+    .bind(auditKey).first<{ detailsJson: string; targetId: string }>();
+  if (replay) {
+    if (replay.targetId !== runId) throw new ApiError(409, "idempotency_key_reused", "This Idempotency-Key was already used for another workflow run.");
+    const details = parsedJson<Record<string, unknown>>(replay.detailsJson, {});
+    return {
+      runId,
+      state: String(details.state ?? "unknown"),
+      platformState: typeof details.platformState === "string" ? details.platformState : null,
+      repairApplied: details.repairApplied === true,
+      idempotentReplay: true,
+    };
+  }
+  const run = await env.DB.prepare(`SELECT id, state, cloudflare_instance_id AS instanceId FROM workflow_runs WHERE id = ?1 LIMIT 1`)
+    .bind(runId).first<{ id: string; state: string; instanceId: string | null }>();
+  if (!run) throw new ApiError(404, "workflow_run_not_found", "The workflow run does not exist.");
+  const timestamp = now();
+  let state = run.state;
+  let platformState: string | null = null;
+  let repairApplied = false;
+  const terminal = ["succeeded", "failed", "terminated", "superseded"].includes(run.state);
+  const statements: D1PreparedStatement[] = [];
+  if (!run.instanceId) {
+    if (!terminal) {
+      const reset = await env.DB.prepare(`
+        UPDATE workflow_dispatch_outbox SET state = 'pending', attempt_count = 0,
+          available_at = ?2, lease_expires_at = NULL, last_error_code = NULL, updated_at = ?2
+        WHERE run_id = ?1 AND operation = 'start' AND state IN ('failed','dead_letter','leased')
+      `).bind(runId, timestamp).run();
+      repairApplied = reset.meta.changes > 0;
+      if (repairApplied) {
+        state = "dispatch_pending";
+        statements.push(
+          env.DB.prepare(`UPDATE workflow_runs SET state = 'dispatch_pending', error_code = NULL, error_summary = NULL, updated_at = ?2, revision = revision + 1 WHERE id = ?1 AND state NOT IN ('succeeded','failed','terminated','superseded')`)
+            .bind(runId, timestamp),
+        );
+      }
+    }
+  } else {
+    let status: InstanceStatus;
+    try {
+      status = await (await env.WORKFLOWS.get(run.instanceId)).status();
+    } catch {
+      throw new ApiError(503, "workflow_platform_status_unavailable", "The Workflow platform status is temporarily unavailable.");
+    }
+    platformState = status.status;
+    const mapped = status.status === "complete" ? "succeeded"
+      : status.status === "errored" ? "failed"
+        : status.status === "terminated" ? "terminated"
+          : status.status === "waiting" || status.status === "waitingForPause" ? "waiting"
+            : status.status === "queued" || status.status === "running" || status.status === "paused" ? status.status
+              : null;
+    if (mapped && !terminal && mapped !== run.state) {
+      state = mapped;
+      repairApplied = true;
+      statements.push(
+        env.DB.prepare(`
+          UPDATE workflow_runs SET state = ?2,
+            error_code = CASE WHEN ?2 = 'failed' THEN 'platform_instance_errored' ELSE error_code END,
+            error_summary = CASE WHEN ?2 = 'failed' THEN ?3 ELSE error_summary END,
+            waiting_at = CASE WHEN ?2 = 'waiting' THEN COALESCE(waiting_at, ?4) ELSE NULL END,
+            finished_at = CASE WHEN ?2 IN ('succeeded','failed','terminated') THEN COALESCE(finished_at, ?4) ELSE finished_at END,
+            updated_at = ?4, revision = revision + 1 WHERE id = ?1
+        `).bind(runId, mapped, status.error?.message?.slice(0, 500) ?? null, timestamp),
+      );
+    } else if (mapped && terminal && mapped !== run.state) {
+      const incidentKey = `workflow-state-mismatch:${runId}:${run.state}:${mapped}`;
+      statements.push(env.DB.prepare(`
+        INSERT INTO operations_incidents (
+          id, installation_id, incident_key, severity, category, state, title, summary,
+          entity_type, entity_id, occurrence_count, first_seen_at, last_seen_at, metadata_json
+        ) VALUES (?1, 'default', ?2, 'high', 'workflow_state_mismatch', 'open',
+          'Workflow state differs from platform', ?3, 'workflow_run', ?4, 1, ?5, ?5, ?6)
+        ON CONFLICT(installation_id, incident_key) DO UPDATE SET
+          state = 'open', occurrence_count = operations_incidents.occurrence_count + 1,
+          last_seen_at = excluded.last_seen_at, metadata_json = excluded.metadata_json
+      `).bind(
+        crypto.randomUUID(), incidentKey, "A terminal D1 workflow state differs from the Cloudflare Workflow instance state.",
+        runId, timestamp, canonicalJson({ d1State: run.state, platformState: mapped, platformRawState: status.status }),
+      ));
+    }
+  }
+  if (repairApplied) {
+    statements.push(env.DB.prepare(`
+      INSERT OR IGNORE INTO repair_actions (
+        id, action_type, entity_type, entity_id, operation_key, state,
+        before_json, after_json, planned_by, planned_at, applied_at
+      ) VALUES (?1, 'reconcile_workflow_run', 'workflow_run', ?2, ?3, 'applied', ?4, ?5, ?6, ?7, ?7)
+    `).bind(
+      crypto.randomUUID(), runId, `workflow-run-reconcile:${runId}:${idempotencyKey}`,
+      canonicalJson({ state: run.state }), canonicalJson({ state, platformState }), session.actor.id, timestamp,
+    ));
+  }
+  const details = { state, platformState, repairApplied };
+  statements.push(auditStatement(env.DB, {
+    key: auditKey,
+    action: "workflow.run_reconciled",
+    session,
+    targetId: runId,
+    details,
+    createdAt: timestamp,
+  }));
+  await env.DB.batch(statements);
+  return { runId, ...details, idempotentReplay: false };
+}