@osolmaz/pi-workflows 0.14.0 → 0.15.0

package/src/controllers/sqlite.ts

@@ -4,6 +4,7 @@ import path from "node:path";
 import { StateDatabase, workflowStatePath } from "../state/database.js";
 import { canonicalJson } from "../state/json.js";
 import { resourceIdFor, tokenHash } from "../state/mutation.js";
+import { initializeViewerRun, recordViewerDeltas } from "../state/viewer.js";
 import {
   EffectRequestConflictError,
   ResourceConflictError,
@@ -51,6 +52,7 @@ export type WorkflowRunReservationOptions = {
   launchOptions?: unknown;
   runnerId: string;
   originSessionId: string;
+  executionMode?: "interactive" | "headless";
   parentRunId?: string;
   now?: string;
 };
@@ -76,6 +78,7 @@ export type WorkflowRunQueueRecord = {
   claimExpiresAt: string | null;
   affinityRunnerId: string | null;
   originSessionId: string | null;
+  executionMode: "interactive" | "headless";
   parentRunId: string | null;
   errorCode: string | null;
   errorMessage: string | null;
@@ -199,6 +202,7 @@ type RunRow = {
   errorCode: string | null;
   errorHash: Buffer | null;
   originSessionId: string | null;
+  executionMode: "interactive" | "headless";
   parentRunId: string | null;
   createdAt: number;
   updatedAt: number;
@@ -228,6 +232,18 @@ type EffectRow = {
   errorHash: Buffer | null;
 };
 
+type CancelledRunEffectRow = {
+  effectId: string;
+  resourceId: string;
+  status: "pending" | "applying";
+  attemptCount: number;
+};
+
+type ExpiredInteractionRow = {
+  requestId: string;
+  attemptId: string;
+};
+
 type WorkflowRow = {
   requestId: string;
   resourceUid: string;
@@ -249,7 +265,13 @@ export class SqliteControllerStore implements ControllerStore {
 
   constructor(
     filePath: string = workflowStatePath(),
-    options: { readOnly?: boolean; projectPath?: string; state?: StateDatabase } = {},
+    options: {
+      readOnly?: boolean;
+      projectPath?: string;
+      state?: StateDatabase;
+      /** Host-only global view. Project-scoped mutations still require projectPath. */
+      global?: boolean;
+    } = {},
   ) {
     this.ownsState = options.state === undefined;
     this.state =
@@ -260,9 +282,13 @@ export class SqliteControllerStore implements ControllerStore {
         checkLegacyState: filePath === workflowStatePath(),
       });
     this.filePath = this.state.filePath;
-    const projectPath = options.projectPath === undefined ? process.cwd() : options.projectPath;
-    this.projectId =
-      options.readOnly === true ? this.findProject(projectPath) : this.ensureProject(projectPath);
+    if (options.global === true) {
+      this.projectId = null;
+    } else {
+      const projectPath = options.projectPath === undefined ? process.cwd() : options.projectPath;
+      this.projectId =
+        options.readOnly === true ? this.findProject(projectPath) : this.ensureProject(projectPath);
+    }
   }
 
   close(): void {
@@ -530,10 +556,16 @@ export class SqliteControllerStore implements ControllerStore {
     ownerId: string;
     leaseMs: number;
     now?: string;
+    exclude?: ControllerResourceRef[];
   }): ControllerQueueClaim | undefined {
     if (options.controllers.length === 0) return undefined;
     const now = epoch(validTimestamp(options.now));
     const placeholders = options.controllers.map(() => "?").join(", ");
+    const exclusions = options.exclude ?? [];
+    const exclusionSql = exclusions
+      .map(() => "AND NOT (c.controller_name = ? AND c.resource_key = ?)")
+      .join("\n               ");
+    const exclusionParams = exclusions.flatMap((ref) => [ref.controller, ref.key]);
     return this.state.transaction(() => {
       const row = this.state.connection
         .prepare(
@@ -542,12 +574,13 @@ export class SqliteControllerStore implements ControllerStore {
              JOIN leases l ON l.resource_id = c.resource_id
              WHERE c.project_id = ?
                AND c.controller_name IN (${placeholders})
+               ${exclusionSql}
                AND q.available_at <= ?
                AND (l.owner_id IS NULL OR l.expires_at <= ?)
              ORDER BY q.available_at, c.controller_name, c.resource_key
              LIMIT 1`)}`,
         )
-        .get(this.requireProjectId(), ...options.controllers, now, now);
+        .get(this.requireProjectId(), ...options.controllers, ...exclusionParams, now, now);
       if (!isControllerRow(row)) return undefined;
       const lease = this.requireLease(row.resourceId);
       const token = randomBytes(32).toString("base64url");
@@ -1122,13 +1155,14 @@ export class SqliteControllerStore implements ControllerStore {
         now,
         now,
       );
+    initializeViewerRun(this.state, options.runId, now);
     insertQueuedRunSources(this.state, options.runId, queuedSource);
     this.state.connection
       .prepare(
         `INSERT INTO run_bindings(run_id, origin_session_id, execution_mode, created_at)
-           VALUES (?, ?, 'interactive', ?)`,
+           VALUES (?, ?, ?, ?)`,
       )
-      .run(options.runId, options.originSessionId, now);
+      .run(options.runId, options.originSessionId, options.executionMode ?? "interactive", now);
     this.state.connection
       .prepare(
         `INSERT INTO run_queue(
@@ -1204,6 +1238,16 @@ export class SqliteControllerStore implements ControllerStore {
     return row === undefined ? undefined : this.mapWorkflowRun(row);
   }
 
+  workflowRunProjectPath(runId: string): string | undefined {
+    const row = this.state.connection
+      .prepare(
+        `SELECT p.canonical_path AS canonicalPath
+         FROM runs r JOIN projects p ON p.project_id = r.project_id WHERE r.run_id = ?`,
+      )
+      .get(runId);
+    return isCanonicalPathRow(row) ? row.canonicalPath : undefined;
+  }
+
   listWorkflowRuns(
     options: {
       statuses?: WorkflowRunLaunchStatus[];
@@ -1211,8 +1255,12 @@ export class SqliteControllerStore implements ControllerStore {
       limit?: number;
     } = {},
   ): WorkflowRunQueueRecord[] {
-    const clauses = ["r.project_id = ?"];
-    const params: unknown[] = [this.requireProjectId()];
+    const clauses: string[] = [];
+    const params: unknown[] = [];
+    if (this.projectId !== null) {
+      clauses.push("r.project_id = ?");
+      params.push(this.projectId);
+    }
     if (options.statuses !== undefined && options.statuses.length > 0) {
       clauses.push(`q.status IN (${options.statuses.map(() => "?").join(", ")})`);
       params.push(...options.statuses);
@@ -1224,7 +1272,9 @@ export class SqliteControllerStore implements ControllerStore {
     params.push(options.limit ?? 100);
     const rows = this.state.connection
       .prepare(
-        workflowRunSelect(`WHERE ${clauses.join(" AND ")} ORDER BY q.created_at DESC LIMIT ?`),
+        workflowRunSelect(
+          `${clauses.length === 0 ? "" : `WHERE ${clauses.join(" AND ")}`} ORDER BY q.created_at DESC LIMIT ?`,
+        ),
       )
       .all(...params);
     return rows.filter(isRunRow).map((row) => this.mapWorkflowRun(row));
@@ -1252,6 +1302,132 @@ export class SqliteControllerStore implements ControllerStore {
     return this.claimRun(options.runId, options.runnerId, options.claimToken, options.leaseMs, now);
   }
 
+  /** Schedule a worker that validates one pending interactive submission. */
+  claimWorkflowRunForInteractionValidation(options: {
+    runId: string;
+    runnerId: string;
+    claimToken: string;
+    leaseMs: number;
+    now?: string;
+  }): WorkflowRunQueueRecord | undefined {
+    const now = epoch(validTimestamp(options.now));
+    return this.claimRun(
+      options.runId,
+      options.runnerId,
+      options.claimToken,
+      options.leaseMs,
+      now,
+      {
+        allowPendingInteraction: true,
+      },
+    );
+  }
+
+  /** Take a short host claim without scheduling a parked interactive run. */
+  claimWorkflowRunForControl(options: {
+    runId: string;
+    runnerId: string;
+    claimToken: string;
+    leaseMs: number;
+    now?: string;
+  }): WorkflowRunQueueRecord | undefined {
+    const now = epoch(validTimestamp(options.now));
+    return this.claimRun(
+      options.runId,
+      options.runnerId,
+      options.claimToken,
+      options.leaseMs,
+      now,
+      { allowPendingInteraction: true, preserveQueueStatus: true },
+    );
+  }
+
+  beginWorkflowRunInteractionTimeout(options: {
+    runId: string;
+    claimToken: string;
+    now?: string;
+  }): boolean {
+    const now = epoch(validTimestamp(options.now));
+    return this.state.transaction(() => {
+      if (
+        !this.verifyWorkflowRunClaim({
+          runId: options.runId,
+          claimToken: options.claimToken,
+          now: new Date(now).toISOString(),
+        })
+      ) {
+        return false;
+      }
+      const row = this.requireWorkflowRunRow(options.runId);
+      const interaction = this.state.connection
+        .prepare(
+          `SELECT i.request_id AS requestId, i.attempt_id AS attemptId
+           FROM interactive_requests i
+           JOIN node_attempts a ON a.attempt_id = i.attempt_id
+           WHERE i.run_id = ? AND i.status IN ('pending', 'presenting')
+             AND a.deadline_at IS NOT NULL AND a.deadline_at <= ?
+           ORDER BY a.deadline_at, i.request_id LIMIT 1`,
+        )
+        .get(options.runId, now);
+      if (!isExpiredInteractionRow(interaction)) return false;
+      const request = this.state.connection
+        .prepare(
+          `UPDATE interactive_requests
+           SET status = 'cancelled', presenter_id = NULL,
+               presentation_claim_expires_at = NULL, revision = revision + 1, updated_at = ?
+           WHERE request_id = ? AND run_id = ? AND status IN ('pending', 'presenting')`,
+        )
+        .run(now, interaction.requestId, options.runId);
+      const run = this.state.connection
+        .prepare(
+          `UPDATE runs
+           SET status = 'running', status_detail = 'finishing expired interaction deadline',
+               updated_at = ?, finished_at = NULL
+           WHERE run_id = ? AND status = 'waiting'`,
+        )
+        .run(now, options.runId);
+      const queue = this.state.connection
+        .prepare(
+          `UPDATE run_queue SET status = 'starting', error_code = NULL, error_hash = NULL,
+                  updated_at = ?, finished_at = NULL
+           WHERE run_id = ? AND status = 'parked'`,
+        )
+        .run(now, options.runId);
+      /* istanbul ignore if -- the expired parked interaction was selected above */
+      if (request.changes !== 1 || run.changes !== 1 || queue.changes !== 1) {
+        throw new Error(`Workflow run ${options.runId} changed during interaction timeout`);
+      }
+      const error = "Workflow node deadline expired before origin-session input arrived";
+      const receiptHash = this.state.putJson({ status: "rejected", error }, now);
+      this.state.connection
+        .prepare(
+          `UPDATE interactive_submissions SET outcome = 'rejected', receipt_hash = ?
+           WHERE request_id = ? AND outcome = 'validating'`,
+        )
+        .run(receiptHash, interaction.requestId);
+      const revision = this.resourceRevision(row.resourceId);
+      const lease = this.requireLease(row.resourceId);
+      this.bumpResource(row.resourceId, revision, now);
+      this.insertEvent(
+        row.resourceId,
+        revision + 1,
+        "run.interaction_timeout_started",
+        lease.ownerType ?? "system",
+        lease.ownerId,
+        { requestId: interaction.requestId, attemptId: interaction.attemptId },
+        now,
+        lease.generation || undefined,
+      );
+      recordViewerDeltas(
+        this.state,
+        options.runId,
+        [{ targetType: "summary" }, { targetType: "replay" }, { targetType: "conversation" }],
+        now,
+      );
+      return true;
+    });
+  }
+
   markWorkflowRunRunning(options: { runId: string; claimToken: string; now?: string }): boolean {
     return this.updateClaimedRunStatus(options.runId, options.claimToken, "running", options.now);
   }
@@ -1266,19 +1442,19 @@ export class SqliteControllerStore implements ControllerStore {
   }): WorkflowRunQueueRecord | undefined {
     const now = epoch(validTimestamp(options.now));
     const clauses = [
-      "r.project_id = ?",
       "q.status IN ('queued', 'parked', 'starting', 'running')",
       "q.available_at <= ?",
       "(l.owner_id IS NULL OR l.expires_at <= ? OR l.owner_id = ?)",
       "(q.affinity_runner_id IS NULL OR q.affinity_runner_id = ? OR q.status IN ('parked', 'starting', 'running'))",
+      "NOT (q.status = 'parked' AND (r.status = 'waiting' OR r.paused = 1))",
+      "NOT EXISTS (SELECT 1 FROM interactive_requests i WHERE i.run_id = r.run_id AND i.status IN ('pending', 'presenting'))",
+      "(q.error_code IS NULL OR q.error_code <> 'workflowSourceChanged')",
     ];
-    const params: unknown[] = [
-      this.requireProjectId(),
-      now,
-      now,
-      options.runnerId,
-      options.runnerId,
-    ];
+    const params: unknown[] = [now, now, options.runnerId, options.runnerId];
+    if (this.projectId !== null) {
+      clauses.unshift("r.project_id = ?");
+      params.unshift(this.projectId);
+    }
     if (options.sessionId !== undefined) {
       clauses.push("b.origin_session_id = ?");
       params.push(options.sessionId);
@@ -1307,22 +1483,37 @@ export class SqliteControllerStore implements ControllerStore {
     now?: string;
   }): boolean {
     const now = epoch(validTimestamp(options.now));
-    const row = this.workflowRunRow(options.runId);
-    if (row === undefined || row.ownerId === null) return false;
-    const result = this.state.connection
-      .prepare(
-        `UPDATE leases SET heartbeat_at = ?, expires_at = ?
-         WHERE resource_id = ? AND token_hash = ? AND owner_id = ? AND expires_at > ?`,
-      )
-      .run(
-        now,
-        now + options.leaseMs,
-        row.resourceId,
-        tokenHash(options.claimToken),
-        row.ownerId,
-        now,
-      );
-    return result.changes === 1;
+    return this.state.transaction(() => {
+      const row = this.workflowRunRow(options.runId);
+      if (row === undefined || row.ownerId === null) return false;
+      const lease = this.requireLease(row.resourceId);
+      const result = this.state.connection
+        .prepare(
+          `UPDATE leases SET heartbeat_at = ?, expires_at = ?
+           WHERE resource_id = ? AND owner_type = ? AND owner_id = ?
+             AND token_hash = ? AND generation = ? AND expires_at > ?`,
+        )
+        .run(
+          now,
+          now + options.leaseMs,
+          row.resourceId,
+          lease.ownerType,
+          row.ownerId,
+          tokenHash(options.claimToken),
+          lease.generation,
+          now,
+        );
+      if (result.changes === 1) {
+        recordViewerDeltas(
+          this.state,
+          options.runId,
+          [{ targetType: "summary" }, { targetType: "replay" }],
+          now,
+        );
+        return true;
+      }
+      return false;
+    });
   }
 
   verifyWorkflowRunClaim(options: { runId: string; claimToken: string; now?: string }): boolean {
@@ -1347,16 +1538,11 @@ export class SqliteControllerStore implements ControllerStore {
         ownerId: string;
         token: string;
         generation: number;
+        leaseMs: number;
       }
     | undefined {
     const row = this.workflowRunRow(runId);
-    if (
-      row === undefined ||
-      row.ownerId === null ||
-      row.claimExpiresAt === null ||
-      row.claimExpiresAt <= Date.now()
-    )
-      return undefined;
+    if (row === undefined || row.ownerId === null || row.claimExpiresAt === null) return undefined;
     const lease = this.requireLease(row.resourceId);
     if (lease.tokenHash === null || !lease.tokenHash.equals(tokenHash(claimToken)))
       return undefined;
@@ -1367,6 +1553,7 @@ export class SqliteControllerStore implements ControllerStore {
       ownerId: row.ownerId,
       token: claimToken,
       generation: row.leaseGeneration,
+      leaseMs: 30_000,
     };
   }
 
@@ -1374,6 +1561,158 @@ export class SqliteControllerStore implements ControllerStore {
     return this.releaseRunClaim(options.runId, options.claimToken, "parked", options.now);
   }
 
+  parkWorkflowRunForSourceChange(options: {
+    runId: string;
+    claimToken: string;
+    detail: string;
+    now?: string;
+  }): boolean {
+    const now = epoch(validTimestamp(options.now));
+    return this.state.transaction(() => {
+      const row = this.workflowRunRow(options.runId);
+      if (
+        row === undefined ||
+        !this.verifyWorkflowRunClaim({
+          runId: options.runId,
+          claimToken: options.claimToken,
+          now: new Date(now).toISOString(),
+        })
+      ) {
+        return false;
+      }
+      const lease = this.requireLease(row.resourceId);
+      const detail = options.detail.slice(0, 8_192);
+      const errorHash = this.state.putText(detail, now);
+      const run = this.state.connection
+        .prepare(
+          `UPDATE runs SET status_detail = ?, updated_at = ?, finished_at = NULL
+           WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
+        )
+        .run(detail, now, options.runId);
+      const queue = this.state.connection
+        .prepare(
+          `UPDATE run_queue
+           SET status = 'parked', error_code = 'workflowSourceChanged', error_hash = ?,
+               available_at = ?, updated_at = ?, finished_at = NULL
+           WHERE run_id = ? AND status IN ('queued', 'starting', 'running', 'parked')`,
+        )
+        .run(errorHash, now, now, options.runId);
+      /* istanbul ignore if -- exact live claim and nonterminal checks make both updates mandatory */
+      if (run.changes !== 1 || queue.changes !== 1) {
+        throw new Error(`Workflow run ${options.runId} has inconsistent source-change state`);
+      }
+      const released = this.state.connection
+        .prepare(
+          `UPDATE leases
+           SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
+               acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
+           WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
+        )
+        .run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
+      /* istanbul ignore if -- the exact live claim is stable in this transaction */
+      if (released.changes !== 1) {
+        throw new Error(`Workflow run ${options.runId} claim changed during source recovery`);
+      }
+      const revision = this.resourceRevision(row.resourceId);
+      this.bumpResource(row.resourceId, revision, now);
+      this.insertEvent(
+        row.resourceId,
+        revision + 1,
+        "run.source_changed",
+        lease.ownerType ?? "system",
+        lease.ownerId,
+        { status: "parked", code: "workflowSourceChanged" },
+        now,
+        lease.generation || undefined,
+      );
+      recordViewerDeltas(
+        this.state,
+        options.runId,
+        [{ targetType: "summary" }, { targetType: "replay" }],
+        now,
+      );
+      return true;
+    });
+  }
+
+  parkWorkflowRunForAmbiguousEffect(options: {
+    runId: string;
+    claimToken: string;
+    now?: string;
+  }): boolean {
+    const now = epoch(validTimestamp(options.now));
+    return this.state.transaction(() => {
+      const row = this.workflowRunRow(options.runId);
+      if (
+        row === undefined ||
+        !this.verifyWorkflowRunClaim({
+          runId: options.runId,
+          claimToken: options.claimToken,
+          now: new Date(now).toISOString(),
+        })
+      ) {
+        return false;
+      }
+      const lease = this.requireLease(row.resourceId);
+      const detail = "effect outcome is ambiguous; explicit recovery is required";
+      const errorHash = this.state.putText(detail, now);
+      this.state.connection
+        .prepare(
+          `UPDATE node_attempts SET status = 'waiting', updated_at = ?
+           WHERE run_id = ? AND status IN ('pending', 'running')`,
+        )
+        .run(now, options.runId);
+      const run = this.state.connection
+        .prepare(
+          `UPDATE runs SET status = 'waiting', status_detail = ?, updated_at = ?, finished_at = ?
+           WHERE run_id = ? AND status = 'running'`,
+        )
+        .run(detail, now, now, options.runId);
+      const queue = this.state.connection
+        .prepare(
+          `UPDATE run_queue
+           SET status = 'parked', error_code = 'effectAmbiguous', error_hash = ?, updated_at = ?
+           WHERE run_id = ? AND status IN ('starting', 'running', 'parked')`,
+        )
+        .run(errorHash, now, options.runId);
+      /* istanbul ignore if -- exact live claim and run checks make both updates mandatory */
+      if (run.changes !== 1 || queue.changes !== 1) {
+        throw new Error(`Workflow run ${options.runId} has inconsistent ambiguous-effect state`);
+      }
+      const released = this.state.connection
+        .prepare(
+          `UPDATE leases
+           SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
+               acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
+           WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`,
+        )
+        .run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
+      /* istanbul ignore if -- the exact live claim is stable in this transaction */
+      if (released.changes !== 1) {
+        throw new Error(`Workflow run ${options.runId} claim changed during effect recovery`);
+      }
+      const revision = this.resourceRevision(row.resourceId);
+      this.bumpResource(row.resourceId, revision, now);
+      this.insertEvent(
+        row.resourceId,
+        revision + 1,
+        "run.effect_ambiguous",
+        lease.ownerType ?? "system",
+        lease.ownerId,
+        { status: "waiting", code: "effectAmbiguous" },
+        now,
+        lease.generation || undefined,
+      );
+      recordViewerDeltas(
+        this.state,
+        options.runId,
+        [{ targetType: "summary" }, { targetType: "replay" }],
+        now,
+      );
+      return true;
+    });
+  }
+
   failWorkflowRun(options: {
     runId: string;
     claimToken?: string;
@@ -1392,7 +1731,7 @@ export class SqliteControllerStore implements ControllerStore {
   }
 
   cancelWorkflowRun(options: { runId: string; claimToken?: string; now?: string }): boolean {
-    return this.terminalRun(
+    const cancelled = this.terminalRun(
       options.runId,
       options.claimToken,
       "cancelled",
@@ -1400,6 +1739,111 @@ export class SqliteControllerStore implements ControllerStore {
       "Workflow run cancelled",
       options.now,
     );
+    if (cancelled || options.claimToken !== undefined) return cancelled;
+    return this.cancelStaleWorkflowRun({
+      runId: options.runId,
+      ...(options.now === undefined ? {} : { now: options.now }),
+    });
+  }
+
+  /** Cancel a nonterminal run only when its claim is absent or expired. */
+  cancelStaleWorkflowRun(options: {
+    runId: string;
+    controlId?: string;
+    claimToken?: string;
+    now?: string;
+  }): boolean {
+    const now = epoch(validTimestamp(options.now));
+    const controlId = options.controlId ?? "workflow-control";
+    const claimToken = options.claimToken ?? randomUUID();
+    return this.state.transaction(() => {
+      const row = this.workflowRunRow(options.runId);
+      if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return false;
+      const lease = this.requireLease(row.resourceId);
+      if (lease.ownerId !== null && lease.expiresAt !== null && lease.expiresAt > now) return false;
+
+      const generation = lease.generation + 1;
+      const claimed = this.state.connection
+        .prepare(
+          `UPDATE leases
+           SET generation = ?, owner_type = 'system', owner_id = ?, token_hash = ?,
+               acquired_at = ?, heartbeat_at = ?, expires_at = ?
+           WHERE resource_id = ? AND generation = ?
+             AND (owner_id IS NULL OR expires_at IS NULL OR expires_at <= ?)`,
+        )
+        .run(
+          generation,
+          controlId,
+          tokenHash(claimToken),
+          now,
+          now,
+          now + 30_000,
+          row.resourceId,
+          lease.generation,
+          now,
+        );
+      if (claimed.changes !== 1) return false;
+
+      const errorHash = this.state.putText("Workflow run cancelled", now);
+      const queue = this.state.connection
+        .prepare(
+          `UPDATE run_queue
+           SET status = 'cancelled', error_code = 'cancelled', error_hash = ?,
+               updated_at = ?, finished_at = ?
+           WHERE run_id = ? AND status NOT IN ('done', 'failed', 'cancelled')`,
+        )
+        .run(errorHash, now, now, options.runId);
+      /* istanbul ignore if -- the control claim selects one nonterminal queue row */
+      if (queue.changes !== 1) {
+        throw new Error(`Workflow run ${options.runId} has inconsistent queue state`);
+      }
+      const run = this.state.connection
+        .prepare(
+          `UPDATE runs
+           SET status = 'cancelled', paused = 0, status_detail = NULL, error_hash = ?,
+               updated_at = ?, finished_at = ?
+           WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`,
+        )
+        .run(errorHash, now, now, options.runId);
+      /* istanbul ignore if -- the control claim selects one nonterminal durable run */
+      if (run.changes !== 1) {
+        throw new Error(`Workflow run ${options.runId} has inconsistent durable state`);
+      }
+      this.cancelWorkflowRunDependents(options.runId, controlId, errorHash, now);
+
+      const released = this.state.connection
+        .prepare(
+          `UPDATE leases
+           SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
+               acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
+           WHERE resource_id = ? AND owner_type = 'system' AND owner_id = ?
+             AND token_hash = ? AND generation = ? AND expires_at > ?`,
+        )
+        .run(row.resourceId, controlId, tokenHash(claimToken), generation, now);
+      /* istanbul ignore if -- the exact control claim was written in this transaction */
+      if (released.changes !== 1) {
+        throw new Error(`Workflow run ${options.runId} control claim changed during cancellation`);
+      }
+      const revision = this.resourceRevision(row.resourceId);
+      this.bumpResource(row.resourceId, revision, now);
+      this.insertEvent(
+        row.resourceId,
+        revision + 1,
+        "run.queue_cancelled",
+        "control",
+        controlId,
+        { status: "cancelled", code: "cancelled", staleControl: true },
+        now,
+        generation,
+      );
+      recordViewerDeltas(
+        this.state,
+        options.runId,
+        [{ targetType: "summary" }, { targetType: "replay" }],
+        now,
+      );
+      return true;
+    });
   }
 
   deleteWorkflowRun(options: { runId: string; claimToken: string }): boolean {
@@ -1445,34 +1889,36 @@ export class SqliteControllerStore implements ControllerStore {
     runnerId?: string;
     now?: string;
   }): RunEventRecord {
-    const row = this.requireWorkflowRunRow(options.runId);
-    const now = epoch(validTimestamp(options.now));
-    const revision = this.resourceRevision(row.resourceId) + 1;
-    this.bumpResource(row.resourceId, revision - 1, now);
-    const eventId = this.insertEvent(
-      row.resourceId,
-      revision,
-      options.type,
-      options.runnerId?.startsWith("host-") ? "host" : "session",
-      options.runnerId ?? null,
-      options.payload ?? {},
-      now,
-      row.leaseGeneration || undefined,
-    );
-    const seq = this.state.connection
-      .prepare("SELECT event_seq AS seq FROM events WHERE event_id = ?")
-      .get(eventId);
-    /* istanbul ignore if -- exact schema and internal query shape */
-    if (!isSequenceRow(seq)) throw new Error("Run event was not recorded");
-    return {
-      seq: seq.seq,
-      recordedAt: new Date(now).toISOString(),
-      runId: options.runId,
-      workflowRef: options.workflowRef,
-      type: options.type,
-      runnerId: options.runnerId ?? null,
-      payload: options.payload ?? {},
-    };
+    return this.state.transaction(() => {
+      const row = this.requireWorkflowRunRow(options.runId);
+      const now = epoch(validTimestamp(options.now));
+      const revision = this.resourceRevision(row.resourceId) + 1;
+      this.bumpResource(row.resourceId, revision - 1, now);
+      const eventId = this.insertEvent(
+        row.resourceId,
+        revision,
+        options.type,
+        options.runnerId?.startsWith("host-") ? "host" : "session",
+        options.runnerId ?? null,
+        options.payload ?? {},
+        now,
+        row.leaseGeneration || undefined,
+      );
+      const seq = this.state.connection
+        .prepare("SELECT event_seq AS seq FROM events WHERE event_id = ?")
+        .get(eventId);
+      /* istanbul ignore if -- exact schema and internal query shape */
+      if (!isSequenceRow(seq)) throw new Error("Run event was not recorded");
+      return {
+        seq: seq.seq,
+        recordedAt: new Date(now).toISOString(),
+        runId: options.runId,
+        workflowRef: options.workflowRef,
+        type: options.type,
+        runnerId: options.runnerId ?? null,
+        payload: options.payload ?? {},
+      };
+    });
   }
 
   listRunEventsAfter(seq: number, options: { limit?: number } = {}): RunEventRecord[] {
@@ -1794,6 +2240,20 @@ export class SqliteControllerStore implements ControllerStore {
     return this.requireNotification(options.runId, options.attemptId, options.notificationIndex);
   }
 
+  listPendingWorkflowNotifications(options: {
+    targetSessionId: string;
+    limit?: number;
+  }): WorkflowNotificationRecord[] {
+    const now = Date.now();
+    return this.notificationRows(options.targetSessionId, options.limit ?? 20)
+      .map((row) => this.mapNotification(row))
+      .filter(
+        (notification) =>
+          notification.deliveryClaimExpiresAt === null ||
+          Date.parse(notification.deliveryClaimExpiresAt) <= now,
+      );
+  }
+
   claimPendingWorkflowNotifications(options: {
     targetSessionId: string;
     claimToken: string;
@@ -2141,6 +2601,26 @@ export class SqliteControllerStore implements ControllerStore {
       )
       .run(now, resourceId, expectedRevision);
     if (result.changes !== 1) throw new Error("Resource revision conflict");
+    const run = this.state.connection
+      .prepare(
+        `SELECT r.run_id AS runId
+         FROM runs r JOIN viewer_runs v ON v.run_id = r.run_id
+         WHERE r.resource_id = ?`,
+      )
+      .get(resourceId);
+    if (isRecord(run) && typeof run.runId === "string") {
+      recordViewerDeltas(
+        this.state,
+        run.runId,
+        [
+          { targetType: "summary" },
+          { targetType: "graph" },
+          { targetType: "replay" },
+          { targetType: "inspector", targetKey: "run" },
+        ],
+        now,
+      );
+    }
   }
 
   private insertEvent(
@@ -2284,6 +2764,7 @@ export class SqliteControllerStore implements ControllerStore {
       canonicalJson(this.state.readJson(row.launchOptionsHash)) ===
         canonicalJson(options.launchOptions ?? {}) &&
       row.originSessionId === options.originSessionId &&
+      row.executionMode === (options.executionMode ?? "interactive") &&
       row.parentRunId === (options.parentRunId ?? null);
     if (!compatible) {
       throw new Error(`Workflow run preparation conflicts: ${options.runId}`);
@@ -2308,7 +2789,8 @@ export class SqliteControllerStore implements ControllerStore {
       claimExpiresAt:
         row.claimExpiresAt === null ? null : new Date(row.claimExpiresAt).toISOString(),
       affinityRunnerId: row.affinityRunnerId,
-      originSessionId: row.originSessionId,
+      originSessionId: row.executionMode === "headless" ? null : row.originSessionId,
+      executionMode: row.executionMode,
       parentRunId: row.parentRunId,
       errorCode: row.errorCode,
       errorMessage:
@@ -2328,9 +2810,10 @@ export class SqliteControllerStore implements ControllerStore {
     claimToken: string,
     leaseMs: number,
     now: number,
+    options: { allowPendingInteraction?: boolean; preserveQueueStatus?: boolean } = {},
   ): WorkflowRunQueueRecord | undefined {
     return this.state.transaction(() =>
-      this.claimRunInTransaction(runId, runnerId, claimToken, leaseMs, now),
+      this.claimRunInTransaction(runId, runnerId, claimToken, leaseMs, now, options),
     );
   }
 
@@ -2340,9 +2823,21 @@ export class SqliteControllerStore implements ControllerStore {
     claimToken: string,
     leaseMs: number,
     now: number,
+    options: { allowPendingInteraction?: boolean; preserveQueueStatus?: boolean } = {},
   ): WorkflowRunQueueRecord | undefined {
     const row = this.workflowRunRow(runId);
     if (row === undefined || ["done", "failed", "cancelled"].includes(row.status)) return undefined;
+    if (
+      options.allowPendingInteraction !== true &&
+      this.state.connection
+        .prepare(
+          `SELECT 1 FROM interactive_requests
+           WHERE run_id = ? AND status IN ('pending', 'presenting') LIMIT 1`,
+        )
+        .get(runId) !== undefined
+    ) {
+      return undefined;
+    }
     const lease = this.requireLease(row.resourceId);
     if (
       lease.ownerId !== null &&
@@ -2372,9 +2867,15 @@ export class SqliteControllerStore implements ControllerStore {
       );
     /* istanbul ignore if -- impossible after exact schema and transaction checks */
     if (result.changes !== 1) return undefined;
-    this.state.connection
-      .prepare("UPDATE run_queue SET status = 'starting', updated_at = ? WHERE run_id = ?")
-      .run(now, runId);
+    if (options.preserveQueueStatus !== true) {
+      this.state.connection
+        .prepare(
+          `UPDATE run_queue
+           SET status = 'starting', error_code = NULL, error_hash = NULL, updated_at = ?
+           WHERE run_id = ?`,
+        )
+        .run(now, runId);
+    }
     const revision = this.resourceRevision(row.resourceId);
     this.bumpResource(row.resourceId, revision, now);
     this.insertEvent(
@@ -2474,6 +2975,12 @@ export class SqliteControllerStore implements ControllerStore {
         now,
         lease.generation || undefined,
       );
+      recordViewerDeltas(
+        this.state,
+        runId,
+        [{ targetType: "summary" }, { targetType: "replay" }],
+        now,
+      );
       return true;
     });
   }
@@ -2536,6 +3043,14 @@ export class SqliteControllerStore implements ControllerStore {
            WHERE run_id = ?`,
         )
         .run(status, errorHash, now, now, runId);
+      if (status === "cancelled") {
+        this.cancelWorkflowRunDependents(
+          runId,
+          lease.ownerId ?? "workflow-control",
+          errorHash,
+          now,
+        );
+      }
       this.bumpResource(row.resourceId, revision, now);
       this.insertEvent(
         row.resourceId,
@@ -2547,10 +3062,110 @@ export class SqliteControllerStore implements ControllerStore {
         now,
         lease.generation || undefined,
       );
+      recordViewerDeltas(
+        this.state,
+        runId,
+        [{ targetType: "summary" }, { targetType: "replay" }],
+        now,
+      );
       return true;
     });
   }
 
+  private cancelWorkflowRunDependents(
+    runId: string,
+    actorId: string,
+    errorHash: Buffer,
+    now: number,
+  ): void {
+    this.state.connection
+      .prepare(
+        `UPDATE node_attempts
+         SET status = 'cancelled', error_hash = COALESCE(error_hash, ?),
+             updated_at = ?, finished_at = COALESCE(finished_at, ?)
+         WHERE run_id = ? AND status IN ('pending', 'running', 'waiting', 'interrupted')`,
+      )
+      .run(errorHash, now, now, runId);
+
+    const effects = this.state.connection
+      .prepare(
+        `SELECT e.effect_id AS effectId, e.resource_id AS resourceId,
+                e.status, e.attempt_count AS attemptCount
+         FROM effects e JOIN runs r ON r.resource_id = e.source_resource_id
+         WHERE r.run_id = ? AND e.owner_scope = 'run' AND e.status IN ('pending', 'applying')
+         ORDER BY e.effect_id`,
+      )
+      .all(runId)
+      .filter(isCancelledRunEffectRow);
+    for (const effect of effects) {
+      const status = effect.status === "applying" ? "ambiguous" : "cancelled";
+      const revision = this.resourceRevision(effect.resourceId);
+      const changed = this.state.connection
+        .prepare(
+          `UPDATE effects
+           SET status = ?, next_attempt_at = NULL, error_hash = ?, updated_at = ?, settled_at = ?
+           WHERE effect_id = ? AND status = ? AND attempt_count = ?`,
+        )
+        .run(status, errorHash, now, now, effect.effectId, effect.status, effect.attemptCount);
+      /* istanbul ignore if -- cancellation serializes the selected effect transition */
+      if (changed.changes !== 1) {
+        throw new Error(`Workflow effect ${effect.effectId} changed during cancellation`);
+      }
+      if (effect.status === "applying") {
+        this.state.connection
+          .prepare(
+            `UPDATE effect_attempts
+             SET finished_at = ?, outcome = 'interrupted', error_hash = ?
+             WHERE effect_id = ? AND attempt_number = ? AND finished_at IS NULL`,
+          )
+          .run(now, errorHash, effect.effectId, effect.attemptCount);
+      }
+      this.bumpResource(effect.resourceId, revision, now);
+      this.insertEvent(
+        effect.resourceId,
+        revision + 1,
+        `effect.${status}`,
+        "control",
+        actorId,
+        { runId, reason: "workflowCancelled" },
+        now,
+      );
+    }
+
+    this.state.connection
+      .prepare(
+        `INSERT INTO human_decision_resolutions(
+           decision_id, outcome, provenance, response_hash, reason, channel,
+           actor_id, request_digest, resolved_at
+         )
+         SELECT d.decision_id, 'cancelled', 'explicit_cancel', NULL,
+                'Workflow run cancelled', NULL, ?, d.request_digest, ?
+         FROM human_decisions d
+         LEFT JOIN human_decision_resolutions r ON r.decision_id = d.decision_id
+         WHERE d.run_id = ? AND r.decision_id IS NULL`,
+      )
+      .run(actorId, now, runId);
+    const receiptHash = this.state.putJson(
+      { status: "rejected", error: "Workflow run cancelled" },
+      now,
+    );
+    this.state.connection
+      .prepare(
+        `UPDATE interactive_submissions SET outcome = 'rejected', receipt_hash = ?
+         WHERE outcome = 'validating'
+           AND request_id IN (SELECT request_id FROM interactive_requests WHERE run_id = ?)`,
+      )
+      .run(receiptHash, runId);
+    this.state.connection
+      .prepare(
+        `UPDATE interactive_requests
+         SET status = 'cancelled', presenter_id = NULL,
+             presentation_claim_expires_at = NULL, revision = revision + 1, updated_at = ?
+         WHERE run_id = ? AND status IN ('pending', 'presenting')`,
+      )
+      .run(now, runId);
+  }
+
   private turnIntent(intentId: string): WorkflowTurnIntentRecord | undefined {
     const row = this.turnIntentRow(intentId);
     return row === undefined ? undefined : this.mapTurnIntent(row);
@@ -2843,7 +3458,8 @@ function workflowRunSelect(clause: string): string {
     q.status, q.available_at AS availableAt, q.affinity_runner_id AS affinityRunnerId,
     q.consecutive_errors AS consecutiveErrors, q.error_code AS errorCode,
     q.error_hash AS errorHash, b.origin_session_id AS originSessionId,
-    r.parent_run_id AS parentRunId, q.created_at AS createdAt, q.updated_at AS updatedAt,
+    b.execution_mode AS executionMode, r.parent_run_id AS parentRunId,
+    q.created_at AS createdAt, q.updated_at AS updatedAt,
     q.started_at AS startedAt, q.finished_at AS finishedAt,
     l.generation AS leaseGeneration, l.owner_id AS ownerId, l.expires_at AS claimExpiresAt
     FROM runs r JOIN workflow_definitions d ON d.definition_digest = r.definition_digest
@@ -2952,6 +3568,20 @@ function isRunSourceIdentityRow(value: unknown): value is RunSourceIdentityRow {
 function isEffectRow(value: unknown): value is EffectRow {
   return isRecord(value);
 }
+function isCancelledRunEffectRow(value: unknown): value is CancelledRunEffectRow {
+  return (
+    isRecord(value) &&
+    typeof value.effectId === "string" &&
+    typeof value.resourceId === "string" &&
+    (value.status === "pending" || value.status === "applying") &&
+    typeof value.attemptCount === "number"
+  );
+}
+function isExpiredInteractionRow(value: unknown): value is ExpiredInteractionRow {
+  return (
+    isRecord(value) && typeof value.requestId === "string" && typeof value.attemptId === "string"
+  );
+}
 function isWorkflowRow(value: unknown): value is WorkflowRow {
   return isRecord(value);
 }
@@ -2973,6 +3603,9 @@ function isNotificationRow(value: unknown): value is NotificationRow {
 function isProjectRow(value: unknown): value is { projectId: string } {
   return isRecord(value);
 }
+function isCanonicalPathRow(value: unknown): value is { canonicalPath: string } {
+  return isRecord(value) && typeof value.canonicalPath === "string";
+}
 function isFinalizerRow(value: unknown): value is { finalizer: string } {
   return isRecord(value);
 }