@osolmaz/pi-workflows 0.16.1 → 0.16.3

package/dist/{controllers → resource-managers}/sqlite.js

@@ -5,10 +5,12 @@ import { StateDatabase, workflowStatePath } from "../state/database.js";
 import { canonicalJson } from "../state/json.js";
 import { resourceIdFor, tokenHash } from "../state/mutation.js";
 import { initializeViewerRun, recordViewerDeltas } from "../state/viewer.js";
-import { EffectRequestConflictError, ResourceConflictError, ResourceNotFoundError, WorkflowRequestConflictError, } from "./errors.js";
-export class SqliteControllerStore {
+import { WorkflowMessageStore } from "../state/workflow-messages.js";
+import { EffectRequestConflictError, ManagedResourceConflictError, ManagedResourceNotFoundError, WorkflowRequestConflictError, } from "./errors.js";
+export class SqliteResourceManagerStore {
     filePath;
     state;
+    workflowMessages;
     ownsState;
     projectId;
     closed = false;
@@ -22,6 +24,7 @@ export class SqliteControllerStore {
                     checkLegacyState: filePath === workflowStatePath(),
                 });
         this.filePath = this.state.filePath;
+        this.workflowMessages = new WorkflowMessageStore(this.state);
         if (options.global === true) {
             this.projectId = null;
         }
@@ -40,27 +43,30 @@ export class SqliteControllerStore {
     }
     putResource(options) {
         this.assertWritable();
-        validateName(options.controller, "controller");
+        validateName(options.resourceManager, "resource manager");
         validateKey(options.key, "resource key");
         const now = epoch(validTimestamp(options.now));
         const specHash = this.state.putJson(options.spec, now);
         const initialStatus = {
             observedGeneration: 0,
             conditions: [],
-            controllerStatus: options.initialStatus,
+            resourceManagerStatus: options.initialStatus,
         };
         const statusHash = this.state.putJson(initialStatus, now);
         return this.state.transaction(() => {
-            const existing = this.controllerRow({ controller: options.controller, key: options.key });
+            const existing = this.resourceManagerRow({
+                resourceManager: options.resourceManager,
+                key: options.key,
+            });
             if (existing === undefined) {
                 const uid = randomUUID();
-                const controllerResourceId = `controller-resource-${randomUUID()}`;
-                const resourceId = resourceIdFor("controller", `${this.requireProjectId()}:${options.controller}:${options.key}`);
+                const managedResourceId = `controller-resource-${randomUUID()}`;
+                const resourceId = resourceIdFor("controller", `${this.requireProjectId()}:${options.resourceManager}:${options.key}`);
                 this.state.connection
                     .prepare(`INSERT INTO resources(
                resource_id, resource_type, aggregate_key, revision, created_at, updated_at
              ) VALUES (?, 'controller', ?, 1, ?, ?)`)
-                    .run(resourceId, controllerResourceId, now, now);
+                    .run(resourceId, managedResourceId, now, now);
                 this.state.connection
                     .prepare("INSERT INTO leases(resource_id, generation) VALUES (?, 0)")
                     .run(resourceId);
@@ -69,8 +75,8 @@ export class SqliteControllerStore {
                controller_resource_id, resource_id, project_id, controller_name,
                resource_key, uid, generation, spec_hash, status_hash, created_at, updated_at
              ) VALUES (?, ?, ?, ?, ?, ?, 1, ?, ?, ?, ?)`)
-                    .run(controllerResourceId, resourceId, this.requireProjectId(), options.controller, options.key, uid, specHash, statusHash, now, now);
-                this.enqueueControllerRow(controllerResourceId, now, now);
+                    .run(managedResourceId, resourceId, this.requireProjectId(), options.resourceManager, options.key, uid, specHash, statusHash, now, now);
+                this.enqueueResourceManagerRow(managedResourceId, now, now);
                 this.insertEvent(resourceId, 1, "resource.created", "control", null, { generation: 1 }, now);
             }
             else if (!existing.specHash.equals(specHash)) {
@@ -79,26 +85,26 @@ export class SqliteControllerStore {
                     .prepare(`UPDATE controller_resources
              SET spec_hash = ?, generation = generation + 1, updated_at = ?
              WHERE controller_resource_id = ?`)
-                    .run(specHash, now, existing.controllerResourceId);
+                    .run(specHash, now, existing.managedResourceId);
                 this.bumpResource(existing.resourceId, existing.resourceVersion, now);
-                this.enqueueControllerRow(existing.controllerResourceId, now, now);
+                this.enqueueResourceManagerRow(existing.managedResourceId, now, now);
                 this.insertEvent(existing.resourceId, revision, "resource.spec_updated", "control", null, { generation: existing.generation + 1 }, now);
             }
             return this.requireResource({
-                controller: options.controller,
+                resourceManager: options.resourceManager,
                 key: options.key,
             });
         });
     }
     getResource(ref) {
-        const row = this.controllerRow(ref);
+        const row = this.resourceManagerRow(ref);
         return row === undefined
             ? undefined
-            : this.mapControllerResource(row);
+            : this.mapManagedResource(row);
     }
     getResourceByUid(uid) {
-        const row = this.state.connection.prepare(controllerSelect("WHERE c.uid = ?")).get(uid);
-        return isControllerRow(row) ? this.mapControllerResource(row) : undefined;
+        const row = this.state.connection.prepare(resourceManagerSelect("WHERE c.uid = ?")).get(uid);
+        return isResourceManagerRow(row) ? this.mapManagedResource(row) : undefined;
     }
     listResources(options = {}) {
         const clauses = ["1 = 1"];
@@ -107,31 +113,31 @@ export class SqliteControllerStore {
             clauses.push("c.project_id = ?");
             params.push(this.projectId);
         }
-        if (options.controller !== undefined) {
+        if (options.resourceManager !== undefined) {
             clauses.push("c.controller_name = ?");
-            params.push(options.controller);
+            params.push(options.resourceManager);
         }
         const rows = this.state.connection
-            .prepare(controllerSelect(`WHERE ${clauses.join(" AND ")} ORDER BY c.controller_name, c.resource_key`))
+            .prepare(resourceManagerSelect(`WHERE ${clauses.join(" AND ")} ORDER BY c.controller_name, c.resource_key`))
             .all(...params);
         return rows
-            .filter(isControllerRow)
-            .map((row) => this.mapControllerResource(row));
+            .filter(isResourceManagerRow)
+            .map((row) => this.mapManagedResource(row));
     }
     updateStatus(options) {
         const now = epoch(validTimestamp(options.now));
         return this.state.transaction(() => {
-            const row = this.requireControllerRow(options.ref);
-            this.assertControllerClaim(row, options.claim, now);
+            const row = this.requireResourceManagerRow(options.ref);
+            this.assertResourceManagerClaim(row, options.claim, now);
             if (row.resourceVersion !== options.expectedResourceVersion) {
-                throw new ResourceConflictError(options.ref.controller, options.ref.key);
+                throw new ManagedResourceConflictError(options.ref.resourceManager, options.ref.key);
             }
             const statusHash = this.state.putJson(options.status, now);
             this.state.connection
                 .prepare("UPDATE controller_resources SET status_hash = ?, updated_at = ? WHERE controller_resource_id = ?")
-                .run(statusHash, now, row.controllerResourceId);
+                .run(statusHash, now, row.managedResourceId);
             if (options.finalizers !== undefined)
-                this.replaceFinalizers(row.controllerResourceId, options.finalizers);
+                this.replaceFinalizers(row.managedResourceId, options.finalizers);
             this.bumpResource(row.resourceId, row.resourceVersion, now);
             this.insertEvent(row.resourceId, row.resourceVersion + 1, "resource.status_updated", "controller", options.claim.ownerId, {}, now, options.claim.generation);
             options.claim.resourceVersion = row.resourceVersion + 1;
@@ -141,13 +147,13 @@ export class SqliteControllerStore {
     requestDeletion(ref, nowValue) {
         const now = epoch(validTimestamp(nowValue));
         return this.state.transaction(() => {
-            const row = this.requireControllerRow(ref);
+            const row = this.requireResourceManagerRow(ref);
             if (row.deletionRequestedAt === null) {
                 this.state.connection
                     .prepare("UPDATE controller_resources SET deletion_requested_at = ?, updated_at = ? WHERE controller_resource_id = ?")
-                    .run(now, now, row.controllerResourceId);
+                    .run(now, now, row.managedResourceId);
                 this.bumpResource(row.resourceId, row.resourceVersion, now);
-                this.enqueueControllerRow(row.controllerResourceId, now, now);
+                this.enqueueResourceManagerRow(row.managedResourceId, now, now);
                 this.insertEvent(row.resourceId, row.resourceVersion + 1, "resource.deletion_requested", "control", null, {}, now);
             }
             return this.requireResource(ref);
@@ -156,14 +162,14 @@ export class SqliteControllerStore {
     updateFinalizers(options) {
         const now = epoch(validTimestamp(options.now));
         return this.state.transaction(() => {
-            const row = this.requireControllerRow(options.ref);
+            const row = this.requireResourceManagerRow(options.ref);
             if (row.resourceVersion !== options.expectedResourceVersion) {
-                throw new ResourceConflictError(options.ref.controller, options.ref.key);
+                throw new ManagedResourceConflictError(options.ref.resourceManager, options.ref.key);
             }
-            this.replaceFinalizers(row.controllerResourceId, options.finalizers);
+            this.replaceFinalizers(row.managedResourceId, options.finalizers);
             this.state.connection
                 .prepare("UPDATE controller_resources SET updated_at = ? WHERE controller_resource_id = ?")
-                .run(now, row.controllerResourceId);
+                .run(now, row.managedResourceId);
             this.bumpResource(row.resourceId, row.resourceVersion, now);
             this.insertEvent(row.resourceId, row.resourceVersion + 1, "resource.finalizers_updated", "control", null, {}, now);
             return this.requireResource(options.ref);
@@ -171,37 +177,37 @@ export class SqliteControllerStore {
     }
     deleteResource(ref, expectedResourceVersion, claim) {
         return this.state.transaction(() => {
-            const row = this.requireControllerRow(ref);
-            this.assertControllerClaim(row, claim, Date.now());
+            const row = this.requireResourceManagerRow(ref);
+            this.assertResourceManagerClaim(row, claim, Date.now());
             if (row.resourceVersion !== expectedResourceVersion) {
-                throw new ResourceConflictError(ref.controller, ref.key);
+                throw new ManagedResourceConflictError(ref.resourceManager, ref.key);
             }
-            if (row.deletionRequestedAt === null || this.finalizers(row.controllerResourceId).length > 0)
+            if (row.deletionRequestedAt === null || this.finalizers(row.managedResourceId).length > 0)
                 return false;
             this.state.connection
                 .prepare("DELETE FROM controller_resources WHERE controller_resource_id = ?")
-                .run(row.controllerResourceId);
+                .run(row.managedResourceId);
             return true;
         });
     }
     enqueue(ref, availableAt) {
-        const row = this.requireControllerRow(ref);
+        const row = this.requireResourceManagerRow(ref);
         const now = Date.now();
-        this.enqueueControllerRow(row.controllerResourceId, epoch(validTimestamp(availableAt)), now);
+        this.enqueueResourceManagerRow(row.managedResourceId, epoch(validTimestamp(availableAt)), now);
     }
     claimNext(options) {
-        if (options.controllers.length === 0)
+        if (options.resourceManagers.length === 0)
             return undefined;
         const now = epoch(validTimestamp(options.now));
-        const placeholders = options.controllers.map(() => "?").join(", ");
+        const placeholders = options.resourceManagers.map(() => "?").join(", ");
         const exclusions = options.exclude ?? [];
         const exclusionSql = exclusions
             .map(() => "AND NOT (c.controller_name = ? AND c.resource_key = ?)")
             .join("\n               ");
-        const exclusionParams = exclusions.flatMap((ref) => [ref.controller, ref.key]);
+        const exclusionParams = exclusions.flatMap((ref) => [ref.resourceManager, ref.key]);
         return this.state.transaction(() => {
             const row = this.state.connection
-                .prepare(`${controllerSelect(`
+                .prepare(`${resourceManagerSelect(`
              JOIN controller_queue q ON q.controller_resource_id = c.controller_resource_id
              JOIN leases l ON l.resource_id = c.resource_id
              WHERE c.project_id = ?
@@ -211,8 +217,8 @@ export class SqliteControllerStore {
                AND (l.owner_id IS NULL OR l.expires_at <= ?)
              ORDER BY q.available_at, c.controller_name, c.resource_key
              LIMIT 1`)}`)
-                .get(this.requireProjectId(), ...options.controllers, ...exclusionParams, now, now);
-            if (!isControllerRow(row))
+                .get(this.requireProjectId(), ...options.resourceManagers, ...exclusionParams, now, now);
+            if (!isResourceManagerRow(row))
                 return undefined;
             const lease = this.requireLease(row.resourceId);
             const token = randomBytes(32).toString("base64url");
@@ -231,14 +237,14 @@ export class SqliteControllerStore {
             this.bumpResource(row.resourceId, row.resourceVersion, now);
             this.insertEvent(row.resourceId, row.resourceVersion + 1, "lease.claimed", "controller", options.ownerId, { expiresAt }, now, generation);
             return {
-                controller: row.controllerName,
+                resourceManager: row.resourceManagerName,
                 key: row.resourceKey,
                 ownerId: options.ownerId,
                 token,
                 generation,
-                queueVersion: this.queueVersion(row.controllerResourceId),
+                queueVersion: this.queueVersion(row.managedResourceId),
                 resourceVersion: row.resourceVersion + 1,
-                consecutiveErrors: this.queueErrors(row.controllerResourceId),
+                consecutiveErrors: this.queueErrors(row.managedResourceId),
                 expiresAt: new Date(expiresAt).toISOString(),
             };
         });
@@ -246,7 +252,7 @@ export class SqliteControllerStore {
     renewClaim(claim, leaseMs, nowValue) {
         const now = epoch(validTimestamp(nowValue));
         const expiresAt = now + leaseMs;
-        const row = this.controllerRow({ controller: claim.controller, key: claim.key });
+        const row = this.resourceManagerRow({ resourceManager: claim.resourceManager, key: claim.key });
         if (row === undefined)
             return false;
         const result = this.state.connection
@@ -259,14 +265,14 @@ export class SqliteControllerStore {
         return result.changes === 1;
     }
     settleClaim(claim, nowValue) {
-        return this.settleControllerClaim(claim, undefined, nowValue);
+        return this.settleResourceManagerClaim(claim, undefined, nowValue);
     }
     requeueClaim(claim, options, nowValue) {
-        return this.settleControllerClaim(claim, options, nowValue);
+        return this.settleResourceManagerClaim(claim, options, nowValue);
     }
     listQueue() {
         const rows = this.state.connection
-            .prepare(`SELECT c.controller_name AS controller, c.resource_key AS resourceKey,
+            .prepare(`SELECT c.controller_name AS resourceManager, c.resource_key AS resourceKey,
                 q.available_at AS availableAt, q.consecutive_errors AS consecutiveErrors,
                 l.expires_at AS claimExpiresAt
          FROM controller_queue q
@@ -275,7 +281,7 @@ export class SqliteControllerStore {
          WHERE c.project_id = ? ORDER BY q.available_at, c.controller_name, c.resource_key`)
             .all(this.requireProjectId());
         return rows.filter(isQueueListRow).map((row) => ({
-            controller: row.controller,
+            resourceManager: row.resourceManager,
             key: row.resourceKey,
             availableAt: new Date(row.availableAt).toISOString(),
             consecutiveErrors: row.consecutiveErrors,
@@ -287,8 +293,8 @@ export class SqliteControllerStore {
     reserveEffect(options) {
         const now = epoch(validTimestamp(options.now));
         return this.state.transaction(() => {
-            const source = this.requireControllerRowByUid(options.resourceUid);
-            this.assertControllerClaim(source, options.claim, now);
+            const source = this.requireResourceManagerRowByUid(options.resourceUid);
+            this.assertResourceManagerClaim(source, options.claim, now);
             const effectId = effectIdFor(source.resourceId, options.key);
             const existing = this.effectRow(options.resourceUid, options.key);
             if (existing !== undefined) {
@@ -335,8 +341,8 @@ export class SqliteControllerStore {
     updateEffect(options) {
         const now = epoch(validTimestamp(options.now));
         return this.state.transaction(() => {
-            const source = this.requireControllerRowByUid(options.resourceUid);
-            this.assertControllerClaim(source, options.claim, now);
+            const source = this.requireResourceManagerRowByUid(options.resourceUid);
+            this.assertResourceManagerClaim(source, options.claim, now);
             const row = this.requireEffectRow(options.resourceUid, options.key);
             const effectResourceId = resourceIdFor("effect", row.effectId);
             const revision = this.resourceRevision(effectResourceId);
@@ -360,8 +366,8 @@ export class SqliteControllerStore {
     reserveWorkflow(options) {
         const now = Date.now();
         return this.state.transaction(() => {
-            const source = this.requireControllerRowByUid(options.resourceUid);
-            this.assertControllerClaim(source, options.claim, now);
+            const source = this.requireResourceManagerRowByUid(options.resourceUid);
+            this.assertResourceManagerClaim(source, options.claim, now);
             const existing = this.workflowRow(options.resourceUid, options.requestKey);
             if (existing !== undefined) {
                 if (existing.workflowName !== options.workflow ||
@@ -376,7 +382,7 @@ export class SqliteControllerStore {
              request_id, controller_resource_id, request_key, workflow_name,
              input_fingerprint, status, attempt_count, created_at, updated_at
            ) VALUES (?, ?, ?, ?, ?, 'pending', 0, ?, ?)`)
-                .run(requestId, source.controllerResourceId, options.requestKey, options.workflow, Buffer.from(options.inputFingerprint, "hex"), now, now);
+                .run(requestId, source.managedResourceId, options.requestKey, options.workflow, Buffer.from(options.inputFingerprint, "hex"), now, now);
             return { record: this.mapWorkflow(this.requireWorkflowRow(requestId)), created: true };
         });
     }
@@ -392,8 +398,8 @@ export class SqliteControllerStore {
         const now = Date.now();
         return this.state.transaction(() => {
             const row = this.requireWorkflowRow(requestId);
-            const source = this.requireControllerRowByUid(row.resourceUid);
-            this.assertControllerClaim(source, claim, now);
+            const source = this.requireResourceManagerRowByUid(row.resourceUid);
+            this.assertResourceManagerClaim(source, claim, now);
             const errorHash = update.error === undefined || update.error === null
                 ? null
                 : this.state.putText(update.error, now);
@@ -419,7 +425,7 @@ export class SqliteControllerStore {
         if (update.runId !== undefined && row.runId !== null && update.runId !== row.runId) {
             throw new Error("Scheduler completion run ID does not match the reserved child run");
         }
-        const source = this.requireControllerRowByUid(row.resourceUid);
+        const source = this.requireResourceManagerRowByUid(row.resourceUid);
         const now = Date.now();
         return this.state.transaction(() => {
             const errorHash = update.error === undefined || update.error === null
@@ -450,9 +456,12 @@ export class SqliteControllerStore {
     recordEvent(options) {
         const now = epoch(validTimestamp(options.now));
         return this.state.transaction(() => {
-            const row = this.requireControllerRow({ controller: options.controller, key: options.key });
+            const row = this.requireResourceManagerRow({
+                resourceManager: options.resourceManager,
+                key: options.key,
+            });
             if (options.claim !== undefined)
-                this.assertControllerClaim(row, options.claim, now);
+                this.assertResourceManagerClaim(row, options.claim, now);
             const revision = this.resourceRevision(row.resourceId) + 1;
             this.bumpResource(row.resourceId, revision - 1, now);
             const eventId = this.insertEvent(row.resourceId, revision, options.type, options.claim === undefined ? "control" : "controller", options.claim?.ownerId ?? null, options.payload ?? {}, now, options.claim?.generation);
@@ -463,11 +472,11 @@ export class SqliteControllerStore {
                 .get(eventId);
             /* istanbul ignore if -- exact schema and internal query shape */
             if (!isSequenceRow(event))
-                throw new Error("Controller event was not recorded");
+                throw new Error("ResourceManager event was not recorded");
             return {
                 seq: event.seq,
                 recordedAt: new Date(now).toISOString(),
-                controller: options.controller,
+                resourceManager: options.resourceManager,
                 key: options.key,
                 type: options.type,
                 payload: options.payload ?? {},
@@ -481,9 +490,9 @@ export class SqliteControllerStore {
             clauses.push("c.project_id = ?");
             params.push(this.projectId);
         }
-        if (options.controller !== undefined) {
+        if (options.resourceManager !== undefined) {
             clauses.push("c.controller_name = ?");
-            params.push(options.controller);
+            params.push(options.resourceManager);
         }
         if (options.key !== undefined) {
             clauses.push("c.resource_key = ?");
@@ -492,7 +501,7 @@ export class SqliteControllerStore {
         params.push(options.limit ?? 100);
         const rows = this.state.connection
             .prepare(`SELECT e.event_seq AS seq, e.recorded_at AS recordedAt,
-                c.controller_name AS controller, c.resource_key AS resourceKey,
+                c.controller_name AS resourceManager, c.resource_key AS resourceKey,
                 e.event_type AS eventType, e.payload_hash AS payloadHash
          FROM events e
          JOIN resources r ON r.resource_id = e.resource_id
@@ -500,10 +509,10 @@ export class SqliteControllerStore {
          WHERE ${clauses.join(" AND ")}
          ORDER BY e.event_seq DESC LIMIT ?`)
             .all(...params);
-        return rows.filter(isControllerEventRow).map((row) => ({
+        return rows.filter(isResourceManagerEventRow).map((row) => ({
             seq: row.seq,
             recordedAt: new Date(row.recordedAt).toISOString(),
-            controller: row.controller,
+            resourceManager: row.resourceManager,
             key: row.resourceKey,
             type: row.eventType,
             payload: row.payloadHash === null ? {} : this.state.readJson(row.payloadHash),
@@ -747,14 +756,14 @@ export class SqliteControllerStore {
         const now = epoch(validTimestamp(options.now));
         return this.claimRun(options.runId, options.runnerId, options.claimToken, options.leaseMs, now);
     }
-    /** Schedule a worker that validates one pending interactive submission. */
+    /** Schedule a runner that validates one pending interactive submission. */
     claimWorkflowRunForInteractionValidation(options) {
         const now = epoch(validTimestamp(options.now));
         return this.claimRun(options.runId, options.runnerId, options.claimToken, options.leaseMs, now, {
             allowPendingInteraction: true,
         });
     }
-    /** Take a short host claim without scheduling a parked interactive run. */
+    /** Take a short server claim without scheduling a parked interactive run. */
     claimWorkflowRunForControl(options) {
         const now = epoch(validTimestamp(options.now));
         return this.claimRun(options.runId, options.runnerId, options.claimToken, options.leaseMs, now, { allowPendingInteraction: true, preserveQueueStatus: true });
@@ -833,7 +842,7 @@ export class SqliteControllerStore {
             "(q.affinity_runner_id IS NULL OR q.affinity_runner_id = ? OR q.status IN ('parked', 'starting', 'running'))",
             "NOT (q.status = 'parked' AND (r.status = 'waiting' OR r.paused = 1))",
             "NOT EXISTS (SELECT 1 FROM interactive_requests i WHERE i.run_id = r.run_id AND i.status = 'pending')",
-            "(q.error_code IS NULL OR q.error_code <> 'workflowSourceChanged')",
+            "(q.error_code IS NULL OR q.error_code NOT IN ('workflowSourceChanged', 'runnerNoProgress'))",
         ];
         const params = [now, now, options.runnerId, options.runnerId];
         if (this.projectId !== null) {
@@ -907,6 +916,50 @@ export class SqliteControllerStore {
     parkWorkflowRun(options) {
         return this.releaseRunClaim(options.runId, options.claimToken, "parked", options.now);
     }
+    parkWorkflowRunForRunnerNoProgress(options) {
+        const now = epoch(validTimestamp(options.now));
+        return this.state.transaction(() => {
+            if (!this.verifyWorkflowRunClaim({
+                runId: options.runId,
+                claimToken: options.claimToken,
+                now: new Date(now).toISOString(),
+            })) {
+                return false;
+            }
+            const row = this.requireWorkflowRunRow(options.runId);
+            const lease = this.requireLease(row.resourceId);
+            const detail = options.detail.slice(0, 8_192);
+            const errorHash = this.state.putText(detail, now);
+            const run = this.state.connection
+                .prepare(`UPDATE runs SET status_detail = ?, updated_at = ?, finished_at = NULL
+           WHERE run_id = ? AND status NOT IN ('completed', 'failed', 'timed_out', 'cancelled')`)
+                .run(detail, now, options.runId);
+            const queue = this.state.connection
+                .prepare(`UPDATE run_queue
+           SET status = 'parked', error_code = 'runnerNoProgress', error_hash = ?,
+               available_at = ?, updated_at = ?, finished_at = NULL
+           WHERE run_id = ? AND status IN ('queued', 'starting', 'running', 'parked')`)
+                .run(errorHash, now, now, options.runId);
+            if (run.changes !== 1 || queue.changes !== 1) {
+                throw new Error(`Workflow run ${options.runId} changed during runner recovery`);
+            }
+            const released = this.state.connection
+                .prepare(`UPDATE leases
+           SET owner_type = NULL, owner_id = NULL, token_hash = NULL,
+               acquired_at = NULL, heartbeat_at = NULL, expires_at = NULL
+           WHERE resource_id = ? AND token_hash = ? AND generation = ? AND expires_at > ?`)
+                .run(row.resourceId, tokenHash(options.claimToken), lease.generation, now);
+            /* istanbul ignore if -- the exact live claim is stable in this transaction */
+            if (released.changes !== 1) {
+                throw new Error(`Workflow run ${options.runId} claim changed during runner recovery`);
+            }
+            const revision = this.resourceRevision(row.resourceId);
+            this.bumpResource(row.resourceId, revision, now);
+            this.insertEvent(row.resourceId, revision + 1, "run.worker_no_progress", lease.ownerType ?? "system", lease.ownerId, { status: "parked", code: "runnerNoProgress" }, now, lease.generation || undefined);
+            recordViewerDeltas(this.state, options.runId, [{ targetType: "summary" }, { targetType: "replay" }], now);
+            return true;
+        });
+    }
     pauseParkedWorkflowRun(options) {
         const now = epoch(validTimestamp(options.now));
         return this.state.transaction(() => {
@@ -1125,6 +1178,7 @@ export class SqliteControllerStore {
             if (run.changes !== 1) {
                 throw new Error(`Workflow run ${options.runId} has inconsistent durable state`);
             }
+            this.settleWorkflowRunMessages(options.runId, now);
             this.cancelWorkflowRunDependents(options.runId, controlId, errorHash, now);
             const released = this.state.connection
                 .prepare(`UPDATE leases
@@ -1261,73 +1315,73 @@ export class SqliteControllerStore {
     }
     requireProjectId() {
         if (this.projectId === null)
-            throw new Error("Controller project is not registered in the state database");
+            throw new Error("ResourceManager project is not registered in the state database");
         return this.projectId;
     }
-    controllerRow(ref) {
+    resourceManagerRow(ref) {
         if (this.projectId === null)
             return undefined;
         const row = this.state.connection
-            .prepare(controllerSelect("WHERE c.project_id = ? AND c.controller_name = ? AND c.resource_key = ?"))
-            .get(this.projectId, ref.controller, ref.key);
-        return isControllerRow(row) ? row : undefined;
+            .prepare(resourceManagerSelect("WHERE c.project_id = ? AND c.controller_name = ? AND c.resource_key = ?"))
+            .get(this.projectId, ref.resourceManager, ref.key);
+        return isResourceManagerRow(row) ? row : undefined;
     }
-    requireControllerRow(ref) {
-        const row = this.controllerRow(ref);
+    requireResourceManagerRow(ref) {
+        const row = this.resourceManagerRow(ref);
         if (row === undefined)
-            throw new ResourceNotFoundError(ref.controller, ref.key);
+            throw new ManagedResourceNotFoundError(ref.resourceManager, ref.key);
         return row;
     }
-    requireControllerRowByUid(uid) {
-        const row = this.state.connection.prepare(controllerSelect("WHERE c.uid = ?")).get(uid);
+    requireResourceManagerRowByUid(uid) {
+        const row = this.state.connection.prepare(resourceManagerSelect("WHERE c.uid = ?")).get(uid);
         /* istanbul ignore if -- impossible after exact schema and transaction checks */
-        if (!isControllerRow(row))
-            throw new Error(`Controller resource not found: ${uid}`);
+        if (!isResourceManagerRow(row))
+            throw new Error(`Managed resource not found: ${uid}`);
         return row;
     }
     requireResource(ref) {
         const resource = this.getResource(ref);
         if (resource === undefined)
-            throw new ResourceNotFoundError(ref.controller, ref.key);
+            throw new ManagedResourceNotFoundError(ref.resourceManager, ref.key);
         return resource;
     }
     /* istanbul ignore next -- pure projection covered by integration tests */
-    mapControllerResource(row) {
+    mapManagedResource(row) {
         const spec = this.state.readJson(row.specHash);
         const status = this.state.readJson(row.statusHash);
         return {
             metadata: {
                 uid: row.uid,
-                controller: row.controllerName,
+                resourceManager: row.resourceManagerName,
                 key: row.resourceKey,
                 resourceVersion: row.resourceVersion,
                 generation: row.generation,
                 ...(row.deletionRequestedAt === null
                     ? {}
                     : { deletionTimestamp: new Date(row.deletionRequestedAt).toISOString() }),
-                finalizers: this.finalizers(row.controllerResourceId),
+                finalizers: this.finalizers(row.managedResourceId),
             },
             spec,
             status,
         };
     }
-    finalizers(controllerResourceId) {
+    finalizers(managedResourceId) {
         const rows = this.state.connection
             .prepare("SELECT finalizer FROM controller_finalizers WHERE controller_resource_id = ? ORDER BY position")
-            .all(controllerResourceId);
+            .all(managedResourceId);
         return rows.flatMap((row) => (isFinalizerRow(row) ? [row.finalizer] : []));
     }
-    replaceFinalizers(controllerResourceId, finalizers) {
+    replaceFinalizers(managedResourceId, finalizers) {
         const unique = [...new Set(finalizers)];
         if (unique.length !== finalizers.length)
-            throw new Error("Controller finalizers must be unique");
+            throw new Error("ResourceManager finalizers must be unique");
         this.state.connection
             .prepare("DELETE FROM controller_finalizers WHERE controller_resource_id = ?")
-            .run(controllerResourceId);
+            .run(managedResourceId);
         const insert = this.state.connection.prepare("INSERT INTO controller_finalizers(controller_resource_id, finalizer, position) VALUES (?, ?, ?)");
-        unique.forEach((finalizer, index) => insert.run(controllerResourceId, finalizer, index));
+        unique.forEach((finalizer, index) => insert.run(managedResourceId, finalizer, index));
     }
-    enqueueControllerRow(controllerResourceId, availableAt, now) {
+    enqueueResourceManagerRow(managedResourceId, availableAt, now) {
         this.state.connection
             .prepare(`INSERT INTO controller_queue(
            controller_resource_id, available_at, queue_version,
@@ -1337,42 +1391,45 @@ export class SqliteControllerStore {
            available_at = MIN(controller_queue.available_at, excluded.available_at),
            queue_version = controller_queue.queue_version + 1,
            updated_at = excluded.updated_at`)
-            .run(controllerResourceId, availableAt, now, now);
+            .run(managedResourceId, availableAt, now, now);
     }
-    queueErrors(controllerResourceId) {
+    queueErrors(managedResourceId) {
         const row = this.state.connection
             .prepare("SELECT consecutive_errors AS consecutiveErrors FROM controller_queue WHERE controller_resource_id = ?")
-            .get(controllerResourceId);
+            .get(managedResourceId);
         return isErrorCountRow(row) ? row.consecutiveErrors : 0;
     }
-    queueVersion(controllerResourceId) {
+    queueVersion(managedResourceId) {
         const row = this.state.connection
             .prepare("SELECT queue_version AS queueVersion FROM controller_queue WHERE controller_resource_id = ?")
-            .get(controllerResourceId);
+            .get(managedResourceId);
         /* istanbul ignore if -- exact schema and internal query shape */
         /* istanbul ignore if -- impossible after exact schema and transaction checks */
         if (!isQueueVersionRow(row))
-            throw new Error("Controller queue item is missing");
+            throw new Error("ResourceManager queue item is missing");
         return row.queueVersion;
     }
-    settleControllerClaim(claim, requeue, nowValue) {
+    settleResourceManagerClaim(claim, requeue, nowValue) {
         const now = epoch(validTimestamp(nowValue));
         return this.state.transaction(() => {
-            const row = this.controllerRow({ controller: claim.controller, key: claim.key });
+            const row = this.resourceManagerRow({
+                resourceManager: claim.resourceManager,
+                key: claim.key,
+            });
             if (row === undefined)
                 return false;
             try {
-                this.assertControllerClaim(row, claim, now, false);
+                this.assertResourceManagerClaim(row, claim, now, false);
             }
             catch {
                 return false;
             }
-            const currentQueueVersion = this.queueVersion(row.controllerResourceId);
+            const currentQueueVersion = this.queueVersion(row.managedResourceId);
             if (requeue === undefined) {
                 if (currentQueueVersion === claim.queueVersion) {
                     this.state.connection
                         .prepare("DELETE FROM controller_queue WHERE controller_resource_id = ?")
-                        .run(row.controllerResourceId);
+                        .run(row.managedResourceId);
                 }
             }
             else {
@@ -1382,15 +1439,15 @@ export class SqliteControllerStore {
              SET available_at = ?, queue_version = queue_version + 1,
                  consecutive_errors = consecutive_errors + 1,
                  last_error_hash = ?, updated_at = ? WHERE controller_resource_id = ?`)
-                    .run(epoch(requeue.availableAt), errorHash, now, row.controllerResourceId);
+                    .run(epoch(requeue.availableAt), errorHash, now, row.managedResourceId);
             }
             this.releaseLease(row.resourceId, claim, now);
             return true;
         });
     }
-    assertControllerClaim(row, claim, now, requireUnexpired = true) {
-        if (row.controllerName !== claim.controller || row.resourceKey !== claim.key)
-            throw new Error("Controller claim targets another resource");
+    assertResourceManagerClaim(row, claim, now, requireUnexpired = true) {
+        if (row.resourceManagerName !== claim.resourceManager || row.resourceKey !== claim.key)
+            throw new Error("ResourceManager claim targets another resource");
         const lease = this.requireLease(row.resourceId);
         if (lease.ownerType !== "controller" ||
             lease.ownerId !== claim.ownerId ||
@@ -1398,7 +1455,7 @@ export class SqliteControllerStore {
             lease.tokenHash === null ||
             !lease.tokenHash.equals(tokenHash(claim.token)) ||
             (requireUnexpired && (lease.expiresAt === null || lease.expiresAt <= now))) {
-            throw new Error("Controller claim is stale");
+            throw new Error("ResourceManager claim is stale");
         }
     }
     requireLease(resourceId) {
@@ -1414,7 +1471,7 @@ export class SqliteControllerStore {
         return row;
     }
     releaseLease(resourceId, claim, now) {
-        const row = this.controllerRow({ controller: claim.controller, key: claim.key });
+        const row = this.resourceManagerRow({ resourceManager: claim.resourceManager, key: claim.key });
         if (row === undefined)
             return;
         const revision = this.resourceRevision(resourceId);
@@ -1489,7 +1546,7 @@ export class SqliteControllerStore {
             resourceUid: row.resourceUid,
             generation: row.generation,
             kind: row.kind,
-            state: controllerEffectState(row.state),
+            state: resourceManagerEffectState(row.state),
             requestFingerprint: row.requestFingerprint,
             startedAt: new Date(row.startedAt).toISOString(),
             ...(row.completedAt === null ? {} : { completedAt: new Date(row.completedAt).toISOString() }),
@@ -1769,6 +1826,7 @@ export class SqliteControllerStore {
                updated_at = ?, finished_at = ?
            WHERE run_id = ?`)
                 .run(status, errorHash, now, now, runId);
+            this.settleWorkflowRunMessages(runId, now);
             if (status === "cancelled") {
                 this.cancelWorkflowRunDependents(runId, lease.ownerId ?? "workflow-control", errorHash, now);
             }
@@ -1778,6 +1836,10 @@ export class SqliteControllerStore {
             return true;
         });
     }
+    settleWorkflowRunMessages(runId, now) {
+        this.workflowMessages.settleOpenTurnsForRun(runId, "lost", now);
+        this.workflowMessages.cancelPendingForRun(runId, now);
+    }
     cancelWorkflowRunDependents(runId, actorId, errorHash, now) {
         this.state.connection
             .prepare(`UPDATE node_attempts
@@ -1840,13 +1902,13 @@ export class SqliteControllerStore {
     }
     assertWritable() {
         if (this.state.mode === "read-only")
-            throw new Error("Controller store is read-only");
+            throw new Error("ResourceManager store is read-only");
     }
 }
-function controllerSelect(clause) {
-    return `SELECT c.controller_resource_id AS controllerResourceId,
+function resourceManagerSelect(clause) {
+    return `SELECT c.controller_resource_id AS managedResourceId,
     c.resource_id AS resourceId, r.revision AS resourceVersion,
-    c.controller_name AS controllerName, c.resource_key AS resourceKey,
+    c.controller_name AS resourceManagerName, c.resource_key AS resourceKey,
     c.uid, c.generation, c.spec_hash AS specHash, c.status_hash AS statusHash,
     c.deletion_requested_at AS deletionRequestedAt
     FROM controller_resources c JOIN resources r ON r.resource_id = c.resource_id ${clause}`;
@@ -1912,7 +1974,7 @@ function workflowRunViewSelect(clause) {
 function isRecord(value) {
     return typeof value === "object" && value !== null && !Array.isArray(value);
 }
-function isControllerRow(value) {
+function isResourceManagerRow(value) {
     return isRecord(value);
 }
 function isLeaseRow(value) {
@@ -2007,7 +2069,7 @@ function isWorkflowRow(value) {
 function isQueueListRow(value) {
     return isRecord(value);
 }
-function isControllerEventRow(value) {
+function isResourceManagerEventRow(value) {
     return isRecord(value);
 }
 function isRunEventRow(value) {
@@ -2151,7 +2213,7 @@ function rowToSource(row) {
 function effectStatus(value) {
     return value === "indeterminate" ? "ambiguous" : value;
 }
-function controllerEffectState(value) {
+function resourceManagerEffectState(value) {
     return value === "ambiguous" ? "indeterminate" : value;
 }
 function validTimestamp(value) {