@astrosheep/keiyaku 2.9.6 → 2.9.8

package/build/core/projection/projection-wake.js

@@ -2,16 +2,17 @@ import { FlowError } from "../../flow-error.js";
 import { parseResolvedAkumaLaunchSnapshot } from "../../agents/launch-snapshot/model.js";
 import { isRecoverableSessionEnvelope } from "../../agents/harness/index.js";
 import { ProjectionStateError } from "../atomic-publish.js";
-import { applyContinuationEffort, snapshotPendingTells, } from "./generation/projection-generation-continuation.js";
+import { applyContinuationEffort, readProjectionGenerationContinuationSnapshot, } from "./generation/projection-generation-continuation.js";
 import { createProjectionExecutionId } from "./generation/projection-generation-identity.js";
 import { PROJECTION_ADOPTION_POLL_MS, PROJECTION_ADOPTION_TIMEOUT_MS, startProjectionGeneration, } from "./generation/projection-generation-launcher.js";
 import { openProjectionGenerationStore, openProjectionGenerationStoreReadOnly, } from "./generation/store.js";
 import { observeProjectionLifeSnapshot } from "./projection-life-observer.js";
-import { observeProjectionRunnerLock } from "./projection-runner-lock.js";
+import { acquireProjectionRunnerLock, acquireProjectionRunnerSettlementLock, ProjectionRunnerLockUnavailableError, } from "./projection-runner-lock.js";
 import { setTimeout as delay } from "node:timers/promises";
 function incompatibleProjectionError(projectionId) {
     const subject = projectionId ? `Projection '${projectionId}'` : "Projection";
-    return new FlowError("PROJECTION_INCOMPATIBLE", `${subject} has incompatible historical storage; keiyaku status --all can inspect it, but tell cannot change it.`);
+    const statusCoordinate = projectionId ?? "<projection-id>";
+    return new FlowError("PROJECTION_INCOMPATIBLE", `${subject} has incompatible historical storage; inspect it with keiyaku status ${statusCoordinate}, but tell cannot change it.`);
 }
 /**
  * Tell is a current-schema mutation only. This gate runs before the handler
@@ -98,10 +99,24 @@ export function assertTellEffortSupported(projectionDirectory, effort) {
         throw new FlowError("INVALID_EFFORT_OVERRIDE", `Effort '${effort}' is not permitted; legal values: ${effortOptions.join(", ")}.`);
     }
 }
-async function startCommittedSuccessor(projectionDirectory, projectionId, executionId, startGeneration) {
-    const started = await startGeneration(projectionDirectory, executionId);
+async function startCommittedSuccessor(projectionDirectory, projectionId, executionId, startGeneration, dependencies = {}) {
+    let started;
+    try {
+        started = await startGeneration(projectionDirectory, executionId, dependencies);
+    }
+    finally {
+        dependencies.launchOwnership?.close();
+    }
     if (started.status === "adopted")
         return "woken";
+    if (started.status === "pending")
+        return "pending";
+    const reconciled = readCurrentGeneration(projectionDirectory);
+    if (reconciled?.launch.executionId === executionId && reconciled.adoption)
+        return "woken";
+    if (reconciled && !reconciled.verdict && reconciled.launch.executionId !== executionId) {
+        return reconciled.adoption ? "active" : "pending";
+    }
     throw new FlowError("PROJECTION_WAKE_FAILED", `The tell was saved for '${projectionId}', but its successor could not be started (cause: ${started.cause}).`, {
         facts: {
             kind: "projection_wake_failure",
@@ -122,6 +137,31 @@ function sessionWakeFailure(projectionId, cause) {
         },
     });
 }
+export function persistInterruptWakeFailure(projectionDirectory, error, nowMs = Date.now()) {
+    if (!(error instanceof FlowError)
+        || error.code !== "PROJECTION_WAKE_FAILED"
+        || error.facts?.kind !== "projection_wake_failure")
+        return;
+    const continuation = readProjectionGenerationContinuationSnapshot(projectionDirectory, 10);
+    const current = continuation.current;
+    if (current?.verdict?.facts.state !== "interrupted" || !continuation.requiresWake)
+        return;
+    const successorExecutionId = createProjectionExecutionId(nowMs);
+    const facts = successorFacts(current, continuation.successor, nowMs, false);
+    const store = openProjectionGenerationStore(projectionDirectory);
+    try {
+        store.failInterruptWakeIfSettled({
+            predecessorExecutionId: current.launch.executionId,
+            successorExecutionId,
+            successorFacts: facts,
+            cause: error.facts.cause,
+            terminalAt: new Date(nowMs).toISOString(),
+        });
+    }
+    finally {
+        store.close();
+    }
+}
 export async function tellProjection(projectionDirectory, projectionId, effort, dependencies = {}) {
     const startGeneration = dependencies.startGeneration ?? startProjectionGeneration;
     const sleep = dependencies.sleep ?? ((milliseconds) => delay(milliseconds));
@@ -148,12 +188,15 @@ export async function tellProjection(projectionDirectory, projectionId, effort,
         if (life.phase === "unknown") {
             return "pending";
         }
-        const current = readCurrentGeneration(projectionDirectory);
+        const continuation = readProjectionGenerationContinuationSnapshot(projectionDirectory, 10);
+        const current = continuation.current;
         if (!current)
             return "pending";
         if (current.killIntent && !current.verdict)
             return "pending";
-        const snapshot = snapshotPendingTells(projectionDirectory);
+        if (!continuation.requiresWake)
+            return "pending";
+        const snapshot = continuation.successor;
         const successorExecutionId = createProjectionExecutionId(nowMs);
         let facts;
         try {
@@ -169,56 +212,144 @@ export async function tellProjection(projectionDirectory, projectionId, effort,
         try {
             if (life.phase === "startup-timeout") {
                 const terminalAt = new Date(nowMs).toISOString();
-                const transition = store.supersedeStartupTimeout({
-                    executionId: current.launch.executionId,
-                    observedAtMs: nowMs,
-                    startupTimeoutMs: PROJECTION_ADOPTION_TIMEOUT_MS,
-                    diagnostic: { detail: "runner did not adopt before startup timeout" },
-                    successorExecutionId,
-                    successorFacts: facts,
-                    terminalAt,
-                });
-                if (transition.status === "committed") {
-                    return await startCommittedSuccessor(projectionDirectory, projectionId, successorExecutionId, startGeneration);
+                let deathEvidence;
+                try {
+                    deathEvidence = acquireProjectionRunnerSettlementLock(projectionDirectory);
                 }
-                if (transition.reason === "kill-requested")
+                catch (error) {
+                    if (error instanceof ProjectionRunnerLockUnavailableError)
+                        return "pending";
                     return "pending";
-                await sleep(PROJECTION_ADOPTION_POLL_MS);
-                continue;
+                }
+                try {
+                    store.beginWriteTransaction();
+                    const protectedCurrent = store.readCurrentGeneration();
+                    if (protectedCurrent?.launch.executionId !== current.launch.executionId
+                        || protectedCurrent.adoption
+                        || protectedCurrent.verdict) {
+                        store.rollbackWriteTransaction();
+                        return "pending";
+                    }
+                    if (protectedCurrent.killIntent) {
+                        const settled = store.settleKillIfRunnerDead({
+                            executionId: current.launch.executionId,
+                            releasedLockEvidence: { observedAt: terminalAt, state: "acquired" },
+                        });
+                        if (settled.status === "committed")
+                            store.commitWriteTransaction();
+                        else
+                            store.rollbackWriteTransaction();
+                        return "pending";
+                    }
+                    deathEvidence.close();
+                    let launchOwnership;
+                    try {
+                        launchOwnership = acquireProjectionRunnerLock(projectionDirectory);
+                    }
+                    catch (error) {
+                        store.rollbackWriteTransaction();
+                        if (error instanceof ProjectionRunnerLockUnavailableError)
+                            return "pending";
+                        return "pending";
+                    }
+                    return await startCommittedSuccessor(projectionDirectory, projectionId, successorExecutionId, startGeneration, {
+                        commitLaunch: () => {
+                            const transition = store.supersedeStartupTimeout({
+                                executionId: current.launch.executionId,
+                                releasedLockEvidence: { observedAt: terminalAt, state: "acquired" },
+                                diagnostic: { detail: "runner did not adopt before startup timeout" },
+                                successorExecutionId,
+                                successorFacts: facts,
+                                terminalAt,
+                            });
+                            if (transition.status === "committed")
+                                store.commitWriteTransaction();
+                            else
+                                store.rollbackWriteTransaction();
+                            return transition.status === "committed";
+                        },
+                        launchOwnership,
+                    });
+                }
+                finally {
+                    store.rollbackWriteTransaction();
+                    deathEvidence.close();
+                }
             }
             if (life.phase === "lost" && !current.verdict) {
-                const lock = observeProjectionRunnerLock(projectionDirectory);
-                if (lock.state !== "released")
-                    return lock.state === "held" ? "active" : "pending";
-                const terminalAt = new Date(nowMs).toISOString();
-                const transition = store.replaceDeadRunner({
-                    executionId: current.launch.executionId,
-                    releasedLockEvidence: { observedAt: terminalAt },
-                    successorExecutionId,
-                    successorFacts: facts,
-                    terminalAt,
-                });
-                if (transition.status === "committed") {
-                    return await startCommittedSuccessor(projectionDirectory, projectionId, successorExecutionId, startGeneration);
+                let deathEvidence;
+                try {
+                    deathEvidence = acquireProjectionRunnerSettlementLock(projectionDirectory);
                 }
-                if (transition.reason === "kill-requested")
+                catch (error) {
+                    if (error instanceof ProjectionRunnerLockUnavailableError)
+                        return "active";
                     return "pending";
-                await sleep(PROJECTION_ADOPTION_POLL_MS);
-                continue;
+                }
+                try {
+                    store.beginWriteTransaction();
+                    const protectedCurrent = store.readCurrentGeneration();
+                    if (protectedCurrent?.launch.executionId !== current.launch.executionId
+                        || !protectedCurrent.adoption
+                        || protectedCurrent.verdict) {
+                        store.rollbackWriteTransaction();
+                        return "pending";
+                    }
+                    const terminalAt = new Date(nowMs).toISOString();
+                    if (protectedCurrent.killIntent) {
+                        const settled = store.settleKillIfRunnerDead({
+                            executionId: current.launch.executionId,
+                            releasedLockEvidence: { observedAt: terminalAt, state: "acquired" },
+                        });
+                        if (settled.status === "committed")
+                            store.commitWriteTransaction();
+                        else
+                            store.rollbackWriteTransaction();
+                        return "pending";
+                    }
+                    deathEvidence.close();
+                    let launchOwnership;
+                    try {
+                        launchOwnership = acquireProjectionRunnerLock(projectionDirectory);
+                    }
+                    catch (error) {
+                        store.rollbackWriteTransaction();
+                        if (error instanceof ProjectionRunnerLockUnavailableError)
+                            return "active";
+                        return "pending";
+                    }
+                    return await startCommittedSuccessor(projectionDirectory, projectionId, successorExecutionId, startGeneration, {
+                        commitLaunch: () => {
+                            const transition = store.replaceDeadRunner({
+                                executionId: current.launch.executionId,
+                                releasedLockEvidence: { observedAt: terminalAt, state: "acquired" },
+                                successorExecutionId,
+                                successorFacts: facts,
+                                terminalAt,
+                            });
+                            if (transition.status === "committed")
+                                store.commitWriteTransaction();
+                            else
+                                store.rollbackWriteTransaction();
+                            return transition.status === "committed";
+                        },
+                        launchOwnership,
+                    });
+                }
+                finally {
+                    store.rollbackWriteTransaction();
+                    deathEvidence.close();
+                }
             }
             if (!["done", "failed", "lost", "dismissed", "killed"].includes(life.phase)) {
                 return "pending";
             }
-            const transition = store.launchIfSettled({
-                executionId: successorExecutionId,
-                facts,
+            return await startCommittedSuccessor(projectionDirectory, projectionId, successorExecutionId, startGeneration, {
+                commitLaunch: () => store.launchIfSettled({
+                    executionId: successorExecutionId,
+                    facts,
+                }).status === "committed",
             });
-            if (transition.status === "committed") {
-                return await startCommittedSuccessor(projectionDirectory, projectionId, successorExecutionId, startGeneration);
-            }
-            if (transition.reason === "kill-requested")
-                return "pending";
-            await sleep(PROJECTION_ADOPTION_POLL_MS);
         }
         finally {
             store.close();

package/build/core/projection/tell/database.js

@@ -42,6 +42,24 @@ export function withRead(projectionDirectory, read) {
         database.close();
     }
 }
+export function withReadSnapshot(projectionDirectory, read) {
+    const database = openDatabase(projectionDirectory, true);
+    try {
+        database.exec("BEGIN");
+        try {
+            const result = read(database);
+            database.exec("COMMIT");
+            return result;
+        }
+        catch (error) {
+            rollbackQuietly(database);
+            throw error;
+        }
+    }
+    finally {
+        database.close();
+    }
+}
 export function withWrite(projectionDirectory, write) {
     const database = openDatabase(projectionDirectory, false);
     try {

package/build/core/projection/tell/model.js

@@ -6,6 +6,7 @@ export const PROJECTION_TELL_STATES = [
     "consumed",
     "delivered",
 ];
+export const PENDING_PROJECTION_TELL_STATES = ["inbox", "inflight", "submitted"];
 export function requireExecutionId(value) {
     if (typeof value !== "string" || value.trim() === "") {
         throw new ProjectionStateError("submitted tell executionId must be a non-empty string");

package/build/core/projection/tell/store.js

@@ -1,7 +1,6 @@
 import { ProjectionStateError, randomHex8 } from "../../atomic-publish.js";
-import { withRead, withWrite } from "./database.js";
-import { assertTellIdRandomHex, originalFromRow, parseFence, requireExecutionId, submittedFromRow, } from "./model.js";
-const PENDING_TELL_LIMIT = 10_000;
+import { withRead, withReadSnapshot, withWrite } from "./database.js";
+import { assertTellIdRandomHex, originalFromRow, PENDING_PROJECTION_TELL_STATES, parseFence, requireExecutionId, submittedFromRow, } from "./model.js";
 function selectTell(database, tellId) {
     return database.prepare(`
     SELECT
@@ -26,6 +25,77 @@ export function listTellIds(projectionDirectory, state) {
       ORDER BY tell_id
     `).all(state).map((row) => row.tell_id));
 }
+/** List continuation fence metadata without loading operator tell bodies. */
+export function listTellMetadataFromDatabase(database, states) {
+    const placeholders = states.map(() => "?").join(", ");
+    return database.prepare(`
+    SELECT tell_id, state, execution_id, effort
+    FROM projection_tells
+    WHERE state IN (${placeholders})
+    ORDER BY tell_id
+  `).all(...states).map((row) => ({
+        tellId: row.tell_id,
+        state: row.state,
+        ...(row.execution_id ? { executionId: row.execution_id } : {}),
+        ...(row.effort ? { effort: row.effort } : {}),
+    }));
+}
+/** List continuation fence metadata without loading operator tell bodies. */
+export function listTellMetadata(projectionDirectory, states) {
+    return withRead(projectionDirectory, (database) => listTellMetadataFromDatabase(database, states));
+}
+export function readPendingTellWindowFromDatabase(database, limit) {
+    if (!Number.isSafeInteger(limit) || limit < 0) {
+        throw new ProjectionStateError("pending tell window limit must be a non-negative safe integer");
+    }
+    const pendingPlaceholders = PENDING_PROJECTION_TELL_STATES.map(() => "?").join(", ");
+    const total = database.prepare(`
+    SELECT COUNT(*) AS count
+    FROM projection_tells
+    WHERE state IN (${pendingPlaceholders})
+  `).get(...PENDING_PROJECTION_TELL_STATES);
+    const rows = database.prepare(`
+    SELECT tell_id, text, created_at, state
+    FROM projection_tells
+    WHERE state IN (${pendingPlaceholders})
+    ORDER BY tell_id
+    LIMIT ?
+  `).all(...PENDING_PROJECTION_TELL_STATES, limit);
+    return {
+        totals: { undelivered: Number(total.count) },
+        window: rows.map((row) => ({ tellId: row.tell_id, text: row.text, createdAt: row.created_at, state: row.state })),
+    };
+}
+export function readPendingTellWindow(projectionDirectory, limit) {
+    return withReadSnapshot(projectionDirectory, (database) => readPendingTellWindowFromDatabase(database, limit));
+}
+/** Whether a failed successor fence still retains delivered mail for replay. */
+export function hasRetainedDeliveredTellFromDatabase(database, retainedFence) {
+    if (retainedFence.length === 0)
+        return false;
+    const statement = database.prepare(`
+    SELECT 1
+    FROM projection_tells
+    WHERE tell_id = ? AND state = 'delivered'
+    LIMIT 1
+  `);
+    return retainedFence.some((tellId) => statement.get(tellId) !== undefined);
+}
+export function hasRetainedDeliveredTell(projectionDirectory, retainedFence) {
+    return withRead(projectionDirectory, (database) => hasRetainedDeliveredTellFromDatabase(database, retainedFence));
+}
+/** Whether interrupted replay has any delivered tell left to return to inbox. */
+export function hasDeliveredTellFromDatabase(database, executionId) {
+    return database.prepare(`
+    SELECT 1
+    FROM projection_tells
+    WHERE state = 'delivered' AND execution_id = ?
+    LIMIT 1
+  `).get(executionId) !== undefined;
+}
+export function hasDeliveredTell(projectionDirectory, executionId) {
+    return withRead(projectionDirectory, (database) => hasDeliveredTellFromDatabase(database, executionId));
+}
 export function readTellOriginal(projectionDirectory, state, tellId) {
     return withRead(projectionDirectory, (database) => {
         const row = selectTell(database, tellId);
@@ -49,14 +119,6 @@ export function writeInboxTell(projectionDirectory, text, nowMs, options) {
     }
     const tellId = `${nowMs}-${assertTellIdRandomHex((options?.randomHex ?? randomHex8)())}`;
     return withWrite(projectionDirectory, (database) => {
-        const pending = database.prepare(`
-      SELECT COUNT(*) AS count
-      FROM projection_tells
-      WHERE state IN ('inbox', 'inflight', 'submitted', 'delivered')
-    `).get();
-        if (Number(pending.count) >= PENDING_TELL_LIMIT) {
-            throw new ProjectionStateError(`projection has reached the ${PENDING_TELL_LIMIT} pending tell limit`);
-        }
         try {
             database.prepare(`
         INSERT INTO projection_tells (
@@ -123,6 +185,31 @@ export function claimFencedTells(projectionDirectory, fence, executionId) {
                 claimed.push(tellId);
                 continue;
             }
+            if (row.state === "submitted" || row.state === "delivered") {
+                const submitted = submittedFromRow(row);
+                if (submitted.executionId === owner) {
+                    throw new ProjectionStateError(`submitted tell belongs to current execution: ${tellId}`);
+                }
+                const replayedFrom = [
+                    ...(submitted.replayedFrom ?? []),
+                    { executionId: submitted.executionId, fence: submitted.fence },
+                ];
+                const result = database.prepare(`
+          UPDATE projection_tells
+          SET
+            state = 'inflight',
+            execution_id = NULL,
+            fence_json = NULL,
+            replayed_from_json = ?,
+            delivered_at = NULL
+          WHERE tell_id = ? AND state = ?
+        `).run(JSON.stringify(replayedFrom), tellId, row.state);
+                if (Number(result.changes) !== 1) {
+                    throw new ProjectionStateError(`generation tell fence member changed while claiming: ${tellId}`);
+                }
+                claimed.push(tellId);
+                continue;
+            }
             if (row.state === "consumed" && row.execution_id === owner)
                 continue;
             throw new ProjectionStateError(`generation tell fence member is not claimable: ${tellId}`);
@@ -152,7 +239,7 @@ export function markTellSubmitted(projectionDirectory, tellId, fields) {
         state = 'submitted',
         execution_id = ?,
         fence_json = ?,
-        replayed_from_json = ?
+        replayed_from_json = COALESCE(?, replayed_from_json)
       WHERE tell_id = ? AND state = 'inflight'
     `).run(executionId, JSON.stringify(fence), replayedFrom.length > 0 ? JSON.stringify(replayedFrom) : null, tellId);
     });
@@ -190,12 +277,13 @@ export function readProjectionTellCounts(projectionDirectory) {
 }
 export function snapshotPendingTells(projectionDirectory) {
     return withRead(projectionDirectory, (database) => {
+        const pendingPlaceholders = PENDING_PROJECTION_TELL_STATES.map(() => "?").join(", ");
         const rows = database.prepare(`
       SELECT tell_id, effort
       FROM projection_tells
-      WHERE state IN ('inbox', 'inflight', 'submitted', 'delivered')
+      WHERE state IN (${pendingPlaceholders})
       ORDER BY tell_id
-    `).all();
+    `).all(...PENDING_PROJECTION_TELL_STATES);
         const effort = rows.reduce((selected, row) => row.effort ?? selected, undefined);
         return {
             tellIds: rows.map((row) => row.tell_id),
@@ -203,44 +291,3 @@ export function snapshotPendingTells(projectionDirectory) {
         };
     });
 }
-export function demoteSubmittedTellsForReplay(projectionDirectory, executionId) {
-    const currentExecutionId = requireExecutionId(executionId);
-    return withWrite(projectionDirectory, (database) => {
-        const rows = database.prepare(`
-      SELECT
-        tell_id,
-        text,
-        created_at,
-        effort,
-        state,
-        execution_id,
-        fence_json,
-        replayed_from_json,
-        delivered_at
-      FROM projection_tells
-      WHERE state IN ('submitted', 'delivered')
-      ORDER BY tell_id
-    `).all();
-        for (const row of rows) {
-            const submitted = submittedFromRow(row);
-            if (submitted.executionId === currentExecutionId) {
-                throw new ProjectionStateError(`submitted tell belongs to current execution: ${row.tell_id}`);
-            }
-            const replayedFrom = [
-                ...(submitted.replayedFrom ?? []),
-                { executionId: submitted.executionId, fence: submitted.fence },
-            ];
-            database.prepare(`
-        UPDATE projection_tells
-        SET
-          state = 'inflight',
-          execution_id = NULL,
-          fence_json = NULL,
-          replayed_from_json = ?,
-          delivered_at = NULL
-        WHERE tell_id = ? AND state IN ('submitted', 'delivered')
-      `).run(JSON.stringify(replayedFrom), row.tell_id);
-        }
-        return rows.map((row) => row.tell_id);
-    });
-}