narraeon 0.7.1 → 0.7.4

package/dist/node/runtime/play/FileNativePlayTimelineStore.js

@@ -2,10 +2,10 @@ import { isResolutionReceipt } from "../../protocol/resolution.js";
 import { legacyModelBindingFailure, projectLegacyModelBindingFailure, } from "./LegacyModelBindingFailure.js";
 import { isPlayFailureKind, } from "../../protocol/playFailure.js";
 import { isNarrativeOrigin, } from "../history/HistoryBackground.js";
-import { encodePlayPromptRun, decodePlayPromptRun, } from "./PlayPromptRun.js";
+import { contextPromptBasis, decodePlayPromptRun, encodePlayPromptRunRecord, isPlayPromptRunReference, promptRunCompiled, promptRunFromReference, sharePromptCompiled, sharePromptRun, } from "./PlayPromptRun.js";
 import { createHash, randomUUID } from "node:crypto";
 import { constants } from "node:fs";
-import { chmod, copyFile, mkdir, readFile, rename, rm, writeFile, } from "node:fs/promises";
+import { chmod, copyFile, mkdir, readFile, readdir, rename, rm, stat, writeFile, } from "node:fs/promises";
 import { linkExclusive } from "../FileNativeExclusiveLink.js";
 import { dirname, join, resolve } from "node:path";
 import { isDeepStrictEqual } from "node:util";
@@ -43,16 +43,10 @@ export class FileNativePlayTimelineStore {
         const transcriptCount = prefixLength(input.source.transcript, input.target.transcript);
         const eventCount = prefixLength(input.source.events, input.target.events);
         const completedToolCount = prefixLength(input.source.completedTools, input.target.completedTools);
-        if (input.retainCompleteContext ||
-            input.target.events.some((event) => event.kind === "assistant" && event.status === "completed")) {
-            const encoding = await this.readInitialEncoding(input.sourceWorldId, input.source.chainId).catch(() => null);
+        for (const exchange of (await recordedEncodingExchanges(sourceRoot)).slice(0, 2)) {
+            const encoding = await this.readInitialEncoding(input.sourceWorldId, input.source.chainId, exchange).catch(() => null);
             if (encoding !== null)
-                await publishImmutableJson(join(targetRoot, "initial-request-encoding.json"), encoding);
-        }
-        for (const run of input.target.promptRuns ?? []) {
-            const encoding = await this.readInitialEncoding(input.sourceWorldId, input.source.chainId, run.exchange).catch(() => null);
-            if (encoding !== null)
-                await publishImmutableJson(join(targetRoot, `request-encoding-${run.exchange}.json`), encoding);
+                await publishImmutableJson(join(targetRoot, encodingFileName(exchange)), encoding);
         }
         await Promise.all([
             cloneNumberedPrefix({
@@ -120,7 +114,7 @@ export class FileNativePlayTimelineStore {
         };
         const lastRequestDigest = value.lastRequest === null
             ? null
-            : await this.#publishRequest(runtimeRoot, value.lastRequest);
+            : await this.#publishRequest(root, value.lastRequest);
         const continuation = {
             schemaVersion: 3,
             kind: "play_context_continuation",
@@ -148,8 +142,9 @@ export class FileNativePlayTimelineStore {
         for (let index = previous.completedToolCount; index < value.completedTools.length; index += 1)
             await publishImmutableJson(join(root, "completed-tools", numbered(index + 1)), value.completedTools[index]);
         const promptRuns = value.promptRuns ?? [];
+        const promptBase = contextPromptBasis(value);
         for (let index = supplied.promptRunCount ?? 0; index < promptRuns.length; index += 1) {
-            await publishImmutableJson(join(root, "prompt-runs", numbered(index + 1)), encodePlayPromptRun(promptRuns[index]));
+            await publishImmutableJson(join(root, "prompt-runs", numbered(index + 1)), encodePlayPromptRunRecord(promptRuns[index], promptRunBasisAt(promptRuns, index, promptBase)));
         }
         const state = {
             schemaVersion: 3,
@@ -173,6 +168,17 @@ export class FileNativePlayTimelineStore {
             updatedAt: value.updatedAt,
         };
         await publishJson(join(root, "state.json"), state);
+        // Only after state.json names the new continuation and request do the
+        // superseded ones go; a reader holding the old state retries (readContext).
+        await pruneSupersededContextFiles(root, {
+            continuationDigest,
+            lastRequestDigest,
+        });
+        // Every write path upgrades the world first (ensureCurrentStorage), so a
+        // context written here is already in the current layout.
+        if ((await readOptionalJson(join(runtimeRoot, LAYOUT_MARKER))) ===
+            null)
+            await publishJson(join(runtimeRoot, LAYOUT_MARKER), layoutMarker());
         if (publishHead)
             await publishJson(join(runtimeRoot, "play-timeline-head.json"), {
                 schemaVersion: 3,
@@ -214,20 +220,33 @@ export class FileNativePlayTimelineStore {
     async readContext(worldId, chainId) {
         assertIdentity(worldId, "World ID");
         assertIdentity(chainId, "Call-chain ID");
+        // Reads take no lock. A writer prunes the superseded continuation and request
+        // only after publishing the state that names their replacements, so a reader
+        // that loaded the older state retries from the current one.
+        for (let attempt = 1;; attempt += 1)
+            try {
+                return await this.#readContextOnce(worldId, chainId);
+            }
+            catch (error) {
+                if (attempt === 3 || !isNodeError(error) || error.code !== "ENOENT")
+                    throw error;
+            }
+    }
+    async #readContextOnce(worldId, chainId) {
         const metadata = await this.#readContextMetadata(worldId, chainId);
         if (metadata === null)
             return null;
         const { index: indexValue, state: stateValue, root } = metadata;
-        const [baseValue, continuationValue, transcript, events, completedTools, promptRuns,] = await Promise.all([
+        const [baseValue, continuationValue, transcript, events, completedTools] = await Promise.all([
             readJson(join(root, "base.json")),
             readJson(join(root, "continuations", `${stateValue.continuationDigest}.json`)),
             readNumbered(join(root, "transcript"), stateValue.transcriptCount),
             readNumberedValidated(join(root, "events"), stateValue.eventCount, assertPlayEvent),
             readNumbered(join(root, "completed-tools"), stateValue.completedToolCount),
-            readNumbered(join(root, "prompt-runs"), stateValue.promptRunCount ?? 0).then((records) => records.map(decodePlayPromptRun)),
         ]);
         assertContextBase(baseValue);
         assertContextContinuation(continuationValue);
+        const promptRuns = await readPromptRuns(join(root, "prompt-runs"), stateValue.promptRunCount ?? 0, contextPromptBasis({ ...baseValue, playPreset: indexValue.playPreset }));
         if (baseValue.worldId !== worldId ||
             baseValue.chainId !== chainId ||
             continuationValue.chainId !== chainId ||
@@ -237,7 +256,7 @@ export class FileNativePlayTimelineStore {
             throw new Error("Play context durable identity is inconsistent");
         const lastRequest = continuationValue.lastRequestDigest === null
             ? null
-            : await readJson(join(this.#worldRuntimeRoot(worldId), "play-requests", `${continuationValue.lastRequestDigest}.json`));
+            : await readJson(join(root, "requests", `${continuationValue.lastRequestDigest}.json`));
         if (lastRequest !== null &&
             (!isRecord(lastRequest) ||
                 createHash("sha256")
@@ -316,17 +335,18 @@ export class FileNativePlayTimelineStore {
             },
         };
     }
-    /** The first dispatched request encoding is diagnostic evidence, not Authority. */
+    /**
+     * The first dispatched request encoding is diagnostic evidence, not Authority.
+     * A context keeps only the encodings of its two latest exchanges, which are
+     * what the AI-reading diagnostics compare.
+     */
     async recordInitialEncoding(worldId, chainId, encoding, exchange = 1) {
-        const path = join(this.#contextRoot(worldId, chainId), exchange === 1
-            ? "initial-request-encoding.json"
-            : `request-encoding-${exchange}.json`);
-        await publishImmutableJson(path, encoding);
+        const root = this.#contextRoot(worldId, chainId);
+        await publishImmutableJson(join(root, encodingFileName(exchange)), encoding);
+        await pruneEncodings(root);
     }
     async readInitialEncoding(worldId, chainId, exchange = 1) {
-        const value = await readOptionalJson(join(this.#contextRoot(worldId, chainId), exchange === 1
-            ? "initial-request-encoding.json"
-            : `request-encoding-${exchange}.json`));
+        const value = await readOptionalJson(join(this.#contextRoot(worldId, chainId), encodingFileName(exchange)));
         if (value === null)
             return null;
         if (!isRecord(value) ||
@@ -354,6 +374,26 @@ export class FileNativePlayTimelineStore {
         }
         return reverse.reverse();
     }
+    /**
+     * The timeline's context IDs, oldest first, and its generation, read from the
+     * small context indexes only, never their transcripts or prompt runs.
+     */
+    async readTimelineOutline(worldId) {
+        assertIdentity(worldId, "World ID");
+        const head = await this.#readTimelineHead(worldId);
+        if (head === null)
+            return null;
+        const reverse = [];
+        let chainId = head.chainId;
+        while (chainId !== null) {
+            const metadata = await this.#readContextMetadata(worldId, chainId);
+            if (metadata === null)
+                throw new Error("Play timeline points to a missing context");
+            reverse.push(chainId);
+            chainId = metadata.index.previousChainId;
+        }
+        return { generation: head.generation, chainIds: reverse.reverse() };
+    }
     /** Resolve only requested reply summaries; decoration reads never load Provider transcripts. */
     async resolveReplies(worldId, targets) {
         if (targets.length === 0)
@@ -503,6 +543,47 @@ export class FileNativePlayTimelineStore {
     newGeneration() {
         return `timeline:${randomUUID()}`;
     }
+    /**
+     * Whether the world's play timeline is in the current layout (ADR-0034). A world
+     * with no play timeline has nothing to upgrade, so reads never write its marker.
+     */
+    async layoutCurrent(worldId) {
+        assertIdentity(worldId, "World ID");
+        const runtimeRoot = this.#worldRuntimeRoot(worldId);
+        if ((await readOptionalJson(this.#layoutMarkerPath(worldId))) !==
+            null)
+            return true;
+        return ((await pathExists(join(runtimeRoot, "play-contexts"))) === false &&
+            (await pathExists(join(runtimeRoot, "play-requests"))) === false);
+    }
+    /**
+     * Upgrade a released play timeline in place: each context takes its own copy
+     * of its frozen request, snapshots that repeat their basis become references,
+     * and superseded encodings and records go. The released request directory is
+     * removed only after every context is complete, and the marker is written
+     * last, so an interrupted run starts again from the released files.
+     */
+    async migrateLayout(worldId) {
+        assertIdentity(worldId, "World ID");
+        if (await this.layoutCurrent(worldId))
+            return;
+        const runtimeRoot = this.#worldRuntimeRoot(worldId);
+        const contextsRoot = join(runtimeRoot, "play-contexts");
+        for (const entry of await readDirectoryNames(contextsRoot)) {
+            const root = join(contextsRoot, entry);
+            if ((await readOptionalJson(join(root, "state.json"))) === null)
+                continue;
+            await migrateContextLayout(runtimeRoot, root);
+        }
+        await rm(join(runtimeRoot, "play-requests"), {
+            recursive: true,
+            force: true,
+        });
+        await publishJson(this.#layoutMarkerPath(worldId), layoutMarker());
+    }
+    #layoutMarkerPath(worldId) {
+        return join(this.#worldRuntimeRoot(worldId), LAYOUT_MARKER);
+    }
     #worldRuntimeRoot(worldId) {
         return join(this.#worldsRoot, worldId, "runtime");
     }
@@ -522,11 +603,12 @@ export class FileNativePlayTimelineStore {
         assertIdentity(value.chainId, "Call-chain ID");
         return value;
     }
-    async #publishRequest(runtimeRoot, request) {
+    /** The context's frozen request lives in the context: forks copy the one they keep. */
+    async #publishRequest(contextRoot, request) {
         const digest = createHash("sha256")
             .update(JSON.stringify(request))
             .digest("hex");
-        await publishImmutableJson(join(runtimeRoot, "play-requests", `${digest}.json`), request);
+        await publishImmutableJson(join(contextRoot, "requests", `${digest}.json`), request);
         return digest;
     }
     async #initialCursor(worldId, generation, chainId) {
@@ -1062,6 +1144,134 @@ async function assertSameFileBytes(source, target) {
 async function readNumbered(root, count) {
     return Promise.all(Array.from({ length: count }, (_, index) => readJson(join(root, numbered(index + 1)))));
 }
+/**
+ * Runs are read one at a time and share their strings as they are decoded, so
+ * a long context never holds every run's raw text and parsed copy at once. A
+ * reference repeats an earlier compiled prompt: the context base (0) or a run
+ * decoded before it.
+ */
+async function readPromptRuns(root, count, base) {
+    const strings = new Map();
+    const sharedBase = base === null ? null : sharePromptCompiled(base, strings);
+    const runs = [];
+    for (let index = 0; index < count; index += 1) {
+        const record = await readJson(join(root, numbered(index + 1)));
+        runs.push(isPlayPromptRunReference(record)
+            ? promptRunFromReference(record, (basisRun) => basisRun === 0
+                ? sharedBase
+                : basisRun <= runs.length
+                    ? runs[basisRun - 1]
+                    : null)
+            : sharePromptRun(decodePlayPromptRun(record), strings));
+    }
+    return runs;
+}
+/** The basis a run repeats: the context base for the first run, else the run before it. */
+function promptRunBasisAt(runs, index, base) {
+    if (index === 0)
+        return base === null ? null : { index: 0, compiled: base };
+    return { index, compiled: promptRunCompiled(runs[index - 1]) };
+}
+const LAYOUT_MARKER = "play-timeline-layout.json";
+function layoutMarker() {
+    return { schemaVersion: 1, kind: "play_timeline_layout" };
+}
+async function pathExists(path) {
+    return stat(path).then(() => true, (error) => {
+        if (isNodeError(error) && error.code === "ENOENT")
+            return false;
+        throw error;
+    });
+}
+function encodingFileName(exchange) {
+    return exchange === 1
+        ? "initial-request-encoding.json"
+        : `request-encoding-${exchange}.json`;
+}
+/** The exchanges with a recorded encoding in a context, latest first. */
+async function recordedEncodingExchanges(contextRoot) {
+    return (await readDirectoryNames(contextRoot))
+        .map((name) => {
+        if (name === "initial-request-encoding.json")
+            return 1;
+        const match = /^request-encoding-([1-9][0-9]*)\.json$/u.exec(name);
+        return match === null ? null : Number(match[1]);
+    })
+        .filter((exchange) => exchange !== null)
+        .sort((left, right) => right - left);
+}
+/**
+ * Keep the encodings of the two latest exchanges, the pair the AI-reading
+ * diagnostics compare. A context without runs is read at exchange 1.
+ */
+async function pruneEncodings(contextRoot) {
+    for (const exchange of (await recordedEncodingExchanges(contextRoot)).slice(2))
+        await rm(join(contextRoot, encodingFileName(exchange)), { force: true });
+}
+/**
+ * Bring one released context to the current layout. Every step is idempotent:
+ * an existing request copy is kept, and a record already in its current form is
+ * left alone, so a rerun after an interruption converges on the same files.
+ */
+async function migrateContextLayout(runtimeRoot, root) {
+    const indexValue = await readJson(join(root, "index.json"));
+    const stateValue = await readJson(join(root, "state.json"));
+    assertContextIndex(indexValue);
+    assertContextState(stateValue);
+    const baseValue = await readJson(join(root, "base.json"));
+    assertContextBase(baseValue);
+    const continuationValue = await readJson(join(root, "continuations", `${stateValue.continuationDigest}.json`));
+    assertContextContinuation(continuationValue);
+    const digest = continuationValue.lastRequestDigest;
+    if (digest !== null) {
+        const target = join(root, "requests", `${digest}.json`);
+        if ((await readOptionalJson(target)) === null) {
+            const request = await readJson(join(runtimeRoot, "play-requests", `${digest}.json`));
+            if (!isRecord(request) ||
+                createHash("sha256").update(JSON.stringify(request)).digest("hex") !==
+                    digest)
+                throw new Error("Frozen play request does not match its durable identity");
+            await publishImmutableJson(target, request);
+        }
+    }
+    const basis = contextPromptBasis({
+        ...baseValue,
+        playPreset: indexValue.playPreset,
+    });
+    const promptRuns = await readPromptRuns(join(root, "prompt-runs"), stateValue.promptRunCount ?? 0, basis);
+    for (let index = 0; index < promptRuns.length; index += 1) {
+        const path = join(root, "prompt-runs", numbered(index + 1));
+        const record = encodePlayPromptRunRecord(promptRuns[index], promptRunBasisAt(promptRuns, index, basis));
+        if (!isDeepStrictEqual(await readJson(path), record))
+            await publishJson(path, record);
+    }
+    await pruneEncodings(root);
+    await pruneSupersededContextFiles(root, {
+        continuationDigest: stateValue.continuationDigest,
+        lastRequestDigest: digest,
+    });
+}
+/** Remove continuations and requests that the current state no longer names. */
+async function pruneSupersededContextFiles(root, current) {
+    for (const [directory, keep] of [
+        ["continuations", current.continuationDigest],
+        ["requests", current.lastRequestDigest],
+    ]) {
+        for (const name of await readDirectoryNames(join(root, directory)))
+            if (name !== `${keep}.json`)
+                await rm(join(root, directory, name), { force: true });
+    }
+}
+async function readDirectoryNames(directory) {
+    try {
+        return await readdir(directory);
+    }
+    catch (error) {
+        if (isNodeError(error) && error.code === "ENOENT")
+            return [];
+        throw error;
+    }
+}
 async function readNumberedValidated(root, count, assertValue) {
     const values = await readNumbered(root, count);
     const validated = [];