narraeon 0.4.0 → 0.5.0

package/dist/node/runtime/world-revision/WorldRevisionWorkspace.js

@@ -0,0 +1,494 @@
+import { createHash } from "node:crypto";
+import { contentTreeFingerprint } from "../content/ContentTreeFingerprint.js";
+import { readDeclaredWorldClock } from "../prompt/WorldMaintenanceReport.js";
+import { WorldOperationClaimLostError } from "../world/WorldOperationCoordinator.js";
+import { WorldDocumentStore } from "../world/WorldDocumentStore.js";
+import { assertUsableWorldRevisionFiles, inspectWorldRevisionFiles, } from "./WorldRevisionAuthoringTransaction.js";
+export class WorldRevisionWorkspaceError extends Error {
+    constructor(message, options) {
+        super(message, options);
+        this.name = "WorldRevisionWorkspaceError";
+    }
+}
+/** Owns the one persistent state/control worktree for a locked world. */
+export class WorldRevisionWorkspace {
+    #store;
+    #worlds;
+    constructor(input) {
+        this.#store = input.store;
+        this.#worlds = input.worlds;
+    }
+    async open(worldId) {
+        const locked = await this.#worlds.readWorldRevisionLock(worldId);
+        if (locked !== null) {
+            const resumed = await this.#resumeLocked(locked);
+            return resumed.lifecycle === "active" ? resumed : this.open(worldId);
+        }
+        const epochId = this.#store.createEpochId();
+        const acquired = await this.#worlds.acquireWorldRevisionLock(worldId, epochId);
+        if (acquired.kind === "busy") {
+            const current = await this.#worlds.readWorldRevisionLock(worldId);
+            if (current === null)
+                throw new WorldRevisionWorkspaceError("Another world operation is still finishing; retry opening the revision");
+            const resumed = await this.#resumeLocked(current);
+            return resumed.lifecycle === "active" ? resumed : this.open(worldId);
+        }
+        try {
+            return await this.#resumeLocked(acquired.handle);
+        }
+        catch (error) {
+            await this.#worlds.releaseWorldRevisionLock(acquired.handle);
+            throw error;
+        }
+    }
+    async active(worldId) {
+        const handle = await this.#worlds.readWorldRevisionLock(worldId);
+        if (handle === null)
+            return null;
+        const resumed = await this.#resumeLocked(handle);
+        return resumed.lifecycle === "active" ? resumed : null;
+    }
+    async latest(worldId) {
+        return (await this.#store.listEpochs(worldId))[0] ?? null;
+    }
+    async sealed(worldId) {
+        return (await this.#store.listEpochs(worldId)).flatMap((epoch) => {
+            if ((epoch.lifecycle !== "applied" && epoch.lifecycle !== "discarded") ||
+                epoch.finishedAt === null)
+                return [];
+            return [
+                {
+                    epochId: epoch.epochId,
+                    worldId: epoch.worldId,
+                    lifecycle: epoch.lifecycle,
+                    baseHead: epoch.baseHead,
+                    diff: fileDiff(epoch.baseFiles, epoch.files),
+                    changes: structuredClone(epoch.changes),
+                    createdAt: epoch.createdAt,
+                    finishedAt: epoch.finishedAt,
+                    appliedHead: epoch.apply?.committedHead ?? null,
+                },
+            ];
+        });
+    }
+    async replace(input) {
+        const handle = revisionHandle(input);
+        return this.#withMutableEpoch(handle, async (epoch) => {
+            assertExpectedRevision(epoch, input.expectedRevision);
+            const files = cloneFiles(input.files);
+            assertUsableWorldRevisionFiles(files, epoch.baseFiles);
+            const changes = fileDiff(epoch.files, files);
+            if (changes.length === 0)
+                return epoch;
+            epoch.files = files;
+            epoch.revision = contentTreeFingerprint(files);
+            epoch.changes.push({
+                changeSetId: this.#store.createChangeSetId(),
+                source: "manual",
+                createdAt: Date.now(),
+                changes,
+            });
+            epoch.updatedAt = Date.now();
+            await this.#store.saveEpoch(epoch);
+            return epoch;
+        });
+    }
+    async publishAi(input) {
+        const handle = revisionHandle(input);
+        return this.#withMutableEpoch(handle, async (epoch) => {
+            if (epoch.revision === input.afterRevision) {
+                const ids = input.toolResults.flatMap(({ changeSetId }) => changeSetId === null ? [] : [changeSetId]);
+                if (ids.every((id) => epoch.changes.some(({ changeSetId }) => changeSetId === id)))
+                    return epoch;
+            }
+            assertExpectedRevision(epoch, input.expectedRevision);
+            const files = cloneFiles(input.files);
+            assertUsableWorldRevisionFiles(files, epoch.baseFiles);
+            if (contentTreeFingerprint(files) !== input.afterRevision)
+                throw new WorldRevisionWorkspaceError("Prepared AI publication does not match its revision fingerprint");
+            for (const result of input.toolResults) {
+                if (result.isError ||
+                    result.changeSetId === null ||
+                    result.changes.length === 0)
+                    continue;
+                epoch.changes.push({
+                    changeSetId: result.changeSetId,
+                    source: "ai",
+                    createdAt: Date.now(),
+                    changes: structuredClone(result.changes),
+                });
+            }
+            epoch.files = files;
+            epoch.revision = input.afterRevision;
+            epoch.updatedAt = Date.now();
+            await this.#store.saveEpoch(epoch);
+            return epoch;
+        });
+    }
+    async rollback(input) {
+        const handle = revisionHandle(input);
+        return this.#withMutableEpoch(handle, async (epoch) => {
+            const set = epoch.changes.find(({ changeSetId }) => changeSetId === input.changeSetId);
+            const matches = set?.changes.filter(({ path }) => path === input.path);
+            const change = matches?.[0];
+            if (set === undefined || matches?.length !== 1 || change === undefined)
+                throw new WorldRevisionWorkspaceError("The requested file change does not exist in this revision epoch");
+            if (fileMatchesBefore(epoch.files, change))
+                return {
+                    status: "already_rolled_back",
+                    changeSetId: input.changeSetId,
+                    path: input.path,
+                    changes: [],
+                    epoch,
+                };
+            const inverse = inverseDiff(epoch.files, change);
+            const files = restoreBefore(epoch.files, change);
+            assertUsableWorldRevisionFiles(files, epoch.baseFiles);
+            epoch.files = files;
+            epoch.revision = contentTreeFingerprint(files);
+            epoch.changes.push({
+                changeSetId: this.#store.createChangeSetId(),
+                source: "rollback",
+                createdAt: Date.now(),
+                changes: [inverse],
+            });
+            epoch.updatedAt = Date.now();
+            await this.#store.saveEpoch(epoch);
+            return {
+                status: "rolled_back",
+                changeSetId: input.changeSetId,
+                path: input.path,
+                changes: [inverse],
+                epoch,
+            };
+        });
+    }
+    async discard(input) {
+        const handle = revisionHandle(input);
+        const epoch = await this.#withMutableEpoch(handle, async (current) => {
+            current.lifecycle = "discarded";
+            current.finishedAt = Date.now();
+            current.updatedAt = current.finishedAt;
+            await this.#store.saveEpoch(current);
+            return current;
+        });
+        await this.#worlds.releaseWorldRevisionLock(handle);
+        return epoch;
+    }
+    async apply(input) {
+        const handle = revisionHandle(input);
+        let epoch = await this.#prepareApply(handle, input.expectedRevision);
+        const overallDiff = fileDiff(epoch.baseFiles, epoch.files);
+        if (overallDiff.length === 0) {
+            epoch = await this.#finishEmptyApply(handle);
+            await this.#worlds.releaseWorldRevisionLock(handle);
+            return epoch;
+        }
+        if (epoch.apply?.phase === "prepared") {
+            const binding = await this.#worlds.bindWorldRevision(handle);
+            const worldClock = readDeclaredWorldClock(WorldDocumentStore.open({
+                layout: "world_state",
+                files: epoch.files,
+            }));
+            const outcome = await this.#worlds.commitCorrection({
+                ...(worldClock === undefined ? {} : { worldClock }),
+                operationId: epoch.apply.operationId,
+                worldId: epoch.worldId,
+                parentHead: epoch.baseHead,
+                nextMaterials: binding.additionalMaterials,
+                stateChanges: stateChanges(epoch.baseFiles, epoch.files),
+                worldRevisionLock: handle,
+                validationControl: controlFiles(epoch.files),
+            });
+            epoch = await this.#recordStateCommit(handle, outcome);
+        }
+        if (epoch.apply?.phase === "state_committed") {
+            await this.#worlds.replaceWorldRevisionControl({
+                handle,
+                expectedFingerprint: epoch.baseControlFingerprint,
+                files: controlFiles(epoch.files),
+            });
+            await this.#recordControlPublication(handle);
+        }
+        epoch = await this.#finishApply(handle);
+        await this.#worlds.releaseWorldRevisionLock(handle);
+        return epoch;
+    }
+    view(epoch, locked) {
+        const inspection = inspectWorldRevisionFiles(epoch.files, epoch.baseFiles);
+        return {
+            epochId: epoch.epochId,
+            worldId: epoch.worldId,
+            lifecycle: epoch.lifecycle,
+            locked,
+            baseHead: epoch.baseHead,
+            revision: epoch.revision,
+            files: cloneFiles(epoch.files),
+            diff: fileDiff(epoch.baseFiles, epoch.files),
+            diagnostics: structuredClone(inspection.diagnostics),
+            changes: structuredClone(epoch.changes),
+            createdAt: epoch.createdAt,
+            updatedAt: epoch.updatedAt,
+            finishedAt: epoch.finishedAt,
+            appliedHead: epoch.apply?.committedHead ?? null,
+        };
+    }
+    async #resumeLocked(handle) {
+        let epoch = await this.#store.readEpochIfPresent(handle.epochId);
+        if (epoch === null) {
+            const binding = await this.#worlds.bindWorldRevision(handle);
+            const files = Object.entries(binding.files)
+                .map(([path, contents]) => ({ path, contents }))
+                .sort((left, right) => left.path.localeCompare(right.path));
+            assertUsableWorldRevisionFiles(files, files);
+            const now = Date.now();
+            epoch = {
+                schemaVersion: 1,
+                epochId: handle.epochId,
+                worldId: handle.worldId,
+                lifecycle: "active",
+                baseHead: binding.parentHead,
+                baseControlFingerprint: controlFingerprint(controlFiles(files)),
+                baseFiles: cloneFiles(files),
+                files: cloneFiles(files),
+                revision: contentTreeFingerprint(files),
+                changes: [],
+                apply: null,
+                createdAt: now,
+                updatedAt: now,
+                finishedAt: null,
+            };
+            await this.#store.saveEpoch(epoch);
+        }
+        if (epoch.worldId !== handle.worldId)
+            throw new WorldRevisionWorkspaceError("World-revision lock points to another world's epoch");
+        if (epoch.lifecycle === "applying")
+            return this.apply({
+                worldId: epoch.worldId,
+                epochId: epoch.epochId,
+                expectedRevision: epoch.apply?.expectedRevision ?? epoch.revision,
+            });
+        if (epoch.lifecycle !== "active") {
+            await this.#worlds.releaseWorldRevisionLock(handle);
+            return epoch;
+        }
+        return epoch;
+    }
+    async #withMutableEpoch(handle, action) {
+        return this.#withRevisionOwnership(handle, async () => {
+            const epoch = await this.#store.readEpoch(handle.epochId);
+            if (epoch.worldId !== handle.worldId || epoch.lifecycle !== "active")
+                throw new WorldRevisionWorkspaceError("World-revision epoch is no longer editable");
+            return action(epoch);
+        });
+    }
+    async #prepareApply(handle, expectedRevision) {
+        return this.#withRevisionOwnership(handle, async () => {
+            const epoch = await this.#store.readEpoch(handle.epochId);
+            if (epoch.lifecycle === "applying")
+                return epoch;
+            if (epoch.lifecycle !== "active")
+                throw new WorldRevisionWorkspaceError("World-revision epoch is no longer applicable");
+            assertExpectedRevision(epoch, expectedRevision);
+            assertUsableWorldRevisionFiles(epoch.files, epoch.baseFiles);
+            epoch.lifecycle = "applying";
+            epoch.apply = {
+                operationId: `world-revision-${epoch.epochId}`,
+                expectedRevision,
+                phase: "prepared",
+                committedHead: null,
+            };
+            epoch.updatedAt = Date.now();
+            await this.#store.saveEpoch(epoch);
+            return epoch;
+        });
+    }
+    async #finishEmptyApply(handle) {
+        return this.#withRevisionOwnership(handle, async () => {
+            const epoch = await this.#store.readEpoch(handle.epochId);
+            if (epoch.lifecycle !== "applying" || epoch.apply === null)
+                throw new WorldRevisionWorkspaceError("Apply recovery state is damaged");
+            epoch.lifecycle = "applied";
+            epoch.apply.phase = "control_published";
+            epoch.apply.committedHead = epoch.baseHead;
+            epoch.finishedAt = Date.now();
+            epoch.updatedAt = epoch.finishedAt;
+            await this.#store.saveEpoch(epoch);
+            return epoch;
+        });
+    }
+    async #recordStateCommit(handle, outcome) {
+        return this.#withRevisionOwnership(handle, async () => {
+            const epoch = await this.#store.readEpoch(handle.epochId);
+            if (epoch.lifecycle !== "applying" || epoch.apply === null)
+                throw new WorldRevisionWorkspaceError("Apply recovery state is damaged");
+            epoch.apply.phase = "state_committed";
+            epoch.apply.committedHead = outcome.head;
+            epoch.updatedAt = Date.now();
+            await this.#store.saveEpoch(epoch);
+            return epoch;
+        });
+    }
+    async #recordControlPublication(handle) {
+        return this.#withRevisionOwnership(handle, async () => {
+            const epoch = await this.#store.readEpoch(handle.epochId);
+            if (epoch.lifecycle !== "applying" || epoch.apply === null)
+                throw new WorldRevisionWorkspaceError("Apply recovery state is damaged");
+            epoch.apply.phase = "control_published";
+            epoch.updatedAt = Date.now();
+            await this.#store.saveEpoch(epoch);
+            return epoch;
+        });
+    }
+    async #finishApply(handle) {
+        return this.#withRevisionOwnership(handle, async () => {
+            const epoch = await this.#store.readEpoch(handle.epochId);
+            if (epoch.lifecycle !== "applying" ||
+                epoch.apply?.phase !== "control_published")
+                throw new WorldRevisionWorkspaceError("Apply recovery state is damaged");
+            epoch.lifecycle = "applied";
+            epoch.finishedAt = Date.now();
+            epoch.updatedAt = epoch.finishedAt;
+            await this.#store.saveEpoch(epoch);
+            return epoch;
+        });
+    }
+    async #withRevisionOwnership(handle, action) {
+        try {
+            return await this.#worlds.operations.withWorldRevisionLock(handle, action);
+        }
+        catch (error) {
+            if (error instanceof WorldOperationClaimLostError)
+                throw new WorldRevisionWorkspaceError("The world-revision epoch no longer owns this world's lock", { cause: error });
+            throw error;
+        }
+    }
+}
+function revisionHandle(input) {
+    return { worldId: input.worldId, epochId: input.epochId };
+}
+function assertExpectedRevision(epoch, expected) {
+    if (epoch.revision !== expected)
+        throw new WorldRevisionWorkspaceError("The world-revision worktree changed; refresh it before writing");
+}
+function stateChanges(baseFiles, files) {
+    const base = new Map(baseFiles
+        .filter(({ path }) => path.startsWith("state/"))
+        .map((file) => [file.path, file]));
+    const snapshot = WorldDocumentStore.open({
+        layout: "world_state",
+        files: files.filter(({ path }) => path.startsWith("state/")),
+    });
+    if (snapshot.status !== "usable")
+        throw new WorldRevisionWorkspaceError("Only a usable world-state snapshot can be applied");
+    return files
+        .filter(({ path }) => path.startsWith("state/"))
+        .flatMap((file) => {
+        const previous = base.get(file.path);
+        if (previous?.contents === file.contents)
+            return [];
+        const result = snapshot.query({
+            kind: "read_document",
+            document: { logicalPath: file.path },
+        });
+        if (result.kind !== "read_document" || !result.ok)
+            throw new WorldRevisionWorkspaceError(`Cannot resolve changed world document: ${file.path}`);
+        if (previous !== undefined) {
+            const baseSnapshot = WorldDocumentStore.open({
+                layout: "world_state",
+                files: baseFiles.filter(({ path }) => path.startsWith("state/")),
+            });
+            const before = baseSnapshot.query({
+                kind: "read_document",
+                document: { logicalPath: file.path },
+            });
+            if (before.kind !== "read_document" ||
+                !before.ok ||
+                before.document.documentId !== result.document.documentId)
+                throw new WorldRevisionWorkspaceError(`Existing world-document identity changed: ${file.path}`);
+        }
+        return [
+            {
+                kind: previous === undefined ? "create" : "replace",
+                documentId: result.document.documentId,
+                stableShortRef: result.document.shortRef,
+                relativePath: file.path.slice("state/".length),
+                codec: result.document.codec,
+                expectedPreviousHash: previous === undefined ? null : sha256(previous.contents),
+                nextHash: sha256(file.contents),
+                canonicalNextBytes: file.contents,
+            },
+        ];
+    });
+}
+function controlFiles(files) {
+    return files
+        .filter(({ path }) => path.startsWith("control/"))
+        .map(({ path, contents, encoding }) => ({
+        path: path.slice("control/".length),
+        contents,
+        ...(encoding === undefined ? {} : { encoding }),
+    }));
+}
+function controlFingerprint(files) {
+    const hash = createHash("sha256");
+    for (const file of [...files].sort((left, right) => left.path.localeCompare(right.path)))
+        hash.update(file.path).update("\0").update(file.contents).update("\0");
+    return `sha256:${hash.digest("hex")}`;
+}
+function sha256(contents) {
+    return `sha256:${createHash("sha256").update(contents).digest("hex")}`;
+}
+function fileDiff(beforeFiles, afterFiles) {
+    const before = new Map(beforeFiles.map((file) => [file.path, file]));
+    const after = new Map(afterFiles.map((file) => [file.path, file]));
+    return [...new Set([...before.keys(), ...after.keys()])]
+        .sort()
+        .flatMap((path) => {
+        const left = before.get(path);
+        const right = after.get(path);
+        if (left?.contents === right?.contents &&
+            left?.encoding === right?.encoding)
+            return [];
+        return [
+            {
+                path,
+                kind: left === undefined
+                    ? "create"
+                    : right === undefined
+                        ? "delete"
+                        : "modify",
+                before: left?.contents ?? null,
+                after: right?.contents ?? null,
+            },
+        ];
+    });
+}
+function fileMatchesBefore(files, change) {
+    const current = files.find(({ path }) => path === change.path);
+    return change.before === null
+        ? current === undefined
+        : current?.contents === change.before && current.encoding === undefined;
+}
+function restoreBefore(files, change) {
+    const restored = new Map(files.map((file) => [file.path, structuredClone(file)]));
+    if (change.before === null)
+        restored.delete(change.path);
+    else
+        restored.set(change.path, { path: change.path, contents: change.before });
+    return [...restored.values()].sort((left, right) => left.path.localeCompare(right.path));
+}
+function inverseDiff(files, change) {
+    const before = files.find(({ path }) => path === change.path)?.contents ?? null;
+    return {
+        path: change.path,
+        kind: before === null ? "create" : change.before === null ? "delete" : "modify",
+        before,
+        after: change.before,
+    };
+}
+function cloneFiles(files) {
+    return files
+        .map((file) => structuredClone(file))
+        .sort((left, right) => left.path.localeCompare(right.path));
+}

package/dist/node/server/ConversationStream.js

@@ -0,0 +1,109 @@
+import { randomUUID } from "node:crypto";
+import { PassThrough } from "node:stream";
+import { conversationUpdate, } from "../protocol/conversationObservation.js";
+import { v1Protocol } from "../protocol/v1.js";
+/** A connection observes an independently owned Runtime task. No connection
+ * teardown path has access to the model's cancellation signal. */
+export function registerConversationStream(server, runtime) {
+    const connections = new Set();
+    server.addHook("preClose", () => {
+        for (const stream of connections)
+            stream.destroy();
+        return Promise.resolve();
+    });
+    server.get("/api/runtime/v1/events", (request, reply) => {
+        const { kind, id, sessionId } = request.query;
+        if ((kind !== "play" && kind !== "setting" && kind !== "revision") ||
+            typeof id !== "string" ||
+            !/^[A-Za-z0-9_-]{1,200}$/.test(id) ||
+            (sessionId !== undefined &&
+                (kind === "play" ||
+                    typeof sessionId !== "string" ||
+                    !/^[A-Za-z0-9_-]{1,200}$/.test(sessionId)))) {
+            return reply.code(400).send({ error: "Invalid conversation target" });
+        }
+        const target = kind === "play"
+            ? { kind, id }
+            : { kind, id, ...(sessionId === undefined ? {} : { sessionId }) };
+        const stream = new PassThrough();
+        connections.add(stream);
+        const epoch = randomUUID();
+        let sequence = 0;
+        let changes = 0;
+        let durableChanged = true;
+        let reading = false;
+        let previous = null;
+        let timer;
+        const write = (text) => {
+            if (stream.destroyed)
+                return;
+            // A slow renderer may reconnect from a snapshot; it cannot backpressure
+            // Provider execution or accumulate unbounded queued token events.
+            if (stream.readableLength + stream.writableLength > 1024 * 1024) {
+                stream.destroy();
+                return;
+            }
+            stream.write(text);
+        };
+        const read = async () => {
+            if (reading || stream.destroyed)
+                return;
+            reading = true;
+            const revision = changes;
+            const reload = durableChanged;
+            durableChanged = false;
+            try {
+                const next = await runtime.readConversation(target, reload ? undefined : (previous ?? undefined));
+                if (stream.destroyed)
+                    return;
+                // Subscribe-before-read plus a revision check closes the initialization
+                // and asynchronous snapshot race without replaying transient deltas.
+                if (revision !== changes)
+                    schedule();
+                if (previous === null ||
+                    JSON.stringify(previous) !== JSON.stringify(next)) {
+                    const update = conversationUpdate(previous, next);
+                    write(`id: ${epoch}:${++sequence}\nevent: observation\ndata: ${JSON.stringify({ protocol: v1Protocol, update })}\n\n`);
+                    previous = next;
+                }
+            }
+            catch (error) {
+                write(`event: failure\ndata: ${JSON.stringify({ message: error instanceof Error ? error.message : "Conversation observation failed" })}\n\n`);
+                stream.end();
+                cleanup();
+            }
+            finally {
+                reading = false;
+            }
+        };
+        const schedule = () => {
+            if (timer !== undefined || stream.destroyed)
+                return;
+            timer = setTimeout(() => {
+                timer = undefined;
+                void read();
+            }, 25);
+        };
+        const unsubscribe = runtime.subscribeConversation(target, (durable) => {
+            changes += 1;
+            durableChanged ||= durable;
+            schedule();
+        });
+        const heartbeat = setInterval(() => write(": heartbeat\n\n"), 15_000);
+        const cleanup = () => {
+            unsubscribe();
+            clearInterval(heartbeat);
+            clearTimeout(timer);
+            connections.delete(stream);
+        };
+        stream.on("close", cleanup);
+        reply.raw.on("close", () => stream.destroy());
+        write(": connected\nretry: 1000\n\n");
+        void read();
+        return reply
+            .header("Cache-Control", "no-store")
+            .header("X-Accel-Buffering", "no")
+            .type("text/event-stream; charset=utf-8")
+            .send(stream);
+    });
+}

package/dist/node/server/createServer.js

@@ -1,3 +1,4 @@
+import { registerConversationStream } from "./ConversationStream.js";
 import { readFile } from "node:fs/promises";
 import { join } from "node:path";
 import { PassThrough } from "node:stream";
@@ -27,7 +28,9 @@ function isPollingRequest(request) {
         return false;
     const runtimeRequest = body.request;
     return (isRecord(runtimeRequest) &&
-        (runtimeRequest.type === "setting-improvement.progress" ||
+        (runtimeRequest.type === "setting-improvement.read" ||
+            runtimeRequest.type === "setting-improvement.status" ||
+            runtimeRequest.type === "setting-improvement.overview" ||
             runtimeRequest.type === "play.chain.inspect"));
 }
 function isRuntimeApiRequest(request) {
@@ -77,6 +80,7 @@ export async function createServer(input) {
             .header("X-Frame-Options", "DENY");
         done(null, payload);
     });
+    registerConversationStream(server, input.runtime);
     server.get("/health", () => ({
         status: "ok",
         protocol: "narraeon.runtime/v1",