narraeon 0.7.10 → 0.8.1

package/dist/node/runtime/setting/FileNativeSettingImprovementStore.js

@@ -1,11 +1,14 @@
 import { validPromptCompilation, validLogicalMessages, validPromptTools, validPlayFollowup, validProviderKind, validCacheStrategy, validToolStrategy, } from "../prompt/PromptCompilationCodec.js";
 import { validAuthoringRequests, } from "../authoring/AuthoringRequestSnapshot.js";
 import { randomUUID } from "node:crypto";
+import { readGoogleHops, writeGoogleHop } from "../model/GoogleHopJournal.js";
 import { mkdir, open, readFile, readdir, rename, rm, stat, } from "node:fs/promises";
 import { linkExclusive } from "../FileNativeExclusiveLink.js";
 import { dirname, join } from "node:path";
 import { isDeepStrictEqual } from "node:util";
 import { fictionGameplay } from "../../protocol/gameplay.js";
+import { validGoogleInteractionsSteps } from "../model/ProviderExchangeState.js";
+import { hasDamagedGoogleSteps, withoutDamagedGoogleSteps, } from "../authoring/DamagedGoogleContinuation.js";
 import { ContentWorkspace } from "../content/ContentWorkspace.js";
 import { contentTreeFingerprint } from "../content/ContentTreeFingerprint.js";
 import { isPlayPresetBinding, } from "../play/FileNativePlayPresetStore.js";
@@ -28,19 +31,38 @@ export class FileNativeSettingImprovementStore {
     createId() {
         return `setting-${randomUUID()}`;
     }
+    get sessionScope() {
+        return this.#sessionsRoot;
+    }
+    recordGoogleHop(sessionId, requestId, hop) {
+        assertSessionId(sessionId);
+        return writeGoogleHop(join(this.#sessionsRoot, ".google-hops", sessionId), requestId, hop, () => stat(this.#path(sessionId)).then(() => true, () => false));
+    }
     async read(sessionId) {
         await this.#mutationTail;
-        return this.#read(sessionId);
+        try {
+            const session = await this.#read(sessionId);
+            if (hasDamagedGoogleSteps(session) &&
+                (await this.#damagedSummary(sessionId)))
+                throw new Error("The historical conversation has damaged Google continuation; start a fresh model context");
+            return session;
+        }
+        catch (error) {
+            if (await this.#damagedSummary(sessionId))
+                throw new Error("The historical conversation has damaged Google continuation; start a fresh model context", { cause: error });
+            throw error;
+        }
     }
     async findLatestByPackage(packageId) {
         const summaries = await this.listSummariesByPackage(packageId);
-        return summaries[0] === undefined
-            ? null
-            : this.read(summaries[0].sessionId);
+        const latest = summaries.find((summary) => !summary.damagedContinuation);
+        return latest === undefined ? null : this.read(latest.sessionId);
     }
     async findByCreationRequest(packageId, requestId) {
         const summaries = await this.listSummariesByPackage(packageId);
         const match = summaries.find(({ creationRequestId }) => creationRequestId === requestId);
+        if (match?.damagedContinuation)
+            throw new Error("The historical conversation has damaged Google continuation; start a fresh model context with a new request ID");
         return match === undefined ? null : this.read(match.sessionId);
     }
     async listSummariesByPackage(packageId) {
@@ -65,10 +87,65 @@ export class FileNativeSettingImprovementStore {
     save(session) {
         return this.#mutate(async () => {
             validateStoredSession(session);
+            await this.#preservePreviousSchema(session);
             await publishJson(this.#path(session.sessionId), session);
             await this.#recordSavedSession(session);
         });
     }
+    async preparePendingEdit(session) {
+        if (session.pendingEdit === undefined)
+            throw new Error("No pending setting conversation edit");
+        await publishJson(this.#pendingEditPath(session.packageId), {
+            schemaVersion: 1,
+            packageId: session.packageId,
+            sessionId: session.sessionId,
+        });
+        await this.save(session);
+    }
+    async recoverPendingEdits(packageId, lease) {
+        const path = this.#pendingEditPath(packageId);
+        let marker;
+        try {
+            marker = JSON.parse(await readFile(path, "utf8"));
+        }
+        catch (error) {
+            if (isNodeError(error) && error.code === "ENOENT")
+                return;
+            throw error;
+        }
+        if (!isRecord(marker) ||
+            !hasExactKeys(marker, ["schemaVersion", "packageId", "sessionId"]) ||
+            marker.schemaVersion !== 1 ||
+            marker.packageId !== packageId ||
+            typeof marker.sessionId !== "string")
+            throw new Error("Pending authoring edit marker is damaged");
+        assertSessionId(marker.sessionId);
+        const raw = JSON.parse(await readFile(this.#path(marker.sessionId), "utf8"));
+        const session = validateStoredSession(raw);
+        if (session.sessionId !== marker.sessionId ||
+            session.packageId !== packageId)
+            throw new Error("Pending edit identity does not match its package");
+        if (session.pendingEdit === undefined) {
+            await rm(path, { force: true });
+            return;
+        }
+        await this.finishPendingEdit(session, lease);
+    }
+    async finishPendingEdit(session, lease) {
+        const pending = session.pendingEdit;
+        if (pending === undefined)
+            throw new Error("No pending setting conversation edit");
+        const fingerprint = contentTreeFingerprint(lease.package.files);
+        if (fingerprint === pending.beforeFingerprint &&
+            fingerprint !== pending.afterFingerprint)
+            await lease.replace(pending.afterFiles);
+        else if (fingerprint !== pending.afterFingerprint)
+            throw new Error("Cannot recover the conversation edit because the content package changed; no further changes were made");
+        await publishJson(this.#path(session.sessionId), pending.next);
+        await rm(this.#pendingEditPath(session.packageId), { force: true });
+        await this.#recordSavedSession(pending.next);
+        return structuredClone(pending.next);
+    }
     saveIfUnchanged(expected, replacement) {
         return this.#mutate(async () => {
             if (expected.sessionId !== replacement.sessionId)
@@ -77,6 +154,7 @@ export class FileNativeSettingImprovementStore {
             const current = await this.#read(expected.sessionId);
             if (!isDeepStrictEqual(current, expected))
                 return false;
+            await this.#preservePreviousSchema(replacement);
             await publishJson(this.#path(replacement.sessionId), replacement);
             await this.#recordSavedSession(replacement);
             return true;
@@ -88,9 +166,15 @@ export class FileNativeSettingImprovementStore {
             if (session.packageId !== packageId)
                 throw new Error("The setting-improvement conversation does not belong to this content package");
             await rm(this.#path(sessionId));
+            await rm(join(this.#sessionsRoot, ".google-hops", sessionId), {
+                recursive: true,
+                force: true,
+            });
             await rm(join(this.#sessionsRoot, `${sessionId}.schema-v1.json`), {
                 force: true,
             });
+            for (const version of [2, 3, 4])
+                await rm(join(this.#sessionsRoot, `${sessionId}.schema-v${version}.json`), { force: true });
             await this.#recordDeletedSession(session);
         });
     }
@@ -117,8 +201,20 @@ export class FileNativeSettingImprovementStore {
             for (const entry of entries) {
                 if (!entry.isFile() || !sessionFilePattern.test(entry.name))
                     continue;
-                const session = await this.#read(entry.name.slice(0, -5));
-                summaries.set(session.sessionId, summarizeStoredSettingImprovementSession(session));
+                const sessionId = entry.name.slice(0, -5);
+                try {
+                    const session = await this.#read(sessionId);
+                    summaries.set(session.sessionId, (!hasDamagedGoogleSteps(session)
+                        ? null
+                        : await this.#damagedSummary(sessionId)) ??
+                        summarizeStoredSettingImprovementSession(session));
+                }
+                catch (error) {
+                    const summary = await this.#damagedSummary(sessionId);
+                    if (summary === null)
+                        throw error;
+                    summaries.set(sessionId, summary);
+                }
             }
             const completedRevision = await directoryRevision(this.#sessionsRoot);
             if (completedRevision !== revision)
@@ -127,6 +223,41 @@ export class FileNativeSettingImprovementStore {
             return;
         }
     }
+    async #damagedSummary(sessionId) {
+        let session;
+        try {
+            const raw = JSON.parse(await readFile(this.#path(sessionId), "utf8"));
+            const copy = withoutDamagedGoogleSteps(raw);
+            if (copy === null) {
+                if (hasDamagedGoogleSteps(raw))
+                    throw new Error("Cannot safely identify the damaged historical conversation; settlement or identity is unverified");
+                return null;
+            }
+            session = validateStoredSession(copy);
+        }
+        catch (error) {
+            if (error instanceof Error &&
+                error.message.startsWith("Cannot safely identify"))
+                throw error;
+            return null;
+        }
+        if (session.sessionId !== sessionId ||
+            session.googleIdentity?.sessionId !== sessionId ||
+            session.googleIdentity.packageId !== session.packageId ||
+            session.googleIdentity.creationRequestId !== session.creationRequestId ||
+            (session.requests?.[0]?.requestId !== session.creationRequestId &&
+                !(session.schemaVersion === 5 &&
+                    session.editReceipts?.some((receipt) => receipt.requestId === session.requests?.[0]?.requestId))) ||
+            !/^package-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u.test(session.packageId) ||
+            !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$/u.test(session.creationRequestId) ||
+            hasPendingToolCalls(session))
+            throw new Error("Cannot safely identify the damaged historical conversation; its owner or creation identity is unverified");
+        return {
+            ...summarizeStoredSettingImprovementSession(session),
+            runStatus: "interrupted",
+            damagedContinuation: true,
+        };
+    }
     async #recordSavedSession(session) {
         if (this.#summaryIndex !== null)
             this.#cacheSummary(summarizeStoredSettingImprovementSession(session));
@@ -219,8 +350,32 @@ export class FileNativeSettingImprovementStore {
         const session = validateStoredSession(value);
         if (session.sessionId !== sessionId)
             throw new Error("Setting-improvement conversation identity does not match its file");
+        for (const request of session.requests ?? [])
+            if (request.wireRequest !== undefined)
+                request.wireRequest = await readGoogleHops(join(this.#sessionsRoot, ".google-hops", sessionId), request.requestId, request.wireRequest);
         return session;
     }
+    async #preservePreviousSchema(next) {
+        if (next.schemaVersion < 4)
+            return;
+        let source;
+        try {
+            source = await readFile(this.#path(next.sessionId), "utf8");
+        }
+        catch (error) {
+            if (isNodeError(error) && error.code === "ENOENT")
+                return;
+            throw error;
+        }
+        const previous = validateStoredSession(JSON.parse(source));
+        if (previous.sessionId !== next.sessionId)
+            throw new Error("Setting-improvement conversation identity does not match its file");
+        if (previous.schemaVersion === next.schemaVersion)
+            return;
+        if (previous.schemaVersion > next.schemaVersion)
+            throw new Error("Cannot downgrade a setting-improvement conversation");
+        await publishImmutableSource(join(this.#sessionsRoot, `${next.sessionId}.schema-v${previous.schemaVersion}.json`), source);
+    }
     async #migrateLegacy(sessionId, source, value) {
         if (!validLegacySession(value))
             throw new Error("Legacy setting-improvement conversation is damaged");
@@ -267,6 +422,11 @@ export class FileNativeSettingImprovementStore {
         assertSessionId(sessionId);
         return join(this.#sessionsRoot, `${sessionId}.json`);
     }
+    #pendingEditPath(packageId) {
+        if (!/^package-[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/u.test(packageId))
+            throw new Error("Invalid content package ID");
+        return join(this.#sessionsRoot, ".pending-edits", `${packageId}.json`);
+    }
 }
 function compareSummaries(left, right) {
     return (right.updatedAt - left.updatedAt ||
@@ -309,7 +469,8 @@ export function summarizeStoredSettingImprovementSession(session) {
         changedFileCount: changedPaths.size,
         exchange: session.exchange,
         toolCalls: session.toolCalls,
-        needsRecovery: session.pendingSettlement !== null ||
+        needsRecovery: session.pendingEdit !== undefined ||
+            session.pendingSettlement !== null ||
             session.runStatus === "running" ||
             hasPendingToolCalls(session),
     };
@@ -362,15 +523,49 @@ function validateStoredSession(value) {
             "authorization",
             "pendingSettlement",
             "legacyDraft",
-        ], ["contentPackageTitle", "playPreset", "requests"]) ||
-        (value.schemaVersion !== 2 && value.schemaVersion !== 3) ||
+        ], [
+            "contentPackageTitle",
+            "playPreset",
+            "requests",
+            "googleOutcomeUnknown",
+            "googleIdentity",
+            "retryDispatch",
+            "pendingEdit",
+            "editReceipts",
+        ]) ||
+        (value.schemaVersion !== 2 &&
+            value.schemaVersion !== 3 &&
+            value.schemaVersion !== 4 &&
+            value.schemaVersion !== 5) ||
+        (value.schemaVersion !== 4 &&
+            value.schemaVersion !== 5 &&
+            value.retryDispatch !== undefined) ||
+        (value.schemaVersion !== 5 && value.pendingEdit !== undefined) ||
+        (value.schemaVersion !== 5 && value.editReceipts !== undefined) ||
+        (value.editReceipts !== undefined &&
+            !validEditReceipts(value.editReceipts)) ||
+        (value.pendingEdit !== undefined &&
+            !validPendingEdit(value.pendingEdit, value)) ||
         (value.schemaVersion === 2
             ? value.requests !== undefined
-            : !validAuthoringRequests(value.requests, validPromptCompilation, {
-                modelItems: value.modelItems,
-                activeRequestId: value.activeRequestId,
-                completedRequestIds: value.completedRequestIds,
-            })) ||
+            : (value.schemaVersion === 4 || value.schemaVersion === 5) &&
+                Array.isArray(value.requests) &&
+                value.requests.length === 0 &&
+                ((value.schemaVersion === 5 && value.pendingEdit !== undefined) ||
+                    (value.runStatus === "ready" &&
+                        Array.isArray(value.modelItems) &&
+                        value.modelItems.length === 0 &&
+                        Array.isArray(value.messages) &&
+                        value.messages.length === 0 &&
+                        value.activeRequestId === null &&
+                        Array.isArray(value.completedRequestIds) &&
+                        value.completedRequestIds.length === 0))
+                ? false
+                : !validAuthoringRequests(value.requests, validPromptCompilation, {
+                    modelItems: value.modelItems,
+                    activeRequestId: value.activeRequestId,
+                    completedRequestIds: value.completedRequestIds,
+                })) ||
         typeof value.sessionId !== "string" ||
         typeof value.packageId !== "string" ||
         !validOptionalTitle(value.contentPackageTitle) ||
@@ -381,6 +576,27 @@ function validateStoredSession(value) {
         typeof value.creationRequestId !== "string" ||
         !validPromptCompilation(value.bootstrap) ||
         !validModelBinding(value.modelBinding) ||
+        (value.retryDispatch !== undefined &&
+            (!isRecord(value.retryDispatch) ||
+                !hasExactKeys(value.retryDispatch, [
+                    "requestId",
+                    "exchange",
+                    "modelItemCount",
+                    "state",
+                ]) ||
+                typeof value.retryDispatch.requestId !== "string" ||
+                !validCount(value.retryDispatch.exchange) ||
+                !validCount(value.retryDispatch.modelItemCount) ||
+                !["dispatching", "retryable", "blocked"].includes(String(value.retryDispatch.state)) ||
+                (value.schemaVersion !== 4 && value.schemaVersion !== 5) ||
+                lastRequestId(value.requests) !== value.retryDispatch.requestId ||
+                (value.runStatus !== "running" && value.runStatus !== "interrupted") ||
+                value.pendingSettlement !== null ||
+                !validCount(value.exchange) ||
+                (value.retryDispatch.exchange !== value.exchange &&
+                    value.retryDispatch.exchange !== Number(value.exchange) + 1) ||
+                !Array.isArray(value.modelItems) ||
+                value.retryDispatch.modelItemCount !== value.modelItems.length)) ||
         (value.playPreset !== undefined &&
             !isPlayPresetBinding(value.playPreset)) ||
         !Array.isArray(value.modelItems) ||
@@ -392,18 +608,102 @@ function validateStoredSession(value) {
             typeof value.activeRequestId !== "string") ||
         !Array.isArray(value.completedRequestIds) ||
         (value.lastFailure !== null && typeof value.lastFailure !== "string") ||
+        (value.googleOutcomeUnknown !== undefined &&
+            (typeof value.googleOutcomeUnknown !== "boolean" ||
+                !isRecord(value.modelBinding) ||
+                value.modelBinding.provider !== "google_interactions")) ||
+        (value.googleIdentity !== undefined &&
+            (!isRecord(value.googleIdentity) ||
+                !hasExactKeys(value.googleIdentity, [
+                    "sessionId",
+                    "packageId",
+                    "creationRequestId",
+                ]) ||
+                typeof value.googleIdentity.sessionId !== "string" ||
+                typeof value.googleIdentity.packageId !== "string" ||
+                typeof value.googleIdentity.creationRequestId !== "string" ||
+                !isRecord(value.modelBinding) ||
+                value.modelBinding.provider !== "google_interactions")) ||
         !validAuthorization(value.authorization) ||
         !validPendingSettlement(value.pendingSettlement) ||
         !validPendingSettlementForTranscript(value.pendingSettlement, value.modelItems, value.runStatus) ||
         !validLegacyDraft(value.legacyDraft))
         throw new Error("Setting-improvement conversation does not match its durable schema");
     assertSessionId(value.sessionId);
+    if (isRecord(value.googleIdentity) &&
+        (value.googleIdentity.sessionId !== value.sessionId ||
+            value.googleIdentity.packageId !== value.packageId ||
+            value.googleIdentity.creationRequestId !== value.creationRequestId))
+        throw new Error("Cannot safely identify the historical conversation; its owner or creation identity is inconsistent");
     if (!value.completedRequestIds.every((id) => typeof id === "string") ||
         !value.messages.every(validMessage) ||
         !value.modelItems.every(validModelItem))
         throw new Error("Setting-improvement transcript is damaged");
     return structuredClone(value);
 }
+function lastRequestId(requests) {
+    if (!Array.isArray(requests))
+        return undefined;
+    const last = requests.at(-1);
+    return isRecord(last) ? last.requestId : undefined;
+}
+function validEditReceipts(value) {
+    if (!Array.isArray(value))
+        return false;
+    const ids = new Set();
+    return value.every((entry) => {
+        if (!isRecord(entry) ||
+            !hasExactKeys(entry, [
+                "requestId",
+                "messageId",
+                "message",
+                "filePolicy",
+            ]) ||
+            typeof entry.requestId !== "string" ||
+            !/^[A-Za-z0-9][A-Za-z0-9._:-]{0,159}$/u.test(entry.requestId) ||
+            ids.has(entry.requestId) ||
+            typeof entry.messageId !== "string" ||
+            typeof entry.message !== "string" ||
+            entry.message.trim().length === 0 ||
+            entry.message.length > 64 * 1024 ||
+            (entry.filePolicy !== "keep" && entry.filePolicy !== "revert"))
+            return false;
+        ids.add(entry.requestId);
+        return true;
+    });
+}
+function validPendingEdit(value, original) {
+    if (!isRecord(value) ||
+        !hasExactKeys(value, [
+            "beforeFingerprint",
+            "afterFingerprint",
+            "afterFiles",
+            "next",
+        ]) ||
+        typeof value.beforeFingerprint !== "string" ||
+        !/^[0-9a-f]{64}$/u.test(value.beforeFingerprint) ||
+        typeof value.afterFingerprint !== "string" ||
+        !/^[0-9a-f]{64}$/u.test(value.afterFingerprint) ||
+        !validContentTreeFiles(value.afterFiles) ||
+        contentTreeFingerprint(value.afterFiles) !== value.afterFingerprint ||
+        !isRecord(value.next) ||
+        value.next.pendingEdit !== undefined ||
+        value.next.schemaVersion !== 5 ||
+        value.next.sessionId !== original.sessionId ||
+        value.next.packageId !== original.packageId ||
+        value.next.creationRequestId !== original.creationRequestId ||
+        value.next.runStatus !== "running" ||
+        value.next.activeRequestId === null ||
+        original.pendingSettlement !== null)
+        return false;
+    try {
+        validateStoredSession(value.next);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
 function migrateLegacySession(value, applyOutcome) {
     if (!validLegacySession(value))
         throw new Error("Legacy setting-improvement conversation is damaged");
@@ -823,6 +1123,9 @@ function validModelToolCall(value) {
 function validProviderState(value) {
     if (!isRecord(value))
         return false;
+    if (value.protocol === "google_interactions")
+        return (hasExactKeys(value, ["protocol", "steps"]) &&
+            validGoogleInteractionsSteps(value.steps));
     if (value.protocol === "chat_completions")
         return (hasExactKeys(value, ["protocol", "assistantMessage"]) &&
             Object.hasOwn(value, "assistantMessage"));
@@ -850,7 +1153,10 @@ function validModelUsage(value) {
         "totalTokens",
     ];
     if (!isRecord(value) ||
-        !hasExactKeys(value, [...fields, "provenance"]) ||
+        !hasExactKeys(value, [...fields, "provenance"], ["providerRaw", "accounting"]) ||
+        (value.providerRaw !== undefined && !isRecord(value.providerRaw)) ||
+        (value.accounting !== undefined &&
+            value.accounting !== "google_interactions") ||
         !fields.every((field) => validNullableCount(value[field])) ||
         !isRecord(value.provenance) ||
         !hasExactKeys(value.provenance, fields))