narraeon 0.3.1 → 0.5.0

package/dist/node/runtime/prompt/WorldMaintenanceReport.js

@@ -0,0 +1,251 @@
+import { worldWritePositionText } from "../../protocol/worldMaintenance.js";
+import { parseDocument } from "yaml";
+export function worldMaintenanceSettings(frame) {
+    const slotIds = new Set();
+    for (const entry of Array.isArray(frame.context) ? frame.context : []) {
+        const slot = record(entry) && record(entry.slot) ? entry.slot : null;
+        if (slot === null ||
+            (slot.id === undefined && slot.advisoryBytes === undefined))
+            continue;
+        if (typeof slot.id !== "string" ||
+            !/^[a-z][a-z0-9_-]{0,63}$/u.test(slot.id) ||
+            slotIds.has(slot.id))
+            throw new Error("Advisory slots require unique stable slot.id values");
+        slotIds.add(slot.id);
+        if (slot.advisoryBytes !== undefined && !positiveBytes(slot.advisoryBytes))
+            throw new Error("slot.advisoryBytes must be a positive UTF-8 byte limit");
+    }
+    const value = frame.maintenance;
+    if (value === undefined)
+        return { documents: {} };
+    if (!record(value) ||
+        Object.keys(value).some((key) => key !== "documents" && key !== "clock"))
+        throw new Error("world frame.maintenance accepts only documents and clock");
+    const documents = value.documents ?? {};
+    if (!record(documents) ||
+        Object.entries(documents).some(([key, limit]) => !/^@[a-z][a-z0-9-]*$/u.test(key) || !positiveBytes(limit)))
+        throw new Error("maintenance.documents maps document @short-refs to positive advisory UTF-8 byte limits");
+    if (value.clock === undefined)
+        return { documents: documents };
+    const clock = value.clock;
+    if (!record(clock) ||
+        Object.keys(clock).length !== 2 ||
+        typeof clock.document !== "string" ||
+        !clock.document.startsWith("@") ||
+        !record(clock.locator) ||
+        Object.keys(clock.locator).length !== 1)
+        throw new Error("maintenance.clock requires an exact document @short-ref and locator");
+    const path = clock.locator.yaml ?? clock.locator.markdown;
+    if (!Array.isArray(path) ||
+        path.length === 0 ||
+        !path.every((part) => typeof part === "string") ||
+        (!Object.hasOwn(clock.locator, "yaml") &&
+            !Object.hasOwn(clock.locator, "markdown")))
+        throw new Error("maintenance.clock.locator must name an exact YAML or Markdown node");
+    return {
+        documents: documents,
+        clock: {
+            document: clock.document,
+            locator: Object.hasOwn(clock.locator, "yaml")
+                ? { yaml: path }
+                : { markdown: path },
+        },
+    };
+}
+export function listWorldDocuments(snapshot) {
+    const result = [];
+    const pending = [""];
+    const visited = new Set();
+    while (pending.length > 0) {
+        const directory = pending.shift();
+        if (visited.has(directory))
+            continue;
+        visited.add(directory);
+        let cursor = null;
+        do {
+            const page = snapshot.query({
+                kind: "catalog",
+                directory,
+                limit: 100,
+                cursor,
+            });
+            if (page.kind !== "catalog")
+                throw new Error("World-document inventory could not be read");
+            for (const entry of page.entries) {
+                if (entry.kind === "directory") {
+                    if (!entry.logicalPath.startsWith(`${snapshot.logicalRoot}/`))
+                        throw new Error("World-document inventory escaped its logical root");
+                    pending.push(entry.logicalPath.slice(snapshot.logicalRoot.length + 1));
+                }
+                else if (entry.document !== undefined)
+                    result.push(entry.document);
+            }
+            cursor = page.page.nextCursor;
+        } while (cursor !== null);
+    }
+    return result.sort((a, b) => a.logicalPath.localeCompare(b.logicalPath));
+}
+export function measureWorldMaterials(snapshot, frame, materials, facts = {}) {
+    const settings = worldMaintenanceSettings(frame);
+    const contributions = new Map();
+    for (const material of materials)
+        for (const contribution of material.contributions ?? [])
+            contributions.set(contribution.shortRef, (contributions.get(contribution.shortRef) ?? 0) + contribution.bytes);
+    const documents = listWorldDocuments(snapshot).map((document) => {
+        const read = snapshot.query({
+            kind: "read_document",
+            document: { shortRef: document.shortRef },
+        });
+        if (read.kind !== "read_document")
+            throw new Error(`Document @${document.shortRef} could not be measured`);
+        const bodyBytes = Buffer.byteLength(read.body, "utf8");
+        const fact = facts[document.shortRef] ?? {
+            baselineKind: "unavailable",
+            baselineBytes: null,
+            lastBodyWrite: null,
+            lastMetadataWrite: null,
+            bodyChangedContexts: 0,
+            metadataChangedContexts: 0,
+        };
+        const advisoryBytes = settings.documents[`@${document.shortRef}`] ?? null;
+        return {
+            ...fact,
+            shortRef: document.shortRef,
+            title: document.title,
+            bodyBytes,
+            injectedBytes: contributions.get(document.shortRef) ?? 0,
+            growthBytes: fact.baselineBytes === null ? null : bodyBytes - fact.baselineBytes,
+            growthRatio: fact.baselineBytes === null || fact.baselineBytes === 0
+                ? null
+                : bodyBytes / fact.baselineBytes,
+            advisoryBytes,
+            overAdvisory: advisoryBytes !== null && bodyBytes > advisoryBytes,
+        };
+    });
+    const slots = (Array.isArray(frame.context) ? frame.context : []).flatMap((entry) => {
+        const slot = record(entry) && record(entry.slot) ? entry.slot : null;
+        if (slot === null ||
+            (slot.id === undefined && slot.advisoryBytes === undefined))
+            return [];
+        // worldMaintenanceSettings validated all declared slot ids and limits.
+        const id = slot.id;
+        const bytes = materials
+            .filter(({ slotId }) => slotId === slot.id)
+            .reduce((sum, { markdown }) => sum + Buffer.byteLength(markdown, "utf8"), 0);
+        const advisoryBytes = typeof slot.advisoryBytes === "number" ? slot.advisoryBytes : null;
+        return [
+            {
+                id,
+                bytes,
+                advisoryBytes,
+                overAdvisory: advisoryBytes !== null && bytes > advisoryBytes,
+                documents: [
+                    ...new Set(materials
+                        .filter(({ slotId }) => slotId === slot.id)
+                        .flatMap(({ contributions: items }) => items?.map(({ shortRef }) => shortRef) ?? [])),
+                ],
+            },
+        ];
+    });
+    return {
+        unit: "utf8_bytes",
+        worldMaterialsBytes: 0,
+        checkpointHistoryBytes: materials
+            .filter(({ source }) => source.startsWith("runtime:checkpoint-history:"))
+            .reduce((sum, { markdown }) => sum + Buffer.byteLength(markdown, "utf8"), 0),
+        authorInstructionBytes: 0,
+        runtimeInstructionBytes: 0,
+        toolDefinitionBytes: 0,
+        documents,
+        slots,
+    };
+}
+export function refreshMaintenanceTotals(compilation) {
+    const report = compilation.maintenance;
+    if (report === undefined)
+        return;
+    const bytes = (role) => compilation.logicalMessages
+        .filter((message) => message.role === role)
+        .reduce((sum, { markdown }) => sum + Buffer.byteLength(markdown, "utf8"), 0);
+    report.worldMaterialsBytes = bytes("world_context");
+    report.authorInstructionBytes = bytes("author_instruction");
+    report.runtimeInstructionBytes = bytes("runtime_system");
+    report.toolDefinitionBytes = Buffer.byteLength(JSON.stringify(compilation.tools), "utf8");
+}
+export function renderWriteSizeAdvice(report, shortRef, previousBodyBytes, locale) {
+    const document = report.documents.find((entry) => entry.shortRef === shortRef);
+    if (document === undefined)
+        return "";
+    const zh = locale === "zh-CN";
+    const lines = [];
+    if (document.overAdvisory) {
+        const delta = previousBodyBytes === null
+            ? zh
+                ? "本次新建"
+                : "new document"
+            : `${zh ? "本次变化" : "change"} ${document.bodyBytes - previousBodyBytes >= 0 ? "+" : ""}${document.bodyBytes - previousBodyBytes}`;
+        lines.push(zh
+            ? `整理提示：正文 ${document.bodyBytes} UTF-8 字节，建议上限 ${document.advisoryBytes}；${delta}。`
+            : `Maintenance advice: body ${document.bodyBytes} UTF-8 bytes, advisory limit ${document.advisoryBytes}; ${delta}.`);
+    }
+    for (const slot of report.slots.filter((entry) => entry.overAdvisory && entry.documents.includes(shortRef)))
+        lines.push(zh
+            ? `材料槽 ${slot.id}：${slot.bytes} UTF-8 字节，建议上限 ${slot.advisoryBytes}。`
+            : `Material slot ${slot.id}: ${slot.bytes} UTF-8 bytes, advisory limit ${slot.advisoryBytes}.`);
+    return lines.join("\n");
+}
+export function readDeclaredWorldClock(snapshot) {
+    const source = snapshot.files.find(({ path }) => path === "control/frame.yaml")?.contents;
+    if (source === undefined)
+        return undefined;
+    const parsed = parseDocument(source);
+    if (parsed.errors.length > 0)
+        return undefined;
+    const frame = parsed.toJS();
+    if (!record(frame))
+        return undefined;
+    const clock = worldMaintenanceSettings(frame).clock;
+    if (clock === undefined)
+        return undefined;
+    const selected = snapshot.query({
+        kind: "select_node",
+        document: { shortRef: clock.document.slice(1) },
+        locator: clock.locator,
+    });
+    if (selected.kind !== "select_node")
+        return undefined;
+    if (selected.node.codec === "markdown")
+        return selected.node.markdown;
+    const value = selected.node.value;
+    return typeof value === "string" ||
+        typeof value === "number" ||
+        typeof value === "boolean"
+        ? String(value)
+        : undefined;
+}
+function positiveBytes(value) {
+    return Number.isSafeInteger(value) && Number(value) > 0;
+}
+function record(value) {
+    return value !== null && typeof value === "object" && !Array.isArray(value);
+}
+export function renderDocumentWritePosition(shortRef, fact, locale) {
+    if (fact === undefined)
+        return `${locale === "zh-CN" ? "最近写入位置" : "Last committed write"} @${shortRef}: ${locale === "zh-CN" ? "未记录" : "not recorded"}`;
+    return locale === "zh-CN"
+        ? `最近写入位置 @${shortRef}：\n正文：${worldWritePositionText(fact.lastBodyWrite, locale)}\n元信息：${worldWritePositionText(fact.lastMetadataWrite, locale)}`
+        : `Last committed write @${shortRef}:\nBody: ${worldWritePositionText(fact.lastBodyWrite, locale)}\nMetadata: ${worldWritePositionText(fact.lastMetadataWrite, locale)}`;
+}
+export function renderWorldMaintenanceReport(report, locale) {
+    if (report === undefined)
+        return "";
+    const zh = locale === "zh-CN";
+    return [
+        zh
+            ? "# 文档整理报告（UTF-8 字节；超限仅提示）"
+            : "# Document maintenance (UTF-8 bytes; advisory only)",
+        `${zh ? "世界材料 / 其中检查点历史 / 作者提示 / Runtime 提示 / 工具定义" : "World materials / checkpoint history subset / author instructions / Runtime instructions / tool definitions"}: ${report.worldMaterialsBytes} / ${report.checkpointHistoryBytes} / ${report.authorInstructionBytes} / ${report.runtimeInstructionBytes} / ${report.toolDefinitionBytes}`,
+        ...report.documents.map((document) => `@${document.shortRef}: ${zh ? "正文 / 注入贡献 / 建议上限 / 基准 / 增长" : "body / injection contribution / advisory / baseline / growth"}: ${document.bodyBytes} / ${document.injectedBytes} / ${document.advisoryBytes ?? "—"} / ${document.baselineBytes ?? "—"} / ${document.growthBytes ?? "—"}${document.overAdvisory ? (zh ? "（超过建议）" : " (over advisory)") : ""}`),
+        ...report.slots.map((slot) => `${zh ? "材料槽" : "Slot"} ${slot.id}: ${slot.bytes} / ${slot.advisoryBytes ?? "—"}${slot.overAdvisory ? (zh ? "（超过建议）" : " (over advisory)") : ""}`),
+    ].join("\n");
+}

package/dist/node/runtime/prompt/WorldMaterialCoverage.js

@@ -0,0 +1,83 @@
+/** Body scopes and catalog summaries are independent kinds of exposure. */
+export function worldMaterialCoverage(coverage) {
+    const fullInjected = new Set();
+    const nodeInjected = new Set();
+    const catalogSummary = new Set();
+    const injectedSelections = coverage.flatMap((entry) => {
+        for (const ref of entry.catalogEntries ?? [])
+            catalogSummary.add(ref);
+        const scope = entry.readAuthorization;
+        if (scope === undefined)
+            return [];
+        (scope.locator === null ? fullInjected : nodeInjected).add(scope.shortRef);
+        return [scope];
+    });
+    return { fullInjected, nodeInjected, catalogSummary, injectedSelections };
+}
+export function renderFreshContextCoverage(snapshot, coverage, shortRef, locale = "en") {
+    const exposure = worldMaterialCoverage(coverage);
+    const full = exposure.fullInjected.has(shortRef);
+    const nodes = exposure.nodeInjected.has(shortRef);
+    const summary = exposure.catalogSummary.has(shortRef);
+    const zh = locale === "zh-CN";
+    const scope = full
+        ? zh
+            ? "完整正文"
+            : "full body"
+        : nodes
+            ? zh
+                ? "指定节点"
+                : "selected nodes"
+            : summary
+                ? zh
+                    ? "仅目录摘要"
+                    : "catalog summary only"
+                : zh
+                    ? "不直接注入，需要按需读取"
+                    : "not directly injected; read on demand";
+    const lines = [
+        `${zh ? "如果现在新开上下文" : "If a fresh context started now"}: ${scope}${nodes && !full && summary ? (zh ? "，以及目录摘要" : " and catalog summary") : ""}`,
+    ];
+    if (!full && nodes) {
+        const scopes = exposure.injectedSelections.filter((entry) => entry.shortRef === shortRef && entry.locator !== null);
+        lines.push(`${zh ? "节点范围" : "Node scopes"}: ${scopes.map(({ locator }) => JSON.stringify(locator)).join(", ")}`);
+    }
+    if (!full && summary) {
+        const read = snapshot.query({
+            kind: "read_document",
+            document: { shortRef },
+        });
+        if (read.kind === "read_document")
+            lines.push(`${zh ? "摘要" : "Summary"}: ${read.document.summary}`);
+    }
+    if (!full && !nodes && !summary) {
+        const references = exposure.injectedSelections.flatMap((selection) => {
+            const read = snapshot.query({
+                kind: "read_document",
+                document: { shortRef: selection.shortRef },
+            });
+            const locator = selection.locator ??
+                (read.kind === "read_document" && read.codec === "yaml"
+                    ? { yaml: [] }
+                    : null);
+            if (locator === null)
+                return [];
+            const node = snapshot.query({
+                kind: "select_node",
+                document: { shortRef: selection.shortRef },
+                locator,
+            });
+            return node.kind === "select_node" &&
+                node.references.some(({ target }) => target.shortRef === shortRef)
+                ? [`@${selection.shortRef}`]
+                : [];
+        });
+        if (references.length > 0)
+            lines.push(`${zh ? "已注入材料中的直接引用来自" : "Direct reference in injected material"}: ${[...new Set(references)].join(", ")}`);
+        else if (coverage.some(({ status }) => status === "paged_catalog"))
+            lines.push(zh
+                ? "目录仍有后续页，可使用 state_list 继续查找。"
+                : "Catalogs have further pages; use state_list to continue discovery.");
+    }
+    return lines.join("\n");
+}

package/dist/node/runtime/prompt/WorldPromptDiagnostics.js

@@ -0,0 +1,185 @@
+/** Diagnostic comparison only: never edits the compiler or Provider cache routing. */
+export function comparePromptPrefixes(previous, current) {
+    const previousEncoding = previous?.encoding;
+    const currentEncoding = current.encoding;
+    const beforeBlocks = previous === null ? [] : sourceBlocks(previous.bootstrap);
+    const afterBlocks = sourceBlocks(current.bootstrap);
+    const before = new Map(beforeBlocks.map((block) => [block.key, block]));
+    const after = new Map(afterBlocks.map((block) => [block.key, block]));
+    const changedSources = [];
+    if (previous !== null)
+        for (const key of new Set([...before.keys(), ...after.keys()])) {
+            const left = before.get(key);
+            const right = after.get(key);
+            const change = left === undefined
+                ? "added"
+                : right === undefined
+                    ? "removed"
+                    : left.markdown !== right.markdown
+                        ? "changed"
+                        : left.position !== right.position
+                            ? "moved"
+                            : null;
+            if (change !== null)
+                changedSources.push({
+                    source: (right ?? left).source,
+                    role: (right ?? left).role,
+                    change,
+                    previousPosition: left?.position ?? null,
+                    currentPosition: right?.position ?? null,
+                    previousBytes: left?.bytes ?? 0,
+                    currentBytes: right?.bytes ?? 0,
+                });
+        }
+    const previousText = previous === null ? "" : promptText(previous.bootstrap);
+    const currentText = promptText(current.bootstrap);
+    const prefix = commonBytes(previousText, currentText);
+    const completeEncoding = previousEncoding != null && currentEncoding != null;
+    const previousBody = previousEncoding?.body;
+    const currentBody = currentEncoding?.body;
+    const beforeKey = cacheKey(previousBody);
+    const afterKey = cacheKey(currentBody);
+    const suggestions = [];
+    for (const changed of changedSources) {
+        if (changed.role !== "world_context" ||
+            changed.change !== "changed" ||
+            changed.currentPosition === null)
+            continue;
+        const stableLater = afterBlocks.find((block) => block.role === "world_context" &&
+            block.position > changed.currentPosition &&
+            block.bytes >= changed.currentBytes &&
+            before.get(block.key)?.markdown === block.markdown);
+        if (stableLater !== undefined)
+            suggestions.push({
+                source: changed.source,
+                afterSource: stableLater.source,
+            });
+    }
+    return {
+        logical: previous === null
+            ? null
+            : {
+                commonPrefixBytes: prefix,
+                previousBytes: bytes(previousText),
+                currentBytes: bytes(currentText),
+                firstChangedSource: firstChangedSource(current.bootstrap, prefix),
+                changedSources,
+                currentOrder: afterBlocks.map(({ source, role, position, bytes }) => ({
+                    source,
+                    role,
+                    position,
+                    bytes,
+                })),
+            },
+        encoding: {
+            previous: previousEncoding == null ? null : summarizeEncoding(previousEncoding),
+            current: currentEncoding == null ? null : summarizeEncoding(currentEncoding),
+            commonJsonPrefixBytes: completeEncoding
+                ? commonBytes(JSON.stringify(previousBody), JSON.stringify(currentBody))
+                : null,
+            firstChangedPath: completeEncoding
+                ? firstChangedPath(previousBody, currentBody)
+                : null,
+            cacheKey: !completeEncoding
+                ? "unavailable"
+                : beforeKey === undefined && afterKey === undefined
+                    ? "absent"
+                    : beforeKey === afterKey
+                        ? "same"
+                        : "different",
+        },
+        providerUsage: {
+            previous: previous?.usage ?? null,
+            current: current.usage ?? null,
+        },
+        suggestions,
+        cacheBenefit: "not_measured",
+    };
+}
+function sourceBlocks(bootstrap) {
+    const seen = new Map();
+    return bootstrap.logicalMessages
+        .flatMap(({ role, blocks }) => blocks.map(({ source, markdown }) => ({ role, source, markdown })))
+        .map((block, index) => {
+        const identity = `${block.role}\0${block.source}`;
+        const occurrence = seen.get(identity) ?? 0;
+        seen.set(identity, occurrence + 1);
+        return {
+            ...block,
+            key: `${identity}\0${occurrence}`,
+            position: index + 1,
+            bytes: bytes(block.markdown),
+        };
+    });
+}
+function promptText(bootstrap) {
+    return bootstrap.logicalMessages.map(({ markdown }) => markdown).join("\n\n");
+}
+function firstChangedSource(bootstrap, offset) {
+    let start = 0;
+    for (const message of bootstrap.logicalMessages) {
+        let search = 0;
+        for (const block of message.blocks) {
+            const exact = block.markdown.trim();
+            const index = message.markdown.indexOf(exact, search);
+            if (index < 0)
+                continue;
+            const end = start + bytes(message.markdown.slice(0, index + exact.length));
+            if (offset < end)
+                return block.source;
+            search = index + exact.length;
+        }
+        start += bytes(message.markdown) + 2;
+    }
+    return null;
+}
+function summarizeEncoding(request) {
+    const leaves = scalarPaths(request.body);
+    return {
+        provider: request.provider,
+        jsonBytes: bytes(JSON.stringify(request.body)),
+        contentBlocks: leaves
+            .filter(({ path, value }) => typeof value === "string" &&
+            /(?:^|\.)(?:text|content|instructions)$/u.test(path))
+            .map(({ path, value }) => ({ path, bytes: bytes(String(value)) })),
+        cacheBreakpoints: leaves
+            .filter(({ path, value }) => path.endsWith("cache_control.type") && value === "ephemeral")
+            .map(({ path }) => path.slice(0, -".type".length)),
+        hasCacheKey: cacheKey(request.body) !== undefined,
+    };
+}
+function scalarPaths(value, path = "") {
+    if (Array.isArray(value))
+        return value.flatMap((item, index) => scalarPaths(item, `${path}[${index}]`));
+    if (value !== null && typeof value === "object")
+        return Object.entries(value).flatMap(([key, item]) => scalarPaths(item, path === "" ? key : `${path}.${key}`));
+    return [{ path, value }];
+}
+function firstChangedPath(left, right) {
+    const before = scalarPaths(left);
+    const after = scalarPaths(right);
+    for (let index = 0; index < Math.max(before.length, after.length); index += 1)
+        if (before[index]?.path !== after[index]?.path ||
+            JSON.stringify(before[index]?.value) !==
+                JSON.stringify(after[index]?.value))
+            return after[index]?.path ?? before[index]?.path ?? null;
+    return null;
+}
+function cacheKey(value) {
+    return value !== null &&
+        typeof value === "object" &&
+        "prompt_cache_key" in value
+        ? value.prompt_cache_key
+        : undefined;
+}
+function bytes(value) {
+    return Buffer.byteLength(value, "utf8");
+}
+function commonBytes(left, right) {
+    const a = Buffer.from(left, "utf8");
+    const b = Buffer.from(right, "utf8");
+    let index = 0;
+    while (index < Math.min(a.length, b.length) && a[index] === b[index])
+        index += 1;
+    return index;
+}