@noopolis/daimon 0.1.1 → 0.2.0

package/dist/runtime/toolResultSpill.d.ts

@@ -0,0 +1,152 @@
+import { type McpUpstreamResult, type RenderedMcpToolResult } from "./mcpToolResult.js";
+/**
+ * Cap one declared MCP tool result, and spill the rest of it to disk.
+ *
+ * ## Why
+ *
+ * Production agents make 3–37 tool calls per wake, and every tool result stays
+ * in the transcript for every *subsequent* model request of that wake. Context
+ * per request is flat at 23–32k tokens, so one oversized result is not paid
+ * once — it is paid again on every request that follows it, at cache-read rates
+ * for the ones that hit and at fresh rates for the one that does not. A 40 KiB
+ * result returned on call three of thirty is re-billed twenty-seven times.
+ *
+ * Before this, Daimon applied no cap of its own to a declared MCP tool's answer.
+ * `mcpToolResult.ts` bounded it at 61,440 bytes — the receipt bound, not a
+ * context bound — and above that degraded it to a *head only*, with no way to
+ * recover the rest.
+ *
+ * ## The reference pattern
+ *
+ * `references/oh-my-pi` wraps **every** tool, MCP included, with the same
+ * centralized spill (`packages/coding-agent/src/tools/output-meta.ts:724-891`,
+ * mounted in `sdk.ts:2862-2869`):
+ *
+ *   - a byte threshold (`tools.artifactSpillThreshold`, default 50 KiB);
+ *   - **head + tail retention** with the middle elided, so the model keeps both
+ *     ends of the output (`truncateMiddle`,
+ *     `packages/coding-agent/src/session/streaming-output.ts`), defaults
+ *     20 KiB head + 20 KiB tail — an even split at 80% of the threshold;
+ *   - the **full output written to disk** and addressed by an id
+ *     (`session/artifacts.ts`);
+ *   - a notice naming the ranges shown and the re-fetch address:
+ *     `Showing lines X-Y of Z … Read artifact://ID for full output`
+ *     (`formatTruncationMetaNotice`, `formatFullOutputReference`);
+ *   - and the rule that a **failed save still truncates** — the recovery link is
+ *     attached only when the write actually succeeded, because re-exposing the
+ *     full output on a disk error is the one outcome worse than truncating.
+ *
+ * All five are reproduced here. What differs is only the address: oh-my-pi's
+ * `artifact://<id>` is resolved by its own `read` tool, and Daimon has no such
+ * protocol — so the notice names the **absolute path**, which every Daimon agent
+ * can already read with the mounted `bash` tool (`piHarness.ts`) or the engine's
+ * own shell. The file lives under the agent's own runtime home, which is exactly
+ * the tree that home is for and the one tree every sandbox profile leaves the
+ * agent (`grokSandboxProtectedPaths` denies *peers'* homes, never its own).
+ */
+/**
+ * The cap on one lowered MCP tool result, measured the way
+ * `mcpToolResult.ts` measures: the serialized `{content, structuredContent}`
+ * record, both channels included.
+ *
+ * **16 KiB, ≈4k tokens.** oh-my-pi's 50 KiB default is calibrated for an
+ * interactive coding agent with a 200k+ context; Daimon's whole per-request
+ * context is 23–32k tokens, so a 50 KiB result would be *half of it* and would
+ * be replayed on every remaining request of the wake. 16 KiB keeps any single
+ * result under roughly a sixth of a request while leaving a realistic answer
+ * — a page of messages, a document, a query result — entirely untouched: it is
+ * a ceiling on the pathological case, not a working budget.
+ */
+export declare const DEFAULT_TOOL_RESULT_MAX_BYTES: number;
+export declare const TOOL_RESULT_MAX_BYTES_ENV: "DAIMON_TOOL_RESULT_MAX_BYTES";
+/**
+ * Tools that must never be truncated, by mounted tool name
+ * (`mcp_<server>_<tool>`), comma-separated.
+ *
+ * Empty by default. It exists because Daimon cannot know what an organization
+ * declared: if a deployment has one MCP tool whose *whole* answer is load
+ * bearing — a signature, a checksummed document, a payload the agent must
+ * reproduce verbatim — head+tail is not a degradation there, it is a wrong
+ * answer, and no re-fetch instruction fixes a model that did not notice. That
+ * judgement belongs to whoever declared the tool, so it is configuration rather
+ * than a Daimon-side guess. An exempt tool keeps exactly today's behaviour,
+ * including `mcpToolResult.ts`'s 61,440-byte receipt bound.
+ */
+export declare const TOOL_RESULT_EXEMPT_ENV: "DAIMON_TOOL_RESULT_NO_TRUNCATE";
+/**
+ * A result may never be capped below this. Below roughly a kilobyte the head and
+ * tail stop being usable at all and the notice dominates the payload, so a
+ * mistyped bound degrades into "every tool is blind" — worse than no cap.
+ */
+export declare const MIN_TOOL_RESULT_MAX_BYTES = 2048;
+/** Spilled files live here, under the agent's own runtime home. */
+export declare const TOOL_OUTPUT_DIRECTORY_NAME: "tool-output";
+/** Newest-first retention for spilled files, mirroring `pruneReceipts`. */
+export declare const TOOL_OUTPUT_RETAINED_FILES = 256;
+export declare const resolveToolResultMaxBytes: (environment?: NodeJS.ProcessEnv) => number;
+export declare const resolveExemptToolNames: (environment?: NodeJS.ProcessEnv) => ReadonlySet<string>;
+/** Cut on a code-point boundary, from the front. */
+/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */
+export declare const headUtf8: (value: string, maxBytes: number) => string;
+/** Cut on a code-point boundary, from the back. */
+/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */
+export declare const tailUtf8: (value: string, maxBytes: number) => string;
+/**
+ * The complete payload, as the text that gets written to disk and sliced.
+ *
+ * A result that is nothing but text parts is spilled as that text, because a
+ * human-readable log is what the agent will `grep` and `sed`. Anything else — a
+ * structured result, an image part, a mixed result — is spilled as the
+ * serialized upstream record, which is lossless and is the same serialization
+ * `mcpToolResult.ts` already degrades to today.
+ */
+export declare const spillPayload: (result: McpUpstreamResult) => {
+    text: string;
+    format: "text" | "json";
+};
+/**
+ * Spilled files are group-readable and never other-readable.
+ *
+ * A brokered Grok worker runs as its own uid and reads a spill with
+ * `read_file`, so a 0600 file owned by the runtime (uid 2000) was unreadable to
+ * the very agent the notice sends there. The group grant reaches exactly that
+ * agent's worker only when the deployment provisions `tool-output` as
+ * `<runtime uid>:<worker gid> 2750` (setgid, so each file inherits the worker
+ * group; `GROK_ENGINE_BROKER.worker.home.spillDirectory`). A directory Daimon
+ * creates itself stays 0700, so for every other engine nothing new is exposed.
+ */
+export declare const SPILL_FILE_MODE = 416;
+export declare const SPILL_DIRECTORY_MAX_MODE = 1512;
+/** Pure so a foreign owner is testable without root. */
+export declare const assertSpillDirectoryStat: (entry: Readonly<{
+    uid: number;
+    gid: number;
+    mode: number;
+    isDirectory(): boolean;
+}>, runtime: Readonly<{
+    uid: number;
+    gid: number;
+}>) => void;
+export type CappedToolResult = RenderedMcpToolResult & {
+    /** Absolute path of the spilled full result, when one was written. */
+    readonly spillPath?: string;
+};
+/**
+ * Return `rendered` untouched when it fits, or a head+tail view of the full
+ * payload plus a notice naming the file that holds the rest.
+ *
+ * A result under the cap passes through **byte-identical**: the `mcp_*`
+ * passthrough contract — both channels carrying the payload, an upstream
+ * `structuredContent` forwarded verbatim — is unchanged for every result that
+ * does not blow the bound, which is the overwhelming majority of them.
+ */
+export declare const capToolResult: (input: Readonly<{
+    toolName: string;
+    rendered: RenderedMcpToolResult;
+    result: McpUpstreamResult;
+    maxBytes: number;
+    exempt: ReadonlySet<string>;
+    spillDirectory: string;
+    spillId: string;
+}>) => Promise<CappedToolResult>;
+//# sourceMappingURL=toolResultSpill.d.ts.map

package/dist/runtime/toolResultSpill.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"toolResultSpill.d.ts","sourceRoot":"","sources":["../../src/runtime/toolResultSpill.ts"],"names":[],"mappings":"AAIA,OAAO,EAAkB,KAAK,iBAAiB,EAAE,KAAK,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAExG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GA4CG;AAEH;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,6BAA6B,QAAY,CAAC;AAEvD,eAAO,MAAM,yBAAyB,EAAG,8BAAuC,CAAC;AAEjF;;;;;;;;;;;;GAYG;AACH,eAAO,MAAM,sBAAsB,EAAG,gCAAyC,CAAC;AAEhF;;;;GAIG;AACH,eAAO,MAAM,yBAAyB,OAAQ,CAAC;AAE/C,mEAAmE;AACnE,eAAO,MAAM,0BAA0B,EAAG,aAAsB,CAAC;AAEjE,2EAA2E;AAC3E,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,eAAO,MAAM,yBAAyB,GAAI,cAAa,MAAM,CAAC,UAAwB,KAAG,MAQxF,CAAC;AAEF,eAAO,MAAM,sBAAsB,GAAI,cAAa,MAAM,CAAC,UAAwB,KAAG,WAAW,CAAC,MAAM,CACsB,CAAC;AAS/H,oDAAoD;AACpD,mHAAmH;AACnH,eAAO,MAAM,QAAQ,GAAI,OAAO,MAAM,EAAE,UAAU,MAAM,KAAG,MAM1D,CAAC;AAEF,mDAAmD;AACnD,mHAAmH;AACnH,eAAO,MAAM,QAAQ,GAAI,OAAO,MAAM,EAAE,UAAU,MAAM,KAAG,MAM1D,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,YAAY,GAAI,QAAQ,iBAAiB,KAAG;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAA;CAc/F,CAAC;AAyBF;;;;;;;;;;GAUG;AACH,eAAO,MAAM,eAAe,MAAQ,CAAC;AACrC,eAAO,MAAM,wBAAwB,OAAS,CAAC;AA0D/C,wDAAwD;AACxD,eAAO,MAAM,wBAAwB,GAAI,OAAO,QAAQ,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAC;IAAC,IAAI,EAAE,MAAM,CAAC;IAAC,WAAW,IAAI,OAAO,CAAA;CAAE,CAAC,EAAE,SAAS,QAAQ,CAAC;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,GAAG,EAAE,MAAM,CAAA;CAAE,CAAC,KAAG,IAO/K,CAAC;AAoBF,MAAM,MAAM,gBAAgB,GAAG,qBAAqB,GAAG;IACrD,sEAAsE;IACtE,QAAQ,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF;;;;;;;;GAQG;AACH,eAAO,MAAM,aAAa,GAAU,OAAO,QAAQ,CAAC;IAClD,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,qBAAqB,CAAC;IAChC,MAAM,EAAE,iBAAiB,CAAC;IAC1B,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,CAAC;IAC5B,cAAc,EAAE,MAAM,CAAC;IACvB,OAAO,EAAE,MAAM,CAAC;CACjB,CAAC,KAAG,OAAO,CAAC,gBAAgB,CA6D5B,CAAC"}

package/dist/runtime/toolResultSpill.js

@@ -0,0 +1,351 @@
+import { constants } from "node:fs";
+import { lstat, mkdir, open, readdir, rename, unlink } from "node:fs/promises";
+import path from "node:path";
+import { mcpContentText } from "./mcpToolResult.js";
+/**
+ * Cap one declared MCP tool result, and spill the rest of it to disk.
+ *
+ * ## Why
+ *
+ * Production agents make 3–37 tool calls per wake, and every tool result stays
+ * in the transcript for every *subsequent* model request of that wake. Context
+ * per request is flat at 23–32k tokens, so one oversized result is not paid
+ * once — it is paid again on every request that follows it, at cache-read rates
+ * for the ones that hit and at fresh rates for the one that does not. A 40 KiB
+ * result returned on call three of thirty is re-billed twenty-seven times.
+ *
+ * Before this, Daimon applied no cap of its own to a declared MCP tool's answer.
+ * `mcpToolResult.ts` bounded it at 61,440 bytes — the receipt bound, not a
+ * context bound — and above that degraded it to a *head only*, with no way to
+ * recover the rest.
+ *
+ * ## The reference pattern
+ *
+ * `references/oh-my-pi` wraps **every** tool, MCP included, with the same
+ * centralized spill (`packages/coding-agent/src/tools/output-meta.ts:724-891`,
+ * mounted in `sdk.ts:2862-2869`):
+ *
+ *   - a byte threshold (`tools.artifactSpillThreshold`, default 50 KiB);
+ *   - **head + tail retention** with the middle elided, so the model keeps both
+ *     ends of the output (`truncateMiddle`,
+ *     `packages/coding-agent/src/session/streaming-output.ts`), defaults
+ *     20 KiB head + 20 KiB tail — an even split at 80% of the threshold;
+ *   - the **full output written to disk** and addressed by an id
+ *     (`session/artifacts.ts`);
+ *   - a notice naming the ranges shown and the re-fetch address:
+ *     `Showing lines X-Y of Z … Read artifact://ID for full output`
+ *     (`formatTruncationMetaNotice`, `formatFullOutputReference`);
+ *   - and the rule that a **failed save still truncates** — the recovery link is
+ *     attached only when the write actually succeeded, because re-exposing the
+ *     full output on a disk error is the one outcome worse than truncating.
+ *
+ * All five are reproduced here. What differs is only the address: oh-my-pi's
+ * `artifact://<id>` is resolved by its own `read` tool, and Daimon has no such
+ * protocol — so the notice names the **absolute path**, which every Daimon agent
+ * can already read with the mounted `bash` tool (`piHarness.ts`) or the engine's
+ * own shell. The file lives under the agent's own runtime home, which is exactly
+ * the tree that home is for and the one tree every sandbox profile leaves the
+ * agent (`grokSandboxProtectedPaths` denies *peers'* homes, never its own).
+ */
+/**
+ * The cap on one lowered MCP tool result, measured the way
+ * `mcpToolResult.ts` measures: the serialized `{content, structuredContent}`
+ * record, both channels included.
+ *
+ * **16 KiB, ≈4k tokens.** oh-my-pi's 50 KiB default is calibrated for an
+ * interactive coding agent with a 200k+ context; Daimon's whole per-request
+ * context is 23–32k tokens, so a 50 KiB result would be *half of it* and would
+ * be replayed on every remaining request of the wake. 16 KiB keeps any single
+ * result under roughly a sixth of a request while leaving a realistic answer
+ * — a page of messages, a document, a query result — entirely untouched: it is
+ * a ceiling on the pathological case, not a working budget.
+ */
+export const DEFAULT_TOOL_RESULT_MAX_BYTES = 16 * 1024;
+export const TOOL_RESULT_MAX_BYTES_ENV = "DAIMON_TOOL_RESULT_MAX_BYTES";
+/**
+ * Tools that must never be truncated, by mounted tool name
+ * (`mcp_<server>_<tool>`), comma-separated.
+ *
+ * Empty by default. It exists because Daimon cannot know what an organization
+ * declared: if a deployment has one MCP tool whose *whole* answer is load
+ * bearing — a signature, a checksummed document, a payload the agent must
+ * reproduce verbatim — head+tail is not a degradation there, it is a wrong
+ * answer, and no re-fetch instruction fixes a model that did not notice. That
+ * judgement belongs to whoever declared the tool, so it is configuration rather
+ * than a Daimon-side guess. An exempt tool keeps exactly today's behaviour,
+ * including `mcpToolResult.ts`'s 61,440-byte receipt bound.
+ */
+export const TOOL_RESULT_EXEMPT_ENV = "DAIMON_TOOL_RESULT_NO_TRUNCATE";
+/**
+ * A result may never be capped below this. Below roughly a kilobyte the head and
+ * tail stop being usable at all and the notice dominates the payload, so a
+ * mistyped bound degrades into "every tool is blind" — worse than no cap.
+ */
+export const MIN_TOOL_RESULT_MAX_BYTES = 2_048;
+/** Spilled files live here, under the agent's own runtime home. */
+export const TOOL_OUTPUT_DIRECTORY_NAME = "tool-output";
+/** Newest-first retention for spilled files, mirroring `pruneReceipts`. */
+export const TOOL_OUTPUT_RETAINED_FILES = 256;
+export const resolveToolResultMaxBytes = (environment = process.env) => {
+    const raw = environment[TOOL_RESULT_MAX_BYTES_ENV]?.trim();
+    if (raw === undefined || raw.length === 0)
+        return DEFAULT_TOOL_RESULT_MAX_BYTES;
+    const parsed = Number(raw);
+    if (!Number.isSafeInteger(parsed) || parsed < MIN_TOOL_RESULT_MAX_BYTES) {
+        throw new Error(`${TOOL_RESULT_MAX_BYTES_ENV} must be an integer of at least ${MIN_TOOL_RESULT_MAX_BYTES}`);
+    }
+    return parsed;
+};
+export const resolveExemptToolNames = (environment = process.env) => new Set((environment[TOOL_RESULT_EXEMPT_ENV] ?? "").split(",").map((name) => name.trim()).filter((name) => name.length > 0));
+const safeStringify = (value) => {
+    try {
+        return JSON.stringify(value) ?? "";
+    }
+    catch {
+        return "[daimon: the MCP server returned a value that is not JSON-serialisable]";
+    }
+};
+const measure = (content, details) => Buffer.byteLength(safeStringify({ content, structuredContent: details }), "utf8");
+/** Cut on a code-point boundary, from the front. */
+/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */
+export const headUtf8 = (value, maxBytes) => {
+    const bytes = Buffer.from(value, "utf8");
+    if (bytes.byteLength <= maxBytes)
+        return value;
+    let end = Math.max(0, maxBytes);
+    while (end > 0 && (bytes[end] & 0xc0) === 0x80)
+        end -= 1;
+    return bytes.subarray(0, end).toString("utf8");
+};
+/** Cut on a code-point boundary, from the back. */
+/** Exported for the bounded diagnostic window (`cliChildOutput.ts`): one boundary-safe implementation, not two. */
+export const tailUtf8 = (value, maxBytes) => {
+    const bytes = Buffer.from(value, "utf8");
+    if (bytes.byteLength <= maxBytes)
+        return value;
+    let start = Math.max(0, bytes.byteLength - maxBytes);
+    while (start < bytes.byteLength && (bytes[start] & 0xc0) === 0x80)
+        start += 1;
+    return bytes.subarray(start).toString("utf8");
+};
+/**
+ * The complete payload, as the text that gets written to disk and sliced.
+ *
+ * A result that is nothing but text parts is spilled as that text, because a
+ * human-readable log is what the agent will `grep` and `sed`. Anything else — a
+ * structured result, an image part, a mixed result — is spilled as the
+ * serialized upstream record, which is lossless and is the same serialization
+ * `mcpToolResult.ts` already degrades to today.
+ */
+export const spillPayload = (result) => {
+    const content = Array.isArray(result.content) ? result.content : undefined;
+    const textOnly = content !== undefined
+        && content.length > 0
+        && content.every((part) => typeof part === "object" && part !== null && part.type === "text")
+        && result.structuredContent === undefined;
+    if (textOnly)
+        return { text: mcpContentText(content), format: "text" };
+    return {
+        text: safeStringify({
+            ...(result.content === undefined ? {} : { content: result.content }),
+            ...(result.structuredContent === undefined ? {} : { structured_content: result.structuredContent })
+        }),
+        format: "json"
+    };
+};
+const notice = (input) => {
+    const elided = Math.max(0, input.totalBytes - input.headBytes - input.tailBytes);
+    const shape = input.format === "json" ? "the serialized result record" : "the result text";
+    const head = [
+        `\n\n[daimon: truncated — this tool result was ${input.totalBytes} bytes, above the ${input.boundBytes}-byte per-result bound.`,
+        `Showing the first ${input.headBytes} and last ${input.tailBytes} bytes of ${shape}; ${elided} bytes were elided from the middle.`
+    ].join(" ");
+    if (input.spillPath === undefined) {
+        return `${head} The complete result could NOT be written to disk, so the elided bytes are unrecoverable.`
+            + " Repeating this call will truncate again — narrow the call's arguments instead, or ask for the missing part specifically.]";
+    }
+    return `${head} The complete result is saved at ${input.spillPath}.`
+        + ` Read the elided middle from that file with the shell — for example \`sed -n '1,200p' ${input.spillPath}\``
+        + ` or \`grep -n "<pattern>" ${input.spillPath}\` — and do NOT repeat this tool call to see it: an identical call returns this same truncation.]`;
+};
+/**
+ * Spilled files are group-readable and never other-readable.
+ *
+ * A brokered Grok worker runs as its own uid and reads a spill with
+ * `read_file`, so a 0600 file owned by the runtime (uid 2000) was unreadable to
+ * the very agent the notice sends there. The group grant reaches exactly that
+ * agent's worker only when the deployment provisions `tool-output` as
+ * `<runtime uid>:<worker gid> 2750` (setgid, so each file inherits the worker
+ * group; `GROK_ENGINE_BROKER.worker.home.spillDirectory`). A directory Daimon
+ * creates itself stays 0700, so for every other engine nothing new is exposed.
+ */
+export const SPILL_FILE_MODE = 0o640;
+export const SPILL_DIRECTORY_MAX_MODE = 0o2750;
+/**
+ * Write the full payload where the agent can read it, atomically.
+ *
+ * Staged to a sibling and renamed, so a half-written file is never addressable:
+ * a truncation notice that points at a truncated file is worse than one that
+ * admits the save failed.
+ */
+const writeSpill = async (directory, name, text) => {
+    await mkdir(directory, { recursive: true, mode: 0o700 });
+    const pinned = await pinSpillDirectory(directory);
+    const file = path.join(directory, name);
+    const temporary = `${file}.${process.pid}.${Date.now()}.tmp`;
+    try {
+        const handle = await open(temporary, constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW, SPILL_FILE_MODE);
+        let written;
+        // Explicit, so a restrictive umask cannot strip the group read an agent's sandboxed worker needs.
+        try {
+            await handle.chmod(SPILL_FILE_MODE);
+            await handle.writeFile(text, "utf8");
+            await handle.sync();
+            written = await handle.stat();
+        }
+        finally {
+            await handle.close();
+        }
+        if (!written.isFile() || written.nlink !== 1 || written.uid !== process.getuid?.())
+            throw new Error("spill file is not a private regular file");
+        // Node has no openat: re-check that the directory entry still names the pinned inode before publishing into it.
+        await assertSameDirectory(directory, pinned);
+        // rename replaces a pre-existing destination entry (a symlink included) without following it.
+        await rename(temporary, file);
+        const published = await lstat(file);
+        if (!published.isFile() || published.dev !== written.dev || published.ino !== written.ino)
+            throw new Error("spill file was replaced");
+        return file;
+    }
+    catch (error) {
+        await unlink(temporary).catch(() => undefined);
+        throw error;
+    }
+    finally {
+        await pinned.handle.close();
+    }
+};
+/**
+ * The spill directory must be a real directory owned by this runtime and no
+ * wider than `2750`: either Daimon's own `0700`, or a deployment-provisioned
+ * setgid directory whose group is not the runtime's own (a worker group). A
+ * symlinked, foreign-owned, world-accessible, or group-open-without-setgid
+ * directory is refused and nothing is written. Daimon cannot know *which*
+ * worker gid belongs to this agent; that mapping is the deployment's
+ * provisioning contract (`GROK_ENGINE_BROKER.worker.home.spillDirectory`).
+ */
+const pinSpillDirectory = async (directory) => {
+    const before = await lstat(directory);
+    if (before.isSymbolicLink() || !before.isDirectory())
+        throw new Error("spill directory is not a real directory");
+    const handle = await open(directory, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW);
+    try {
+        const opened = await handle.stat();
+        if (opened.dev !== before.dev || opened.ino !== before.ino)
+            throw new Error("spill directory was replaced");
+        assertSpillDirectoryStat(opened, { uid: process.getuid?.() ?? -1, gid: process.getgid?.() ?? -1 });
+        return { handle, dev: Number(opened.dev), ino: Number(opened.ino) };
+    }
+    catch (error) {
+        await handle.close();
+        throw error;
+    }
+};
+/** Pure so a foreign owner is testable without root. */
+export const assertSpillDirectoryStat = (entry, runtime) => {
+    const mode = Number(entry.mode) & 0o7777;
+    const groupOpen = (mode & 0o070) !== 0;
+    if (!entry.isDirectory() || entry.uid !== runtime.uid || (mode & ~SPILL_DIRECTORY_MAX_MODE) !== 0
+        || (groupOpen && ((mode & 0o2000) === 0 || entry.gid === runtime.gid))) {
+        throw new Error("spill directory is not a private or provisioned worker-group directory");
+    }
+};
+const assertSameDirectory = async (directory, pinned) => {
+    const [now, held] = await Promise.all([lstat(directory), pinned.handle.stat()]);
+    if (now.isSymbolicLink() || Number(now.dev) !== pinned.dev || Number(now.ino) !== pinned.ino || Number(held.ino) !== pinned.ino)
+        throw new Error("spill directory was replaced");
+};
+/** Newest-first retention, so a busy agent cannot fill its own runtime home. */
+const pruneSpills = async (directory) => {
+    const names = (await readdir(directory)).filter((name) => /\.log$/u.test(name));
+    const entries = await Promise.all(names.map(async (name) => ({ name, info: await lstat(path.join(directory, name)) })));
+    const files = entries.filter(({ info }) => info.isFile() && info.nlink === 1);
+    for (const candidate of files.sort((left, right) => right.info.mtimeMs - left.info.mtimeMs || right.name.localeCompare(left.name)).slice(TOOL_OUTPUT_RETAINED_FILES)) {
+        await unlink(path.join(directory, candidate.name)).catch(() => undefined);
+    }
+};
+/** `mcp_<server>_<tool>` is already `[A-Za-z0-9_]`, but the filename never trusts that. */
+const safeName = (value) => value.replace(/[^A-Za-z0-9_-]+/gu, "_").slice(0, 64) || "tool";
+/**
+ * Return `rendered` untouched when it fits, or a head+tail view of the full
+ * payload plus a notice naming the file that holds the rest.
+ *
+ * A result under the cap passes through **byte-identical**: the `mcp_*`
+ * passthrough contract — both channels carrying the payload, an upstream
+ * `structuredContent` forwarded verbatim — is unchanged for every result that
+ * does not blow the bound, which is the overwhelming majority of them.
+ */
+export const capToolResult = async (input) => {
+    if (input.exempt.has(input.toolName))
+        return input.rendered;
+    if (measure(input.rendered.content, input.rendered.details) <= input.maxBytes)
+        return input.rendered;
+    const payload = spillPayload(input.result);
+    const totalBytes = Buffer.byteLength(payload.text, "utf8");
+    let spillPath;
+    try {
+        spillPath = await writeSpill(input.spillDirectory, `${safeName(input.spillId)}.${safeName(input.toolName)}.log`, payload.text);
+    }
+    catch {
+        // oh-my-pi's rule, and the reason it is a rule: a save failure must never
+        // convert a successful call into an error, and must never re-expose the
+        // full output. Truncate anyway; only the recovery link is withheld.
+        spillPath = undefined;
+    }
+    if (spillPath !== undefined)
+        await pruneSpills(input.spillDirectory).catch(() => undefined);
+    // The notice's own bytes are reserved first, priced with the widest numbers it
+    // could print. Both channels then mirror each other (`mcpToolResult.ts` rule
+    // 1) so the retained body is written twice and the budget is halved to pay for
+    // it — the same arithmetic, and the same reason, as that file's `degrade`. The
+    // fit is then *verified* rather than estimated, because JSON escaping of the
+    // payload's own bytes (a newline costs two) is not predictable from its length.
+    const reserve = Buffer.byteLength(notice({
+        totalBytes, headBytes: totalBytes, tailBytes: totalBytes, boundBytes: input.maxBytes, format: payload.format, ...(spillPath === undefined ? {} : { spillPath })
+    }), "utf8");
+    const build = (budget) => {
+        const head = headUtf8(payload.text, Math.floor(budget / 2));
+        const headBytes = Buffer.byteLength(head, "utf8");
+        const tail = tailUtf8(payload.text, Math.max(0, budget - headBytes));
+        const tailBytes = Buffer.byteLength(tail, "utf8");
+        const elided = Math.max(0, totalBytes - headBytes - tailBytes);
+        const shown = `${head}\n[… ${elided} bytes elided …]\n${tail}`;
+        const text = `${shown}${notice({ totalBytes, headBytes, tailBytes, boundBytes: input.maxBytes, format: payload.format, ...(spillPath === undefined ? {} : { spillPath }) })}`;
+        return {
+            content: [{ type: "text", text }],
+            details: {
+                daimon_truncated: true,
+                bound_bytes: input.maxBytes,
+                result_bytes: totalBytes,
+                shown_head_bytes: headBytes,
+                shown_tail_bytes: tailBytes,
+                elided_bytes: elided,
+                payload_format: payload.format,
+                ...(spillPath === undefined ? { full_output_saved: false } : { full_output_path: spillPath }),
+                result_head_tail: shown
+            },
+            truncated: true,
+            ...(spillPath === undefined ? {} : { spillPath })
+        };
+    };
+    let budget = Math.max(0, Math.floor((input.maxBytes - reserve) / 2));
+    let candidate = build(budget);
+    for (let attempt = 0; attempt < 8 && budget > 0; attempt += 1) {
+        const size = measure(candidate.content, candidate.details);
+        if (size <= input.maxBytes)
+            return candidate;
+        budget = Math.max(0, budget - Math.ceil((size - input.maxBytes) / 2) - 16);
+        candidate = build(budget);
+    }
+    return candidate;
+};

package/dist/runtime/turnRequestLedger.d.ts

@@ -0,0 +1,160 @@
+import { type CodexRequestUsage } from "../pi/codexRolloutUsage.js";
+import type { GrokBrokerModel } from "./grokBrokerModelPolicy.js";
+/**
+ * Per-model-request token accounting, written *beside* the per-wake usage
+ * ledger and never inside it.
+ *
+ * `turnUsageLedger.ts` records one aggregate row per wake. Production measured
+ * 55 wakes at 14,890,263 input tokens against 159,726 output — output is 1.06%
+ * of the spend — with context per model request flat at 23–32k regardless of
+ * how many requests a wake made. That is the signature of a fixed prefix
+ * replayed once per request rather than of a conversation that grows, but the
+ * aggregate row cannot tell the two apart, and every optimisation worth doing
+ * next depends on which one it is. This file is what tells them apart.
+ *
+ * It is a separate stream, not a wider row, for one specific reason. Spawnfile's
+ * reader (`spawnfile/src/runtime/usageLedger.ts`) rejects any line whose `v` is
+ * not `noopolis.daimon.turn-usage.v1` and returns `null` for the whole line, so
+ * a version bump on the existing ledger would make an unreleased Spawnfile drop
+ * every row it can read today. Nothing here changes the shape, version, path, or
+ * field list of that ledger; a reader that knows nothing about this file keeps
+ * working byte for byte.
+ */
+export declare const TURN_REQUEST_LEDGER_VERSION: "noopolis.daimon.turn-requests.v1";
+/**
+ * Default location: the directory Spawnfile already provisions for the usage
+ * ledger, under its own filename. No new mount, no new provisioning step.
+ */
+export declare const TURN_REQUEST_LEDGER: {
+    readonly version: "noopolis.daimon.turn-requests.v1";
+    readonly directoryPath: "/var/lib/spawnfile/daimon/usage";
+    readonly filePath: "/var/lib/spawnfile/daimon/usage/requests.jsonl";
+    readonly rotatedFilePath: "/var/lib/spawnfile/daimon/usage/requests.jsonl.1";
+    readonly fileMode: 416;
+};
+/**
+ * `DAIMON_TURN_REQUESTS_LEDGER_PATH` relocates the stream, exactly as
+ * `DAIMON_TURN_USAGE_LEDGER_PATH` relocates the ledger it sits beside. A
+ * relative or empty value is ignored rather than honoured: a ledger written to
+ * a process-relative path is a ledger nobody can find again.
+ */
+export declare const TURN_REQUEST_LEDGER_PATH_ENV: "DAIMON_TURN_REQUESTS_LEDGER_PATH";
+export declare const resolveTurnRequestLedgerPath: (environment?: NodeJS.ProcessEnv) => string;
+export type TurnRequestEntry = Readonly<{
+    agent: string;
+    wake: string;
+    /** The Codex thread whose rollout these rows were read from. */
+    thread: string;
+    requests: readonly CodexRequestUsage[];
+    at?: string;
+}>;
+/**
+ * One line per model request.
+ *
+ * `fresh_input` is `input − cached_input`, the split the whole exercise exists
+ * to see. It is derived from exactly the relationship Daimon already validates
+ * (`cached_input_tokens` is a subset of `input_tokens`, asserted by Codex's own
+ * reconciled totals and re-checked when the block is decoded). `cache_write` is
+ * emitted raw and never subtracted, because whether Codex counts a cache write
+ * inside `input_tokens` has not been verified — a reader that needs that answer
+ * has the raw number to work from instead of a guess baked into the row.
+ */
+export declare const renderTurnRequestLines: (entry: TurnRequestEntry) => string;
+/** One Grok broker model request: usage from the worker stream, timing from the proxy. */
+/**
+ * `usageSource`: `stream` (the worker's own per-request frame), `upstream`
+ * (the provider response the proxy saw), or `estimated` (no valid usage; the
+ * proxy's conservative charge, see `grokBrokerTurnMeter.ts`).
+ */
+/**
+ * `toolCalls` are the tool-call names that request's response carried, as the
+ * proxy read them (`grokBrokerTurnMeter.ts`): names only, bounded, `[]` for a
+ * response that called nothing, and absent when no response could be decoded.
+ */
+export type GrokTurnRequest = Readonly<{
+    index: number;
+    input: number;
+    cacheRead: number;
+    cacheWrite: number;
+    output: number;
+    total: number;
+    startedAt?: string;
+    endedAt?: string;
+    usageSource?: "stream" | "upstream" | "estimated";
+    toolCalls?: readonly string[];
+}>;
+/**
+ * `requestCount` is the turn's admitted request count when it exceeds the rows:
+ * a killed turn's in-flight request was sent upstream but never reported usage,
+ * so it has no row yet still counts in every row's `requests`.
+ */
+export type GrokTurnRequestEntry = Readonly<{
+    agent: string;
+    wake: string;
+    turn: string;
+    session?: string;
+    model: GrokBrokerModel;
+    requests: readonly GrokTurnRequest[];
+    requestCount?: number;
+    at?: string;
+}>;
+/**
+ * Grok rows share the Codex row's field meaning: `input` is the whole prompt
+ * side the request replayed (`input_tokens + cache_read`), `cached_input` the
+ * cache read, `fresh_input` the uncached remainder. Grok does not separate
+ * reasoning tokens, so `reasoning` is absent rather than zero. `turn` is the
+ * broker idempotency key and `thread` the Grok session id when the stream
+ * named one.
+ *
+ * `tool_calls` is what a turn's rows could not say before: whether the model
+ * ever *tried* to call anything. Two live turns ended with correctly mounted
+ * tools and no visible attempt, and the rows recorded timings and tokens only,
+ * so the question could not be answered after the fact. It is names only —
+ * never arguments, never message content, never a bearer — bounded, `[]` for a
+ * response that called nothing, and absent for a response that could not be
+ * decoded, because a fabricated empty list is byte-identical to a measured one.
+ *
+ * It is an additive field inside the unchanged
+ * `noopolis.daimon.turn-requests.v1` row, deliberately without a version bump:
+ * Spawnfile's usage reader (`spawnfile/src/runtime/usageLedger.ts`) drops every
+ * line whose `v` it does not recognise while ignoring fields it does not know,
+ * and Paideia only relocates this stream's path
+ * (`DAIMON_TURN_REQUESTS_LEDGER_PATH`). A bump is what would blind them; a new
+ * field is not.
+ */
+export declare const renderGrokTurnRequestLines: (entry: GrokTurnRequestEntry) => string;
+/**
+ * Append already-rendered, newline-terminated ledger lines in one write, with the
+ * same rotation and file mode as both ledgers. Advisory: never rejects. The
+ * broker uses it to append the exact bytes it sealed into a turn record.
+ */
+export declare const recordLedgerLines: (file: string, lines: string) => Promise<boolean>;
+/** Advisory and never rejects, like {@link recordTurnRequests}; an empty turn writes nothing. */
+export declare const recordGrokTurnRequests: (file: string, entry: GrokTurnRequestEntry) => Promise<boolean>;
+/**
+ * Advisory. Never rejects, and never writes an empty wake.
+ *
+ * A wake whose rollout was absent, unreadable, or undecodable contributes no
+ * rows at all rather than a zero row — the same invariant the usage ledger
+ * holds, for the same reason: a fabricated zero is byte-identical to a measured
+ * one, and this stream exists to be believed.
+ */
+export declare const recordTurnRequests: (file: string, entry: TurnRequestEntry) => Promise<boolean>;
+/**
+ * The whole instrumentation path for one Codex wake: find that thread's
+ * rollout, decode its per-request usage, append it. Never throws.
+ *
+ * A wake that produced no rows says so on stderr exactly once. Silence would
+ * make a permanently blind instrument indistinguishable from a working one —
+ * `--ephemeral`, a relocated `CODEX_HOME`, or a Codex version that renamed the
+ * usage frame all end here, and each is worth one line rather than a
+ * quietly-empty file nobody questions.
+ */
+export declare const recordCodexTurnRequests: (input: Readonly<{
+    agent: string;
+    wake: string;
+    codexHome: string;
+    threadId: string;
+    file?: string;
+}>) => Promise<boolean>;
+//# sourceMappingURL=turnRequestLedger.d.ts.map