@kynetic-ai/spec 0.7.0 → 0.9.0

package/dist/sessions/store.js

@@ -14,6 +14,8 @@
 import * as fs from "node:fs";
 import * as fsPromises from "node:fs/promises";
 import * as path from "node:path";
+import { createHash, randomUUID } from "node:crypto";
+import { parse as parseTOML, stringify as stringifyTOML } from "smol-toml";
 import * as YAML from "yaml";
 import { SessionEventSchema, SessionMetadataSchema, TaskBudgetSchema, } from "./types.js";
 // ─── Constants ───────────────────────────────────────────────────────────────
@@ -21,6 +23,12 @@ const SESSIONS_DIR = "sessions";
 const METADATA_FILE = "session.yaml";
 const EVENTS_FILE = "events.jsonl";
 const BUDGET_FILE = "budget.json";
+const BLOBS_DIR = "blobs";
+// Event persistence guardrails: keep single-line events bounded in size while
+// preserving full payloads via externalized blob files.
+const EVENT_LINE_MAX_BYTES = 256 * 1024;
+const EVENT_FIELD_EXTERNALIZE_BYTES = 16 * 1024;
+const EVENT_PREVIEW_MAX_BYTES = 512;
 // ─── Path Helpers ────────────────────────────────────────────────────────────
 /**
  * Get the sessions directory path within a spec directory.
@@ -59,6 +67,12 @@ export function getSessionContextPath(specDir, sessionId, iteration) {
 export function getSessionBudgetPath(specDir, sessionId) {
     return path.join(getSessionDir(specDir, sessionId), BUDGET_FILE);
 }
+/**
+ * Get the path to a session's blob directory.
+ */
+export function getSessionBlobDir(specDir, sessionId) {
+    return path.join(getSessionDir(specDir, sessionId), BLOBS_DIR);
+}
 // ─── Session CRUD ────────────────────────────────────────────────────────────
 /**
  * Create a new session with metadata.
@@ -264,7 +278,171 @@ export async function closeSession(specDir, sessionId, status, reason) {
     await fsPromises.writeFile(metadataPath, content, "utf-8");
     return updated;
 }
-// ─── Event Storage ───────────────────────────────────────────────────────────
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+export function isSessionBlobPointer(value) {
+    if (!isRecord(value))
+        return false;
+    return (typeof value.path === "string" &&
+        typeof value.bytes === "number" &&
+        typeof value.sha256 === "string" &&
+        value.truncated === true &&
+        typeof value.preview === "string");
+}
+function stringifyPayload(value) {
+    if (typeof value === "string")
+        return value;
+    try {
+        return JSON.stringify(value);
+    }
+    catch {
+        return String(value);
+    }
+}
+function payloadBytes(value) {
+    return Buffer.byteLength(stringifyPayload(value), "utf-8");
+}
+function toPreview(content) {
+    const totalBytes = Buffer.byteLength(content, "utf-8");
+    if (totalBytes <= EVENT_PREVIEW_MAX_BYTES) {
+        return content;
+    }
+    let preview = "";
+    let usedBytes = 0;
+    for (const char of content) {
+        const charBytes = Buffer.byteLength(char, "utf-8");
+        if (usedBytes + charBytes > EVENT_PREVIEW_MAX_BYTES)
+            break;
+        preview += char;
+        usedBytes += charBytes;
+    }
+    return `${preview}...`;
+}
+function normalizeFieldLabel(pathSegments) {
+    const joined = pathSegments.length > 0 ? pathSegments.join("-") : "event-data";
+    const cleaned = joined
+        .replace(/[^a-zA-Z0-9._-]+/g, "-")
+        .replace(/-+/g, "-")
+        .replace(/^-|-$/g, "")
+        .slice(0, 48);
+    return cleaned || "event-data";
+}
+function shouldExternalizeField(pathSegments, value) {
+    if (value === null || value === undefined)
+        return false;
+    const keyPath = pathSegments.join(".");
+    if (keyPath === "update.rawOutput") {
+        return payloadBytes(value) > EVENT_FIELD_EXTERNALIZE_BYTES;
+    }
+    if (keyPath === "update._meta.claudeCode.toolResponse.stdout" ||
+        keyPath === "update._meta.claudeCode.toolResponse.stderr") {
+        return payloadBytes(value) > EVENT_FIELD_EXTERNALIZE_BYTES;
+    }
+    if (pathSegments[pathSegments.length - 1] === "text") {
+        const hasChunkContext = pathSegments.includes("content") ||
+            pathSegments.includes("chunk") ||
+            pathSegments.includes("delta");
+        if (hasChunkContext) {
+            return payloadBytes(value) > EVENT_FIELD_EXTERNALIZE_BYTES;
+        }
+    }
+    return false;
+}
+async function createBlobPointer(specDir, sessionId, seq, pathSegments, value, context) {
+    const content = stringifyPayload(value);
+    const bytes = Buffer.byteLength(content, "utf-8");
+    const sha256 = createHash("sha256").update(content).digest("hex");
+    const fieldLabel = normalizeFieldLabel(pathSegments);
+    const dryRunCounter = context.dryRunCounter ?? 0;
+    const fileName = context.dryRun
+        ? `${String(seq).padStart(6, "0")}-${fieldLabel}-dry-run-${String(dryRunCounter).padStart(4, "0")}.blob`
+        : `${String(seq).padStart(6, "0")}-${fieldLabel}-${randomUUID()}.blob`;
+    const relativePath = path.posix.join(BLOBS_DIR, fileName);
+    if (!context.dryRun) {
+        if (!context.ensuredDir) {
+            await fsPromises.mkdir(context.blobDir, { recursive: true });
+            context.ensuredDir = true;
+        }
+        const absolutePath = path.join(getSessionDir(specDir, sessionId), relativePath);
+        await fsPromises.writeFile(absolutePath, content, "utf-8");
+    }
+    context.createdBlobs = (context.createdBlobs ?? 0) + 1;
+    context.dryRunCounter = dryRunCounter + 1;
+    return {
+        path: relativePath,
+        bytes,
+        sha256,
+        truncated: true,
+        preview: toPreview(content),
+    };
+}
+async function externalizeOversizedPayloads(specDir, sessionId, seq, value, pathSegments, context) {
+    if (isSessionBlobPointer(value)) {
+        return value;
+    }
+    if (shouldExternalizeField(pathSegments, value)) {
+        return createBlobPointer(specDir, sessionId, seq, pathSegments, value, context);
+    }
+    if (Array.isArray(value)) {
+        return Promise.all(value.map((entry, idx) => externalizeOversizedPayloads(specDir, sessionId, seq, entry, [
+            ...pathSegments,
+            String(idx),
+        ], context)));
+    }
+    if (isRecord(value)) {
+        const next = {};
+        for (const [key, child] of Object.entries(value)) {
+            next[key] = await externalizeOversizedPayloads(specDir, sessionId, seq, child, [...pathSegments, key], context);
+        }
+        return next;
+    }
+    return value;
+}
+function resolveBlobAbsolutePath(specDir, sessionId, relativePath) {
+    const sessionDir = path.resolve(getSessionDir(specDir, sessionId));
+    const absolutePath = path.resolve(sessionDir, relativePath);
+    if (absolutePath === sessionDir ||
+        absolutePath.startsWith(`${sessionDir}${path.sep}`)) {
+        return absolutePath;
+    }
+    return null;
+}
+async function resolveBlobPointer(specDir, sessionId, pointer) {
+    const absolutePath = resolveBlobAbsolutePath(specDir, sessionId, pointer.path);
+    if (!absolutePath) {
+        return { ...pointer, content: pointer.preview };
+    }
+    try {
+        const content = await fsPromises.readFile(absolutePath, "utf-8");
+        return { ...pointer, content };
+    }
+    catch {
+        return { ...pointer, content: pointer.preview };
+    }
+}
+/**
+ * Resolve all blob pointers in a value tree to include full payload content.
+ *
+ * Default flows keep compact pointer objects (preview-only). This helper powers
+ * explicit on-demand blob resolution in session log commands.
+ */
+export async function resolveSessionBlobPointers(specDir, sessionId, value) {
+    if (isSessionBlobPointer(value)) {
+        return resolveBlobPointer(specDir, sessionId, value);
+    }
+    if (Array.isArray(value)) {
+        return Promise.all(value.map((entry) => resolveSessionBlobPointers(specDir, sessionId, entry)));
+    }
+    if (isRecord(value)) {
+        const next = {};
+        for (const [key, child] of Object.entries(value)) {
+            next[key] = await resolveSessionBlobPointers(specDir, sessionId, child);
+        }
+        return next;
+    }
+    return value;
+}
 /**
  * Get the current event count for a session (for seq assignment).
  *
@@ -320,14 +498,159 @@ export async function appendEvent(specDir, input) {
         trace_id: input.trace_id,
         data: input.data,
     };
+    // AC: @session-events ac-8, ac-9 - Externalize oversized payload fields
+    // before writing to events.jsonl.
+    const externalizedData = await externalizeOversizedPayloads(specDir, input.session_id, seq, event.data, [], {
+        blobDir: getSessionBlobDir(specDir, input.session_id),
+        ensuredDir: false,
+    });
+    const eventWithGuardrails = {
+        ...event,
+        data: externalizedData,
+    };
     // Validate event
-    const validated = SessionEventSchema.parse(event);
+    let validated = SessionEventSchema.parse(eventWithGuardrails);
+    // Event-line safety cap: if a single JSONL line is still too large after
+    // targeted field externalization, externalize the entire data payload.
+    let line = JSON.stringify(validated);
+    if (Buffer.byteLength(line, "utf-8") > EVENT_LINE_MAX_BYTES) {
+        const blobContext = {
+            blobDir: getSessionBlobDir(specDir, input.session_id),
+            ensuredDir: false,
+        };
+        const fullDataPointer = await createBlobPointer(specDir, input.session_id, seq, [], validated.data, blobContext);
+        validated = SessionEventSchema.parse({
+            ...validated,
+            data: fullDataPointer,
+        });
+        line = JSON.stringify(validated);
+    }
     // AC: @session-events ac-3 - Use synchronous append for crash safety
     // This ensures the line is fully written before returning
-    const line = `${JSON.stringify(validated)}\n`;
-    fs.appendFileSync(eventsPath, line, "utf-8");
+    fs.appendFileSync(eventsPath, `${line}\n`, "utf-8");
     return validated;
 }
+/**
+ * Retroactively compact a session event log by externalizing oversized payloads.
+ *
+ * Reuses the same two-stage externalization pipeline as appendEvent():
+ * 1) Field-level externalization for known large payload fields
+ * 2) Full-data externalization if a line still exceeds EVENT_LINE_MAX_BYTES
+ *
+ * Writes are atomic (temp-file then rename). When dryRun is enabled, no files
+ * are modified and no blob files are written.
+ */
+export async function compactSessionEvents(specDir, sessionId, options = {}) {
+    const dryRun = options.dryRun === true;
+    const renameFn = options.renameFn ?? fsPromises.rename;
+    const eventsPath = getSessionEventsPath(specDir, sessionId);
+    let content;
+    try {
+        content = await fsPromises.readFile(eventsPath, "utf-8");
+    }
+    catch (err) {
+        if (err instanceof Error && "code" in err && err.code === "ENOENT") {
+            return {
+                events_processed: 0,
+                blobs_created: 0,
+                bytes_before: 0,
+                bytes_after: 0,
+                bytes_reclaimed: 0,
+                changed: false,
+                reason: "missing_events_file",
+                dry_run: dryRun,
+            };
+        }
+        throw err;
+    }
+    const bytesBefore = Buffer.byteLength(content, "utf-8");
+    const sourceLines = content
+        .split("\n")
+        .filter((line) => line.trim().length > 0);
+    if (sourceLines.length === 0) {
+        return {
+            events_processed: 0,
+            blobs_created: 0,
+            bytes_before: bytesBefore,
+            bytes_after: bytesBefore,
+            bytes_reclaimed: 0,
+            changed: false,
+            reason: "empty_events_file",
+            dry_run: dryRun,
+        };
+    }
+    const blobContext = {
+        blobDir: getSessionBlobDir(specDir, sessionId),
+        ensuredDir: false,
+        dryRun,
+        createdBlobs: 0,
+        dryRunCounter: 0,
+    };
+    const compactedLines = [];
+    for (let i = 0; i < sourceLines.length; i += 1) {
+        const line = sourceLines[i];
+        let parsed;
+        try {
+            parsed = JSON.parse(line);
+        }
+        catch (err) {
+            throw new Error(`Invalid JSON in events log at line ${i + 1}: ${err instanceof Error ? err.message : String(err)}`);
+        }
+        const event = SessionEventSchema.parse(parsed);
+        const externalizedData = await externalizeOversizedPayloads(specDir, sessionId, event.seq, event.data, [], blobContext);
+        let validated = SessionEventSchema.parse({
+            ...event,
+            data: externalizedData,
+        });
+        let compactedLine = JSON.stringify(validated);
+        if (Buffer.byteLength(compactedLine, "utf-8") > EVENT_LINE_MAX_BYTES) {
+            const fullDataPointer = await createBlobPointer(specDir, sessionId, event.seq, [], validated.data, blobContext);
+            validated = SessionEventSchema.parse({
+                ...validated,
+                data: fullDataPointer,
+            });
+            compactedLine = JSON.stringify(validated);
+        }
+        compactedLines.push(compactedLine);
+    }
+    const compactedContent = `${compactedLines.join("\n")}\n`;
+    const changed = compactedContent !== content;
+    const bytesAfter = changed ? Buffer.byteLength(compactedContent, "utf-8") : bytesBefore;
+    if (!changed) {
+        return {
+            events_processed: sourceLines.length,
+            blobs_created: 0,
+            bytes_before: bytesBefore,
+            bytes_after: bytesBefore,
+            bytes_reclaimed: 0,
+            changed: false,
+            reason: "already_compacted",
+            dry_run: dryRun,
+        };
+    }
+    if (!dryRun) {
+        const sessionDir = getSessionDir(specDir, sessionId);
+        const tmpPath = path.join(sessionDir, `.${EVENTS_FILE}.${process.pid}.${Date.now()}.tmp`);
+        try {
+            await fsPromises.writeFile(tmpPath, compactedContent, "utf-8");
+            await renameFn(tmpPath, eventsPath);
+        }
+        catch (err) {
+            await fsPromises.unlink(tmpPath).catch(() => undefined);
+            throw err;
+        }
+    }
+    return {
+        events_processed: sourceLines.length,
+        blobs_created: blobContext.createdBlobs ?? 0,
+        bytes_before: bytesBefore,
+        bytes_after: bytesAfter,
+        bytes_reclaimed: Math.max(0, bytesBefore - bytesAfter),
+        changed: true,
+        reason: dryRun ? "would_compact" : "compacted",
+        dry_run: dryRun,
+    };
+}
 /**
  * Read all events from a session.
  *
@@ -1119,6 +1442,7 @@ export async function searchSessionEvents(specDir, pattern, options = {}) {
     const rawLimit = options.limit ?? 50;
     const limit = Number.isNaN(rawLimit) || rawLimit <= 0 ? 50 : rawLimit;
     const lowerPattern = pattern.toLowerCase();
+    const resolveBlobs = options.resolveBlobs ?? false;
     // Get all session summaries for metadata filtering
     const allSummaries = await getAllSessionLogSummaries(specDir);
     // AC: @session-log-search ac-3 - Pre-filter by --since
@@ -1152,8 +1476,9 @@ export async function searchSessionEvents(specDir, pattern, options = {}) {
         for (const line of lines) {
             if (totalMatches >= limit)
                 break;
-            // Quick substring pre-filter before parsing JSON
-            if (!line.toLowerCase().includes(lowerPattern))
+            // Quick substring pre-filter before parsing JSON.
+            // Disabled when resolving blobs because full content lives outside line.
+            if (!resolveBlobs && !line.toLowerCase().includes(lowerPattern))
                 continue;
             try {
                 const event = JSON.parse(line);
@@ -1172,8 +1497,12 @@ export async function searchSessionEvents(specDir, pattern, options = {}) {
                         }
                     }
                 }
-                // Verify match in stringified data (not just line, in case pattern appears in metadata)
-                const dataStr = JSON.stringify(event.data);
+                const searchableData = resolveBlobs
+                    ? await resolveSessionBlobPointers(specDir, summary.id, event.data)
+                    : event.data;
+                // Verify match in stringified data (not just line, in case pattern
+                // appears in metadata)
+                const dataStr = JSON.stringify(searchableData);
                 if (!dataStr.toLowerCase().includes(lowerPattern))
                     continue;
                 // AC: @session-log-search ac-4 - Create match with excerpt
@@ -1181,7 +1510,7 @@ export async function searchSessionEvents(specDir, pattern, options = {}) {
                     session_id: summary.id,
                     timestamp: event.ts,
                     event_type: event.type,
-                    content_excerpt: extractContentExcerpt(event.data, pattern, 200),
+                    content_excerpt: extractContentExcerpt(searchableData, pattern, 200),
                 });
                 totalMatches++;
             }
@@ -1411,12 +1740,12 @@ async function removeDotenvSessionId(filePath) {
  */
 export async function injectCodexEnv(sessionId) {
     const configDir = path.join(process.env.HOME || process.env.USERPROFILE || "", ".codex");
-    const configPath = path.join(configDir, "config.json");
+    const configPath = path.join(configDir, "config.toml");
     await fsPromises.mkdir(configDir, { recursive: true });
     let config = {};
     try {
         const content = await fsPromises.readFile(configPath, "utf-8");
-        config = JSON.parse(content);
+        config = parseTOML(content);
     }
     catch (err) {
         // Only start fresh for ENOENT; throw on parse errors to avoid overwriting
@@ -1424,10 +1753,17 @@ export async function injectCodexEnv(sessionId) {
             // File doesn't exist, start fresh
         }
         else {
-            throw new Error(`Cannot inject env: ~/.codex/config.json exists but is not valid JSON. ` +
+            throw new Error(`Cannot inject env: ~/.codex/config.toml exists but is not valid TOML. ` +
                 `Fix the file manually or remove it, then retry.`);
         }
     }
+    // Capture previous value before overwriting
+    const previousValue = config.shell_environment_policy &&
+        typeof config.shell_environment_policy === "object" &&
+        config.shell_environment_policy.set &&
+        typeof config.shell_environment_policy.set === "object"
+        ? (config.shell_environment_policy.set.KSPEC_SESSION_ID ?? null)
+        : null;
     // Ensure shell_environment_policy.set exists
     if (!config.shell_environment_policy ||
         typeof config.shell_environment_policy !== "object") {
@@ -1438,14 +1774,56 @@ export async function injectCodexEnv(sessionId) {
         policy.set = {};
     }
     policy.set.KSPEC_SESSION_ID = sessionId;
-    await fsPromises.writeFile(configPath, JSON.stringify(config, null, 2) + "\n", "utf-8");
+    await fsPromises.writeFile(configPath, stringifyTOML(config) + "\n", "utf-8");
     return {
         injected: true,
         method: "codex_config",
         description: `Added KSPEC_SESSION_ID to Codex config shell_environment_policy.set`,
         path: configPath,
+        previousValue,
     };
 }
+/**
+ * Remove or restore KSPEC_SESSION_ID in Codex config.
+ *
+ * Reverses the injection performed by injectCodexEnv().
+ * If previousValue is provided, restores it instead of deleting.
+ * Best-effort: silently ignores missing files or missing keys.
+ *
+ * @param previousValue - Value to restore, or null/undefined to delete
+ */
+export async function removeCodexEnv(previousValue) {
+    const configDir = path.join(process.env.HOME || process.env.USERPROFILE || "", ".codex");
+    const configPath = path.join(configDir, "config.toml");
+    try {
+        const content = await fsPromises.readFile(configPath, "utf-8");
+        const config = parseTOML(content);
+        const rawPolicy = config.shell_environment_policy;
+        if (rawPolicy && typeof rawPolicy === "object") {
+            const policy = rawPolicy;
+            if (policy.set && typeof policy.set === "object") {
+                if (previousValue) {
+                    policy.set.KSPEC_SESSION_ID = previousValue;
+                }
+                else {
+                    delete policy.set.KSPEC_SESSION_ID;
+                    // Remove set section entirely if empty
+                    if (Object.keys(policy.set).length === 0) {
+                        delete policy.set;
+                    }
+                    // Remove shell_environment_policy if empty
+                    if (Object.keys(policy).length === 0) {
+                        delete config.shell_environment_policy;
+                    }
+                }
+            }
+            await fsPromises.writeFile(configPath, stringifyTOML(config) + "\n", "utf-8");
+        }
+    }
+    catch {
+        // Best-effort cleanup — file may not exist or may not be valid TOML
+    }
+}
 /**
  * Inject KSPEC_SESSION_ID into Gemini CLI environment.
  *
@@ -1509,9 +1887,9 @@ export async function injectEnvForAdapter(adapterId, sessionId) {
         case "claude-agent-acp":
         case "claude-code-acp":
             return injectClaudeCodeEnv(sessionId);
+        case "codex-acp":
+            return injectCodexEnv(sessionId);
         // Future harnesses can be added here:
-        // case "codex-acp":
-        //   return injectCodexEnv(sessionId);
         // case "gemini-acp":
         //   return injectGeminiEnv(sessionId);
         default:
@@ -1534,7 +1912,9 @@ export async function removeEnvForAdapter(adapterId, previousValue) {
         case "claude-code-acp":
             await removeClaudeCodeEnv(previousValue);
             break;
-        // Future harnesses can be added here
+        case "codex-acp":
+            await removeCodexEnv(previousValue);
+            break;
     }
 }
 // ─── Session Validation ───────────────────────────────────────────────────────