chainlesschain 0.162.188 → 0.162.193

package/src/harness/prompt-compressor.js

@@ -11,12 +11,107 @@
 
 import { createHash } from "node:crypto";
 import { feature, featureVariant } from "../lib/feature-flags.js";
+import {
+  DURABLE_SYSTEM_MESSAGE_KINDS,
+  markDurableSystemMessage,
+} from "../lib/session-message-provenance.js";
+import {
+  buildExtractiveHandoff,
+  formatStructuredHandoff,
+  parseStructuredHandoff,
+  StructuredHandoffValidationError,
+  STRUCTURED_HANDOFF_FIELDS,
+} from "./structured-handoff.js";
+
+export * from "./structured-handoff.js";
 
 // Bounds for the fuzzy near-dup pass in _deduplicate (see there). Generous
 // enough that real near-dups are still caught; small enough that compaction of
 // a long, tool-heavy history stays sub-second instead of O(n²·content).
 const DEDUP_JACCARD_MAX_CHARS = 4000;
 const DEDUP_FUZZY_WINDOW = 100;
+export const SUMMARY_INPUT_DEFAULT_MAX_CHARS = 24000;
+export const SUMMARY_INPUT_HARD_MAX_CHARS = 32000;
+const SUMMARY_INPUT_MIN_CHARS = 1024;
+
+function boundedPositiveInteger(value, fallback, minimum, maximum) {
+  const number = Math.floor(Number(value));
+  if (!Number.isFinite(number) || number <= 0) return fallback;
+  return Math.min(Math.max(number, minimum), maximum);
+}
+
+function summaryEntry(message, index) {
+  const role = typeof message?.role === "string" ? message.role : "unknown";
+  return `[${index}] ${role}: ${getContent(message)
+    .replace(/\r\n?/g, "\n")
+    .trim()
+    .slice(0, 2000)}`;
+}
+
+/** Keep the earliest user objective plus as much recent context as fits. */
+function buildBoundedHistory(messages, maxChars) {
+  const entries = messages
+    .map((message, index) => ({
+      index,
+      role: message?.role,
+      text: summaryEntry(message, index),
+    }))
+    .filter((entry) => entry.text.trim());
+  const all = entries.map((entry) => entry.text).join("\n");
+  if (all.length <= maxChars) return all;
+
+  const firstUser = entries.find((entry) => entry.role === "user");
+  const selected = new Map();
+  let remaining = maxChars;
+  if (firstUser && remaining > 0) {
+    const objectiveBudget = Math.min(
+      firstUser.text.length,
+      Math.max(1, Math.floor(maxChars / 3)),
+    );
+    selected.set(firstUser.index, firstUser.text.slice(0, objectiveBudget));
+    remaining -= objectiveBudget;
+  }
+
+  for (let index = entries.length - 1; index >= 0 && remaining > 0; index--) {
+    const entry = entries[index];
+    if (selected.has(entry.index)) continue;
+    const separator = selected.size > 0 ? 1 : 0;
+    if (remaining <= separator) break;
+    const text = entry.text.slice(0, remaining - separator);
+    if (!text) break;
+    selected.set(entry.index, text);
+    remaining -= text.length + separator;
+  }
+
+  return [...selected.entries()]
+    .sort(([left], [right]) => left - right)
+    .map(([, text]) => text)
+    .join("\n")
+    .slice(0, maxChars);
+}
+
+function summaryPrompt(messages, maxChars) {
+  const schema = `{${STRUCTURED_HANDOFF_FIELDS.map((field) => `"${field}"`).join(",")}}`;
+  const instructions = [
+    "Create a durable conversation handoff from the history below.",
+    `Return exactly one strict JSON object with all keys in this order: ${schema}.`,
+    '"objective" must be a string; every other value must be an array of strings.',
+    "Preserve constraints, exact file paths, test evidence, unresolved external side effects, blockers, and actionable next steps. Do not use Markdown.",
+    "Conversation history:",
+  ].join("\n");
+  const historyBudget = Math.max(0, maxChars - instructions.length - 1);
+  const history = buildBoundedHistory(messages, historyBudget);
+  return `${instructions}\n${history}`.slice(0, maxChars);
+}
+
+function boundedFailureMessage(error) {
+  const message =
+    error instanceof Error ? error.message : String(error || "unknown error");
+  return message
+    .replace(/[\r\n]+/g, " ")
+    .trim()
+    .slice(0, 240);
+}
 
 export function estimateTokens(text) {
   if (!text) return 0;
@@ -186,7 +281,8 @@ export function sanitizeToolPairs(messages) {
         stage1.push({ ...m, tool_calls: kept });
       } else {
         // No surviving calls: keep the assistant text if any, else drop it.
-        const { tool_calls: _drop, ...rest } = m;
+        const rest = { ...m };
+        delete rest.tool_calls;
         if (rest.content && String(rest.content).trim()) stage1.push(rest);
       }
     } else {
@@ -232,6 +328,12 @@ export class PromptCompressor {
     }
     this.similarityThreshold = options.similarityThreshold || 0.9;
     this.llmQuery = options.llmQuery || null;
+    this.summaryInputMaxChars = boundedPositiveInteger(
+      options.summaryInputMaxChars,
+      SUMMARY_INPUT_DEFAULT_MAX_CHARS,
+      SUMMARY_INPUT_MIN_CHARS,
+      SUMMARY_INPUT_HARD_MAX_CHARS,
+    );
   }
 
   adaptToModel(model, provider) {
@@ -271,6 +373,7 @@ export class PromptCompressor {
     const originalTokens = estimateMessagesTokens(messages);
     let result = [...working];
     const applied = [];
+    let summaryStats = null;
 
     if (feature("CONTEXT_SNIP")) {
       const before = result.length;
@@ -290,21 +393,28 @@ export class PromptCompressor {
       if (result.length < before) applied.push("collapse");
     }
 
-    if (result.length > this.maxMessages) {
-      result = this._truncate(result);
-      applied.push("truncate");
-    }
-
+    // Semantic handoff must run BEFORE destructive count-based truncation;
+    // otherwise long conversations commonly fall under maxTokens only because
+    // the facts we needed to summarize were already discarded.
     const currentTokens = estimateMessagesTokens(result);
-    if (this.llmQuery && currentTokens > this.maxTokens && result.length > 4) {
-      try {
-        result = await this._summarize(result);
+    if (
+      this.llmQuery &&
+      (currentTokens > this.maxTokens || result.length > this.maxMessages) &&
+      result.length > 4
+    ) {
+      const summarized = await this._summarize(result);
+      result = summarized.messages;
+      summaryStats = summarized.stats;
+      if (summaryStats.summaryMode !== "none") {
         applied.push("summarize");
-      } catch (_err) {
-        // Summarization failed — continue with what we have
       }
     }
 
+    if (result.length > this.maxMessages) {
+      result = this._truncate(result);
+      applied.push("truncate");
+    }
+
     // Re-insert pinned facts verbatim (order preserved) right after the leading
     // system messages, so they end up as a sticky facts block at the top.
     if (pinned.length) {
@@ -330,6 +440,7 @@ export class PromptCompressor {
       compressedTokens,
       saved: originalTokens - compressedTokens,
       ratio: originalTokens > 0 ? compressedTokens / originalTokens : 1,
+      ...(summaryStats || {}),
     };
 
     const abVariant = getCompressionVariant();
@@ -417,24 +528,108 @@ export class PromptCompressor {
       (m) => m.role !== "system" && m !== last,
     );
 
-    if (toSummarize.length < 3) return messages;
-
-    const historyText = toSummarize
-      .map((m) => `${m.role}: ${getContent(m).slice(0, 500)}`)
-      .join("\n");
-
-    const summary = await this.llmQuery(
-      `Summarize this conversation history concisely, preserving key facts and decisions:\n\n${historyText}\n\nSummary:`,
-    );
+    if (toSummarize.length < 3) {
+      return {
+        messages,
+        stats: {
+          summaryMode: "none",
+          degraded: false,
+          degradedReason: null,
+          summaryInputChars: 0,
+          summaryInputLimit: this.summaryInputMaxChars,
+        },
+      };
+    }
 
-    if (!summary) return messages;
+    const prompt = summaryPrompt(toSummarize, this.summaryInputMaxChars);
+    let summary;
+    let summaryMode = "llm-structured";
+    let degraded = false;
+    let degradedReason = null;
+    let rawSummary;
+    let summaryUsage = null;
+    let summaryProvider = null;
+    let summaryModel = null;
+    try {
+      const queryResult = await this.llmQuery(prompt);
+      if (
+        queryResult &&
+        typeof queryResult === "object" &&
+        Object.prototype.hasOwnProperty.call(queryResult, "summary")
+      ) {
+        rawSummary = queryResult.summary;
+        const usage = queryResult.usage;
+        if (usage && typeof usage === "object") {
+          summaryUsage = {
+            inputTokens: Number(usage.input_tokens ?? usage.prompt_tokens) || 0,
+            outputTokens:
+              Number(usage.output_tokens ?? usage.completion_tokens) || 0,
+            cacheReadTokens:
+              Number(
+                usage.cache_read_input_tokens ?? usage.cache_read_tokens,
+              ) || 0,
+            cacheCreationTokens:
+              Number(
+                usage.cache_creation_input_tokens ??
+                  usage.cache_creation_tokens,
+              ) || 0,
+          };
+        }
+        summaryProvider = queryResult.provider || null;
+        summaryModel = queryResult.model || null;
+      } else {
+        rawSummary = queryResult;
+      }
+    } catch (error) {
+      degraded = true;
+      summaryMode = "extractive-fallback";
+      degradedReason = `llm_query_failed:${boundedFailureMessage(error)}`;
+    }
+    if (!degraded) {
+      try {
+        summary = parseStructuredHandoff(rawSummary);
+      } catch (error) {
+        degraded = true;
+        summaryMode = "extractive-fallback";
+        degradedReason =
+          error instanceof StructuredHandoffValidationError
+            ? `invalid_llm_summary:${error.code}`
+            : `invalid_llm_summary:normalization_error:${boundedFailureMessage(error)}`;
+      }
+    }
+    if (degraded) {
+      summary = buildExtractiveHandoff(toSummarize, {
+        maxFallbackSourceChars: Math.min(
+          this.summaryInputMaxChars,
+          SUMMARY_INPUT_DEFAULT_MAX_CHARS,
+        ),
+      });
+    }
 
     const result = [
       ...system,
-      { role: "system", content: `[Conversation Summary]\n${summary}` },
+      markDurableSystemMessage(
+        {
+          role: "system",
+          content: `[Conversation Summary]\n${formatStructuredHandoff(summary)}`,
+        },
+        DURABLE_SYSTEM_MESSAGE_KINDS.COMPACT_SUMMARY,
+      ),
     ];
     if (last) result.push(last);
-    return result;
+    return {
+      messages: result,
+      stats: {
+        summaryMode,
+        degraded,
+        degradedReason,
+        summaryInputChars: prompt.length,
+        summaryInputLimit: this.summaryInputMaxChars,
+        ...(summaryUsage ? { summaryUsage } : {}),
+        ...(summaryProvider ? { summaryProvider } : {}),
+        ...(summaryModel ? { summaryModel } : {}),
+      },
+    };
   }
 
   _snipCompact(messages) {
@@ -515,10 +710,15 @@ export class PromptCompressor {
             .filter(Boolean);
           const uniqueTools = [...new Set(toolNames)];
 
-          result.push({
-            role: "system",
-            content: `[Collapsed ${toolGroup.length} tool messages: ${uniqueTools.join(", ")}]`,
-          });
+          result.push(
+            markDurableSystemMessage(
+              {
+                role: "system",
+                content: `[Collapsed ${toolGroup.length} tool messages: ${uniqueTools.join(", ")}]`,
+              },
+              DURABLE_SYSTEM_MESSAGE_KINDS.COMPACT_TOOL_COLLAPSE,
+            ),
+          );
           i = j;
           continue;
         }

package/src/harness/session-index.js

@@ -17,18 +17,12 @@
  * without touching the user's home dir.
  */
 
-import {
-  existsSync,
-  mkdirSync,
-  readdirSync,
-  statSync,
-  readFileSync,
-} from "node:fs";
+import { existsSync, readdirSync, statSync } from "node:fs";
 import { join, basename } from "node:path";
 import { createRequire } from "node:module";
 import { getHomeDir } from "../lib/paths.js";
-import { readEvents, toIsoSafe } from "./jsonl-session-store.js";
-import { latestChainHash } from "./transcript-integrity.js";
+import { iterateFileLinesSync } from "../lib/file-lines.js";
+import { ensurePrivateDirectory, ensurePrivateFile } from "../lib/secure-fs.js";
 
 const requireCjs = createRequire(import.meta.url);
 
@@ -50,7 +44,7 @@ function sessionsDir() {
 
 function indexPath() {
   const dir = getHomeDir();
-  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
+  ensurePrivateDirectory(dir);
   return join(dir, "session-index.db");
 }
 
@@ -82,43 +76,105 @@ CREATE INDEX IF NOT EXISTS idx_sessions_last_ts ON sessions(last_ts DESC);
  */
 export function openIndex({ file } = {}) {
   const Database = _deps.loadDatabase();
-  const db = new Database(file || indexPath());
+  const target = file || indexPath();
+  const db = new Database(target);
   db.pragma("journal_mode = WAL");
   db.exec(SCHEMA);
+  if (target !== ":memory:") {
+    ensurePrivateFile(target);
+    ensurePrivateFile(`${target}-wal`);
+    ensurePrivateFile(`${target}-shm`);
+  }
   return db;
 }
 
 /** Max characters of concatenated message content indexed per session. */
 const CONTENT_CAP = 200_000;
 
-function summarizeEvents(id, events) {
-  const startEvent = events.find((e) => e.type === "session_start");
-  const lastEvent = events[events.length - 1];
-  const messageCount = events.filter(
-    (e) => e.type === "user_message" || e.type === "assistant_message",
-  ).length;
+function toIsoSafe(value) {
+  const number = Number(value);
+  if (!Number.isFinite(number)) return "";
+  const date = new Date(number);
+  return Number.isNaN(date.getTime()) ? "" : date.toISOString();
+}
+
+function summarizeSessionFile(id, filePath) {
+  let startEvent = null;
+  let lastEvent = null;
+  let renamedTitle = null;
+  let messageCount = 0;
+  let eventCount = 0;
+  let chainHash = "";
   const parts = [];
   let used = 0;
-  for (const e of events) {
-    if (e.type !== "user_message" && e.type !== "assistant_message") continue;
-    const c = e.data?.content;
-    const s = typeof c === "string" ? c : c == null ? "" : JSON.stringify(c);
-    if (!s) continue;
-    parts.push(s);
-    used += s.length;
-    if (used >= CONTENT_CAP) break;
+  for (const { line } of iterateFileLinesSync(filePath)) {
+    let event;
+    try {
+      event = JSON.parse(line);
+    } catch {
+      continue;
+    }
+    eventCount += 1;
+    lastEvent = event;
+    if (!startEvent && event.type === "session_start") startEvent = event;
+    if (event.type === "session_rename" && event.data?.title) {
+      renamedTitle = String(event.data.title);
+    }
+    if (event.type === "user_message" || event.type === "assistant_message") {
+      messageCount += 1;
+      if (used < CONTENT_CAP) {
+        const content = event.data?.content;
+        const text =
+          typeof content === "string"
+            ? content
+            : content == null
+              ? ""
+              : JSON.stringify(content);
+        if (text) {
+          const bounded = text.slice(0, CONTENT_CAP - used);
+          parts.push(bounded);
+          used += bounded.length;
+        }
+      }
+    } else if (
+      event.type === "ws_turn" &&
+      event.data?.schemaVersion === 1 &&
+      event.data?.outcome === "completed" &&
+      event.data?.user?.role === "user" &&
+      event.data?.assistant?.role === "assistant"
+    ) {
+      messageCount += 2;
+      for (const message of [event.data?.user, event.data?.assistant]) {
+        if (used >= CONTENT_CAP) break;
+        const content = message?.content;
+        const text =
+          typeof content === "string"
+            ? content
+            : content == null
+              ? ""
+              : JSON.stringify(content);
+        if (text) {
+          const bounded = text.slice(0, CONTENT_CAP - used);
+          parts.push(bounded);
+          used += bounded.length;
+        }
+      }
+    }
+    if (typeof event.hash === "string") chainHash = event.hash;
   }
+  if (!lastEvent) return null;
   return {
     id,
-    title: startEvent?.data?.title || "Untitled",
+    title: renamedTitle || startEvent?.data?.title || "Untitled",
     provider: startEvent?.data?.provider || "",
     model: startEvent?.data?.model || "",
     message_count: messageCount,
-    event_count: events.length,
+    event_count: eventCount,
     created_at: toIsoSafe(startEvent?.timestamp),
     updated_at: toIsoSafe(lastEvent?.timestamp),
     last_ts: lastEvent?.timestamp || 0,
     content: parts.join("\n").slice(0, CONTENT_CAP),
+    chain_hash: chainHash,
   };
 }
 
@@ -137,16 +193,10 @@ ON CONFLICT(id) DO UPDATE SET
  * @returns {boolean} true if a row was written.
  */
 export function indexOneSession(db, id, { mtimeMs = 0, sizeBytes = 0 } = {}) {
-  const events = readEvents(id);
-  if (events.length === 0) return false;
-  const s = summarizeEvents(id, events);
   const filePath = join(sessionsDir(), `${id}.jsonl`);
-  let chainHash = "";
-  try {
-    chainHash = latestChainHash(readFileSync(filePath, "utf-8")) || "";
-  } catch {
-    /* chain hash is best-effort */
-  }
+  if (!existsSync(filePath)) return false;
+  const s = summarizeSessionFile(id, filePath);
+  if (!s) return false;
   db.prepare(UPSERT_SESSION).run({
     id: s.id,
     title: s.title,
@@ -157,7 +207,7 @@ export function indexOneSession(db, id, { mtimeMs = 0, sizeBytes = 0 } = {}) {
     created_at: s.created_at,
     updated_at: s.updated_at,
     last_ts: s.last_ts,
-    chain_hash: chainHash,
+    chain_hash: s.chain_hash,
     mtime_ms: Math.floor(mtimeMs),
     size_bytes: sizeBytes,
   });
@@ -270,7 +320,8 @@ export function searchSessions(db, query, { limit = 20 } = {}) {
             .replace(/\s+/g, " ")
             .trim()
         : "";
-    const { _text, ...rest } = r;
+    const rest = { ...r };
+    delete rest._text;
     return { ...rest, snippet };
   });
 }