@pi-unipi/compactor 2.6.1 → 2.9.0

package/src/compaction/search-entries.ts

@@ -1,16 +1,36 @@
 /**
- * BM25-lite search over normalized message blocks
- *
- * Includes module-level index cache for fast repeated queries.
+ * Hybrid search over normalized message blocks (pi-vcc parity):
+ * regex-pattern detection with keyword BM25 fallback, line snippets,
+ * and match-count ranking. Includes module-level index cache.
  */
 
 import type { NormalizedBlock } from "../types.js";
+import { extractPath } from "./extract/files.js";
 import { createHash } from "node:crypto";
 
+export interface SearchHit {
+  docId: number;
+  score: number;
+  text: string;
+  kind: string;
+}
+
+/** Recall hit with optional snippet + file indicators (format-recall shape) */
+export interface RecallHit {
+  index: number;
+  score: number;
+  text: string;
+  kind: string;
+  snippet?: string;
+  matchCount?: number;
+  files?: string[];
+}
+
 interface SearchDoc {
   id: number;
   text: string;
   kind: string;
+  files: string[];
 }
 
 function tokenize(text: string): string[] {
@@ -21,6 +41,207 @@ function tokenize(text: string): string[] {
     .filter((w) => w.length > 1);
 }
 
+// ── Regex safety (pi-vcc parity) ──────────────────────────────────────────
+
+const escapeRegex = (s: string): string =>
+  s.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+/** Quantifier starting at `i`, if any. Only unbounded forms (+, *, {n,}) can
+ *  drive catastrophic backtracking. */
+const quantifierAt = (p: string, i: number): { len: number; unbounded: boolean } => {
+  const c = p[i];
+  if (c === "+" || c === "*") return { len: 1, unbounded: true };
+  if (c === "{") {
+    const end = p.indexOf("}", i);
+    const body = end === -1 ? "" : p.slice(i + 1, end);
+    if (/^\d+(,\d*)?$/.test(body)) return { len: end - i + 1, unbounded: body.endsWith(",") };
+  }
+  return { len: 0, unbounded: false };
+};
+
+/**
+ * Detect an unbounded quantifier applied to a group that already contains one,
+ * e.g. `(a+)+`. The search budget in `searchEntries` is the backstop.
+ */
+const hasNestedQuantifier = (pattern: string): boolean => {
+  const groups: boolean[] = [];
+  let inClass = false;
+  for (let i = 0; i < pattern.length; i++) {
+    const c = pattern[i];
+    if (c === "\\") { i++; continue; }
+    if (inClass) { if (c === "]") inClass = false; continue; }
+    if (c === "[") { inClass = true; continue; }
+    if (c === "(") { groups.push(false); continue; }
+    if (c === ")") {
+      const inner = groups.pop() ?? false;
+      const q = quantifierAt(pattern, i + 1);
+      if (inner && q.unbounded) return true;
+      if (groups.length) groups[groups.length - 1] ||= inner || q.unbounded;
+      i += q.len;
+      continue;
+    }
+    const q = quantifierAt(pattern, i);
+    if (q.unbounded && groups.length) {
+      groups[groups.length - 1] = true;
+      i += q.len - 1;
+    }
+  }
+  return false;
+};
+
+/** Try to compile as regex; fall back to escaped literal. Patterns with nested
+ *  unbounded quantifiers are treated as literals rather than compiled. */
+const safeRegex = (pattern: string): RegExp => {
+  if (hasNestedQuantifier(pattern)) return new RegExp(escapeRegex(pattern), "i");
+  try {
+    return new RegExp(pattern, "i");
+  } catch {
+    return new RegExp(escapeRegex(pattern), "i");
+  }
+};
+
+/** Wall-clock budget for one search. Per-entry checkpoint, not a hard ceiling. */
+const SEARCH_BUDGET_MS = 3000;
+
+const startBudget = (): (() => void) => {
+  const deadline = Date.now() + SEARCH_BUDGET_MS;
+  return () => {
+    if (Date.now() > deadline) {
+      throw new Error(
+        `Search aborted: query exceeded ${SEARCH_BUDGET_MS}ms. Simplify the pattern — ` +
+        "nested quantifiers such as (a+)+ can make matching blow up.",
+      );
+    }
+  };
+};
+
+/** Detect if the query looks like a single regex pattern (contains metacharacters). */
+const looksLikeRegex = (query: string): boolean =>
+  /[|*+?{}()[\]\\^$.]/.test(query);
+
+/** Build a regex for snippet highlighting — matches first available term. */
+const snippetRegex = (terms: string[]): RegExp => {
+  const alts = terms.map((t) => safeRegex(t).source);
+  return new RegExp(alts.join("|"), "i");
+};
+
+// ── Stopwords for natural language queries ──
+const STOPWORDS = new Set([
+  "the", "a", "an", "is", "are", "was", "were", "be", "been", "being",
+  "have", "has", "had", "do", "does", "did", "will", "would", "could",
+  "should", "may", "might", "can", "shall", "of", "in", "to", "for",
+  "with", "on", "at", "from", "by", "as", "into", "through", "during",
+  "before", "after", "above", "below", "between", "out", "off", "over",
+  "under", "again", "further", "then", "once", "here", "there", "when",
+  "where", "why", "how", "all", "both", "each", "few", "more", "most",
+  "other", "some", "such", "no", "nor", "not", "only", "own", "same",
+  "so", "than", "too", "very", "just", "about", "it", "its", "that",
+  "this", "what", "which", "who", "whom", "these", "those",
+]);
+
+const filterStopwords = (terms: string[]): string[] => {
+  const meaningful = terms.filter((t) => !STOPWORDS.has(t.toLowerCase()) && t.length > 1);
+  return meaningful.length > 0 ? meaningful : terms;
+};
+
+const countMatches = (hay: string, terms: string[]): number => {
+  let count = 0;
+  for (const t of terms) {
+    if (safeRegex(t).test(hay)) count++;
+  }
+  return count;
+};
+
+// ── BM25 scoring ──────────────────────────────────────────────────────────
+const BM25_K = 1.2;
+const BM25_B = 0.75;
+
+const termFreq = (text: string, pattern: RegExp): number => {
+  const matches = text.match(new RegExp(pattern.source, "gi"));
+  return matches ? matches.length : 0;
+};
+
+interface BM25Context {
+  n: number;
+  avgDl: number;
+  df: Map<string, number>;
+}
+
+const buildBM25Context = (docs: string[], terms: string[], checkBudget: () => void): BM25Context => {
+  const n = docs.length;
+  const df = new Map<string, number>();
+  let totalLen = 0;
+
+  for (const doc of docs) {
+    checkBudget();
+    totalLen += doc.split(/\s+/).length;
+    for (const t of terms) {
+      if (safeRegex(t).test(doc)) {
+        df.set(t, (df.get(t) ?? 0) + 1);
+      }
+    }
+  }
+
+  return { n, avgDl: totalLen / Math.max(n, 1), df };
+};
+
+const bm25Score = (doc: string, terms: string[], ctx: BM25Context): number => {
+  const dl = doc.split(/\s+/).length;
+  let score = 0;
+
+  for (const t of terms) {
+    const tf = termFreq(doc, safeRegex(t));
+    if (tf === 0) continue;
+
+    const docFreq = ctx.df.get(t) ?? 0;
+    const idf = Math.log((ctx.n - docFreq + 0.5) / (docFreq + 0.5) + 1);
+    const tfNorm = (tf * (BM25_K + 1)) / (tf + BM25_K * (1 - BM25_B + BM25_B * dl / ctx.avgDl));
+    score += idf * tfNorm;
+  }
+
+  return score;
+};
+
+/** Line-based snippet: ±contextLines around first regex match. */
+const lineSnippet = (text: string, regex: RegExp, contextLines = 2): string | undefined => {
+  const lines = text.split("\n");
+  let matchIdx = -1;
+  for (let i = 0; i < lines.length; i++) {
+    if (regex.test(lines[i])) {
+      matchIdx = i;
+      break;
+    }
+  }
+  if (matchIdx === -1) return undefined;
+
+  const start = Math.max(0, matchIdx - contextLines);
+  const end = Math.min(lines.length, matchIdx + contextLines + 1);
+  const slice = lines.slice(start, end);
+
+  const parts: string[] = [];
+  if (start > 0) parts.push(`...(${start} lines above)`);
+  parts.push(...slice);
+  if (end < lines.length) parts.push(`...(${lines.length - end} lines below)`);
+  return parts.join("\n");
+};
+
+// ── Block → doc text ──────────────────────────────────────────────────────
+
+const blockFiles = (b: NormalizedBlock): string[] => {
+  if (b.kind !== "tool_call") return [];
+  const p = extractPath(b.args);
+  return p ? [p] : [];
+};
+
+const blockText = (b: NormalizedBlock): string =>
+  b.kind === "tool_call" ? `${b.name} ${JSON.stringify(b.args)}` : b.kind === "tool_result" ? `${b.name} ${b.text}` : b.text;
+
+// ── Module-level index cache ──────────────────────────────────────────────
+
+let cachedIndexHash = "";
+let cachedIndex: Map<string, number[]> | null = null;
+let cachedDocs: SearchDoc[] = [];
+
 function buildIndex(docs: SearchDoc[]): Map<string, number[]> {
   const index = new Map<string, number[]>();
   for (const doc of docs) {
@@ -34,115 +255,131 @@ function buildIndex(docs: SearchDoc[]): Map<string, number[]> {
   return index;
 }
 
-function bm25Score(
-  queryTokens: string[],
-  docId: number,
-  index: Map<string, number[]>,
-  docCount: number,
-  avgDocLen: number,
-  docLens: Map<number, number>,
-): number {
-  const k1 = 1.5;
-  const b = 0.75;
-  let score = 0;
-  const docLen = docLens.get(docId) ?? 1;
-
-  for (const token of queryTokens) {
-    const postings = index.get(token) ?? [];
-    const df = new Set(postings).size;
-    if (df === 0) continue;
-    const tf = postings.filter((id) => id === docId).length;
-    const idf = Math.log((docCount - df + 0.5) / (df + 0.5) + 1);
-    score += idf * ((tf * (k1 + 1)) / (tf + k1 * (1 - b + b * (docLen / avgDocLen))));
-  }
-
-  return score;
-}
+function getCachedIndex(docs: SearchDoc[]): { index: Map<string, number[]>; docs: SearchDoc[] } {
+  const hashSource = docs.length > 0
+    ? `${docs.length}:${docs[0].text.slice(0, 80)}:${docs[docs.length - 1].text.slice(-80)}`
+    : "empty";
+  const currentHash = createHash("sha256").update(hashSource).digest("hex");
 
-export interface SearchHit {
-  docId: number;
-  score: number;
-  text: string;
-  kind: string;
+  if (currentHash === cachedIndexHash && cachedIndex && cachedDocs.length === docs.length) {
+    return { index: cachedIndex, docs: cachedDocs };
+  }
+  const index = buildIndex(docs);
+  cachedIndexHash = currentHash;
+  cachedIndex = index;
+  cachedDocs = docs;
+  return { index, docs };
 }
 
-// Module-level index cache
-let cachedIndexHash = "";
-let cachedDocs: SearchDoc[] = [];
-let cachedIndex: Map<string, number[]> | null = null;
-let cachedDocCount = 0;
-let cachedAvgDocLen = 0;
-let cachedDocLens: Map<number, number> = new Map();
-
-export function invalidateSearchCache(): void {
-  cachedIndexHash = "";
-  cachedDocs = [];
-  cachedIndex = null;
-  cachedDocCount = 0;
-  cachedAvgDocLen = 0;
-  cachedDocLens = new Map();
-}
+// ── Main search ───────────────────────────────────────────────────────────
 
-export function searchEntries(
+/**
+ * Search blocks. When `query` is empty/omitted, returns all blocks as hits
+ * (caller decides recent-window slicing).
+ */
+export const searchEntries = (
   blocks: NormalizedBlock[],
-  query: string,
-  opts?: { limit?: number; offset?: number },
-): SearchHit[] {
-  const docs: SearchDoc[] = blocks.map((b, i) => ({
+  query?: string,
+): RecallHit[] => {
+  const allDocs: SearchDoc[] = blocks.map((b, i) => ({
     id: i,
-    text: b.kind === "tool_call" ? `${b.name} ${JSON.stringify(b.args)}` : b.kind === "tool_result" ? `${b.name} ${b.text}` : b.text,
+    text: blockText(b),
     kind: b.kind,
+    files: blockFiles(b),
   }));
 
-  // Compute content hash to detect blocks change
-  const hashSource = docs.length > 0
-    ? `${docs.length}:${docs[0].text.slice(0, 80)}:${docs[docs.length - 1].text.slice(-80)}`
-    : "empty";
-  const currentHash = createHash("sha256").update(hashSource).digest("hex");
-
-  // Use cached index if blocks haven't changed
-  let index: Map<string, number[]>;
-  let docCount: number;
-  let avgDocLen: number;
-  let docLens: Map<number, number>;
-
-  if (currentHash === cachedIndexHash && cachedIndex) {
-    index = cachedIndex;
-    docCount = cachedDocCount;
-    avgDocLen = cachedAvgDocLen;
-    docLens = cachedDocLens;
-  } else {
-    index = buildIndex(docs);
-    docCount = docs.length;
-    docLens = new Map(docs.map((d) => [d.id, tokenize(d.text).length]));
-    avgDocLen = docCount > 0 ? [...docLens.values()].reduce((a, b) => a + b, 0) / docCount : 1;
-
-    // Update cache
-    cachedIndexHash = currentHash;
-    cachedDocs = docs;
-    cachedIndex = index;
-    cachedDocCount = docCount;
-    cachedAvgDocLen = avgDocLen;
-    cachedDocLens = docLens;
+  if (!query?.trim()) {
+    return allDocs.map((d) => ({
+      index: d.id,
+      score: 0,
+      text: d.text,
+      kind: d.kind,
+      files: d.files.length > 0 ? d.files : undefined,
+    }));
   }
 
-  const queryTokens = tokenize(query);
-  if (queryTokens.length === 0) return [];
+  const rawQuery = query.trim();
+  const checkBudget = startBudget();
 
-  const scores = new Map<number, number>();
-  for (const doc of docs) {
-    const score = bm25Score(queryTokens, doc.id, index, docCount, avgDocLen, docLens);
-    if (score > 0) scores.set(doc.id, score);
+  // If the query looks like a single regex pattern (contains metacharacters),
+  // treat the whole thing as one pattern — don't split into terms. Detection
+  // is deliberately loose (ordinary prose trips it), so an empty regex result
+  // falls through to term search below — mode detection must never silently
+  // lose results.
+  if (looksLikeRegex(rawQuery)) {
+    const regex = safeRegex(rawQuery);
+    const hits: RecallHit[] = [];
+    for (let i = 0; i < allDocs.length; i++) {
+      checkBudget();
+      const d = allDocs[i];
+      const hay = `${d.kind} ${d.text} ${d.files.join(" ")}`;
+      if (regex.test(hay)) {
+        const snip = lineSnippet(d.text, regex);
+        hits.push({ index: d.id, score: 1, text: d.text, kind: d.kind, snippet: snip, matchCount: 1, files: d.files.length > 0 ? d.files : undefined });
+      }
+    }
+    if (hits.length > 0) return hits;
   }
 
-  const sorted = [...scores.entries()]
-    .sort((a, b) => b[1] - a[1])
-    .map(([docId, score]) => {
-      const doc = docs[docId];
-      return { docId, score, text: doc.text.slice(0, 300), kind: doc.kind };
+  // Natural language / multi-word query: BM25 scoring
+  const rawTerms = rawQuery.split(/\s+/);
+  const terms = filterStopwords(rawTerms);
+  const snipRe = snippetRegex(terms);
+
+  const docs = allDocs.map((d) => `${d.kind} ${d.text} ${d.files.join(" ")}`);
+  const ctx = buildBM25Context(docs, terms, checkBudget);
+
+  const scored: Array<{ hit: RecallHit; score: number }> = [];
+  for (let i = 0; i < allDocs.length; i++) {
+    checkBudget();
+    const d = allDocs[i];
+    const hay = docs[i];
+    const mc = countMatches(hay, terms);
+    if (mc === 0) continue;
+    const score = bm25Score(hay, terms, ctx);
+    const snip = lineSnippet(d.text, snipRe);
+    scored.push({
+      hit: {
+        index: d.id,
+        score,
+        text: d.text,
+        kind: d.kind,
+        snippet: snip,
+        matchCount: mc,
+        files: d.files.length > 0 ? d.files : undefined,
+      },
+      score,
     });
+  }
+
+  scored.sort((a, b) => b.score - a.score);
+  return scored.map((s) => s.hit);
+};
 
+/** Backwards-compatible BM25-only search over blocks (legacy callers). */
+export function searchEntriesLegacy(
+  blocks: NormalizedBlock[],
+  query: string,
+  opts?: { limit?: number; offset?: number },
+): SearchHit[] {
+  const hits = searchEntries(blocks, query);
   const offset = opts?.offset ?? 0;
   const limit = opts?.limit ?? 10;
-  return sorted.slice(offset, offset + limit);
+  return hits.slice(offset, offset + limit).map((h) => ({
+    docId: h.index,
+    score: h.score,
+    text: h.text.slice(0, 300),
+    kind: h.kind,
+  }));
+}
+
+// Keep the token-index cache warm for repeated legacy queries.
+export function warmIndexCache(blocks: NormalizedBlock[]): void {
+  const docs: SearchDoc[] = blocks.map((b, i) => ({
+    id: i,
+    text: blockText(b),
+    kind: b.kind,
+    files: blockFiles(b),
+  }));
+  getCachedIndex(docs);
 }

package/src/compaction/skill-collapse.ts

@@ -0,0 +1,35 @@
+/** Shared skill-tag collapse utilities (pi-vcc parity) */
+
+const SKILL_TAG_RE = /^-?\s*<skill\s+name="([^"]+)"/;
+const SKILL_CLOSE_RE = /^-?\s*<\/skill>/;
+
+/** Collapse skill tags in an array of lines — dedup by name, drop all content inside block */
+export const collapseSkillLines = (lines: string[]): string[] => {
+  const result: string[] = [];
+  const seenSkills = new Set<string>();
+  let insideSkill = false;
+
+  for (const line of lines) {
+    const skillMatch = line.match(SKILL_TAG_RE);
+    if (skillMatch) {
+      insideSkill = true;
+      const name = skillMatch[1];
+      if (!seenSkills.has(name)) {
+        seenSkills.add(name);
+        result.push(`[skill: ${name}]`);
+      }
+      continue;
+    }
+    if (insideSkill) {
+      if (SKILL_CLOSE_RE.test(line)) insideSkill = false;
+      continue;
+    }
+    result.push(line);
+  }
+  return result;
+};
+
+/** Collapse <skill name="X" ...>...</skill> blocks in raw text */
+const SKILL_BLOCK_RE = /<skill\s+name="([^"]+)"[^>]*>[\s\S]*?(?:<\/skill>|$)/g;
+export const collapseSkillText = (text: string): string =>
+  text.replace(SKILL_BLOCK_RE, (_, name) => `[skill: ${name}]`);

package/src/compaction/summarize.ts

@@ -3,27 +3,58 @@
  */
 
 import type { Message } from "@earendil-works/pi-ai";
-import type { CompileInput, FileOps } from "../types.js";
+import type { CompileInput, FileOps, NormalizedBlock } from "../types.js";
 import { normalizeMessages } from "./normalize.js";
 import { filterNoise } from "./filter-noise.js";
 import { buildSections } from "./build-sections.js";
 import { formatSummary, RECALL_NOTE } from "./format.js";
-import { mergePrevious } from "./merge.js";
+import { mergePrevious, mergeBriefTranscriptWithFreshBudget } from "./merge.js";
+import { selectRankedBriefBlocks, type BriefRankingOptions } from "./rank.js";
 
-export const compile = (input: CompileInput): string => {
+export interface RankedCompileInput extends CompileInput {
+  fileOps?: FileOps;
+  ranking?: BriefRankingOptions;
+}
+
+interface CompileWithBriefBlocksOptions {
+  briefBlocksFor?: (blocks: NormalizedBlock[]) => NormalizedBlock[];
+  capFreshBrief?: boolean;
+  preserveFreshBriefOnMerge?: boolean;
+}
+
+const compileWithBriefBlocks = (input: CompileInput, options: CompileWithBriefBlocksOptions = {}): string => {
   const blocks = filterNoise(normalizeMessages(input.messages));
-  const data = buildSections({ blocks });
-  const fresh = formatSummary(data);
+  const briefBlocks = options.briefBlocksFor?.(blocks);
+  const data = buildSections({ blocks, briefBlocks });
+  const fresh = formatSummary(data, { capBriefTranscript: options.capFreshBrief ?? true });
   const prev = input.previousSummary
     ? stripRecallNote(input.previousSummary)
     : undefined;
-  const merged = prev ? mergePrevious(prev, fresh) : fresh;
+  const merged = prev
+    ? mergePrevious(prev, fresh, { preserveFreshBrief: options.preserveFreshBriefOnMerge })
+    : fresh;
   if (!merged) return "";
   return merged + "\n\n---\n\n" + RECALL_NOTE;
 };
 
+export const compile = (input: CompileInput): string =>
+  compileWithBriefBlocks(input);
+
+export const compileRanked = (input: RankedCompileInput): string =>
+  compileWithBriefBlocks(input, {
+    briefBlocksFor: (blocks) =>
+      selectRankedBriefBlocks(blocks, {
+        ...input.ranking,
+        fileOps: input.ranking?.fileOps ?? input.fileOps,
+      }),
+    capFreshBrief: false,
+    preserveFreshBriefOnMerge: true,
+  });
+
 const stripRecallNote = (text: string): string => {
   const idx = text.lastIndexOf(RECALL_NOTE);
   if (idx < 0) return text;
   return text.slice(0, idx).replace(/\s*(?:\n\n---\n\n)?\s*$/, "").trimEnd();
 };
+
+export { mergeBriefTranscriptWithFreshBudget };

package/src/compaction/token-estimate.ts

@@ -0,0 +1,104 @@
+/**
+ * Token estimation with per-session calibration (ported from pi-vcc token-estimate.ts)
+ */
+
+export const DEFAULT_CHARS_PER_TOKEN = 4;
+export const MIN_CHARS_PER_TOKEN = 2;
+export const MAX_CHARS_PER_TOKEN = 6;
+
+export type TokenEstimateMode = "heuristic" | "calibrated";
+
+export interface TokenEstimateCalibration {
+  mode: TokenEstimateMode;
+  charsPerToken: number;
+  sourceChars?: number;
+  sourceTokens?: number;
+  rawCharsPerToken?: number;
+}
+
+const clamp = (value: number, min: number, max: number): number =>
+  Math.min(max, Math.max(min, value));
+
+export const calibrateCharsPerToken = (
+  sourceChars: number,
+  sourceTokens: number | undefined,
+): TokenEstimateCalibration => {
+  if (!sourceTokens || sourceTokens <= 0 || sourceChars <= 0) {
+    return { mode: "heuristic", charsPerToken: DEFAULT_CHARS_PER_TOKEN };
+  }
+
+  const rawCharsPerToken = sourceChars / sourceTokens;
+  if (!Number.isFinite(rawCharsPerToken) || rawCharsPerToken <= 0) {
+    return { mode: "heuristic", charsPerToken: DEFAULT_CHARS_PER_TOKEN };
+  }
+
+  return {
+    mode: "calibrated",
+    charsPerToken: clamp(rawCharsPerToken, MIN_CHARS_PER_TOKEN, MAX_CHARS_PER_TOKEN),
+    sourceChars,
+    sourceTokens,
+    rawCharsPerToken,
+  };
+};
+
+export const estimateTokensFromChars = (
+  chars: number,
+  charsPerToken = DEFAULT_CHARS_PER_TOKEN,
+): number => Math.ceil(chars / charsPerToken);
+
+/**
+ * Chars attributed to one image part, mirroring pi-agent-core's own
+ * estimateTokens heuristic (4800 chars ≈ 1200 tokens at 4 chars/token).
+ */
+export const IMAGE_CONTENT_CHARS = 4800;
+
+const safeJsonStringify = (value: unknown): string => {
+  try {
+    return JSON.stringify(value ?? "") ?? "";
+  } catch {
+    return "";
+  }
+};
+
+/**
+ * Estimate the char length of a message's content (string or content-parts
+ * array). Counts every token-bearing part that pi-agent-core's harness
+ * estimateTokens counts, so the calibrated chars/token ratio is not deflated:
+ *  - text       → text.length
+ *  - thinking   → thinking.length   (opus emits large reasoning blocks)
+ *  - toolCall   → name + arguments  (Pi uses `arguments`; `input` kept for compat)
+ *  - image      → IMAGE_CONTENT_CHARS
+ *  - toolResult → nested content    (legacy part shape)
+ */
+export const estimateMessageContentChars = (content: unknown): number => {
+  if (typeof content === "string") return content.length;
+  if (!Array.isArray(content)) return 0;
+  return content.reduce((sum: number, part: any) => {
+    if (!part || typeof part !== "object") return sum;
+    switch (part.type) {
+      case "text":
+        return sum + (typeof part.text === "string" ? part.text.length : 0);
+      case "thinking":
+        return sum + (typeof part.thinking === "string" ? part.thinking.length : 0);
+      case "toolCall": {
+        const args = part.arguments ?? part.input;
+        const argLength = typeof args === "string" ? args.length : safeJsonStringify(args).length;
+        return sum + (part.name?.length ?? 0) + argLength;
+      }
+      case "toolResult": {
+        const c = part.content;
+        return sum + (typeof c === "string" ? c.length : safeJsonStringify(c).length);
+      }
+      case "image":
+        return sum + IMAGE_CONTENT_CHARS;
+      default:
+        // Unknown part: fall back to any text field so we never undercount.
+        return sum + (typeof part.text === "string" ? part.text.length : 0);
+    }
+  }, 0);
+};
+
+export const estimateMessageContentTokens = (
+  content: unknown,
+  charsPerToken = DEFAULT_CHARS_PER_TOKEN,
+): number => estimateTokensFromChars(estimateMessageContentChars(content), charsPerToken);