opencode-context-indicator 1.4.0 → 1.5.0

package/lib/estimate.js

@@ -81,6 +81,18 @@ function safeCountOrNull(v) {
   return typeof v === "number" && Number.isFinite(v) && v >= 0 ? v : null;
 }
 
+// A finite, non-negative, SANE number or null — for persisted provider-reported
+// figures (cache read/write). Like safeCountOrNull but also rejects absurd
+// magnitudes, so a poisoned state.json cannot render garbage (fmt(NaN), 1e15).
+function safeSaneOrNull(v) {
+  return typeof v === "number" &&
+    Number.isFinite(v) &&
+    v >= 0 &&
+    v <= MAX_SANE_LIMIT
+    ? v
+    : null;
+}
+
 // Normalise a categories object from untrusted input (state.json) into the 8
 // known numeric fields — never trust its shape. `toolOut` is 0 for legacy
 // entries written before the category existed (their tool results stay folded
@@ -99,6 +111,53 @@ function safeCategories(raw) {
   };
 }
 
+// A session key safe to use as an object property / map key. Rejects the
+// prototype-pollution vectors ("__proto__", "constructor", "prototype").
+// Sum of the 8 canonical category fields (null `system`/`toolSchemas` count as
+// 0). The single place the "total measured tokens" is derived — shared by the
+// residual-`other` computation, the continuity test and the log/summary writers.
+function sumCategories(c) {
+  if (!c || typeof c !== "object") return 0;
+  return (
+    safeCount(c.user) +
+    safeCount(c.assistant) +
+    safeCount(c.reasoning) +
+    safeCount(c.toolArgs) +
+    safeCount(c.toolOut) +
+    safeCount(c.system) +
+    safeCount(c.toolSchemas) +
+    safeCount(c.other)
+  );
+}
+
+// Normalise a breakdown record (cache record with *Chars keys, or a persisted
+// state entry with canonical keys) into the 8 categories, deriving the residual
+// `other` from `inputTokens`. Pure; the one converter shared by the V2 live log
+// (lib/v2.js) and the slash-command report (lib/commands.js).
+function breakdownCategories(rec, inputTokens) {
+  const user = rec?.userChars ?? rec?.user ?? 0;
+  const assistant = rec?.assistantChars ?? rec?.assistant ?? 0;
+  const reasoning = rec?.reasoningChars ?? rec?.reasoning ?? 0;
+  const toolArgs = rec?.toolArgsChars ?? rec?.toolArgs ?? 0;
+  const toolOut = rec?.toolOutChars ?? rec?.toolOut ?? 0;
+  const system =
+    rec?.systemChars !== undefined ? rec.systemChars : (rec?.system ?? null);
+  const toolSchemas =
+    rec?.toolSchemasChars !== undefined
+      ? rec.toolSchemasChars
+      : (rec?.toolSchemas ?? null);
+  const sum =
+    user +
+    assistant +
+    reasoning +
+    toolArgs +
+    toolOut +
+    (system || 0) +
+    (toolSchemas || 0);
+  const other = Math.max(0, (inputTokens || 0) - sum);
+  return { user, assistant, reasoning, toolArgs, toolOut, system, toolSchemas, other };
+}
+
 // A session key safe to use as an object property / map key. Rejects the
 // prototype-pollution vectors ("__proto__", "constructor", "prototype").
 function isSafeSessionKey(k) {
@@ -111,6 +170,250 @@ function isSafeSessionKey(k) {
   );
 }
 
+// Bound on the number of distinct tool names kept in a per-tool tool-result
+// map (state.json / context payload are untrusted and a plugin could define an
+// unbounded tool set). Everything past the cap — plus any unsafe name — is
+// folded into a single "other tools" bucket so the total still reconciles with
+// the toolOut category.
+const TOOL_OUT_MAX_KEYS = 24;
+// Normalise a per-tool tool-result map (tool name -> estimated tokens) from
+// untrusted input. Names become object keys AND display cells, so prototype
+// vectors are dropped and the map is bounded (see TOOL_OUT_MAX_KEYS). The
+// overflow bucket is named "other tools" — a real tool with that exact name
+// would merge into it (accepted: it is a display-only label).
+function safeToolOutByTool(raw, maxKeys = TOOL_OUT_MAX_KEYS) {
+  if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
+  const out = {};
+  let rest = 0;
+  let used = 0;
+  for (const [k, v] of Object.entries(raw)) {
+    const val = safeCount(v);
+    if (!isSafeSessionKey(k) || used >= maxKeys) {
+      rest += val;
+      continue;
+    }
+    out[k] = (out[k] || 0) + val;
+    used++;
+  }
+  if (rest > 0) out["other tools"] = (out["other tools"] || 0) + rest;
+  return Object.keys(out).length > 0 ? out : null;
+}
+
+// Descending [name, tokens] pairs for display, capped at `limit`; empty when
+// the map carries no positive rows.
+function toolOutEntries(map, limit = 8) {
+  if (!map || typeof map !== "object" || Array.isArray(map)) return [];
+  return Object.entries(map)
+    .map(([k, v]) => [k, safeCount(v)])
+    .filter(([, v]) => v > 0)
+    .sort((a, b) => b[1] - a[1])
+    .slice(0, Math.max(0, limit));
+}
+
+// ---------------------------------------------------------------------------
+// Waste detection: find redundant tool output already sitting in the context.
+// Pure transforms — the caller supplies per-call instances captured from the
+// request payload; nothing here touches the fs or the caches.
+// ---------------------------------------------------------------------------
+
+// Best-effort TARGET of a tool call, from the per-call input. A short, stable
+// string used only to group repeated calls (the same file read twice) and to
+// label a finding. Returns null when no suitable string field exists. The value
+// is untrusted (tool args); it is whitespace-collapsed and bounded here, and
+// display layers sanitise it further.
+const TARGET_FIELD_MAP = {
+  webfetch: ["url", "uri"],
+  fetch: ["url", "uri"],
+  bash: ["command", "cmd"],
+  shell: ["command", "cmd"],
+  grep: ["pattern", "query", "regex", "path"],
+  search: ["query", "pattern"],
+  glob: ["pattern", "path"],
+  read: ["filePath", "file", "path", "filename"],
+  read_file: ["filePath", "file", "path", "filename"],
+  write: ["filePath", "file", "path", "filename"],
+  edit: ["filePath", "file", "path", "filename"],
+  list: ["path", "dir"],
+  ls: ["path", "dir"],
+  task: ["description", "prompt"],
+  agent: ["description", "prompt"],
+};
+function extractToolTarget(name, input) {
+  if (!input || typeof input !== "object" || Array.isArray(input)) return null;
+  const n = typeof name === "string" ? name.toLowerCase() : "";
+  const fields = TARGET_FIELD_MAP[n];
+  const pick = (keys) => {
+    for (const k of keys) {
+      const v = input[k];
+      if (typeof v === "string" && v.length > 0) return v;
+    }
+    return null;
+  };
+  let t = fields ? pick(fields) : null;
+  if (t == null) {
+    // Generic fallback: the first non-empty string-valued field. Bounded scan
+    // (the input object is untrusted) so a crafted huge input cannot stall us.
+    let scanned = 0;
+    for (const k of Object.keys(input)) {
+      if (++scanned > 32) break;
+      const v = input[k];
+      if (typeof v === "string" && v.length > 0) {
+        t = v;
+        break;
+      }
+    }
+  }
+  if (typeof t !== "string" || t.length === 0) return null;
+  // Slice BEFORE the whitespace pass: normalising an untrusted multi-MB string
+  // is the expensive part, so cap the work first.
+  return t.slice(0, 400).replace(/\s+/g, " ").trim().slice(0, 200);
+}
+
+// Cheap content fingerprint of a tool RESULT (length + FNV-1a over the first
+// 4KB). Used to detect byte-identical outputs already in context. Not
+// cryptographic — deliberately cheap (runs on every model request).
+function contentFingerprint(text) {
+  if (typeof text !== "string" || text.length === 0) return "";
+  const s = text.length > 4096 ? text.slice(0, 4096) : text;
+  let h = 2166136261;
+  for (let i = 0; i < s.length; i++) {
+    h ^= s.charCodeAt(i);
+    h = Math.imul(h, 16777619);
+  }
+  return `${text.length}:${(h >>> 0).toString(36)}`;
+}
+
+// Group per-call instances and estimate the tool-output tokens that are
+// redundant (the same target invoked repeatedly, or byte-identical outputs).
+// `instances`: [{ name, target, tokens, fp }]. Reclaim = the sum of the
+// NON-largest member of every group of size >= 2, unioned across target- and
+// content-groups so an instance is counted once. This is a LOWER bound of what
+// compaction/trim could reclaim — labelled as an estimate everywhere.
+function computeReclaim(instances, maxFindings = 8) {
+  const list = Array.isArray(instances) ? instances : [];
+  const byTarget = new Map();
+  const byContent = new Map();
+  const push = (map, key, idx) => {
+    const a = map.get(key);
+    if (a) a.push(idx);
+    else map.set(key, [idx]);
+  };
+  for (let i = 0; i < list.length; i++) {
+    const it = list[i];
+    if (safeCount(it?.tokens) <= 0) continue;
+    const name = typeof it?.name === "string" && it.name ? it.name : "?";
+    const target =
+      typeof it?.target === "string" && it.target ? it.target : null;
+    if (target) push(byTarget, `${name}\u0000${target}`, i);
+    const fp = typeof it?.fp === "string" && it.fp ? it.fp : "";
+    if (fp) push(byContent, fp, i);
+  }
+  // Resolve each group's keeper (largest member) once.
+  const buildGroups = (map) => {
+    const out = [];
+    for (const idxs of map.values()) {
+      if (idxs.length < 2) continue;
+      let maxI = idxs[0];
+      for (const i of idxs) {
+        if (safeCount(list[i]?.tokens) > safeCount(list[maxI]?.tokens)) maxI = i;
+      }
+      let groupTokens = 0;
+      for (const i of idxs) {
+        if (i !== maxI) groupTokens += safeCount(list[i]?.tokens);
+      }
+      out.push({ idxs, maxI, groupTokens });
+    }
+    out.sort((a, b) => b.groupTokens - a.groupTokens);
+    return out;
+  };
+  const targetGroups = buildGroups(byTarget);
+  const contentGroups = buildGroups(byContent);
+
+  // The authoritative total: every instance that is NOT the largest member of
+  // at least one duplicate group, counted ONCE (union across group kinds).
+  const removable = new Set();
+  for (const g of targetGroups) {
+    for (const i of g.idxs) if (i !== g.maxI) removable.add(i);
+  }
+  for (const g of contentGroups) {
+    for (const i of g.idxs) if (i !== g.maxI) removable.add(i);
+  }
+  let tokens = 0;
+  for (const i of removable) tokens += safeCount(list[i]?.tokens);
+
+  // Findings: target groups first (they carry the target label), then content
+  // groups only for instances not already covered — so the rendered findings
+  // never sum to more than the headline total.
+  const findings = [];
+  const covered = new Set();
+  for (const g of targetGroups) {
+    const rep = list[g.maxI] || {};
+    findings.push({
+      kind: "target",
+      name: typeof rep.name === "string" ? rep.name : "?",
+      target: typeof rep.target === "string" ? rep.target : null,
+      n: g.idxs.length,
+      tokens: g.groupTokens,
+    });
+    for (const i of g.idxs) covered.add(i);
+  }
+  for (const g of contentGroups) {
+    let t = 0;
+    for (const i of g.idxs) {
+      if (i === g.maxI || covered.has(i)) continue;
+      t += safeCount(list[i]?.tokens);
+    }
+    if (t > 0) {
+      const rep = list[g.maxI] || {};
+      findings.push({
+        kind: "content",
+        name: typeof rep.name === "string" ? rep.name : "?",
+        target: null,
+        n: g.idxs.length,
+        tokens: t,
+      });
+    }
+  }
+  findings.sort((a, b) => b.tokens - a.tokens);
+  return {
+    tokens,
+    count: removable.size,
+    findings: findings.slice(0, Math.max(0, maxFindings)),
+  };
+}
+
+// Sanitise a reclaim summary from a live estimate OR persisted state.json
+// (untrusted). `keepTarget=false` drops the per-finding target so a persisted /
+// exported payload stays free of file paths and URLs. Returns null when there
+// is nothing worth showing.
+function safeReclaim(raw, keepTarget = true, maxFindings = 8) {
+  if (!raw || typeof raw !== "object") return null;
+  const tokens = safeCount(raw.tokens);
+  const count = safeCount(raw.count);
+  if (tokens <= 0 || count <= 0) return null;
+  const findings = [];
+  const src = Array.isArray(raw.findings) ? raw.findings : [];
+  for (const f of src) {
+    if (!f || typeof f !== "object") continue;
+    const name = isSafeSessionKey(f.name) && f.name.length <= 64 ? f.name : "?";
+    const ft = safeCount(f.tokens);
+    const fn = safeCount(f.n);
+    if (ft <= 0 || fn < 2) continue;
+    const item = {
+      kind: f.kind === "content" ? "content" : "target",
+      name,
+      n: fn,
+      tokens: ft,
+    };
+    if (keepTarget && typeof f.target === "string" && f.target.length > 0) {
+      item.target = f.target.slice(0, 200);
+    }
+    findings.push(item);
+    if (findings.length >= Math.max(0, maxFindings)) break;
+  }
+  return { tokens, count, findings };
+}
+
 function fmt(n) {
   return n >= 1000 ? `${(n / 1000).toFixed(1)}k` : String(n);
 }
@@ -215,17 +518,23 @@ function fmtCell(n) {
 }
 
 export {
-  CHARS_PER_TOKEN_OTHER,
-  CHARS_PER_TOKEN_CYRILLIC,
-  CHARS_PER_TOKEN_CJK,
-  MAX_SANE_LIMIT,
   oneLine,
   safeCell,
   finitePos,
   safeCount,
   safeCountOrNull,
+  safeSaneOrNull,
   safeCategories,
+  sumCategories,
+  breakdownCategories,
   isSafeSessionKey,
+  safeToolOutByTool,
+  toolOutEntries,
+  TOOL_OUT_MAX_KEYS,
+  extractToolTarget,
+  contentFingerprint,
+  computeReclaim,
+  safeReclaim,
   fmt,
   estimateTokens,
   estJson,

package/lib/limits.js

@@ -453,7 +453,6 @@ export {
   getModelLimit,
   getUsableContext,
   reserveFor,
-  setModelLimit,
   ensureProviderLimits,
   ensureModelLimits,
   hydrateModelLimitsFromState,

package/lib/state.js

@@ -27,6 +27,10 @@ import {
   oneLine,
   safeCount,
   safeCountOrNull,
+  safeReclaim,
+  safeSaneOrNull,
+  safeToolOutByTool,
+  sumCategories,
 } from "./estimate.js";
 import {
   breakdownCache,
@@ -168,16 +172,6 @@ function writeLog(liveLines) {
   }
 }
 
-// Append a single line to the file (used for final summaries / notes).
-function appendLogLine(line) {
-  try {
-    ensureRegularFile(LOG_FILE); // refuse to follow a symlink planted at the path
-    appendFileSync(LOG_FILE, line + "\n", "utf8");
-  } catch (err) {
-    logErr("appendLogLine failed", err);
-  }
-}
-
 // Synchronous sleep for the lock retry loop. Atomics.wait is the documented
 // non-busy way to block this thread; fall back to a bounded spin if unavailable.
 function sleepSync(ms) {
@@ -316,6 +310,7 @@ function writeStateFile(
   inputTokens,
   ctxTokens,
   reasoningTokens,
+  opts,
 ) {
   try {
     if (!sessionID) return;
@@ -432,15 +427,74 @@ function writeStateFile(
         ? safeCount(bd.toolOutChars)
         : safeCount(pc?.toolOut);
 
-      const sumEstimates =
-        catUser +
-        catAssistant +
-        catReasoning +
-        catToolArgs +
-        catToolOut +
-        (catSystem || 0) +
-        (catToolSchemas || 0);
-      const other = Math.max(0, (inputTokens || 0) - sumEstimates);
+      // Per-tool split of the toolOut category (webfetch / read / bash / …).
+      // Untrusted names -> sanitised + bounded. A present breakdown OWNS the
+      // split (an empty one means "no tool results now" — do NOT revive a stale
+      // peer value); only fall back to the persisted map when this instance has
+      // no live breakdown at all.
+      const toolOutByTool = bd
+        ? safeToolOutByTool(bd.toolOutByTool)
+        : safeToolOutByTool(prevEntry?.toolOutByTool);
+
+      // Redundant tool output (waste detector). Persisted WITHOUT targets
+      // (keepTarget=false): state.json is a local temp file, but keeping it
+      // path-free matches the redacted-export policy. A present breakdown owns
+      // the value; otherwise inherit the persisted one.
+      const reclaim = bd
+        ? safeReclaim(bd.reclaim, false)
+        : safeReclaim(prevEntry?.reclaim, false);
+
+      // Provider-reported prompt cache (EXACT; a subset of input — never added
+      // to the category sum). Inherit when this write carries no fresh figure
+      // (idle / compaction mirror writes must not drop it).
+      const cacheRead =
+        safeSaneOrNull(opts?.cacheRead) ?? safeSaneOrNull(prevEntry?.cacheRead);
+      const cacheWrite =
+        safeSaneOrNull(opts?.cacheWrite) ??
+        safeSaneOrNull(prevEntry?.cacheWrite);
+
+      // Compaction staleness: a `session.compacted` write re-persists the
+      // pre-compaction numbers (the compaction flow fires no context hook), so
+      // the entry is stale BY CONSTRUCTION — even when this instance has no live
+      // breakdown to compare against (a mirror instance must not drop the
+      // signal). Later writes inherit `compactedAt` and clear the mark only once
+      // a FRESH breakdown (bd.capturedAt newer than the compaction) lands.
+      const compactedAt = opts?.compacted
+        ? new Date().toISOString()
+        : typeof prevEntry?.compactedAt === "string"
+          ? prevEntry.compactedAt
+          : null;
+      let compactStale = false;
+      if (opts?.compacted === true) {
+        compactStale = true;
+      } else if (compactedAt) {
+        const cMs = Date.parse(compactedAt);
+        const measuredMs =
+          bd && typeof bd.capturedAt === "number" ? bd.capturedAt : NaN;
+        if (Number.isFinite(cMs) && Number.isFinite(measuredMs)) {
+          compactStale = measuredMs <= cMs;
+        } else {
+          // No live breakdown here: keep the previous verdict (a peer instance
+          // may have measured it).
+          compactStale = prevEntry?.compactStale === true;
+        }
+      }
+
+      // Residual = input minus every measured category (shared sum; null
+      // system/toolSchemas count as 0).
+      const other = Math.max(
+        0,
+        safeCount(inputTokens) -
+          sumCategories({
+            user: catUser,
+            assistant: catAssistant,
+            reasoning: catReasoning,
+            toolArgs: catToolArgs,
+            toolOut: catToolOut,
+            system: catSystem,
+            toolSchemas: catToolSchemas,
+          }),
+      );
       // Parent link for the /context-breakdown command. Prefer the live cache;
       // fall back to the previously persisted value (backward compatible with
       // snapshots written before parentID existed -> null). role mirrors it.
@@ -490,6 +544,12 @@ function writeStateFile(
           toolSchemas: catToolSchemas,
           other,
         },
+        toolOutByTool,
+        reclaim,
+        cacheRead,
+        cacheWrite,
+        compactedAt,
+        compactStale,
         updatedAt: new Date().toISOString(),
       };
       all[sessionID] = entry;
@@ -618,19 +678,9 @@ function pushFinalSummary(claimKey, summaryLines) {
 
 export {
   logErr,
-  renderFile,
   writeLog,
-  appendLogLine,
-  sleepSync,
-  ensureRegularFile,
-  isStaleLock,
-  acquireStateLock,
-  releaseStateLock,
-  renameWithRetry,
   writeStateFile,
-  appendEventLog,
   debugEvent,
   readStateSnapshot,
-  finalSummaries,
   pushFinalSummary,
 };

package/lib/tui-data.d.ts

@@ -29,6 +29,26 @@ export interface StateEntry {
   limit?: number | null
   reasoning?: number
   categories?: Categories
+  /** Per-tool tool-result split: tool name -> estimated tokens. */
+  toolOutByTool?: Record<string, number>
+  /** Provider-reported prompt-cache tokens (exact; a subset of input). */
+  cacheRead?: number | null
+  cacheWrite?: number | null
+  /** Redundant tool output (waste detector). Findings here are path-free. */
+  reclaim?: {
+    tokens: number
+    count: number
+    findings?: Array<{
+      kind?: string
+      name?: string
+      n?: number
+      tokens?: number
+    }>
+  } | null
+  /** ISO time of the last session.compacted (marker only). */
+  compactedAt?: string | null
+  /** True while the entry still shows pre-compaction numbers. */
+  compactStale?: boolean
   updatedAt?: string
 }
 
@@ -44,3 +64,7 @@ export declare function fmt(n: number): string
 export declare function num(v: unknown): number
 export declare function fit(s: string, w: number): string
 export declare function updatedAt(iso: string | undefined): string
+export declare function toolOutEntries(
+  map: Record<string, number> | undefined,
+  limit?: number,
+): Array<[string, number]>

package/lib/tui-data.js

@@ -12,7 +12,8 @@
 //
 // state.json shape (written by writeStateFile() in index.js, untrusted input):
 //   { [sessionID]: { sessionID, parentID, role, agent, model, providerID,
-//     ctx, input, usable, reserve, limit, reasoning, categories{...}, updatedAt } }
+//     ctx, input, usable, reserve, limit, reasoning, categories{...},
+//     toolOutByTool{name->tokens}, compactedAt, compactStale, updatedAt } }
 // The authoritative TS mirror of the entry shape is the `StateEntry` interface
 // in lib/tui-data.d.ts — keep the two structurally compatible.
 
@@ -20,6 +21,11 @@ import { readFileSync } from "node:fs";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { fileURLToPath } from "node:url";
+import { isSafeSessionKey, sumCategories } from "./estimate.js";
+
+// Re-exported so the panel keeps importing it from this module; the
+// implementation is the shared one (lib/estimate.js) — no second copy to drift.
+export { toolOutEntries } from "./estimate.js";
 
 export const STATE_FILE =
   process.env.OPENCODE_CONTEXT_INDICATOR_STATE_FILE ||
@@ -42,19 +48,6 @@ export function isNodeModulesInstall() {
   }
 }
 
-// Reserved object keys must never be treated as session ids: state.json is
-// untrusted input (structural mirror of isSafeSessionKey() in lib/estimate.js,
-// including the string/length requirement).
-function isSafeKey(k) {
-  return (
-    typeof k === "string" &&
-    k.length > 0 &&
-    k !== "__proto__" &&
-    k !== "constructor" &&
-    k !== "prototype"
-  )
-}
-
 // Whole state.json snapshot ({ sessionID: entry }) or {} on any error. The
 // sidebar resolves one entry from it and the breakdown panel walks the
 // parentID family over it. Never throws — a missing / empty / corrupt
@@ -68,7 +61,7 @@ export function readSnapshot() {
     // (__proto__ / constructor / prototype) and non-object entries in one pass.
     const clean = {}
     for (const [k, v] of Object.entries(all)) {
-      if (isSafeKey(k) && v && typeof v === "object") clean[k] = v
+      if (isSafeSessionKey(k) && v && typeof v === "object") clean[k] = v
     }
     return clean
   } catch {
@@ -82,17 +75,11 @@ export function readSnapshot() {
 // a missing categories object is not the only trigger. `system` / `toolSchemas`
 // may legitimately be null, hence the num() coercion.
 function isEmptyEntry(e) {
-  const c = e?.categories
-  const sum =
-    num(c?.user) +
-    num(c?.assistant) +
-    num(c?.reasoning) +
-    num(c?.toolArgs) +
-    num(c?.toolOut) +
-    num(c?.system) +
-    num(c?.toolSchemas) +
-    num(c?.other)
-  return sum === 0 && num(e?.ctx) === 0 && num(e?.input) === 0
+  return (
+    sumCategories(e?.categories) === 0 &&
+    num(e?.ctx) === 0 &&
+    num(e?.input) === 0
+  )
 }
 
 // The panel row set: the root entry (panel.sessionID) plus every entry whose