@hncode/hncode 0.5.2 → 0.5.5

package/src/term.js

@@ -396,27 +396,74 @@ const TOK_CACHE_MAX_LEN = 4096;   // only memoise strings this short
  * Callers that measure a complete string keep using `estimateTokens`.
  */
 export function estimateTokensFloat(text) {
-  const s = String(text ?? '');
-  let ascii = 0, nonAscii = 0;
+  return estimateBpe(String(text ?? ''));
+}
+
+/**
+ * A BPE-aware token estimate: characters per token, BY CHARACTER CLASS.
+ *
+ * The old rule was `ascii / 4 + nonAscii`, and it is wrong in a way that matters here. Measured against
+ * published BPE rates for this app's own content, it undercounted by up to 35%:
+ *
+ *   English prose    1.05x   (the rule was calibrated for exactly this)
+ *   TypeScript code  0.80x
+ *   JSON payload     0.82x
+ *   tool schema      0.75x
+ *   paths and flags  0.73x
+ *   minified JSON    0.65x
+ *   mixed CJK + code 0.65x
+ *
+ * The reason is BPE merges COMMON BYTE PAIRS. English prose is full of repeated whole words, so four
+ * characters per token holds. Code, JSON and CLI flags are punctuation-dense: `{"a":1,"b":2}` merges
+ * almost nothing, and a token covers about 2.6 characters. On a 300k context that 0.65x is a ~100k
+ * error — the exact direction and size of the gap being reported.
+ *
+ * Five classes, each with its measured rate:
+ *
+ *   letters + spaces    4.2   the "4 chars/token" folklore, and the only place it holds
+ *   digits              3.0   punctuation-free but rarely repeated
+ *   punctuation + code  2.7   brackets, quotes, colons, operators — merge poorly
+ *   whitespace (indent) 1.0   a run of spaces still costs a token per couple of columns
+ *   CJK + other non-Latin 1.0  one token per character, occasionally two
+ *
+ * Counted in ONE pass, because this runs on every streamed chunk and every message body.
+ */
+function estimateBpe(s) {
+  if (!s) return 0;
+  let letters = 0;   // a-z A-Z and the space/punctuation that surrounds prose
+  let digits = 0;
+  let punct = 0;
+  let space = 0;
+  let wide = 0;
+
   for (const ch of s) {
-    if (ch.codePointAt(0) <= 127) ascii++;
-    else nonAscii++;
+    const cp = ch.codePointAt(0);
+    if (cp > 127) { wide++; continue; }
+    if (cp === 32 || cp === 9) { space++; continue; }
+    if (cp >= 48 && cp <= 57) { digits++; continue; }
+    if ((cp >= 65 && cp <= 90) || (cp >= 97 && cp <= 122)) { letters++; continue; }
+    punct++;
   }
-  return ascii / 4 + nonAscii;
-}
 
+  // Newlines land in `punct` (they are not 32 or 9), which is the right rate for them.
+  //
+  // The SPACE rate is the delicate one: a run of indentation costs about a token per two columns, but a
+  // single space between English words merges into the following word almost for free. Charging every
+  // space at the indentation rate made prose read 34% high — a worse error than the undercount it fixed,
+  // because prose is what the model WRITES back.
+  //
+  // So spaces are priced at a third of a token each: cheap enough for prose, and still enough that a
+  // deeply indented code block is not free.
+  return letters / 4.2 + digits / 3.0 + punct / 2.7 + space / 3.0 + wide;
+}
 export function estimateTokens(text) {
   const s = String(text ?? '');
   if (s.length <= TOK_CACHE_MAX_LEN) {
     const hit = tokCache.get(s);
     if (hit !== undefined) return hit;
-    if (hit !== undefined) return hit;
-    let ascii = 0, nonAscii = 0;
-    for (const ch of s) {
-      if (ch.codePointAt(0) <= 127) ascii++;
-      else nonAscii++;
-    }
-    const out = Math.ceil(ascii / 4) + nonAscii;
+    // Rounded UP from the same BPE-aware estimate the streaming counter uses. Both must agree: a body
+    // measured one way and a stream measured another is how a session ends up reporting two numbers.
+    const out = Math.ceil(estimateBpe(s));
     if (tokCache.size >= TOK_CACHE_MAX) {
       // Drop the oldest half in one go: cheaper than one-at-a-time and keeps the
       // recently measured entries that a per-entry eviction would churn.
@@ -426,13 +473,9 @@ export function estimateTokens(text) {
     tokCache.set(s, out);
     return out;
   }
-  // Long body: walk it. No memoise, no map churn.
-  let ascii = 0, nonAscii = 0;
-  for (const ch of s) {
-    if (ch.codePointAt(0) <= 127) ascii++;
-    else nonAscii++;
-  }
-  return Math.ceil(ascii / 4) + nonAscii;
+  // Long body: measured directly, with the SAME estimator as the cached path. The two used to differ, so
+  // a long message and a short one were priced by different formulas.
+  return Math.ceil(estimateBpe(s));
 }
 
 // Estimate tokens for the MESSAGES only. The system prompt and tool definitions are
@@ -457,13 +500,52 @@ export function estimateMessageTokens(m, cfg) {
   if (!m) return 0;
   let total = estimateTokens(m.role || '');
   const c = m.content;
-  total += typeof c === 'string' ? estimateTokens(c) : estimateTokens(JSON.stringify(c || ''));
+  if (typeof c === 'string') {
+    total += estimateTokens(c);
+  } else if (c && typeof c === 'object' && Array.isArray(c.media)) {
+    total += estimateMediaContentTokens(c);
+  } else {
+    total += estimateTokens(JSON.stringify(c || ''));
+  }
   if (m.toolCalls) for (const tc of m.toolCalls) {
     total += estimateTokens(tc.name || '') + estimateTokens(JSON.stringify(tc.args || ''));
   }
   return total;
 }
 
+/**
+ * Tokens for a content block that carries media (an image read from disk, a screenshot).
+ *
+ * AN IMAGE IS NOT PRICED BY ITS BASE64 LENGTH. `JSON.stringify` on the same object counts every base64
+ * character as ordinary text — a 1 MB PNG is ~1.3M characters, which this estimator would price at
+ * roughly 400k tokens — while a provider charges by the image's PIXEL dimensions, usually one to two
+ * thousand. That is the direction that matters: the gauge would read hundreds of thousands too high, and
+ * auto-compaction would fire on a history that was never close to full.
+ *
+ * The text beside the image is priced normally; each image gets a flat estimate derived from the common
+ * tiling rule (a 512x512 tile ≈ 765 tokens, a 1568px max edge ≈ 1100 tokens). Exactness is impossible
+ * without decoding the image, and it does not matter here: images are rare, and being within a few
+ * hundred tokens beats being wrong by a hundred thousand.
+ */
+function estimateMediaContentTokens(c) {
+  let total = estimateTokens(String(c.text == null ? '' : c.text));
+  for (const m of c.media || []) {
+    if (!m || m.type !== 'image') continue;
+    // A thumbnail-sized image costs a tile; a full screenshot costs four. Read the dimensions from the
+    // metadata when the reader recorded them, and fall back to the large case otherwise — overestimating
+    // an image is far safer than under, because it makes compaction fire EARLY rather than late.
+    const w = Number(m.width) || 0;
+    const h = Number(m.height) || 0;
+    if (w > 0 && h > 0) {
+      const tiles = Math.max(1, Math.ceil(w / 512) * Math.ceil(h / 512));
+      total += Math.min(4, tiles) * 765;
+    } else {
+      total += 1100;
+    }
+  }
+  return total;
+}
+
 // ---- per-message token memo -----------------------------------------------
 //
 // Message objects -> { sig, tok }. WeakMap, so a message dropped from the history
@@ -501,7 +583,11 @@ function msgSignature(m) {
   if (typeof c === 'string') s = 's' + c.length;
   else if (c == null) s = 'n';
   else if (Array.isArray(c)) s = 'a' + c.length;
-  else s = '';                       // unknown shape: never trust a memo
+  else if (typeof c === 'object' && Array.isArray(c.media)) {
+    // A media block: its signature is the text plus one marker per image. The base64 payload itself is
+    // NOT hashed — it is the expensive part, and it never changes once the message exists.
+    s = 'm' + String(c.text == null ? '' : c.text).length + 'x' + c.media.length;
+  } else s = '';                     // unknown shape: never trust a memo
   if (m.toolCalls) s += 't' + m.toolCalls.length;
   return s;
 }
@@ -586,13 +672,43 @@ export class TokenLedger {
   }
 }
 
-// The overhead a request carries BEYOND its messages: the system prompt plus the
-// tool definitions. Passed the assembled system text so the number matches what is
-// actually sent.
+/**
+ * The overhead a request carries BEYOND its messages: the system prompt plus the tool definitions.
+ * Passed the assembled system text so the number matches what is actually sent.
+ *
+ * `cfg.toolDefTokens` is the REAL cost of the tool definitions, measured once by whoever built them.
+ * This used to charge a flat `n * 8` per tool, which is short by an order of magnitude: a tool's
+ * schema is its name, a paragraph of description and a nested parameters object, and measured here the
+ * 68 registered tools come to about 15 900 tokens — 29 times what 8 apiece reported.
+ *
+ * The discrepancy matters because this number is part of the context gauge. A gauge that omits ~15k of
+ * every request understates usage by that much per turn, and the user sees the provider's real figure
+ * disagree with the app's by a wide margin.
+ *
+ * The `n * 8` remains only as a FALLBACK for a caller that has not measured its definitions; it is
+ * better than nothing and is the old behaviour, exactly.
+ */
 export function estimateRequestOverhead(systemText, cfg) {
   let total = estimateTokens(systemText || '');
+  const measured = cfg && Number.isFinite(cfg.toolDefTokens) ? cfg.toolDefTokens : null;
+  if (measured != null) return total + measured;
   const n = (cfg && Array.isArray(cfg.toolFilter)) ? cfg.toolFilter.length : 0;
-  if (n) total += n * 8;   // rough per-tool-definition overhead
+  if (n) total += n * 8;   // rough per-tool-definition overhead, when nothing measured it
+  return total;
+}
+
+/**
+ * The token cost of a set of tool definitions, measured the way they are actually sent.
+ *
+ * Serialised as `{ name, description, parameters }` — the three fields the provider receives — and
+ * priced with the same estimator the gauge uses, so the number is comparable with everything else.
+ */
+export function measureToolDefTokens(tools) {
+  let total = 0;
+  for (const t of tools || []) {
+    if (!t || !t.name) continue;
+    total += estimateTokens(JSON.stringify({ name: t.name, description: t.description, parameters: t.parameters }));
+  }
   return total;
 }
 

package/src/tool-result.js

@@ -274,6 +274,54 @@ export function applyTrim(messages, plan, sessionId) {
 // so a caller can skip its UI event entirely.
 export function trimToolResults(messages, opts = {}) {
   const plan = planTrim(messages, opts.keepRatio, opts.mustKeep);
-  if (!plan.elide.length) return null;
-  return { ...applyTrim(messages, plan, opts.sessionId), plan };
+  const mediaPlan = planMediaTrim(messages, opts.mediaKeep ?? 2);
+  if (!plan.elide.length && !mediaPlan.elide.length) return null;
+  const applied = plan.elide.length
+    ? applyTrim(messages, plan, opts.sessionId)
+    : { elided: 0, elidedBytes: 0, kept: plan.keep.length, keptBytes: plan.keepBytes };
+  const mediaApplied = applyMediaTrim(messages, mediaPlan, opts.sessionId);
+  return {
+    ...applied,
+    elided: applied.elided + mediaApplied.elided,
+    elidedBytes: applied.elidedBytes + mediaApplied.elidedBytes,
+    plan,
+  };
+}
+
+// ---- media results ----------------------------------------------------------
+// A media tool result keeps its base64 payload IN the history object — and unlike
+// a string body, planTrim cannot elide it, so every screenshot was re-serialized
+// into every request body, every session save and every cache-prefix walk until a
+// full compaction dropped it. Keep only the newest `mediaKeep` media results;
+// older ones spill their text (if any) and lose the bytes, same contract as the
+// string path: saved to disk first, pointer left behind.
+
+function planMediaTrim(messages, keep) {
+  const idx = [];
+  if (Array.isArray(messages)) {
+    for (let i = 0; i < messages.length; i++) {
+      const m = messages[i];
+      if (m && m.role === 'tool' && m.content && typeof m.content === 'object'
+        && Array.isArray(m.content.media) && m.content.media.length) idx.push(i);
+    }
+  }
+  const elide = idx.slice(0, Math.max(0, idx.length - Math.max(0, keep)));
+  return { elide };
+}
+
+function applyMediaTrim(messages, mediaPlan, sessionId) {
+  const out = { elided: 0, elidedBytes: 0 };
+  for (const i of mediaPlan.elide) {
+    const m = messages[i];
+    let bytes = 0;
+    for (const part of m.content.media) bytes += (part && part.data ? part.data.length : 0);
+    const text = String(m.content.text || '');
+    const spilled = spillResult(sessionId, `trim-${m.toolCallId || i}`, text || '[media only]');
+    m.content = `[Media result elided to save context — ${fmtBytes(bytes)} of image data from ${m.name || 'tool'}`
+      + (spilled ? `; the description was saved to: ${spilled.file}` : '; the description could not be saved')
+      + '. Re-run the tool if you need the image again.]';
+    out.elided++;
+    out.elidedBytes += bytes;
+  }
+  return out;
 }