distil-llm 1.41.0 → 1.42.0

package/README.md

@@ -60,3 +60,66 @@ const openai = new OpenAI({ baseURL: distilOpenAIBaseURL() }); // adds /v1
   publishes real, auditable numbers.
 
 Full docs: <https://dshakes.github.io/distil/> · License: Apache-2.0
+
+## In-process compression (lossless tier)
+
+```js
+import { compress } from "distil-llm";
+
+const r = compress(messages);           // never mutates the input
+console.log(`${r.savedPct.toFixed(1)}% smaller`);
+const response = await client.messages.create({ model, messages: r.messages });
+```
+
+Tool and user text get lossless transforms; **the model's own turns and
+non-string content (image blocks, tool_use) are never rewritten**. Unchanged
+messages are returned by identity.
+
+This runs entirely in Node — no daemon, no network — and is **byte-identical to
+the Python engine**, enforced by a conformance suite (`tests/test_ts_conformance.py`)
+that runs both implementations over a shared corpus and fails on any divergence.
+Where JS cannot reproduce Python's bytes (Python renders an integral float as
+`1.0`, JS as `1`; JS objects hoist integer-like keys) the transform **declines**
+rather than emitting output the certificate does not cover.
+
+### Getting the reversible digest tier
+
+`compress()` is lossless-only, on purpose. The digest tier mints restore handles
+whose originals must live in one store shared with the proxy and the MCP server,
+and it is the tier the decision-equivalence certificate measures — so there is
+deliberately no second implementation of it here.
+
+Route through the proxy instead, which is the same engine and the same store:
+
+```js
+import Anthropic from "@anthropic-ai/sdk";
+import { distilBaseURL } from "distil-llm";
+
+const client = new Anthropic({ baseURL: distilBaseURL() });
+```
+
+```bash
+npx distil-llm proxy        # or: distil default --always-on
+```
+
+## Vercel AI SDK middleware
+
+```js
+import { wrapLanguageModel } from "ai";
+import { distilMiddleware } from "distil-llm";
+
+const model = wrapLanguageModel({
+  model: gateway("anthropic/claude-sonnet-5"),
+  middleware: distilMiddleware({
+    onSavings: (s) => console.log(`${s.savedPct.toFixed(1)}% smaller`),
+  }),
+});
+```
+
+Implements `transformParams` only — compression happens on the way **in**, and
+wrapping the response would mean rewriting the model's own output, which distil
+does not do. Tool results (both the v4 `result` and v5 `output.value` shapes),
+user text parts, and system strings are compressed; assistant messages, files,
+and tool calls are passed through untouched.
+
+Lossless tier, like `compress()`. Route through the proxy for the digest tier.

package/index.d.ts

@@ -1,15 +1,92 @@
-export interface DistilURLOptions {
-  /** Proxy host. Default "127.0.0.1". */
+// Type definitions for distil-llm.
+
+export interface Message {
+  role?: string;
+  type?: string;
+  content?: unknown;
+  [k: string]: unknown;
+}
+
+export interface CompressionResult {
+  /** New array. Unchanged messages are returned by identity. */
+  messages: Message[];
+  tokensBefore: number;
+  tokensAfter: number;
+  tokensSaved: number;
+  /**
+   * Percent smaller. Legitimately near 0 on small payloads — savings come from
+   * LARGE tool output, and a request that is mostly system prompt has nothing
+   * to fold.
+   */
+  savedPct: number;
+  tier: "lossless";
+}
+
+export interface BaseURLOptions {
   host?: string;
-  /** Proxy port. Default 8788 (or DISTIL_PORT). */
   port?: number;
 }
 
-/** Base URL of a local distil proxy (honors DISTIL_BASE_URL). */
-export function distilBaseURL(opts?: DistilURLOptions): string;
+/**
+ * Compress a message list in-process, lossless tier only.
+ *
+ * Byte-identical to the Python engine (enforced by a conformance suite). Tool
+ * and user text get lossless transforms; the model's own turns and non-string
+ * content are never rewritten; the input array is never mutated.
+ *
+ * For the reversible DIGEST tier, route your SDK through a distil proxy with
+ * `distilBaseURL()` — that path mints restore handles into the shared store and
+ * is the path the decision-equivalence certificate covers.
+ */
+export function compress(messages: Message[]): CompressionResult;
 
-/** distil proxy base URL with the OpenAI `/v1` segment appended. */
-export function distilOpenAIBaseURL(opts?: DistilURLOptions): string;
+/** Base URL of a local distil proxy (honours DISTIL_BASE_URL / DISTIL_PORT). */
+export function distilBaseURL(opts?: BaseURLOptions): string;
+/** Same, with the `/v1` segment OpenAI-style clients expect. */
+export function distilOpenAIBaseURL(opts?: BaseURLOptions): string;
 
 export const DEFAULT_PORT: number;
 export const DEFAULT_HOST: string;
+
+/** Full Tier-0 pipeline: JSON minify, then run-collapse if it does not inflate. */
+export function applyTier0(text: string): string;
+/** Run-length-collapse consecutive identical lines to `L\n<<xN>>`. */
+export function collapseRuns(text: string): string;
+/**
+ * Textual inverse of `collapseRuns`.
+ * PRECONDITION: the original text contained no line matching /^<<x\d+>>$/ —
+ * a generated marker and content that looks like one are indistinguishable.
+ * distil's real recovery path is the restore store, not this function.
+ */
+export function expandRuns(text: string): string;
+/** Minify JSON, or null when a byte-identical round-trip is not provable. */
+export function minifyJson(text: string): string | null;
+/** Heuristic token count, matching the Python HeuristicTokenizer. */
+export function countTokens(text: string): number;
+
+export interface SavingsInfo {
+  tokensBefore: number;
+  tokensAfter: number;
+  tokensSaved: number;
+  savedPct: number;
+}
+
+export interface DistilMiddlewareOptions {
+  /** Called once per request with the measured delta. */
+  onSavings?: (info: SavingsInfo) => void;
+}
+
+/**
+ * Vercel AI SDK middleware that losslessly compresses the prompt in-process.
+ *
+ * Implements `transformParams` only — compression happens on the way IN, and
+ * wrapping the response would mean rewriting the model's own output, which
+ * distil does not do.
+ *
+ *   const model = wrapLanguageModel({ model: ..., middleware: distilMiddleware() });
+ *
+ * For the reversible digest tier, route through a distil proxy instead.
+ */
+export function distilMiddleware(
+  opts?: DistilMiddlewareOptions
+): { transformParams: (arg: { params: unknown }) => Promise<unknown> };

package/index.js

@@ -17,22 +17,30 @@
 
 "use strict";
 
-const DEFAULT_PORT = 8788;
-const DEFAULT_HOST = "127.0.0.1";
+const {
+  distilBaseURL,
+  distilOpenAIBaseURL,
+  DEFAULT_PORT,
+  DEFAULT_HOST,
+} = require("./lib/baseurl");
+const { compress } = require("./lib/compress");
+const { distilMiddleware } = require("./lib/aisdk");
+const { applyTier0, collapseRuns, expandRuns, minifyJson, countTokens } = require("./lib/tier0");
 
-/** Base URL of a local distil proxy. Reads DISTIL_BASE_URL if set, else builds
- *  http://<host>:<port> (defaults 127.0.0.1:8788). */
-function distilBaseURL(opts) {
-  opts = opts || {};
-  if (process.env.DISTIL_BASE_URL) return process.env.DISTIL_BASE_URL;
-  const host = opts.host || DEFAULT_HOST;
-  const port = opts.port || Number(process.env.DISTIL_PORT) || DEFAULT_PORT;
-  return `http://${host}:${port}`;
-}
-
-/** OpenAI-style clients expect the base URL to include the API version segment. */
-function distilOpenAIBaseURL(opts) {
-  return distilBaseURL(opts).replace(/\/+$/, "") + "/v1";
-}
-
-module.exports = { distilBaseURL, distilOpenAIBaseURL, DEFAULT_PORT, DEFAULT_HOST };
+module.exports = {
+  // in-process lossless compression (byte-identical to the Python engine)
+  compress,
+  // Vercel AI SDK middleware for wrapLanguageModel()
+  distilMiddleware,
+  // proxy routing helpers
+  distilBaseURL,
+  distilOpenAIBaseURL,
+  DEFAULT_PORT,
+  DEFAULT_HOST,
+  // lower-level transforms, exposed for verification and testing
+  applyTier0,
+  collapseRuns,
+  expandRuns,
+  minifyJson,
+  countTokens,
+};

package/lib/aisdk.js

@@ -0,0 +1,132 @@
+// Vercel AI SDK middleware.
+//
+// The proxy route (`createAnthropic({ baseURL: distilBaseURL() })`) is still the
+// way to get the reversible digest tier. This is for the other case: you want
+// lossless savings in-process, with no daemon to keep alive — a serverless
+// function, an edge worker, a CI job that runs once.
+//
+//   import { wrapLanguageModel } from "ai";
+//   import { distilMiddleware } from "distil-llm";
+//
+//   const model = wrapLanguageModel({
+//     model: gateway("anthropic/claude-sonnet-5"),
+//     middleware: distilMiddleware(),
+//   });
+//
+// Implements `transformParams` only. `wrapGenerate`/`wrapStream` are not
+// implemented because compression happens on the way IN — wrapping the response
+// would let us rewrite the model's own output, which distil does not do.
+
+"use strict";
+
+const { applyTier0, countTokens } = require("./tier0");
+
+/**
+ * A `LanguageModelV4Prompt` message carries `content` that is a plain STRING for
+ * `system`, and an ARRAY OF PARTS for `user`, `assistant`, and `tool`. Both are
+ * handled; anything unrecognised is passed through untouched, because a
+ * compressor that mangles a shape it does not understand is worse than one that
+ * saves nothing.
+ */
+
+/** Compress a text-bearing part, preserving every sibling key. */
+function compressPart(part) {
+  if (!part || typeof part !== "object") return part;
+
+  // { type: "text", text: "..." } — user and tool text.
+  if (part.type === "text" && typeof part.text === "string") {
+    const next = applyTier0(part.text);
+    return next === part.text ? part : { ...part, text: next };
+  }
+
+  // Tool results carry the payload under different keys across SDK versions:
+  //   v5: { type: "tool-result", output: { type: "text", value: "..." } }
+  //   v4: { type: "tool-result", result: "..." }
+  // Both are handled rather than pinning one, so an SDK bump does not silently
+  // stop compressing the single largest thing in an agent's context.
+  if (part.type === "tool-result") {
+    if (part.output && typeof part.output.value === "string") {
+      const next = applyTier0(part.output.value);
+      return next === part.output.value
+        ? part
+        : { ...part, output: { ...part.output, value: next } };
+    }
+    if (typeof part.result === "string") {
+      const next = applyTier0(part.result);
+      return next === part.result ? part : { ...part, result: next };
+    }
+  }
+  return part; // file / image / tool-call parts are never rewritten
+}
+
+function compressMessage(message) {
+  if (!message || typeof message !== "object") return message;
+
+  // The model's own words are not distil's to edit.
+  if (message.role === "assistant") return message;
+
+  if (typeof message.content === "string") {
+    const next = applyTier0(message.content);
+    return next === message.content ? message : { ...message, content: next };
+  }
+  if (Array.isArray(message.content)) {
+    const parts = message.content.map(compressPart);
+    const changed = parts.some((p, i) => p !== message.content[i]);
+    return changed ? { ...message, content: parts } : message;
+  }
+  return message;
+}
+
+function textOf(message) {
+  if (typeof message?.content === "string") return message.content;
+  if (Array.isArray(message?.content)) {
+    return message.content
+      .map((p) =>
+        typeof p?.text === "string"
+          ? p.text
+          : typeof p?.output?.value === "string"
+            ? p.output.value
+            : typeof p?.result === "string"
+              ? p.result
+              : ""
+      )
+      .join("");
+  }
+  return "";
+}
+
+/**
+ * Build AI SDK middleware that losslessly compresses the prompt before it is sent.
+ *
+ * @param {{onSavings?: (info: {tokensBefore: number, tokensAfter: number,
+ *   tokensSaved: number, savedPct: number}) => void}} [opts]
+ *   `onSavings` is called once per request with the measured delta. Provided
+ *   because in-process compression has no proxy to report through, and savings
+ *   you cannot see are savings you will not trust.
+ * @returns {{transformParams: (arg: {params: object}) => Promise<object>}}
+ */
+function distilMiddleware(opts) {
+  const onSavings = opts && typeof opts.onSavings === "function" ? opts.onSavings : null;
+  return {
+    transformParams: async ({ params }) => {
+      if (!params || !Array.isArray(params.prompt)) return params;
+
+      const before = onSavings ? params.prompt.reduce((n, m) => n + countTokens(textOf(m)), 0) : 0;
+      const prompt = params.prompt.map(compressMessage);
+      const changed = prompt.some((m, i) => m !== params.prompt[i]);
+
+      if (onSavings) {
+        const after = prompt.reduce((n, m) => n + countTokens(textOf(m)), 0);
+        onSavings({
+          tokensBefore: before,
+          tokensAfter: after,
+          tokensSaved: before - after,
+          savedPct: before > 0 ? (100 * (before - after)) / before : 0,
+        });
+      }
+      return changed ? { ...params, prompt } : params;
+    },
+  };
+}
+
+module.exports = { distilMiddleware };

package/lib/baseurl.js

@@ -0,0 +1,23 @@
+// Base-URL helpers, factored out of index.js so lib/ modules can use them
+// without importing the package entry point (which imports them back).
+"use strict";
+
+const DEFAULT_PORT = 8788;
+const DEFAULT_HOST = "127.0.0.1";
+
+/** Base URL of a local distil proxy. Reads DISTIL_BASE_URL if set, else builds
+ *  http://<host>:<port> (defaults 127.0.0.1:8788). */
+function distilBaseURL(opts) {
+  opts = opts || {};
+  if (process.env.DISTIL_BASE_URL) return process.env.DISTIL_BASE_URL;
+  const host = opts.host || DEFAULT_HOST;
+  const port = opts.port || Number(process.env.DISTIL_PORT) || DEFAULT_PORT;
+  return `http://${host}:${port}`;
+}
+
+/** OpenAI-style clients expect the base URL to include the API version segment. */
+function distilOpenAIBaseURL(opts) {
+  return distilBaseURL(opts).replace(/\/+$/, "") + "/v1";
+}
+
+module.exports = { distilBaseURL, distilOpenAIBaseURL, DEFAULT_PORT, DEFAULT_HOST };

package/lib/compress.js

@@ -0,0 +1,98 @@
+// Public TypeScript/JavaScript compression API.
+//
+// Two tiers, two very different trust stories, and the difference is the whole
+// design of this file:
+//
+//   Tier-0 (lossless)  — implemented natively in lib/tier0.js and proven
+//                        BYTE-IDENTICAL to the Python engine by
+//                        tests/test_ts_conformance.py. Safe to run in-process:
+//                        no daemon, no network, no handles to keep alive.
+//
+//   Tier-1 (digest)    — NOT implemented here. It mints restore handles whose
+//                        originals must live in ONE store shared with the proxy
+//                        and the MCP server, or `expand` silently fails to
+//                        resolve a handle the model can see. It is also the tier
+//                        the decision-equivalence certificate actually measures.
+//                        A second implementation would be a second thing to
+//                        certify, and nobody would notice when they drifted.
+//
+// So `compress()` is lossless-only and always available in-process. The digest
+// tier comes from routing your client through the proxy (see the note at the
+// bottom of this file), which is the same engine, store, and certificate.
+
+"use strict";
+
+const { applyTier0, countTokens } = require("./tier0");
+
+/** Roles whose content distil never rewrites. */
+const NEVER_REWRITE = new Set(["assistant", "ai"]);
+
+function roleOf(m) {
+  if (!m || typeof m !== "object") return "";
+  return String(m.role || m.type || "").toLowerCase();
+}
+
+/**
+ * Compress a message list in-process, lossless tier only.
+ *
+ * Mirrors the Python `distil.compress_messages` contract: the input is never
+ * mutated, unchanged messages pass through by identity, non-string content
+ * (image blocks, tool_use structures) is untouched, and the model's own turns
+ * are never rewritten.
+ *
+ * @param {Array<object>} messages OpenAI/Anthropic-style message objects
+ * @returns {{messages: Array<object>, tokensBefore: number, tokensAfter: number,
+ *            tokensSaved: number, savedPct: number, tier: "lossless"}}
+ */
+function compress(messages) {
+  if (!Array.isArray(messages)) {
+    throw new TypeError("compress(messages): expected an array of messages");
+  }
+  let before = 0;
+  let after = 0;
+  const out = messages.map((m) => {
+    const content = m && typeof m === "object" ? m.content : undefined;
+    if (typeof content !== "string") return m;
+    before += countTokens(content);
+    if (NEVER_REWRITE.has(roleOf(m))) {
+      after += countTokens(content);
+      return m;
+    }
+    const next = applyTier0(content);
+    after += countTokens(next);
+    return next === content ? m : Object.assign({}, m, { content: next });
+  });
+  return {
+    messages: out,
+    tokensBefore: before,
+    tokensAfter: after,
+    tokensSaved: before - after,
+    // Can legitimately be ~0: savings come from LARGE tool output, and a request
+    // that is mostly system prompt has nothing to fold.
+    savedPct: before > 0 ? (100 * (before - after)) / before : 0,
+    tier: "lossless",
+  };
+}
+
+/**
+ * How to get the reversible DIGEST tier from JS.
+ *
+ * There is deliberately no `compressViaProxy()` here. The distil proxy has no
+ * "compress this payload and hand it back" endpoint, and inventing one would add
+ * a new public API surface — accepting arbitrary content, needing its own
+ * authentication story — to get something the existing design already provides.
+ *
+ * The proxy's contract is transparent interception: point your SDK's baseURL at
+ * it and every request is compressed on the way through, by the same engine the
+ * decision-equivalence certificate was measured against, minting handles into the
+ * one restore store that `distil expand` and the MCP server read.
+ *
+ *   import Anthropic from "@anthropic-ai/sdk";
+ *   import { distilBaseURL } from "distil-llm";
+ *   const client = new Anthropic({ baseURL: distilBaseURL() });
+ *
+ * Use `compress()` above when you hold the message list and want lossless
+ * savings with no daemon; route through the proxy when you want the digest.
+ */
+
+module.exports = { compress };

package/lib/tier0.js

@@ -0,0 +1,323 @@
+// Tier-0 lossless transforms — a faithful port of distil/compress/tier0.py.
+//
+// WHY THIS IS A PORT AND NOT A REIMPLEMENTATION
+// ---------------------------------------------
+// distil's whole claim is a decision-equivalence certificate, and that
+// certificate is measured against the PYTHON compressor. A JS implementation
+// that merely "compresses similarly" would sit outside it — users would get a
+// guarantee that does not describe the code they are running.
+//
+// So this file targets BYTE-IDENTICAL output with tier0.py, and
+// tests/test_ts_conformance.py runs both against a shared fixture corpus and
+// fails on any divergence. Where byte-identity is not achievable (see
+// `minifyJson`), the transform DECLINES rather than producing output the
+// certificate does not cover. Declining costs a little compression; diverging
+// silently would cost the guarantee.
+//
+// Tier-1 (the reversible digest) is deliberately NOT ported: it allocates
+// restore handles and must share one store with the proxy and the MCP server.
+// Use `compressViaProxy` in compress.js for that.
+
+"use strict";
+
+// Matches tier0.py's _RUN_MARKER: a generated marker, so a run of markers is
+// left alone (collapsing it would make the new marker indistinguishable from
+// original content when reversing).
+const RUN_MARKER = /^<<x\d+>>$/;
+
+/**
+ * Run-length-encode consecutive identical lines, reversibly.
+ * Two or more identical lines `L` collapse to `L\n<<xN>>\n`, where N is the
+ * exact original count, so the input is fully recoverable.
+ *
+ * Pure string manipulation, so this is exactly portable — no locale, no number
+ * formatting, no object ordering. It is the bulk of Tier-0's real-world win
+ * (repeated log lines, duplicated stack frames).
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function collapseRuns(text) {
+  if (typeof text !== "string" || text.indexOf("\n") === -1) return text;
+
+  const lines = text.split("\n");
+  const out = [];
+  let i = 0;
+  // The Python side uses the regex /^(.*)\n(?:\1\n)+/m, which only ever matches
+  // runs that are newline-TERMINATED. A trailing final line with no newline
+  // after it therefore cannot participate in a run; splitting keeps that
+  // property because the last element has no terminator to consume.
+  const lastIndex = lines.length - 1;
+  while (i < lines.length) {
+    const line = lines[i];
+    let j = i;
+    // Count how many consecutive newline-terminated copies follow.
+    while (j < lastIndex && lines[j] === line) j++;
+    const runLen = j - i;
+    if (runLen >= 2 && !RUN_MARKER.test(line)) {
+      out.push(line, `<<x${runLen}>>`);
+      i = j;
+    } else {
+      out.push(line);
+      i += 1;
+    }
+  }
+  return out.join("\n");
+}
+
+/**
+ * Textual inverse of `collapseRuns`.
+ *
+ * PRECONDITION: the ORIGINAL text contained no line matching /^<<x\d+>>$/.
+ * A generated marker and original content that looks like one are
+ * indistinguishable after the fact, so no textual inverse can separate them.
+ * `collapseRuns` refuses to collapse such runs precisely so nothing is lost —
+ * but expanding text that already contained markers will invent lines.
+ *
+ * This is why distil's real reversibility guarantee comes from the restore
+ * store (the exact original bytes, keyed by handle) rather than from inverting
+ * the transform. Use this for local verification of ordinary text, not as the
+ * recovery path.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function expandRuns(text) {
+  if (typeof text !== "string") return text;
+  const lines = text.split("\n");
+  const out = [];
+  for (let i = 0; i < lines.length; i++) {
+    const m = /^<<x(\d+)>>$/.exec(lines[i]);
+    if (m && out.length > 0) {
+      const prev = out[out.length - 1];
+      const n = parseInt(m[1], 10);
+      for (let k = 1; k < n; k++) out.push(prev);
+    } else {
+      out.push(lines[i]);
+    }
+  }
+  return out.join("\n");
+}
+
+// --- JSON ------------------------------------------------------------------
+//
+// Python's json.dumps and JS's JSON.stringify disagree on two things that
+// matter for byte-identity:
+//
+//   1. NUMBER FORMATTING. Python renders a float that happens to be integral as
+//      "1.0"; JS renders it "1". Nothing in JS can recover the distinction,
+//      because JSON.parse has already collapsed 1.0 and 1 to the same double.
+//   2. KEY ORDER. Python dicts preserve insertion order for every key. JS
+//      objects hoist integer-like keys ("2" before "a", regardless of input).
+//
+// Both are detected below and cause the transform to DECLINE (return null), so
+// the caller keeps the text byte-exact. That is the same posture tier0.py takes
+// for duplicate keys and non-finite numbers: when the round trip is not
+// provably value-preserving, do nothing.
+
+/** True when a key would be reordered by a JS object (array-index-like). */
+function isIntegerLikeKey(k) {
+  return /^(0|[1-9]\d*)$/.test(k) && Number(k) <= 4294967294;
+}
+
+/**
+ * Scan the raw JSON source for constructs we cannot round-trip byte-identically
+ * with Python. Returns a reason string, or null when it is safe.
+ */
+function unsafeJsonReason(src) {
+  // Strip string contents FIRST. Scanning raw source treats `"line1"` as
+  // exponent notation ("e1") and `"v1.0"` as an integral float, which made this
+  // decline JSON that Python happily minifies — a conformance divergence in the
+  // safe direction, but a divergence.
+  const bare = stripJsonStrings(src);
+
+  // Python renders an integral float as "1.0"; JS renders it "1". After parsing
+  // the distinction is gone, so it has to be decided on the literal.
+  const nums = bare.match(/-?\d+\.\d+/g) || [];
+  for (const n of nums) {
+    if (Number.isInteger(Number(n))) {
+      return "integral float renders differently in Python and JS";
+    }
+  }
+  if (/\d[eE][+-]?\d+/.test(bare)) return "exponent notation renders differently";
+  if (/\b(Infinity|-Infinity|NaN)\b/.test(bare)) return "non-finite constant";
+  return null;
+}
+
+/** Replace every JSON string literal with "" so scans see structure only. */
+function stripJsonStrings(src) {
+  let out = "";
+  let inStr = false;
+  let esc = false;
+  for (let i = 0; i < src.length; i++) {
+    const c = src[i];
+    if (inStr) {
+      if (esc) esc = false;
+      else if (c === "\\") esc = true;
+      else if (c === '"') {
+        inStr = false;
+        out += '""';
+      }
+      continue;
+    }
+    if (c === '"') inStr = true;
+    else out += c;
+  }
+  return out;
+}
+
+/** Depth-first check for duplicate or reorder-prone keys, on the raw source. */
+function objectKeysAreSafe(obj, seenPath) {
+  if (Array.isArray(obj)) return obj.every((v) => objectKeysAreSafe(v, seenPath));
+  if (obj && typeof obj === "object") {
+    for (const k of Object.keys(obj)) {
+      if (isIntegerLikeKey(k)) return false; // JS would hoist it
+      if (!objectKeysAreSafe(obj[k], seenPath)) return false;
+    }
+  }
+  return true;
+}
+
+/**
+ * Re-encode JSON with no incidental whitespace, or null when that would not be
+ * provably value-preserving *and* byte-identical to the Python implementation.
+ *
+ * @param {string} text
+ * @returns {string|null}
+ */
+function minifyJson(text) {
+  if (typeof text !== "string") return null;
+  const s = text.trim();
+  const first = s.charAt(0);
+  const last = s.charAt(s.length - 1);
+  // Same gate as tier0.py: only whole objects/arrays.
+  if (!((first === "{" || first === "[") && (last === "}" || last === "]"))) return null;
+
+  if (unsafeJsonReason(s) !== null) return null;
+
+  let obj;
+  try {
+    obj = JSON.parse(s);
+  } catch (_e) {
+    return null;
+  }
+  if (!objectKeysAreSafe(obj)) return null;
+
+  // Duplicate keys: JSON.parse silently keeps the last. tier0.py rejects such
+  // input outright, so detect it by counting keys in the source.
+  if (hasDuplicateKeys(s)) return null;
+
+  try {
+    return JSON.stringify(obj);
+  } catch (_e) {
+    return null; // circular refs cannot occur from JSON.parse, but never throw
+  }
+}
+
+/** Cheap duplicate-key detector over the raw source (string-aware). */
+function hasDuplicateKeys(src) {
+  const stack = [];
+  let inStr = false;
+  let esc = false;
+  let cur = null;
+  let pendingKey = null;
+  for (let i = 0; i < src.length; i++) {
+    const c = src[i];
+    if (inStr) {
+      if (esc) {
+        esc = false;
+      } else if (c === "\\") {
+        esc = true;
+      } else if (c === '"') {
+        inStr = false;
+        pendingKey = src.slice(cur + 1, i);
+      }
+      continue;
+    }
+    if (c === '"') {
+      inStr = true;
+      cur = i;
+    } else if (c === "{") {
+      stack.push(new Set());
+      pendingKey = null;
+    } else if (c === "}") {
+      stack.pop();
+      pendingKey = null;
+    } else if (c === ":" && stack.length && pendingKey !== null) {
+      const top = stack[stack.length - 1];
+      if (top.has(pendingKey)) return true;
+      top.add(pendingKey);
+      pendingKey = null;
+    } else if (c === ",") {
+      pendingKey = null;
+    }
+  }
+  return false;
+}
+
+// --- heuristic tokenizer ----------------------------------------------------
+// Port of distil/tokenizer.py:HeuristicTokenizer. Needed because the production
+// Tier-0 path is reject-if-bigger *by tokens*, not by characters: collapsing a
+// run of near-free blank lines into a "<<xN>>" marker can cost MORE tokens than
+// it removes, and Tier-0 must never inflate.
+
+// Python: re.compile(r"\w+|[^\w\s]", re.UNICODE). The `u` flag plus explicit
+// Unicode property escapes reproduce Python's \w/\s semantics closely enough
+// for counting; the conformance suite is what proves it on real inputs.
+const PIECE = /[\p{L}\p{N}_]+|[^\p{L}\p{N}_\s]/gu;
+const SUBWORD_FACTOR = 1.33;
+
+/**
+ * Approximate token count, matching HeuristicTokenizer.count byte-for-byte.
+ * @param {string} text
+ * @returns {number}
+ */
+function countTokens(text) {
+  if (!text) return 0;
+  const pieces = text.match(PIECE);
+  const n = pieces ? pieces.length : 0;
+  return Math.max(1, bankersRound(n * SUBWORD_FACTOR));
+}
+
+/**
+ * Python's round() is round-half-to-EVEN; JS Math.round is round-half-UP.
+ * They differ on exact .5, which n * 1.33 can land on, so the naive version
+ * would silently diverge from Python on specific input lengths.
+ */
+function bankersRound(x) {
+  const floor = Math.floor(x);
+  const diff = x - floor;
+  if (diff > 0.5) return floor + 1;
+  if (diff < 0.5) return floor;
+  return floor % 2 === 0 ? floor : floor + 1;
+}
+
+/**
+ * Apply every Tier-0 transform in the same order as the production path
+ * (distil/adapters/anthropic.py:_apply_tier0): JSON minification first, then
+ * run-collapse on the RESULT, keeping the collapse only when it does not
+ * increase the token count.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function applyTier0(text) {
+  if (typeof text !== "string" || text.length === 0) return text;
+  const mj = minifyJson(text);
+  const base = mj !== null ? mj : text;
+  const collapsed = collapseRuns(base);
+  if (collapsed !== base && countTokens(collapsed) <= countTokens(base)) {
+    return collapsed;
+  }
+  return base;
+}
+
+module.exports = {
+  applyTier0,
+  countTokens,
+  collapseRuns,
+  expandRuns,
+  minifyJson,
+  // exported for the conformance harness
+  _internals: { hasDuplicateKeys, unsafeJsonReason, isIntegerLikeKey },
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "distil-llm",
-  "version": "1.41.0",
+  "version": "1.42.0",
   "description": "Compress your AI agent's context and prove its decisions don't change \u2014 cache-aware, reversible, certified. JS/TS bridge to the distil CLI + proxy helpers.",
   "keywords": [
     "llm",
@@ -38,6 +38,6 @@
     "node": ">=18"
   },
   "scripts": {
-    "test": "node --test test/"
+    "test": "node --test test/*.test.mjs"
   }
 }