@sohqureshi/tokenwise 1.0.7 → 1.0.9

package/README.md

@@ -210,10 +210,24 @@ LLMs charge and reason over tokens. Sending raw JSON often includes repeated key
 node demo.js
 ```
 
-Use `--analyze` to compare serialized input and output with TokenWise's
-four-characters-per-token heuristic. For model-accurate counts, pass
-`exact: true` and a `model` name to `estimateTokens()` or `analyze()`.
-This is still only as exact as the tokenizer implementation you use.
+Use `--analyze` to compare serialized input and output with the model's
+`tiktoken` encoding. `estimateTokens()` and `analyze()` use exact
+model-aware counts by default; pass `exact: false` to opt into the
+four-characters-per-token heuristic. Use `model` to select the tokenizer
+used by `tiktoken`.
+
+---
+
+## Release Notes
+
+### v1.0.8 — 2026-09-06
+
+- Expose exact tokenizer metadata when available (model + encoding), and fall back to a clear, model-aware estimator in browser demos.
+- Demo updated to show selected model and expected encoding when the exact tokenizer (tiktoken) is not available in-browser.
+- Updated analyze() to surface tokenizer encoding in analysis output so the demo shows "Estimator: model=gpt-4 expected_encoding=cl100k_base" even when using the heuristic fallback.
+- Misc: build artifacts updated and docs demo import bumped to v1.0.7 CDN bundle.
+
+If you want the browser demo to display truly exact token counts, run the demo against a small Node endpoint (or local server) that has tiktoken installed and uses analyze(..., { exact: true }). The estimator will display the real tokenizer encoding when tiktoken is present.
 
 ---
 

package/dist/{chunk-IOMOVRC7.js → chunk-2TDVL6BJ.js}

@@ -13,12 +13,12 @@ import {
 import {
   estimateTokensWithMeta,
   serializeForTokenEstimate
-} from "./chunk-U433WUUT.js";
+} from "./chunk-L3HY6AWL.js";
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
   const {
-    exact = false,
+    exact = true,
     model = "gpt-4o-mini",
     fallbackToHeuristic = true
   } = options;

package/dist/chunk-L3HY6AWL.js

@@ -0,0 +1,69 @@
+// src/core/token.ts
+import { createRequire } from "module";
+var require2 = createRequire(import.meta.url);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokensHeuristic(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
+}
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch {
+        encodingName = void 0;
+      }
+    }
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
+    if (!fallbackToHeuristic) {
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
+    }
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
+  }
+}
+
+export {
+  serializeForTokenEstimate,
+  estimateTokens,
+  estimateTokensWithMeta
+};

package/dist/{chunk-P6KXISNT.js → chunk-QHCUNOUH.js}

@@ -1,6 +1,6 @@
 import {
   analyze
-} from "./chunk-IOMOVRC7.js";
+} from "./chunk-2TDVL6BJ.js";
 import {
   toTOON
 } from "./chunk-6VRLDRJK.js";

package/dist/cli.cjs

@@ -153,8 +153,18 @@ function serializeForTokenEstimate(value) {
 function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
 function estimateTokensWithMeta(value, options = {}) {
-  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
   const heuristic = estimateTokensHeuristic(value);
   const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
   if (!exact) {
@@ -163,36 +173,40 @@ function estimateTokensWithMeta(value, options = {}) {
   try {
     const tiktoken = require2("tiktoken");
     const encoder = tiktoken.encoding_for_model(model);
-    const text = serializeForTokenEstimate(value);
-    const count = encoder.encode(text).length;
-    let encodingName = null;
-    if (encoder.name) encodingName = encoder.name;
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
     if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
       try {
         encodingName = tiktoken.model_to_encoding(model);
-      } catch (e) {
-        encodingName = null;
+      } catch {
+        encodingName = void 0;
       }
     }
-    if (!encodingName) {
-      const m = String(model || "").toLowerCase();
-      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
-      else encodingName = "cl100k_base";
-    }
-    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
-    return { count, estimator };
-  } catch (e) {
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
     if (!fallbackToHeuristic) {
-      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
     }
-    return { count: heuristic, estimator: heuristicEstimator };
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
   }
 }
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
   const {
-    exact = false,
+    exact = true,
     model = "gpt-4o-mini",
     fallbackToHeuristic = true
   } = options;
@@ -395,7 +409,7 @@ Usage:
 Options:
   --toon       Convert JSON to TOON format
   --compact    Convert JSON to compact format
-  --analyze    Show token analysis (heuristic estimate)
+  --analyze    Show token analysis (tiktoken estimate)
 `);
   process.exit(0);
 }

package/dist/cli.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 import {
   AIChain
-} from "./chunk-P6KXISNT.js";
-import "./chunk-IOMOVRC7.js";
+} from "./chunk-QHCUNOUH.js";
+import "./chunk-2TDVL6BJ.js";
 import "./chunk-6VRLDRJK.js";
 import "./chunk-XE36GLJP.js";
 import "./chunk-L7BC62MT.js";
 import "./chunk-D4CFTFM3.js";
 import "./chunk-ZD536GZF.js";
-import "./chunk-U433WUUT.js";
+import "./chunk-L3HY6AWL.js";
 
 // src/cli.ts
 import fs from "fs";
@@ -22,7 +22,7 @@ Usage:
 Options:
   --toon       Convert JSON to TOON format
   --compact    Convert JSON to compact format
-  --analyze    Show token analysis (heuristic estimate)
+  --analyze    Show token analysis (tiktoken estimate)
 `);
   process.exit(0);
 }

package/dist/core/analyze.cjs

@@ -150,8 +150,18 @@ function serializeForTokenEstimate(value) {
 function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
 function estimateTokensWithMeta(value, options = {}) {
-  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
   const heuristic = estimateTokensHeuristic(value);
   const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
   if (!exact) {
@@ -160,36 +170,40 @@ function estimateTokensWithMeta(value, options = {}) {
   try {
     const tiktoken = require2("tiktoken");
     const encoder = tiktoken.encoding_for_model(model);
-    const text = serializeForTokenEstimate(value);
-    const count = encoder.encode(text).length;
-    let encodingName = null;
-    if (encoder.name) encodingName = encoder.name;
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
     if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
       try {
         encodingName = tiktoken.model_to_encoding(model);
-      } catch (e) {
-        encodingName = null;
+      } catch {
+        encodingName = void 0;
       }
     }
-    if (!encodingName) {
-      const m = String(model || "").toLowerCase();
-      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
-      else encodingName = "cl100k_base";
-    }
-    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
-    return { count, estimator };
-  } catch (e) {
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
     if (!fallbackToHeuristic) {
-      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
     }
-    return { count: heuristic, estimator: heuristicEstimator };
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
   }
 }
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
   const {
-    exact = false,
+    exact = true,
     model = "gpt-4o-mini",
     fallbackToHeuristic = true
   } = options;

package/dist/core/analyze.js

@@ -1,11 +1,11 @@
 import {
   analyze
-} from "../chunk-IOMOVRC7.js";
+} from "../chunk-2TDVL6BJ.js";
 import "../chunk-6VRLDRJK.js";
 import "../chunk-XE36GLJP.js";
 import "../chunk-L7BC62MT.js";
 import "../chunk-ZD536GZF.js";
-import "../chunk-U433WUUT.js";
+import "../chunk-L3HY6AWL.js";
 export {
   analyze
 };

package/dist/core/token.cjs

@@ -36,11 +36,21 @@ function serializeForTokenEstimate(value) {
 function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
 function estimateTokens(value, options = {}) {
   return estimateTokensWithMeta(value, options).count;
 }
 function estimateTokensWithMeta(value, options = {}) {
-  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
   const heuristic = estimateTokensHeuristic(value);
   const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
   if (!exact) {
@@ -49,29 +59,33 @@ function estimateTokensWithMeta(value, options = {}) {
   try {
     const tiktoken = require2("tiktoken");
     const encoder = tiktoken.encoding_for_model(model);
-    const text = serializeForTokenEstimate(value);
-    const count = encoder.encode(text).length;
-    let encodingName = null;
-    if (encoder.name) encodingName = encoder.name;
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
     if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
       try {
         encodingName = tiktoken.model_to_encoding(model);
-      } catch (e) {
-        encodingName = null;
+      } catch {
+        encodingName = void 0;
       }
     }
-    if (!encodingName) {
-      const m = String(model || "").toLowerCase();
-      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
-      else encodingName = "cl100k_base";
-    }
-    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
-    return { count, estimator };
-  } catch (e) {
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
     if (!fallbackToHeuristic) {
-      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
     }
-    return { count: heuristic, estimator: heuristicEstimator };
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
   }
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/core/token.d.cts

@@ -4,20 +4,14 @@ interface TokenEstimateOptions {
     fallbackToHeuristic?: boolean;
 }
 declare function serializeForTokenEstimate(value: unknown): string;
-/**
- * Estimates tokens using a transparent four-characters-per-token heuristic.
- * JSON values are serialized first, matching the compact form normally sent
- * to an API. Set exact=true to attempt a model-specific tokenizer count when
- * the runtime has the tokenizer installed.
- */
 type TokenEstimateResult = {
     count: number;
     estimator: string;
 };
 declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
 /**
- * Returns both a token count and an estimator string describing how the count was
- * obtained (exact tokenizer + encoding when available, or heuristic otherwise).
+ * Returns an exact model tokenizer count by default. Set exact=false to use the
+ * lightweight four-characters-per-token fallback explicitly.
  */
 declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
 

package/dist/core/token.d.ts

@@ -4,20 +4,14 @@ interface TokenEstimateOptions {
     fallbackToHeuristic?: boolean;
 }
 declare function serializeForTokenEstimate(value: unknown): string;
-/**
- * Estimates tokens using a transparent four-characters-per-token heuristic.
- * JSON values are serialized first, matching the compact form normally sent
- * to an API. Set exact=true to attempt a model-specific tokenizer count when
- * the runtime has the tokenizer installed.
- */
 type TokenEstimateResult = {
     count: number;
     estimator: string;
 };
 declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
 /**
- * Returns both a token count and an estimator string describing how the count was
- * obtained (exact tokenizer + encoding when available, or heuristic otherwise).
+ * Returns an exact model tokenizer count by default. Set exact=false to use the
+ * lightweight four-characters-per-token fallback explicitly.
  */
 declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
 

package/dist/core/token.js

@@ -2,7 +2,7 @@ import {
   estimateTokens,
   estimateTokensWithMeta,
   serializeForTokenEstimate
-} from "../chunk-U433WUUT.js";
+} from "../chunk-L3HY6AWL.js";
 export {
   estimateTokens,
   estimateTokensWithMeta,

package/dist/index.cjs

@@ -159,11 +159,21 @@ function serializeForTokenEstimate(value) {
 function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
 function estimateTokens(value, options = {}) {
   return estimateTokensWithMeta(value, options).count;
 }
 function estimateTokensWithMeta(value, options = {}) {
-  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
   const heuristic = estimateTokensHeuristic(value);
   const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
   if (!exact) {
@@ -172,36 +182,40 @@ function estimateTokensWithMeta(value, options = {}) {
   try {
     const tiktoken = require2("tiktoken");
     const encoder = tiktoken.encoding_for_model(model);
-    const text = serializeForTokenEstimate(value);
-    const count = encoder.encode(text).length;
-    let encodingName = null;
-    if (encoder.name) encodingName = encoder.name;
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
     if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
       try {
         encodingName = tiktoken.model_to_encoding(model);
-      } catch (e) {
-        encodingName = null;
+      } catch {
+        encodingName = void 0;
       }
     }
-    if (!encodingName) {
-      const m = String(model || "").toLowerCase();
-      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
-      else encodingName = "cl100k_base";
-    }
-    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
-    return { count, estimator };
-  } catch (e) {
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
     if (!fallbackToHeuristic) {
-      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
     }
-    return { count: heuristic, estimator: heuristicEstimator };
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
   }
 }
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
   const {
-    exact = false,
+    exact = true,
     model = "gpt-4o-mini",
     fallbackToHeuristic = true
   } = options;

package/dist/index.js

@@ -1,9 +1,9 @@
 import {
   AIChain
-} from "./chunk-P6KXISNT.js";
+} from "./chunk-QHCUNOUH.js";
 import {
   analyze
-} from "./chunk-IOMOVRC7.js";
+} from "./chunk-2TDVL6BJ.js";
 import {
   toTOON
 } from "./chunk-6VRLDRJK.js";
@@ -22,7 +22,7 @@ import {
 import {
   estimateTokens,
   serializeForTokenEstimate
-} from "./chunk-U433WUUT.js";
+} from "./chunk-L3HY6AWL.js";
 
 // src/index.ts
 function ai(data) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sohqureshi/tokenwise",
-  "version": "1.0.7",
+  "version": "1.0.9",
   "description": "Optimize JSON data for AI by reducing token usage",
   "repository": {
     "type": "git",

package/dist/chunk-U433WUUT.js

@@ -1,55 +0,0 @@
-// src/core/token.ts
-import { createRequire } from "module";
-var require2 = createRequire(import.meta.url);
-function serializeForTokenEstimate(value) {
-  if (typeof value === "string") return value;
-  const serialized = JSON.stringify(value);
-  return serialized ?? "";
-}
-function estimateTokensHeuristic(value) {
-  return Math.ceil(serializeForTokenEstimate(value).length / 4);
-}
-function estimateTokens(value, options = {}) {
-  return estimateTokensWithMeta(value, options).count;
-}
-function estimateTokensWithMeta(value, options = {}) {
-  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
-  const heuristic = estimateTokensHeuristic(value);
-  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
-  if (!exact) {
-    return { count: heuristic, estimator: heuristicEstimator };
-  }
-  try {
-    const tiktoken = require2("tiktoken");
-    const encoder = tiktoken.encoding_for_model(model);
-    const text = serializeForTokenEstimate(value);
-    const count = encoder.encode(text).length;
-    let encodingName = null;
-    if (encoder.name) encodingName = encoder.name;
-    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
-      try {
-        encodingName = tiktoken.model_to_encoding(model);
-      } catch (e) {
-        encodingName = null;
-      }
-    }
-    if (!encodingName) {
-      const m = String(model || "").toLowerCase();
-      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
-      else encodingName = "cl100k_base";
-    }
-    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
-    return { count, estimator };
-  } catch (e) {
-    if (!fallbackToHeuristic) {
-      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
-    }
-    return { count: heuristic, estimator: heuristicEstimator };
-  }
-}
-
-export {
-  serializeForTokenEstimate,
-  estimateTokens,
-  estimateTokensWithMeta
-};