@sohqureshi/tokenwise 1.0.4 → 1.0.9

package/README.md

@@ -3,8 +3,8 @@
 TokenWise is a lightweight utility for preparing JSON before sending it to AI models. It helps reduce payload noise, shrink token usage, and turn structured data into formats that are easier for LLMs to consume.
 
 
-![npm version](https://img.shields.io/npm/v/tokenwise)
-![downloads](https://img.shields.io/npm/dw/tokenwise)
+[![npm version](https://img.shields.io/npm/v/%40sohqureshi%2Ftokenwise)](https://www.npmjs.com/package/@sohqureshi/tokenwise)
+[![downloads](https://img.shields.io/npm/dw/%40sohqureshi%2Ftokenwise)](https://www.npmjs.com/package/@sohqureshi/tokenwise)
 ![license](https://img.shields.io/github/license/sohqureshi/tokenwise)
 ![stars](https://img.shields.io/github/stars/sohqureshi/tokenwise?style=social)
 [![Demo](https://img.shields.io/badge/Live%20Demo-Visit-brightgreen)](https://sohqureshi.github.io/tokenwise/)
@@ -36,6 +36,8 @@ Raw JSON is:
 npm install @sohqureshi/tokenwise
 ```
 
+Available on [npm](https://www.npmjs.com/package/@sohqureshi/tokenwise).
+
 ## Quick Usage
 
 ```js
@@ -68,7 +70,7 @@ console.log(ai(product).compact().value());
 Removes fields you do not want to send to the model. By default it also removes `null`, `undefined`, and empty objects.
 
 ```js
-import { prune } from "tokenwise";
+import { prune } from "@sohqureshi/tokenwise";
 
 const input = {
   user: { name: "John", age: 28 },
@@ -208,9 +210,24 @@ LLMs charge and reason over tokens. Sending raw JSON often includes repeated key
 node demo.js
 ```
 
-Use `--analyze` to compare serialized input and output with TokenWise's
-four-characters-per-token heuristic. This is an estimate, not a replacement
-for a model-specific tokenizer or a provider's billed token count.
+Use `--analyze` to compare serialized input and output with the model's
+`tiktoken` encoding. `estimateTokens()` and `analyze()` use exact
+model-aware counts by default; pass `exact: false` to opt into the
+four-characters-per-token heuristic. Use `model` to select the tokenizer
+used by `tiktoken`.
+
+---
+
+## Release Notes
+
+### v1.0.8 — 2026-09-06
+
+- Expose exact tokenizer metadata when available (model + encoding), and fall back to a clear, model-aware estimator in browser demos.
+- Demo updated to show selected model and expected encoding when the exact tokenizer (tiktoken) is not available in-browser.
+- Updated analyze() to surface tokenizer encoding in analysis output so the demo shows "Estimator: model=gpt-4 expected_encoding=cl100k_base" even when using the heuristic fallback.
+- Misc: build artifacts updated and docs demo import bumped to v1.0.7 CDN bundle.
+
+If you want the browser demo to display truly exact token counts, run the demo against a small Node endpoint (or local server) that has tiktoken installed and uses analyze(..., { exact: true }). The estimator will display the real tokenizer encoding when tiktoken is present.
 
 ---
 

package/dist/{chunk-ZVQRYJU5.js → chunk-2TDVL6BJ.js}

@@ -11,12 +11,17 @@ import {
   prune
 } from "./chunk-ZD536GZF.js";
 import {
-  estimateTokens,
+  estimateTokensWithMeta,
   serializeForTokenEstimate
-} from "./chunk-7CL7GVRF.js";
+} from "./chunk-L3HY6AWL.js";
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = true,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -27,10 +32,15 @@ function analyze(input, options = {}) {
       reductionRatio: 1,
       originalCharacters: 0,
       optimizedCharacters: 0,
-      estimator: "heuristic: 1 token \u2248 4 characters"
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  const originalTokens = estimateTokens(input);
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -44,10 +54,16 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  const optimizedTokens = estimateTokens(optimizedData);
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
@@ -57,7 +73,7 @@ function analyze(input, options = {}) {
     reductionRatio,
     originalCharacters: serializeForTokenEstimate(input).length,
     optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
-    estimator: "heuristic: 1 token \u2248 4 characters"
+    estimator: estimatorLabel
   };
 }
 

package/dist/chunk-L3HY6AWL.js

@@ -0,0 +1,69 @@
+// src/core/token.ts
+import { createRequire } from "module";
+var require2 = createRequire(import.meta.url);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokensHeuristic(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
+}
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch {
+        encodingName = void 0;
+      }
+    }
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
+    if (!fallbackToHeuristic) {
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
+    }
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
+  }
+}
+
+export {
+  serializeForTokenEstimate,
+  estimateTokens,
+  estimateTokensWithMeta
+};

package/dist/{chunk-HFMW3K22.js → chunk-QHCUNOUH.js}

@@ -1,6 +1,6 @@
 import {
   analyze
-} from "./chunk-ZVQRYJU5.js";
+} from "./chunk-2TDVL6BJ.js";
 import {
   toTOON
 } from "./chunk-6VRLDRJK.js";
@@ -41,8 +41,8 @@ var AIChain = class {
   toNatural() {
     return toNatural(this.data);
   }
-  analyze() {
-    return analyze(this.data);
+  analyze(options) {
+    return analyze(this.data, options);
   }
   value() {
     return this.data;

package/dist/cli.cjs

@@ -142,17 +142,74 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
 function serializeForTokenEstimate(value) {
   if (typeof value === "string") return value;
   const serialized = JSON.stringify(value);
   return serialized ?? "";
 }
-function estimateTokens(value) {
+function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch {
+        encodingName = void 0;
+      }
+    }
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
+    if (!fallbackToHeuristic) {
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
+    }
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
+  }
+}
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = true,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -163,10 +220,15 @@ function analyze(input, options = {}) {
       reductionRatio: 1,
       originalCharacters: 0,
       optimizedCharacters: 0,
-      estimator: "heuristic: 1 token \u2248 4 characters"
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  const originalTokens = estimateTokens(input);
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -180,10 +242,16 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  const optimizedTokens = estimateTokens(optimizedData);
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
@@ -193,7 +261,7 @@ function analyze(input, options = {}) {
     reductionRatio,
     originalCharacters: serializeForTokenEstimate(input).length,
     optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
-    estimator: "heuristic: 1 token \u2248 4 characters"
+    estimator: estimatorLabel
   };
 }
 
@@ -323,8 +391,8 @@ var AIChain = class {
   toNatural() {
     return toNatural(this.data);
   }
-  analyze() {
-    return analyze(this.data);
+  analyze(options) {
+    return analyze(this.data, options);
   }
   value() {
     return this.data;
@@ -341,7 +409,7 @@ Usage:
 Options:
   --toon       Convert JSON to TOON format
   --compact    Convert JSON to compact format
-  --analyze    Show token analysis (heuristic estimate)
+  --analyze    Show token analysis (tiktoken estimate)
 `);
   process.exit(0);
 }

package/dist/cli.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 import {
   AIChain
-} from "./chunk-HFMW3K22.js";
-import "./chunk-ZVQRYJU5.js";
+} from "./chunk-QHCUNOUH.js";
+import "./chunk-2TDVL6BJ.js";
 import "./chunk-6VRLDRJK.js";
 import "./chunk-XE36GLJP.js";
 import "./chunk-L7BC62MT.js";
 import "./chunk-D4CFTFM3.js";
 import "./chunk-ZD536GZF.js";
-import "./chunk-7CL7GVRF.js";
+import "./chunk-L3HY6AWL.js";
 
 // src/cli.ts
 import fs from "fs";
@@ -22,7 +22,7 @@ Usage:
 Options:
   --toon       Convert JSON to TOON format
   --compact    Convert JSON to compact format
-  --analyze    Show token analysis (heuristic estimate)
+  --analyze    Show token analysis (tiktoken estimate)
 `);
   process.exit(0);
 }

package/dist/core/analyze.cjs

@@ -139,17 +139,74 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
 function serializeForTokenEstimate(value) {
   if (typeof value === "string") return value;
   const serialized = JSON.stringify(value);
   return serialized ?? "";
 }
-function estimateTokens(value) {
+function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch {
+        encodingName = void 0;
+      }
+    }
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
+    if (!fallbackToHeuristic) {
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
+    }
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
+  }
+}
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = true,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -160,10 +217,15 @@ function analyze(input, options = {}) {
       reductionRatio: 1,
       originalCharacters: 0,
       optimizedCharacters: 0,
-      estimator: "heuristic: 1 token \u2248 4 characters"
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  const originalTokens = estimateTokens(input);
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -177,10 +239,16 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  const optimizedTokens = estimateTokens(optimizedData);
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
@@ -190,7 +258,7 @@ function analyze(input, options = {}) {
     reductionRatio,
     originalCharacters: serializeForTokenEstimate(input).length,
     optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
-    estimator: "heuristic: 1 token \u2248 4 characters"
+    estimator: estimatorLabel
   };
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/core/analyze.d.cts

@@ -7,6 +7,9 @@ type AnalyzeOptions = {
     flatten?: boolean;
     toTOON?: boolean;
     toon?: boolean;
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
 };
 declare function analyze(input: unknown, options?: AnalyzeOptions): {
     originalTokens: number;

package/dist/core/analyze.d.ts

@@ -7,6 +7,9 @@ type AnalyzeOptions = {
     flatten?: boolean;
     toTOON?: boolean;
     toon?: boolean;
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
 };
 declare function analyze(input: unknown, options?: AnalyzeOptions): {
     originalTokens: number;

package/dist/core/analyze.js

@@ -1,11 +1,11 @@
 import {
   analyze
-} from "../chunk-ZVQRYJU5.js";
+} from "../chunk-2TDVL6BJ.js";
 import "../chunk-6VRLDRJK.js";
 import "../chunk-XE36GLJP.js";
 import "../chunk-L7BC62MT.js";
 import "../chunk-ZD536GZF.js";
-import "../chunk-7CL7GVRF.js";
+import "../chunk-L3HY6AWL.js";
 export {
   analyze
 };

package/dist/core/token.cjs

@@ -21,19 +21,76 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var token_exports = {};
 __export(token_exports, {
   estimateTokens: () => estimateTokens,
+  estimateTokensWithMeta: () => estimateTokensWithMeta,
   serializeForTokenEstimate: () => serializeForTokenEstimate
 });
 module.exports = __toCommonJS(token_exports);
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
 function serializeForTokenEstimate(value) {
   if (typeof value === "string") return value;
   const serialized = JSON.stringify(value);
   return serialized ?? "";
 }
-function estimateTokens(value) {
+function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch {
+        encodingName = void 0;
+      }
+    }
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
+    if (!fallbackToHeuristic) {
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
+    }
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
+  }
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   estimateTokens,
+  estimateTokensWithMeta,
   serializeForTokenEstimate
 });

package/dist/core/token.d.cts

@@ -1,18 +1,18 @@
-/**
- * Estimates token count from string length.
- *
- * Approximation:
- * 1 token ≈ 4 characters (common for GPT models)
- *
- * @param text - Input string
- * @returns Estimated token count
- */
+interface TokenEstimateOptions {
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
+}
 declare function serializeForTokenEstimate(value: unknown): string;
+type TokenEstimateResult = {
+    count: number;
+    estimator: string;
+};
+declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
 /**
- * Estimates tokens using a transparent four-characters-per-token heuristic.
- * JSON values are serialized first, matching the compact form normally sent
- * to an API. Use a model-specific tokenizer for billing-accurate counts.
+ * Returns an exact model tokenizer count by default. Set exact=false to use the
+ * lightweight four-characters-per-token fallback explicitly.
  */
-declare function estimateTokens(value: unknown): number;
+declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
 
-export { estimateTokens, serializeForTokenEstimate };
+export { type TokenEstimateOptions, type TokenEstimateResult, estimateTokens, estimateTokensWithMeta, serializeForTokenEstimate };

package/dist/core/token.d.ts

@@ -1,18 +1,18 @@
-/**
- * Estimates token count from string length.
- *
- * Approximation:
- * 1 token ≈ 4 characters (common for GPT models)
- *
- * @param text - Input string
- * @returns Estimated token count
- */
+interface TokenEstimateOptions {
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
+}
 declare function serializeForTokenEstimate(value: unknown): string;
+type TokenEstimateResult = {
+    count: number;
+    estimator: string;
+};
+declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
 /**
- * Estimates tokens using a transparent four-characters-per-token heuristic.
- * JSON values are serialized first, matching the compact form normally sent
- * to an API. Use a model-specific tokenizer for billing-accurate counts.
+ * Returns an exact model tokenizer count by default. Set exact=false to use the
+ * lightweight four-characters-per-token fallback explicitly.
  */
-declare function estimateTokens(value: unknown): number;
+declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
 
-export { estimateTokens, serializeForTokenEstimate };
+export { type TokenEstimateOptions, type TokenEstimateResult, estimateTokens, estimateTokensWithMeta, serializeForTokenEstimate };

package/dist/core/token.js

@@ -1,8 +1,10 @@
 import {
   estimateTokens,
+  estimateTokensWithMeta,
   serializeForTokenEstimate
-} from "../chunk-7CL7GVRF.js";
+} from "../chunk-L3HY6AWL.js";
 export {
   estimateTokens,
+  estimateTokensWithMeta,
   serializeForTokenEstimate
 };

package/dist/index.cjs

@@ -148,17 +148,77 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
 function serializeForTokenEstimate(value) {
   if (typeof value === "string") return value;
   const serialized = JSON.stringify(value);
   return serialized ?? "";
 }
-function estimateTokens(value) {
+function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function expectedEncodingForModel(model) {
+  const normalizedModel = model.toLowerCase();
+  if (normalizedModel.includes("gpt-4o") || normalizedModel.includes("gpt-4.1") || normalizedModel.includes("o1") || normalizedModel.includes("o3") || normalizedModel.includes("o4")) {
+    return "o200k_base";
+  }
+  if (normalizedModel.includes("davinci") || normalizedModel.startsWith("text-") || normalizedModel.includes("babbage") || normalizedModel.includes("curie")) {
+    return "r50k_base";
+  }
+  return "cl100k_base";
+}
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = true, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const count = encoder.encode(serializeForTokenEstimate(value)).length;
+    let encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch {
+        encodingName = void 0;
+      }
+    }
+    encodingName ?? (encodingName = expectedEncodingForModel(model));
+    encoder.free?.();
+    return {
+      count,
+      estimator: `exact tokenizer: model=${model} encoding=${encodingName}`
+    };
+  } catch {
+    const expectedEncoding = expectedEncodingForModel(model);
+    if (!fallbackToHeuristic) {
+      return {
+        count: heuristic,
+        estimator: `exact requested but tokenizer unavailable (expected encoding=${expectedEncoding} for model=${model})`
+      };
+    }
+    return {
+      count: heuristic,
+      estimator: `${heuristicEstimator} (model=${model} expected_encoding=${expectedEncoding})`
+    };
+  }
+}
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = true,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -169,10 +229,15 @@ function analyze(input, options = {}) {
       reductionRatio: 1,
       originalCharacters: 0,
       optimizedCharacters: 0,
-      estimator: "heuristic: 1 token \u2248 4 characters"
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  const originalTokens = estimateTokens(input);
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -186,10 +251,16 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  const optimizedTokens = estimateTokens(optimizedData);
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
@@ -199,7 +270,7 @@ function analyze(input, options = {}) {
     reductionRatio,
     originalCharacters: serializeForTokenEstimate(input).length,
     optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
-    estimator: "heuristic: 1 token \u2248 4 characters"
+    estimator: estimatorLabel
   };
 }
 
@@ -329,8 +400,8 @@ var AIChain = class {
   toNatural() {
     return toNatural(this.data);
   }
-  analyze() {
-    return analyze(this.data);
+  analyze(options) {
+    return analyze(this.data, options);
   }
   value() {
     return this.data;

package/dist/index.d.cts

@@ -14,7 +14,7 @@ declare class AIChain {
     flatten(): this;
     toTOON(): this;
     toNatural(): string;
-    analyze(): {
+    analyze(options?: any): {
         originalTokens: number;
         optimizedTokens: number;
         savings: number;

package/dist/index.d.ts

@@ -14,7 +14,7 @@ declare class AIChain {
     flatten(): this;
     toTOON(): this;
     toNatural(): string;
-    analyze(): {
+    analyze(options?: any): {
         originalTokens: number;
         optimizedTokens: number;
         savings: number;

package/dist/index.js

@@ -1,9 +1,9 @@
 import {
   AIChain
-} from "./chunk-HFMW3K22.js";
+} from "./chunk-QHCUNOUH.js";
 import {
   analyze
-} from "./chunk-ZVQRYJU5.js";
+} from "./chunk-2TDVL6BJ.js";
 import {
   toTOON
 } from "./chunk-6VRLDRJK.js";
@@ -22,7 +22,7 @@ import {
 import {
   estimateTokens,
   serializeForTokenEstimate
-} from "./chunk-7CL7GVRF.js";
+} from "./chunk-L3HY6AWL.js";
 
 // src/index.ts
 function ai(data) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sohqureshi/tokenwise",
-  "version": "1.0.4",
+  "version": "1.0.9",
   "description": "Optimize JSON data for AI by reducing token usage",
   "repository": {
     "type": "git",
@@ -75,6 +75,9 @@
   ],
   "author": "Mohammad Sohail",
   "license": "MIT",
+  "dependencies": {
+    "tiktoken": "^1.0.22"
+  },
   "devDependencies": {
     "@types/node": "^25.6.0",
     "tsup": "^8.0.0",

package/dist/chunk-7CL7GVRF.js

@@ -1,14 +0,0 @@
-// src/core/token.ts
-function serializeForTokenEstimate(value) {
-  if (typeof value === "string") return value;
-  const serialized = JSON.stringify(value);
-  return serialized ?? "";
-}
-function estimateTokens(value) {
-  return Math.ceil(serializeForTokenEstimate(value).length / 4);
-}
-
-export {
-  serializeForTokenEstimate,
-  estimateTokens
-};