@sohqureshi/tokenwise 1.0.4 → 1.0.7

package/README.md

@@ -3,8 +3,8 @@
 TokenWise is a lightweight utility for preparing JSON before sending it to AI models. It helps reduce payload noise, shrink token usage, and turn structured data into formats that are easier for LLMs to consume.
 
 
-![npm version](https://img.shields.io/npm/v/tokenwise)
-![downloads](https://img.shields.io/npm/dw/tokenwise)
+[![npm version](https://img.shields.io/npm/v/%40sohqureshi%2Ftokenwise)](https://www.npmjs.com/package/@sohqureshi/tokenwise)
+[![downloads](https://img.shields.io/npm/dw/%40sohqureshi%2Ftokenwise)](https://www.npmjs.com/package/@sohqureshi/tokenwise)
 ![license](https://img.shields.io/github/license/sohqureshi/tokenwise)
 ![stars](https://img.shields.io/github/stars/sohqureshi/tokenwise?style=social)
 [![Demo](https://img.shields.io/badge/Live%20Demo-Visit-brightgreen)](https://sohqureshi.github.io/tokenwise/)
@@ -36,6 +36,8 @@ Raw JSON is:
 npm install @sohqureshi/tokenwise
 ```
 
+Available on [npm](https://www.npmjs.com/package/@sohqureshi/tokenwise).
+
 ## Quick Usage
 
 ```js
@@ -68,7 +70,7 @@ console.log(ai(product).compact().value());
 Removes fields you do not want to send to the model. By default it also removes `null`, `undefined`, and empty objects.
 
 ```js
-import { prune } from "tokenwise";
+import { prune } from "@sohqureshi/tokenwise";
 
 const input = {
   user: { name: "John", age: 28 },
@@ -209,8 +211,9 @@ node demo.js
 ```
 
 Use `--analyze` to compare serialized input and output with TokenWise's
-four-characters-per-token heuristic. This is an estimate, not a replacement
-for a model-specific tokenizer or a provider's billed token count.
+four-characters-per-token heuristic. For model-accurate counts, pass
+`exact: true` and a `model` name to `estimateTokens()` or `analyze()`.
+This is still only as exact as the tokenizer implementation you use.
 
 ---
 

package/dist/{chunk-ZVQRYJU5.js → chunk-IOMOVRC7.js}

@@ -11,12 +11,17 @@ import {
   prune
 } from "./chunk-ZD536GZF.js";
 import {
-  estimateTokens,
+  estimateTokensWithMeta,
   serializeForTokenEstimate
-} from "./chunk-7CL7GVRF.js";
+} from "./chunk-U433WUUT.js";
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = false,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -27,10 +32,15 @@ function analyze(input, options = {}) {
       reductionRatio: 1,
       originalCharacters: 0,
       optimizedCharacters: 0,
-      estimator: "heuristic: 1 token \u2248 4 characters"
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  const originalTokens = estimateTokens(input);
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -44,10 +54,16 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  const optimizedTokens = estimateTokens(optimizedData);
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
@@ -57,7 +73,7 @@ function analyze(input, options = {}) {
     reductionRatio,
     originalCharacters: serializeForTokenEstimate(input).length,
     optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
-    estimator: "heuristic: 1 token \u2248 4 characters"
+    estimator: estimatorLabel
   };
 }
 

package/dist/{chunk-HFMW3K22.js → chunk-P6KXISNT.js}

@@ -1,6 +1,6 @@
 import {
   analyze
-} from "./chunk-ZVQRYJU5.js";
+} from "./chunk-IOMOVRC7.js";
 import {
   toTOON
 } from "./chunk-6VRLDRJK.js";
@@ -41,8 +41,8 @@ var AIChain = class {
   toNatural() {
     return toNatural(this.data);
   }
-  analyze() {
-    return analyze(this.data);
+  analyze(options) {
+    return analyze(this.data, options);
   }
   value() {
     return this.data;

package/dist/chunk-U433WUUT.js

@@ -0,0 +1,55 @@
+// src/core/token.ts
+import { createRequire } from "module";
+var require2 = createRequire(import.meta.url);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokensHeuristic(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
+}
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+}
+
+export {
+  serializeForTokenEstimate,
+  estimateTokens,
+  estimateTokensWithMeta
+};

package/dist/cli.cjs

@@ -142,17 +142,60 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
 function serializeForTokenEstimate(value) {
   if (typeof value === "string") return value;
   const serialized = JSON.stringify(value);
   return serialized ?? "";
 }
-function estimateTokens(value) {
+function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+}
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = false,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -163,10 +206,15 @@ function analyze(input, options = {}) {
       reductionRatio: 1,
       originalCharacters: 0,
       optimizedCharacters: 0,
-      estimator: "heuristic: 1 token \u2248 4 characters"
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  const originalTokens = estimateTokens(input);
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -180,10 +228,16 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  const optimizedTokens = estimateTokens(optimizedData);
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
@@ -193,7 +247,7 @@ function analyze(input, options = {}) {
     reductionRatio,
     originalCharacters: serializeForTokenEstimate(input).length,
     optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
-    estimator: "heuristic: 1 token \u2248 4 characters"
+    estimator: estimatorLabel
   };
 }
 
@@ -323,8 +377,8 @@ var AIChain = class {
   toNatural() {
     return toNatural(this.data);
   }
-  analyze() {
-    return analyze(this.data);
+  analyze(options) {
+    return analyze(this.data, options);
   }
   value() {
     return this.data;

package/dist/cli.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 import {
   AIChain
-} from "./chunk-HFMW3K22.js";
-import "./chunk-ZVQRYJU5.js";
+} from "./chunk-P6KXISNT.js";
+import "./chunk-IOMOVRC7.js";
 import "./chunk-6VRLDRJK.js";
 import "./chunk-XE36GLJP.js";
 import "./chunk-L7BC62MT.js";
 import "./chunk-D4CFTFM3.js";
 import "./chunk-ZD536GZF.js";
-import "./chunk-7CL7GVRF.js";
+import "./chunk-U433WUUT.js";
 
 // src/cli.ts
 import fs from "fs";

package/dist/core/analyze.cjs

@@ -139,17 +139,60 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
 function serializeForTokenEstimate(value) {
   if (typeof value === "string") return value;
   const serialized = JSON.stringify(value);
   return serialized ?? "";
 }
-function estimateTokens(value) {
+function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+}
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = false,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -160,10 +203,15 @@ function analyze(input, options = {}) {
       reductionRatio: 1,
       originalCharacters: 0,
       optimizedCharacters: 0,
-      estimator: "heuristic: 1 token \u2248 4 characters"
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  const originalTokens = estimateTokens(input);
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -177,10 +225,16 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  const optimizedTokens = estimateTokens(optimizedData);
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
@@ -190,7 +244,7 @@ function analyze(input, options = {}) {
     reductionRatio,
     originalCharacters: serializeForTokenEstimate(input).length,
     optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
-    estimator: "heuristic: 1 token \u2248 4 characters"
+    estimator: estimatorLabel
   };
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/core/analyze.d.cts

@@ -7,6 +7,9 @@ type AnalyzeOptions = {
     flatten?: boolean;
     toTOON?: boolean;
     toon?: boolean;
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
 };
 declare function analyze(input: unknown, options?: AnalyzeOptions): {
     originalTokens: number;

package/dist/core/analyze.d.ts

@@ -7,6 +7,9 @@ type AnalyzeOptions = {
     flatten?: boolean;
     toTOON?: boolean;
     toon?: boolean;
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
 };
 declare function analyze(input: unknown, options?: AnalyzeOptions): {
     originalTokens: number;

package/dist/core/analyze.js

@@ -1,11 +1,11 @@
 import {
   analyze
-} from "../chunk-ZVQRYJU5.js";
+} from "../chunk-IOMOVRC7.js";
 import "../chunk-6VRLDRJK.js";
 import "../chunk-XE36GLJP.js";
 import "../chunk-L7BC62MT.js";
 import "../chunk-ZD536GZF.js";
-import "../chunk-7CL7GVRF.js";
+import "../chunk-U433WUUT.js";
 export {
   analyze
 };

package/dist/core/token.cjs

@@ -21,19 +21,62 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 var token_exports = {};
 __export(token_exports, {
   estimateTokens: () => estimateTokens,
+  estimateTokensWithMeta: () => estimateTokensWithMeta,
   serializeForTokenEstimate: () => serializeForTokenEstimate
 });
 module.exports = __toCommonJS(token_exports);
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
 function serializeForTokenEstimate(value) {
   if (typeof value === "string") return value;
   const serialized = JSON.stringify(value);
   return serialized ?? "";
 }
-function estimateTokens(value) {
+function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   estimateTokens,
+  estimateTokensWithMeta,
   serializeForTokenEstimate
 });

package/dist/core/token.d.cts

@@ -1,18 +1,24 @@
-/**
- * Estimates token count from string length.
- *
- * Approximation:
- * 1 token ≈ 4 characters (common for GPT models)
- *
- * @param text - Input string
- * @returns Estimated token count
- */
+interface TokenEstimateOptions {
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
+}
 declare function serializeForTokenEstimate(value: unknown): string;
 /**
  * Estimates tokens using a transparent four-characters-per-token heuristic.
  * JSON values are serialized first, matching the compact form normally sent
- * to an API. Use a model-specific tokenizer for billing-accurate counts.
+ * to an API. Set exact=true to attempt a model-specific tokenizer count when
+ * the runtime has the tokenizer installed.
+ */
+type TokenEstimateResult = {
+    count: number;
+    estimator: string;
+};
+declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
+/**
+ * Returns both a token count and an estimator string describing how the count was
+ * obtained (exact tokenizer + encoding when available, or heuristic otherwise).
  */
-declare function estimateTokens(value: unknown): number;
+declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
 
-export { estimateTokens, serializeForTokenEstimate };
+export { type TokenEstimateOptions, type TokenEstimateResult, estimateTokens, estimateTokensWithMeta, serializeForTokenEstimate };

package/dist/core/token.d.ts

@@ -1,18 +1,24 @@
-/**
- * Estimates token count from string length.
- *
- * Approximation:
- * 1 token ≈ 4 characters (common for GPT models)
- *
- * @param text - Input string
- * @returns Estimated token count
- */
+interface TokenEstimateOptions {
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
+}
 declare function serializeForTokenEstimate(value: unknown): string;
 /**
  * Estimates tokens using a transparent four-characters-per-token heuristic.
  * JSON values are serialized first, matching the compact form normally sent
- * to an API. Use a model-specific tokenizer for billing-accurate counts.
+ * to an API. Set exact=true to attempt a model-specific tokenizer count when
+ * the runtime has the tokenizer installed.
+ */
+type TokenEstimateResult = {
+    count: number;
+    estimator: string;
+};
+declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
+/**
+ * Returns both a token count and an estimator string describing how the count was
+ * obtained (exact tokenizer + encoding when available, or heuristic otherwise).
  */
-declare function estimateTokens(value: unknown): number;
+declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
 
-export { estimateTokens, serializeForTokenEstimate };
+export { type TokenEstimateOptions, type TokenEstimateResult, estimateTokens, estimateTokensWithMeta, serializeForTokenEstimate };

package/dist/core/token.js

@@ -1,8 +1,10 @@
 import {
   estimateTokens,
+  estimateTokensWithMeta,
   serializeForTokenEstimate
-} from "../chunk-7CL7GVRF.js";
+} from "../chunk-U433WUUT.js";
 export {
   estimateTokens,
+  estimateTokensWithMeta,
   serializeForTokenEstimate
 };

package/dist/index.cjs

@@ -148,17 +148,63 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
 function serializeForTokenEstimate(value) {
   if (typeof value === "string") return value;
   const serialized = JSON.stringify(value);
   return serialized ?? "";
 }
-function estimateTokens(value) {
+function estimateTokensHeuristic(value) {
   return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+}
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = false,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -169,10 +215,15 @@ function analyze(input, options = {}) {
       reductionRatio: 1,
       originalCharacters: 0,
       optimizedCharacters: 0,
-      estimator: "heuristic: 1 token \u2248 4 characters"
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  const originalTokens = estimateTokens(input);
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -186,10 +237,16 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  const optimizedTokens = estimateTokens(optimizedData);
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
@@ -199,7 +256,7 @@ function analyze(input, options = {}) {
     reductionRatio,
     originalCharacters: serializeForTokenEstimate(input).length,
     optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
-    estimator: "heuristic: 1 token \u2248 4 characters"
+    estimator: estimatorLabel
   };
 }
 
@@ -329,8 +386,8 @@ var AIChain = class {
   toNatural() {
     return toNatural(this.data);
   }
-  analyze() {
-    return analyze(this.data);
+  analyze(options) {
+    return analyze(this.data, options);
   }
   value() {
     return this.data;

package/dist/index.d.cts

@@ -14,7 +14,7 @@ declare class AIChain {
     flatten(): this;
     toTOON(): this;
     toNatural(): string;
-    analyze(): {
+    analyze(options?: any): {
         originalTokens: number;
         optimizedTokens: number;
         savings: number;

package/dist/index.d.ts

@@ -14,7 +14,7 @@ declare class AIChain {
     flatten(): this;
     toTOON(): this;
     toNatural(): string;
-    analyze(): {
+    analyze(options?: any): {
         originalTokens: number;
         optimizedTokens: number;
         savings: number;

package/dist/index.js

@@ -1,9 +1,9 @@
 import {
   AIChain
-} from "./chunk-HFMW3K22.js";
+} from "./chunk-P6KXISNT.js";
 import {
   analyze
-} from "./chunk-ZVQRYJU5.js";
+} from "./chunk-IOMOVRC7.js";
 import {
   toTOON
 } from "./chunk-6VRLDRJK.js";
@@ -22,7 +22,7 @@ import {
 import {
   estimateTokens,
   serializeForTokenEstimate
-} from "./chunk-7CL7GVRF.js";
+} from "./chunk-U433WUUT.js";
 
 // src/index.ts
 function ai(data) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sohqureshi/tokenwise",
-  "version": "1.0.4",
+  "version": "1.0.7",
   "description": "Optimize JSON data for AI by reducing token usage",
   "repository": {
     "type": "git",
@@ -75,6 +75,9 @@
   ],
   "author": "Mohammad Sohail",
   "license": "MIT",
+  "dependencies": {
+    "tiktoken": "^1.0.22"
+  },
   "devDependencies": {
     "@types/node": "^25.6.0",
     "tsup": "^8.0.0",

package/dist/chunk-7CL7GVRF.js

@@ -1,14 +0,0 @@
-// src/core/token.ts
-function serializeForTokenEstimate(value) {
-  if (typeof value === "string") return value;
-  const serialized = JSON.stringify(value);
-  return serialized ?? "";
-}
-function estimateTokens(value) {
-  return Math.ceil(serializeForTokenEstimate(value).length / 4);
-}
-
-export {
-  serializeForTokenEstimate,
-  estimateTokens
-};