@sohqureshi/tokenwise 1.0.2 → 1.0.4

package/README.md

@@ -33,13 +33,13 @@ Raw JSON is:
 ## Installation
 
 ```bash
-npm install tokenwise
+npm install @sohqureshi/tokenwise
 ```
 
 ## Quick Usage
 
 ```js
-import ai, { compact, flatten, toNatural, toTOON } from "tokenwise";
+import ai, { compact, flatten, toNatural, toTOON } from "@sohqureshi/tokenwise";
 
 const product = {
   product: {
@@ -208,14 +208,16 @@ LLMs charge and reason over tokens. Sending raw JSON often includes repeated key
 node demo.js
 ```
 
-You can visualize token optimization results using planned CLI/Web visual tools.
+Use `--analyze` to compare serialized input and output with TokenWise's
+four-characters-per-token heuristic. This is an estimate, not a replacement
+for a model-specific tokenizer or a provider's billed token count.
 
 ---
 
 ## 🚀 Roadmap
 
-* [x] CLI support *(Coming Soon)*
-* [ ] NPM Support
+* [x] CLI support
+* [x] NPM Support
 * [ ] Streaming support (GB+ data)
 * [ ] Schema-aware optimization
 * [ ] SaaS API
@@ -259,4 +261,4 @@ If it helps you, consider supporting its development:
 
 ## 💡 Vision
 
-Make AI cheaper and faster by optimizing data before it reaches the model.
+Make AI cheaper and faster by optimizing data before it reaches the model.

package/dist/{chunk-3ECVBELU.js → chunk-6VRLDRJK.js}

@@ -14,11 +14,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -37,7 +37,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {

package/dist/chunk-7CL7GVRF.js

@@ -0,0 +1,14 @@
+// src/core/token.ts
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokens(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
+}
+
+export {
+  serializeForTokenEstimate,
+  estimateTokens
+};

package/dist/{chunk-2IQAGI7Q.js → chunk-HFMW3K22.js}

@@ -1,21 +1,21 @@
 import {
   analyze
-} from "./chunk-GQ62V6ZK.js";
+} from "./chunk-ZVQRYJU5.js";
 import {
   toTOON
-} from "./chunk-3ECVBELU.js";
+} from "./chunk-6VRLDRJK.js";
 import {
   compact
-} from "./chunk-EUFLW42L.js";
+} from "./chunk-XE36GLJP.js";
 import {
   flatten
-} from "./chunk-YMOSWQGE.js";
+} from "./chunk-L7BC62MT.js";
 import {
   toNatural
 } from "./chunk-D4CFTFM3.js";
 import {
   prune
-} from "./chunk-IAV75SZA.js";
+} from "./chunk-ZD536GZF.js";
 
 // src/chain.ts
 var AIChain = class {

package/dist/{chunk-YMOSWQGE.js → chunk-L7BC62MT.js}

@@ -2,20 +2,28 @@
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 
 export {
   flatten

package/dist/{chunk-EUFLW42L.js → chunk-XE36GLJP.js}

@@ -1,6 +1,6 @@
 import {
   prune
-} from "./chunk-IAV75SZA.js";
+} from "./chunk-ZD536GZF.js";
 
 // src/core/compact.ts
 function compact(obj) {

package/dist/{chunk-IAV75SZA.js → chunk-ZD536GZF.js}

@@ -19,7 +19,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;

package/dist/{chunk-GQ62V6ZK.js → chunk-ZVQRYJU5.js}

@@ -1,18 +1,19 @@
 import {
   toTOON
-} from "./chunk-3ECVBELU.js";
+} from "./chunk-6VRLDRJK.js";
 import {
   compact
-} from "./chunk-EUFLW42L.js";
+} from "./chunk-XE36GLJP.js";
 import {
   flatten
-} from "./chunk-YMOSWQGE.js";
+} from "./chunk-L7BC62MT.js";
 import {
   prune
-} from "./chunk-IAV75SZA.js";
+} from "./chunk-ZD536GZF.js";
 import {
-  estimateTokens
-} from "./chunk-TM6L7YF2.js";
+  estimateTokens,
+  serializeForTokenEstimate
+} from "./chunk-7CL7GVRF.js";
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
@@ -23,11 +24,13 @@ function analyze(input, options = {}) {
       savings: 0,
       savingsPercent: 0,
       optimizedData: null,
-      reductionRatio: 1
+      reductionRatio: 1,
+      originalCharacters: 0,
+      optimizedCharacters: 0,
+      estimator: "heuristic: 1 token \u2248 4 characters"
     };
   }
-  let originalTokens = estimateTokens(input);
-  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  const originalTokens = estimateTokens(input);
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -41,8 +44,7 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  let optimizedTokens = estimateTokens(optimizedData);
-  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const optimizedTokens = estimateTokens(optimizedData);
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
@@ -52,7 +54,10 @@ function analyze(input, options = {}) {
     savings,
     savingsPercent,
     optimizedData,
-    reductionRatio
+    reductionRatio,
+    originalCharacters: serializeForTokenEstimate(input).length,
+    optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
+    estimator: "heuristic: 1 token \u2248 4 characters"
   };
 }
 

package/dist/cli.cjs

@@ -48,7 +48,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;
@@ -64,20 +64,28 @@ function prune(obj, options = {}) {
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 
 // src/core/compact.ts
 function compact(obj) {
@@ -101,11 +109,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -124,7 +132,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {
@@ -132,9 +142,13 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokens(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
 
 // src/core/analyze.ts
@@ -146,11 +160,13 @@ function analyze(input, options = {}) {
       savings: 0,
       savingsPercent: 0,
       optimizedData: null,
-      reductionRatio: 1
+      reductionRatio: 1,
+      originalCharacters: 0,
+      optimizedCharacters: 0,
+      estimator: "heuristic: 1 token \u2248 4 characters"
     };
   }
-  let originalTokens = estimateTokens(input);
-  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  const originalTokens = estimateTokens(input);
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -164,8 +180,7 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  let optimizedTokens = estimateTokens(optimizedData);
-  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const optimizedTokens = estimateTokens(optimizedData);
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
@@ -175,7 +190,10 @@ function analyze(input, options = {}) {
     savings,
     savingsPercent,
     optimizedData,
-    reductionRatio
+    reductionRatio,
+    originalCharacters: serializeForTokenEstimate(input).length,
+    optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
+    estimator: "heuristic: 1 token \u2248 4 characters"
   };
 }
 
@@ -323,7 +341,7 @@ Usage:
 Options:
   --toon       Convert JSON to TOON format
   --compact    Convert JSON to compact format
-  --analyze    Show token analysis
+  --analyze    Show token analysis (heuristic estimate)
 `);
   process.exit(0);
 }

package/dist/cli.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 import {
   AIChain
-} from "./chunk-2IQAGI7Q.js";
-import "./chunk-GQ62V6ZK.js";
-import "./chunk-3ECVBELU.js";
-import "./chunk-EUFLW42L.js";
-import "./chunk-YMOSWQGE.js";
+} from "./chunk-HFMW3K22.js";
+import "./chunk-ZVQRYJU5.js";
+import "./chunk-6VRLDRJK.js";
+import "./chunk-XE36GLJP.js";
+import "./chunk-L7BC62MT.js";
 import "./chunk-D4CFTFM3.js";
-import "./chunk-IAV75SZA.js";
-import "./chunk-TM6L7YF2.js";
+import "./chunk-ZD536GZF.js";
+import "./chunk-7CL7GVRF.js";
 
 // src/cli.ts
 import fs from "fs";
@@ -22,7 +22,7 @@ Usage:
 Options:
   --toon       Convert JSON to TOON format
   --compact    Convert JSON to compact format
-  --analyze    Show token analysis
+  --analyze    Show token analysis (heuristic estimate)
 `);
   process.exit(0);
 }

package/dist/core/analyze.cjs

@@ -45,7 +45,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;
@@ -67,20 +67,28 @@ function compact(obj) {
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 
 // src/core/toon.ts
 function toTOON(data, indent = 0) {
@@ -98,11 +106,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -121,7 +129,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {
@@ -129,9 +139,13 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokens(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
 
 // src/core/analyze.ts
@@ -143,11 +157,13 @@ function analyze(input, options = {}) {
       savings: 0,
       savingsPercent: 0,
       optimizedData: null,
-      reductionRatio: 1
+      reductionRatio: 1,
+      originalCharacters: 0,
+      optimizedCharacters: 0,
+      estimator: "heuristic: 1 token \u2248 4 characters"
     };
   }
-  let originalTokens = estimateTokens(input);
-  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  const originalTokens = estimateTokens(input);
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -161,8 +177,7 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  let optimizedTokens = estimateTokens(optimizedData);
-  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const optimizedTokens = estimateTokens(optimizedData);
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
@@ -172,7 +187,10 @@ function analyze(input, options = {}) {
     savings,
     savingsPercent,
     optimizedData,
-    reductionRatio
+    reductionRatio,
+    originalCharacters: serializeForTokenEstimate(input).length,
+    optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
+    estimator: "heuristic: 1 token \u2248 4 characters"
   };
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/core/analyze.d.cts

@@ -1,13 +1,23 @@
 /**
  * Safe version of analyze() - Prevents NaN values
  */
-declare function analyze(input: any, options?: any): {
+type AnalyzeOptions = {
+    prune?: string[];
+    compact?: boolean;
+    flatten?: boolean;
+    toTOON?: boolean;
+    toon?: boolean;
+};
+declare function analyze(input: unknown, options?: AnalyzeOptions): {
     originalTokens: number;
     optimizedTokens: number;
     savings: number;
     savingsPercent: number;
-    optimizedData: any;
+    optimizedData: unknown;
     reductionRatio: number;
+    originalCharacters: number;
+    optimizedCharacters: number;
+    estimator: string;
 };
 
-export { analyze };
+export { type AnalyzeOptions, analyze };

package/dist/core/analyze.d.ts

@@ -1,13 +1,23 @@
 /**
  * Safe version of analyze() - Prevents NaN values
  */
-declare function analyze(input: any, options?: any): {
+type AnalyzeOptions = {
+    prune?: string[];
+    compact?: boolean;
+    flatten?: boolean;
+    toTOON?: boolean;
+    toon?: boolean;
+};
+declare function analyze(input: unknown, options?: AnalyzeOptions): {
     originalTokens: number;
     optimizedTokens: number;
     savings: number;
     savingsPercent: number;
-    optimizedData: any;
+    optimizedData: unknown;
     reductionRatio: number;
+    originalCharacters: number;
+    optimizedCharacters: number;
+    estimator: string;
 };
 
-export { analyze };
+export { type AnalyzeOptions, analyze };

package/dist/core/analyze.js

@@ -1,11 +1,11 @@
 import {
   analyze
-} from "../chunk-GQ62V6ZK.js";
-import "../chunk-3ECVBELU.js";
-import "../chunk-EUFLW42L.js";
-import "../chunk-YMOSWQGE.js";
-import "../chunk-IAV75SZA.js";
-import "../chunk-TM6L7YF2.js";
+} from "../chunk-ZVQRYJU5.js";
+import "../chunk-6VRLDRJK.js";
+import "../chunk-XE36GLJP.js";
+import "../chunk-L7BC62MT.js";
+import "../chunk-ZD536GZF.js";
+import "../chunk-7CL7GVRF.js";
 export {
   analyze
 };

package/dist/core/compact.cjs

@@ -45,7 +45,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;

package/dist/core/compact.js

@@ -1,7 +1,7 @@
 import {
   compact
-} from "../chunk-EUFLW42L.js";
-import "../chunk-IAV75SZA.js";
+} from "../chunk-XE36GLJP.js";
+import "../chunk-ZD536GZF.js";
 export {
   compact
 };

package/dist/core/flatten.cjs

@@ -26,20 +26,28 @@ module.exports = __toCommonJS(flatten_exports);
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   flatten

package/dist/core/flatten.d.cts

@@ -13,6 +13,6 @@
  * flatten({ user: { name: "Ali" } })
  * -> { "user.name": "Ali" }
  */
-declare function flatten(obj: any, prefix?: string, res?: any): any;
+declare function flatten(obj: any, prefix?: string, res?: Record<string, unknown>): Record<string, unknown>;
 
 export { flatten };

package/dist/core/flatten.d.ts

@@ -13,6 +13,6 @@
  * flatten({ user: { name: "Ali" } })
  * -> { "user.name": "Ali" }
  */
-declare function flatten(obj: any, prefix?: string, res?: any): any;
+declare function flatten(obj: any, prefix?: string, res?: Record<string, unknown>): Record<string, unknown>;
 
 export { flatten };

package/dist/core/flatten.js

@@ -1,6 +1,6 @@
 import {
   flatten
-} from "../chunk-YMOSWQGE.js";
+} from "../chunk-L7BC62MT.js";
 export {
   flatten
 };

package/dist/core/prune.cjs

@@ -43,7 +43,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;

package/dist/core/prune.js

@@ -1,6 +1,6 @@
 import {
   prune
-} from "../chunk-IAV75SZA.js";
+} from "../chunk-ZD536GZF.js";
 export {
   prune
 };

package/dist/core/token.cjs

@@ -20,14 +20,20 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/core/token.ts
 var token_exports = {};
 __export(token_exports, {
-  estimateTokens: () => estimateTokens
+  estimateTokens: () => estimateTokens,
+  serializeForTokenEstimate: () => serializeForTokenEstimate
 });
 module.exports = __toCommonJS(token_exports);
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokens(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  estimateTokens
+  estimateTokens,
+  serializeForTokenEstimate
 });

package/dist/core/token.d.cts

@@ -7,6 +7,12 @@
  * @param text - Input string
  * @returns Estimated token count
  */
-declare function estimateTokens(text: string): number;
+declare function serializeForTokenEstimate(value: unknown): string;
+/**
+ * Estimates tokens using a transparent four-characters-per-token heuristic.
+ * JSON values are serialized first, matching the compact form normally sent
+ * to an API. Use a model-specific tokenizer for billing-accurate counts.
+ */
+declare function estimateTokens(value: unknown): number;
 
-export { estimateTokens };
+export { estimateTokens, serializeForTokenEstimate };

package/dist/core/token.d.ts

@@ -7,6 +7,12 @@
  * @param text - Input string
  * @returns Estimated token count
  */
-declare function estimateTokens(text: string): number;
+declare function serializeForTokenEstimate(value: unknown): string;
+/**
+ * Estimates tokens using a transparent four-characters-per-token heuristic.
+ * JSON values are serialized first, matching the compact form normally sent
+ * to an API. Use a model-specific tokenizer for billing-accurate counts.
+ */
+declare function estimateTokens(value: unknown): number;
 
-export { estimateTokens };
+export { estimateTokens, serializeForTokenEstimate };

package/dist/core/token.js

@@ -1,6 +1,8 @@
 import {
-  estimateTokens
-} from "../chunk-TM6L7YF2.js";
+  estimateTokens,
+  serializeForTokenEstimate
+} from "../chunk-7CL7GVRF.js";
 export {
-  estimateTokens
+  estimateTokens,
+  serializeForTokenEstimate
 };

package/dist/core/toon.cjs

@@ -38,11 +38,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -61,7 +61,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {

package/dist/core/toon.js

@@ -1,6 +1,6 @@
 import {
   toTOON
-} from "../chunk-3ECVBELU.js";
+} from "../chunk-6VRLDRJK.js";
 export {
   toTOON
 };

package/dist/index.cjs

@@ -27,6 +27,7 @@ __export(index_exports, {
   estimateTokens: () => estimateTokens,
   flatten: () => flatten,
   prune: () => prune,
+  serializeForTokenEstimate: () => serializeForTokenEstimate,
   toNatural: () => toNatural,
   toTOON: () => toTOON
 });
@@ -53,7 +54,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;
@@ -69,20 +70,28 @@ function prune(obj, options = {}) {
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 
 // src/core/compact.ts
 function compact(obj) {
@@ -106,11 +115,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -129,7 +138,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {
@@ -137,9 +148,13 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokens(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
 }
 
 // src/core/analyze.ts
@@ -151,11 +166,13 @@ function analyze(input, options = {}) {
       savings: 0,
       savingsPercent: 0,
       optimizedData: null,
-      reductionRatio: 1
+      reductionRatio: 1,
+      originalCharacters: 0,
+      optimizedCharacters: 0,
+      estimator: "heuristic: 1 token \u2248 4 characters"
     };
   }
-  let originalTokens = estimateTokens(input);
-  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  const originalTokens = estimateTokens(input);
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -169,8 +186,7 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  let optimizedTokens = estimateTokens(optimizedData);
-  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const optimizedTokens = estimateTokens(optimizedData);
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
@@ -180,7 +196,10 @@ function analyze(input, options = {}) {
     savings,
     savingsPercent,
     optimizedData,
-    reductionRatio
+    reductionRatio,
+    originalCharacters: serializeForTokenEstimate(input).length,
+    optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
+    estimator: "heuristic: 1 token \u2248 4 characters"
   };
 }
 
@@ -329,6 +348,7 @@ ai.toTOON = toTOON;
 ai.analyze = analyze;
 ai.toNatural = toNatural;
 ai.estimateTokens = estimateTokens;
+ai.serializeForTokenEstimate = serializeForTokenEstimate;
 ai.AIChain = AIChain;
 var index_default = ai;
 // Annotate the CommonJS export names for ESM import in node:
@@ -339,6 +359,7 @@ var index_default = ai;
   estimateTokens,
   flatten,
   prune,
+  serializeForTokenEstimate,
   toNatural,
   toTOON
 });

package/dist/index.d.cts

@@ -1,10 +1,10 @@
-import { estimateTokens } from './core/token.cjs';
-import { toNatural } from './core/natural.cjs';
-import { analyze } from './core/analyze.cjs';
-import { toTOON } from './core/toon.cjs';
-import { flatten } from './core/flatten.cjs';
-import { compact } from './core/compact.cjs';
 import { prune } from './core/prune.cjs';
+import { compact } from './core/compact.cjs';
+import { flatten } from './core/flatten.cjs';
+import { toTOON } from './core/toon.cjs';
+import { analyze } from './core/analyze.cjs';
+import { toNatural } from './core/natural.cjs';
+import { estimateTokens, serializeForTokenEstimate } from './core/token.cjs';
 
 declare class AIChain {
     private data;
@@ -19,8 +19,11 @@ declare class AIChain {
         optimizedTokens: number;
         savings: number;
         savingsPercent: number;
-        optimizedData: any;
+        optimizedData: unknown;
         reductionRatio: number;
+        originalCharacters: number;
+        optimizedCharacters: number;
+        estimator: string;
     };
     value(): any;
 }
@@ -34,7 +37,8 @@ declare namespace ai {
     var analyze: typeof analyze;
     var toNatural: typeof toNatural;
     var estimateTokens: typeof estimateTokens;
+    var serializeForTokenEstimate: typeof serializeForTokenEstimate;
     var AIChain: typeof AIChain;
 }
 
-export { AIChain, analyze, compact, ai as default, estimateTokens, flatten, prune, toNatural, toTOON };
+export { AIChain, analyze, compact, ai as default, estimateTokens, flatten, prune, serializeForTokenEstimate, toNatural, toTOON };

package/dist/index.d.ts

@@ -1,10 +1,10 @@
-import { estimateTokens } from './core/token.js';
-import { toNatural } from './core/natural.js';
-import { analyze } from './core/analyze.js';
-import { toTOON } from './core/toon.js';
-import { flatten } from './core/flatten.js';
-import { compact } from './core/compact.js';
 import { prune } from './core/prune.js';
+import { compact } from './core/compact.js';
+import { flatten } from './core/flatten.js';
+import { toTOON } from './core/toon.js';
+import { analyze } from './core/analyze.js';
+import { toNatural } from './core/natural.js';
+import { estimateTokens, serializeForTokenEstimate } from './core/token.js';
 
 declare class AIChain {
     private data;
@@ -19,8 +19,11 @@ declare class AIChain {
         optimizedTokens: number;
         savings: number;
         savingsPercent: number;
-        optimizedData: any;
+        optimizedData: unknown;
         reductionRatio: number;
+        originalCharacters: number;
+        optimizedCharacters: number;
+        estimator: string;
     };
     value(): any;
 }
@@ -34,7 +37,8 @@ declare namespace ai {
     var analyze: typeof analyze;
     var toNatural: typeof toNatural;
     var estimateTokens: typeof estimateTokens;
+    var serializeForTokenEstimate: typeof serializeForTokenEstimate;
     var AIChain: typeof AIChain;
 }
 
-export { AIChain, analyze, compact, ai as default, estimateTokens, flatten, prune, toNatural, toTOON };
+export { AIChain, analyze, compact, ai as default, estimateTokens, flatten, prune, serializeForTokenEstimate, toNatural, toTOON };

package/dist/index.js

@@ -1,27 +1,28 @@
 import {
   AIChain
-} from "./chunk-2IQAGI7Q.js";
+} from "./chunk-HFMW3K22.js";
 import {
   analyze
-} from "./chunk-GQ62V6ZK.js";
+} from "./chunk-ZVQRYJU5.js";
 import {
   toTOON
-} from "./chunk-3ECVBELU.js";
+} from "./chunk-6VRLDRJK.js";
 import {
   compact
-} from "./chunk-EUFLW42L.js";
+} from "./chunk-XE36GLJP.js";
 import {
   flatten
-} from "./chunk-YMOSWQGE.js";
+} from "./chunk-L7BC62MT.js";
 import {
   toNatural
 } from "./chunk-D4CFTFM3.js";
 import {
   prune
-} from "./chunk-IAV75SZA.js";
+} from "./chunk-ZD536GZF.js";
 import {
-  estimateTokens
-} from "./chunk-TM6L7YF2.js";
+  estimateTokens,
+  serializeForTokenEstimate
+} from "./chunk-7CL7GVRF.js";
 
 // src/index.ts
 function ai(data) {
@@ -34,6 +35,7 @@ ai.toTOON = toTOON;
 ai.analyze = analyze;
 ai.toNatural = toNatural;
 ai.estimateTokens = estimateTokens;
+ai.serializeForTokenEstimate = serializeForTokenEstimate;
 ai.AIChain = AIChain;
 var index_default = ai;
 export {
@@ -44,6 +46,7 @@ export {
   estimateTokens,
   flatten,
   prune,
+  serializeForTokenEstimate,
   toNatural,
   toTOON
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sohqureshi/tokenwise",
-  "version": "1.0.2",
+  "version": "1.0.4",
   "description": "Optimize JSON data for AI by reducing token usage",
   "repository": {
     "type": "git",
@@ -60,6 +60,8 @@
   "scripts": {
     "build": "tsup src/index.ts src/cli.ts src/core/analyze.ts src/core/compact.ts src/core/flatten.ts src/core/natural.ts src/core/prune.ts src/core/token.ts src/core/toon.ts --format esm,cjs --dts",
     "test": "vitest",
+    "test:run": "vitest run",
+    "typecheck": "tsc --noEmit",
     "prepare": "npm run build"
   },
   "keywords": [

package/dist/chunk-TM6L7YF2.js

@@ -1,9 +0,0 @@
-// src/core/token.ts
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
-}
-
-export {
-  estimateTokens
-};