@sohqureshi/tokenwise 1.0.3 → 1.0.7

package/README.md

@@ -3,8 +3,8 @@
 TokenWise is a lightweight utility for preparing JSON before sending it to AI models. It helps reduce payload noise, shrink token usage, and turn structured data into formats that are easier for LLMs to consume.
 
 
-![npm version](https://img.shields.io/npm/v/tokenwise)
-![downloads](https://img.shields.io/npm/dw/tokenwise)
+[![npm version](https://img.shields.io/npm/v/%40sohqureshi%2Ftokenwise)](https://www.npmjs.com/package/@sohqureshi/tokenwise)
+[![downloads](https://img.shields.io/npm/dw/%40sohqureshi%2Ftokenwise)](https://www.npmjs.com/package/@sohqureshi/tokenwise)
 ![license](https://img.shields.io/github/license/sohqureshi/tokenwise)
 ![stars](https://img.shields.io/github/stars/sohqureshi/tokenwise?style=social)
 [![Demo](https://img.shields.io/badge/Live%20Demo-Visit-brightgreen)](https://sohqureshi.github.io/tokenwise/)
@@ -36,10 +36,12 @@ Raw JSON is:
 npm install @sohqureshi/tokenwise
 ```
 
+Available on [npm](https://www.npmjs.com/package/@sohqureshi/tokenwise).
+
 ## Quick Usage
 
 ```js
-import ai, { compact, flatten, toNatural, toTOON } from "tokenwise";
+import ai, { compact, flatten, toNatural, toTOON } from "@sohqureshi/tokenwise";
 
 const product = {
   product: {
@@ -68,7 +70,7 @@ console.log(ai(product).compact().value());
 Removes fields you do not want to send to the model. By default it also removes `null`, `undefined`, and empty objects.
 
 ```js
-import { prune } from "tokenwise";
+import { prune } from "@sohqureshi/tokenwise";
 
 const input = {
   user: { name: "John", age: 28 },
@@ -208,14 +210,17 @@ LLMs charge and reason over tokens. Sending raw JSON often includes repeated key
 node demo.js
 ```
 
-You can visualize token optimization results using planned CLI/Web visual tools.
+Use `--analyze` to compare serialized input and output with TokenWise's
+four-characters-per-token heuristic. For model-accurate counts, pass
+`exact: true` and a `model` name to `estimateTokens()` or `analyze()`.
+This is still only as exact as the tokenizer implementation you use.
 
 ---
 
 ## 🚀 Roadmap
 
-* [x] CLI support *(Coming Soon)*
-* [ ] NPM Support
+* [x] CLI support
+* [x] NPM Support
 * [ ] Streaming support (GB+ data)
 * [ ] Schema-aware optimization
 * [ ] SaaS API
@@ -259,4 +264,4 @@ If it helps you, consider supporting its development:
 
 ## 💡 Vision
 
-Make AI cheaper and faster by optimizing data before it reaches the model.
+Make AI cheaper and faster by optimizing data before it reaches the model.

package/dist/{chunk-3ECVBELU.js → chunk-6VRLDRJK.js}

@@ -14,11 +14,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -37,7 +37,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {

package/dist/{chunk-GQ62V6ZK.js → chunk-IOMOVRC7.js}

@@ -1,21 +1,27 @@
 import {
   toTOON
-} from "./chunk-3ECVBELU.js";
+} from "./chunk-6VRLDRJK.js";
 import {
   compact
-} from "./chunk-EUFLW42L.js";
+} from "./chunk-XE36GLJP.js";
 import {
   flatten
-} from "./chunk-YMOSWQGE.js";
+} from "./chunk-L7BC62MT.js";
 import {
   prune
-} from "./chunk-IAV75SZA.js";
+} from "./chunk-ZD536GZF.js";
 import {
-  estimateTokens
-} from "./chunk-TM6L7YF2.js";
+  estimateTokensWithMeta,
+  serializeForTokenEstimate
+} from "./chunk-U433WUUT.js";
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = false,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -23,11 +29,18 @@ function analyze(input, options = {}) {
       savings: 0,
       savingsPercent: 0,
       optimizedData: null,
-      reductionRatio: 1
+      reductionRatio: 1,
+      originalCharacters: 0,
+      optimizedCharacters: 0,
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  let originalTokens = estimateTokens(input);
-  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -41,18 +54,26 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  let optimizedTokens = estimateTokens(optimizedData);
-  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
     savings,
     savingsPercent,
     optimizedData,
-    reductionRatio
+    reductionRatio,
+    originalCharacters: serializeForTokenEstimate(input).length,
+    optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
+    estimator: estimatorLabel
   };
 }
 

package/dist/{chunk-YMOSWQGE.js → chunk-L7BC62MT.js}

@@ -2,20 +2,28 @@
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 
 export {
   flatten

package/dist/{chunk-2IQAGI7Q.js → chunk-P6KXISNT.js}

@@ -1,21 +1,21 @@
 import {
   analyze
-} from "./chunk-GQ62V6ZK.js";
+} from "./chunk-IOMOVRC7.js";
 import {
   toTOON
-} from "./chunk-3ECVBELU.js";
+} from "./chunk-6VRLDRJK.js";
 import {
   compact
-} from "./chunk-EUFLW42L.js";
+} from "./chunk-XE36GLJP.js";
 import {
   flatten
-} from "./chunk-YMOSWQGE.js";
+} from "./chunk-L7BC62MT.js";
 import {
   toNatural
 } from "./chunk-D4CFTFM3.js";
 import {
   prune
-} from "./chunk-IAV75SZA.js";
+} from "./chunk-ZD536GZF.js";
 
 // src/chain.ts
 var AIChain = class {
@@ -41,8 +41,8 @@ var AIChain = class {
   toNatural() {
     return toNatural(this.data);
   }
-  analyze() {
-    return analyze(this.data);
+  analyze(options) {
+    return analyze(this.data, options);
   }
   value() {
     return this.data;

package/dist/chunk-U433WUUT.js

@@ -0,0 +1,55 @@
+// src/core/token.ts
+import { createRequire } from "module";
+var require2 = createRequire(import.meta.url);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokensHeuristic(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
+}
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+}
+
+export {
+  serializeForTokenEstimate,
+  estimateTokens,
+  estimateTokensWithMeta
+};

package/dist/{chunk-EUFLW42L.js → chunk-XE36GLJP.js}

@@ -1,6 +1,6 @@
 import {
   prune
-} from "./chunk-IAV75SZA.js";
+} from "./chunk-ZD536GZF.js";
 
 // src/core/compact.ts
 function compact(obj) {

package/dist/{chunk-IAV75SZA.js → chunk-ZD536GZF.js}

@@ -19,7 +19,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;

package/dist/cli.cjs

@@ -48,7 +48,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;
@@ -64,20 +64,28 @@ function prune(obj, options = {}) {
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 
 // src/core/compact.ts
 function compact(obj) {
@@ -101,11 +109,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -124,7 +132,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {
@@ -132,13 +142,60 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokensHeuristic(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
 }
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = false,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -146,11 +203,18 @@ function analyze(input, options = {}) {
       savings: 0,
       savingsPercent: 0,
       optimizedData: null,
-      reductionRatio: 1
+      reductionRatio: 1,
+      originalCharacters: 0,
+      optimizedCharacters: 0,
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  let originalTokens = estimateTokens(input);
-  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -164,18 +228,26 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  let optimizedTokens = estimateTokens(optimizedData);
-  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
     savings,
     savingsPercent,
     optimizedData,
-    reductionRatio
+    reductionRatio,
+    originalCharacters: serializeForTokenEstimate(input).length,
+    optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
+    estimator: estimatorLabel
   };
 }
 
@@ -305,8 +377,8 @@ var AIChain = class {
   toNatural() {
     return toNatural(this.data);
   }
-  analyze() {
-    return analyze(this.data);
+  analyze(options) {
+    return analyze(this.data, options);
   }
   value() {
     return this.data;
@@ -323,7 +395,7 @@ Usage:
 Options:
   --toon       Convert JSON to TOON format
   --compact    Convert JSON to compact format
-  --analyze    Show token analysis
+  --analyze    Show token analysis (heuristic estimate)
 `);
   process.exit(0);
 }

package/dist/cli.js

@@ -1,14 +1,14 @@
 #!/usr/bin/env node
 import {
   AIChain
-} from "./chunk-2IQAGI7Q.js";
-import "./chunk-GQ62V6ZK.js";
-import "./chunk-3ECVBELU.js";
-import "./chunk-EUFLW42L.js";
-import "./chunk-YMOSWQGE.js";
+} from "./chunk-P6KXISNT.js";
+import "./chunk-IOMOVRC7.js";
+import "./chunk-6VRLDRJK.js";
+import "./chunk-XE36GLJP.js";
+import "./chunk-L7BC62MT.js";
 import "./chunk-D4CFTFM3.js";
-import "./chunk-IAV75SZA.js";
-import "./chunk-TM6L7YF2.js";
+import "./chunk-ZD536GZF.js";
+import "./chunk-U433WUUT.js";
 
 // src/cli.ts
 import fs from "fs";
@@ -22,7 +22,7 @@ Usage:
 Options:
   --toon       Convert JSON to TOON format
   --compact    Convert JSON to compact format
-  --analyze    Show token analysis
+  --analyze    Show token analysis (heuristic estimate)
 `);
   process.exit(0);
 }