@sohqureshi/tokenwise 1.0.3 → 1.0.7

package/dist/index.cjs

@@ -27,6 +27,7 @@ __export(index_exports, {
   estimateTokens: () => estimateTokens,
   flatten: () => flatten,
   prune: () => prune,
+  serializeForTokenEstimate: () => serializeForTokenEstimate,
   toNatural: () => toNatural,
   toTOON: () => toTOON
 });
@@ -53,7 +54,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;
@@ -69,20 +70,28 @@ function prune(obj, options = {}) {
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 
 // src/core/compact.ts
 function compact(obj) {
@@ -106,11 +115,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -129,7 +138,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {
@@ -137,13 +148,63 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokensHeuristic(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
+}
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
 }
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = false,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -151,11 +212,18 @@ function analyze(input, options = {}) {
       savings: 0,
       savingsPercent: 0,
       optimizedData: null,
-      reductionRatio: 1
+      reductionRatio: 1,
+      originalCharacters: 0,
+      optimizedCharacters: 0,
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  let originalTokens = estimateTokens(input);
-  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -169,18 +237,26 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  let optimizedTokens = estimateTokens(optimizedData);
-  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
     savings,
     savingsPercent,
     optimizedData,
-    reductionRatio
+    reductionRatio,
+    originalCharacters: serializeForTokenEstimate(input).length,
+    optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
+    estimator: estimatorLabel
   };
 }
 
@@ -310,8 +386,8 @@ var AIChain = class {
   toNatural() {
     return toNatural(this.data);
   }
-  analyze() {
-    return analyze(this.data);
+  analyze(options) {
+    return analyze(this.data, options);
   }
   value() {
     return this.data;
@@ -329,6 +405,7 @@ ai.toTOON = toTOON;
 ai.analyze = analyze;
 ai.toNatural = toNatural;
 ai.estimateTokens = estimateTokens;
+ai.serializeForTokenEstimate = serializeForTokenEstimate;
 ai.AIChain = AIChain;
 var index_default = ai;
 // Annotate the CommonJS export names for ESM import in node:
@@ -339,6 +416,7 @@ var index_default = ai;
   estimateTokens,
   flatten,
   prune,
+  serializeForTokenEstimate,
   toNatural,
   toTOON
 });

package/dist/index.d.cts

@@ -1,10 +1,10 @@
-import { estimateTokens } from './core/token.cjs';
-import { toNatural } from './core/natural.cjs';
-import { analyze } from './core/analyze.cjs';
-import { toTOON } from './core/toon.cjs';
-import { flatten } from './core/flatten.cjs';
-import { compact } from './core/compact.cjs';
 import { prune } from './core/prune.cjs';
+import { compact } from './core/compact.cjs';
+import { flatten } from './core/flatten.cjs';
+import { toTOON } from './core/toon.cjs';
+import { analyze } from './core/analyze.cjs';
+import { toNatural } from './core/natural.cjs';
+import { estimateTokens, serializeForTokenEstimate } from './core/token.cjs';
 
 declare class AIChain {
     private data;
@@ -14,13 +14,16 @@ declare class AIChain {
     flatten(): this;
     toTOON(): this;
     toNatural(): string;
-    analyze(): {
+    analyze(options?: any): {
         originalTokens: number;
         optimizedTokens: number;
         savings: number;
         savingsPercent: number;
-        optimizedData: any;
+        optimizedData: unknown;
         reductionRatio: number;
+        originalCharacters: number;
+        optimizedCharacters: number;
+        estimator: string;
     };
     value(): any;
 }
@@ -34,7 +37,8 @@ declare namespace ai {
     var analyze: typeof analyze;
     var toNatural: typeof toNatural;
     var estimateTokens: typeof estimateTokens;
+    var serializeForTokenEstimate: typeof serializeForTokenEstimate;
     var AIChain: typeof AIChain;
 }
 
-export { AIChain, analyze, compact, ai as default, estimateTokens, flatten, prune, toNatural, toTOON };
+export { AIChain, analyze, compact, ai as default, estimateTokens, flatten, prune, serializeForTokenEstimate, toNatural, toTOON };

package/dist/index.d.ts

@@ -1,10 +1,10 @@
-import { estimateTokens } from './core/token.js';
-import { toNatural } from './core/natural.js';
-import { analyze } from './core/analyze.js';
-import { toTOON } from './core/toon.js';
-import { flatten } from './core/flatten.js';
-import { compact } from './core/compact.js';
 import { prune } from './core/prune.js';
+import { compact } from './core/compact.js';
+import { flatten } from './core/flatten.js';
+import { toTOON } from './core/toon.js';
+import { analyze } from './core/analyze.js';
+import { toNatural } from './core/natural.js';
+import { estimateTokens, serializeForTokenEstimate } from './core/token.js';
 
 declare class AIChain {
     private data;
@@ -14,13 +14,16 @@ declare class AIChain {
     flatten(): this;
     toTOON(): this;
     toNatural(): string;
-    analyze(): {
+    analyze(options?: any): {
         originalTokens: number;
         optimizedTokens: number;
         savings: number;
         savingsPercent: number;
-        optimizedData: any;
+        optimizedData: unknown;
         reductionRatio: number;
+        originalCharacters: number;
+        optimizedCharacters: number;
+        estimator: string;
     };
     value(): any;
 }
@@ -34,7 +37,8 @@ declare namespace ai {
     var analyze: typeof analyze;
     var toNatural: typeof toNatural;
     var estimateTokens: typeof estimateTokens;
+    var serializeForTokenEstimate: typeof serializeForTokenEstimate;
     var AIChain: typeof AIChain;
 }
 
-export { AIChain, analyze, compact, ai as default, estimateTokens, flatten, prune, toNatural, toTOON };
+export { AIChain, analyze, compact, ai as default, estimateTokens, flatten, prune, serializeForTokenEstimate, toNatural, toTOON };

package/dist/index.js

@@ -1,27 +1,28 @@
 import {
   AIChain
-} from "./chunk-2IQAGI7Q.js";
+} from "./chunk-P6KXISNT.js";
 import {
   analyze
-} from "./chunk-GQ62V6ZK.js";
+} from "./chunk-IOMOVRC7.js";
 import {
   toTOON
-} from "./chunk-3ECVBELU.js";
+} from "./chunk-6VRLDRJK.js";
 import {
   compact
-} from "./chunk-EUFLW42L.js";
+} from "./chunk-XE36GLJP.js";
 import {
   flatten
-} from "./chunk-YMOSWQGE.js";
+} from "./chunk-L7BC62MT.js";
 import {
   toNatural
 } from "./chunk-D4CFTFM3.js";
 import {
   prune
-} from "./chunk-IAV75SZA.js";
+} from "./chunk-ZD536GZF.js";
 import {
-  estimateTokens
-} from "./chunk-TM6L7YF2.js";
+  estimateTokens,
+  serializeForTokenEstimate
+} from "./chunk-U433WUUT.js";
 
 // src/index.ts
 function ai(data) {
@@ -34,6 +35,7 @@ ai.toTOON = toTOON;
 ai.analyze = analyze;
 ai.toNatural = toNatural;
 ai.estimateTokens = estimateTokens;
+ai.serializeForTokenEstimate = serializeForTokenEstimate;
 ai.AIChain = AIChain;
 var index_default = ai;
 export {
@@ -44,6 +46,7 @@ export {
   estimateTokens,
   flatten,
   prune,
+  serializeForTokenEstimate,
   toNatural,
   toTOON
 };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sohqureshi/tokenwise",
-  "version": "1.0.3",
+  "version": "1.0.7",
   "description": "Optimize JSON data for AI by reducing token usage",
   "repository": {
     "type": "git",
@@ -60,6 +60,8 @@
   "scripts": {
     "build": "tsup src/index.ts src/cli.ts src/core/analyze.ts src/core/compact.ts src/core/flatten.ts src/core/natural.ts src/core/prune.ts src/core/token.ts src/core/toon.ts --format esm,cjs --dts",
     "test": "vitest",
+    "test:run": "vitest run",
+    "typecheck": "tsc --noEmit",
     "prepare": "npm run build"
   },
   "keywords": [
@@ -73,6 +75,9 @@
   ],
   "author": "Mohammad Sohail",
   "license": "MIT",
+  "dependencies": {
+    "tiktoken": "^1.0.22"
+  },
   "devDependencies": {
     "@types/node": "^25.6.0",
     "tsup": "^8.0.0",

package/dist/chunk-TM6L7YF2.js

@@ -1,9 +0,0 @@
-// src/core/token.ts
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
-}
-
-export {
-  estimateTokens
-};