@sohqureshi/tokenwise 1.0.3 → 1.0.7

package/dist/core/analyze.cjs

@@ -45,7 +45,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;
@@ -67,20 +67,28 @@ function compact(obj) {
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 
 // src/core/toon.ts
 function toTOON(data, indent = 0) {
@@ -98,11 +106,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -121,7 +129,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {
@@ -129,13 +139,60 @@ function isPlainObject(value) {
 }
 
 // src/core/token.ts
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokensHeuristic(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
 }
 
 // src/core/analyze.ts
 function analyze(input, options = {}) {
+  const {
+    exact = false,
+    model = "gpt-4o-mini",
+    fallbackToHeuristic = true
+  } = options;
   if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
     return {
       originalTokens: 0,
@@ -143,11 +200,18 @@ function analyze(input, options = {}) {
       savings: 0,
       savingsPercent: 0,
       optimizedData: null,
-      reductionRatio: 1
+      reductionRatio: 1,
+      originalCharacters: 0,
+      optimizedCharacters: 0,
+      estimator: exact ? `exact tokenizer: model=${model}` : "heuristic: 1 token \u2248 4 characters"
     };
   }
-  let originalTokens = estimateTokens(input);
-  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  const originalMeta = estimateTokensWithMeta(input, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const originalTokens = originalMeta.count;
   let optimizedData = input;
   if (options.prune && Array.isArray(options.prune)) {
     optimizedData = prune(optimizedData, options.prune);
@@ -161,18 +225,26 @@ function analyze(input, options = {}) {
   if (options.toTOON === true || options.toon === true) {
     optimizedData = toTOON(optimizedData);
   }
-  let optimizedTokens = estimateTokens(optimizedData);
-  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const optimizedMeta = estimateTokensWithMeta(optimizedData, {
+    exact,
+    model,
+    fallbackToHeuristic
+  });
+  const optimizedTokens = optimizedMeta.count;
   const savings = Math.max(0, originalTokens - optimizedTokens);
   const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
   const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  const estimatorLabel = originalMeta && originalMeta.estimator ? originalMeta.estimator : optimizedMeta.estimator;
   return {
     originalTokens,
     optimizedTokens,
     savings,
     savingsPercent,
     optimizedData,
-    reductionRatio
+    reductionRatio,
+    originalCharacters: serializeForTokenEstimate(input).length,
+    optimizedCharacters: serializeForTokenEstimate(optimizedData).length,
+    estimator: estimatorLabel
   };
 }
 // Annotate the CommonJS export names for ESM import in node:

package/dist/core/analyze.d.cts

@@ -1,13 +1,26 @@
 /**
  * Safe version of analyze() - Prevents NaN values
  */
-declare function analyze(input: any, options?: any): {
+type AnalyzeOptions = {
+    prune?: string[];
+    compact?: boolean;
+    flatten?: boolean;
+    toTOON?: boolean;
+    toon?: boolean;
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
+};
+declare function analyze(input: unknown, options?: AnalyzeOptions): {
     originalTokens: number;
     optimizedTokens: number;
     savings: number;
     savingsPercent: number;
-    optimizedData: any;
+    optimizedData: unknown;
     reductionRatio: number;
+    originalCharacters: number;
+    optimizedCharacters: number;
+    estimator: string;
 };
 
-export { analyze };
+export { type AnalyzeOptions, analyze };

package/dist/core/analyze.d.ts

@@ -1,13 +1,26 @@
 /**
  * Safe version of analyze() - Prevents NaN values
  */
-declare function analyze(input: any, options?: any): {
+type AnalyzeOptions = {
+    prune?: string[];
+    compact?: boolean;
+    flatten?: boolean;
+    toTOON?: boolean;
+    toon?: boolean;
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
+};
+declare function analyze(input: unknown, options?: AnalyzeOptions): {
     originalTokens: number;
     optimizedTokens: number;
     savings: number;
     savingsPercent: number;
-    optimizedData: any;
+    optimizedData: unknown;
     reductionRatio: number;
+    originalCharacters: number;
+    optimizedCharacters: number;
+    estimator: string;
 };
 
-export { analyze };
+export { type AnalyzeOptions, analyze };

package/dist/core/analyze.js

@@ -1,11 +1,11 @@
 import {
   analyze
-} from "../chunk-GQ62V6ZK.js";
-import "../chunk-3ECVBELU.js";
-import "../chunk-EUFLW42L.js";
-import "../chunk-YMOSWQGE.js";
-import "../chunk-IAV75SZA.js";
-import "../chunk-TM6L7YF2.js";
+} from "../chunk-IOMOVRC7.js";
+import "../chunk-6VRLDRJK.js";
+import "../chunk-XE36GLJP.js";
+import "../chunk-L7BC62MT.js";
+import "../chunk-ZD536GZF.js";
+import "../chunk-U433WUUT.js";
 export {
   analyze
 };

package/dist/core/compact.cjs

@@ -45,7 +45,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;

package/dist/core/compact.js

@@ -1,7 +1,7 @@
 import {
   compact
-} from "../chunk-EUFLW42L.js";
-import "../chunk-IAV75SZA.js";
+} from "../chunk-XE36GLJP.js";
+import "../chunk-ZD536GZF.js";
 export {
   compact
 };

package/dist/core/flatten.cjs

@@ -26,20 +26,28 @@ module.exports = __toCommonJS(flatten_exports);
 function flatten(obj, prefix = "", res = {}) {
   if (obj === null || obj === void 0) return res;
   if (typeof obj !== "object") {
-    res[prefix] = obj;
+    setValue(res, prefix, obj);
     return res;
   }
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     const value = obj[key];
     const newKey = prefix ? `${prefix}.${key}` : key;
     if (typeof value === "object" && value !== null) {
       flatten(value, newKey, res);
     } else {
-      res[newKey] = value;
+      setValue(res, newKey, value);
     }
   }
   return res;
 }
+function setValue(res, key, value) {
+  if (Object.prototype.hasOwnProperty.call(res, key)) {
+    throw new Error(
+      `Cannot flatten input: the path "${key}" collides with an existing key. Use keys without dots or rename one of the conflicting properties.`
+    );
+  }
+  res[key] = value;
+}
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
   flatten

package/dist/core/flatten.d.cts

@@ -13,6 +13,6 @@
  * flatten({ user: { name: "Ali" } })
  * -> { "user.name": "Ali" }
  */
-declare function flatten(obj: any, prefix?: string, res?: any): any;
+declare function flatten(obj: any, prefix?: string, res?: Record<string, unknown>): Record<string, unknown>;
 
 export { flatten };

package/dist/core/flatten.d.ts

@@ -13,6 +13,6 @@
  * flatten({ user: { name: "Ali" } })
  * -> { "user.name": "Ali" }
  */
-declare function flatten(obj: any, prefix?: string, res?: any): any;
+declare function flatten(obj: any, prefix?: string, res?: Record<string, unknown>): Record<string, unknown>;
 
 export { flatten };

package/dist/core/flatten.js

@@ -1,6 +1,6 @@
 import {
   flatten
-} from "../chunk-YMOSWQGE.js";
+} from "../chunk-L7BC62MT.js";
 export {
   flatten
 };

package/dist/core/prune.cjs

@@ -43,7 +43,7 @@ function prune(obj, options = {}) {
     return arr;
   }
   const result = {};
-  for (const key in obj) {
+  for (const key of Object.keys(obj)) {
     if (removeKeys.includes(key)) continue;
     const value = prune(obj[key], normalizedOptions);
     if (value === void 0) continue;

package/dist/core/prune.js

@@ -1,6 +1,6 @@
 import {
   prune
-} from "../chunk-IAV75SZA.js";
+} from "../chunk-ZD536GZF.js";
 export {
   prune
 };

package/dist/core/token.cjs

@@ -20,14 +20,63 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
 // src/core/token.ts
 var token_exports = {};
 __export(token_exports, {
-  estimateTokens: () => estimateTokens
+  estimateTokens: () => estimateTokens,
+  estimateTokensWithMeta: () => estimateTokensWithMeta,
+  serializeForTokenEstimate: () => serializeForTokenEstimate
 });
 module.exports = __toCommonJS(token_exports);
-function estimateTokens(text) {
-  if (!text) return 0;
-  return Math.ceil(text.length / 4);
+var import_node_module = require("module");
+var import_meta = {};
+var require2 = (0, import_node_module.createRequire)(import_meta.url);
+function serializeForTokenEstimate(value) {
+  if (typeof value === "string") return value;
+  const serialized = JSON.stringify(value);
+  return serialized ?? "";
+}
+function estimateTokensHeuristic(value) {
+  return Math.ceil(serializeForTokenEstimate(value).length / 4);
+}
+function estimateTokens(value, options = {}) {
+  return estimateTokensWithMeta(value, options).count;
+}
+function estimateTokensWithMeta(value, options = {}) {
+  const { exact = false, model = "gpt-4o-mini", fallbackToHeuristic = true } = options;
+  const heuristic = estimateTokensHeuristic(value);
+  const heuristicEstimator = "heuristic: 1 token \u2248 4 characters";
+  if (!exact) {
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
+  try {
+    const tiktoken = require2("tiktoken");
+    const encoder = tiktoken.encoding_for_model(model);
+    const text = serializeForTokenEstimate(value);
+    const count = encoder.encode(text).length;
+    let encodingName = null;
+    if (encoder.name) encodingName = encoder.name;
+    if (!encodingName && typeof tiktoken.model_to_encoding === "function") {
+      try {
+        encodingName = tiktoken.model_to_encoding(model);
+      } catch (e) {
+        encodingName = null;
+      }
+    }
+    if (!encodingName) {
+      const m = String(model || "").toLowerCase();
+      if (m.includes("davinci") || m.startsWith("text-")) encodingName = "r50k_base";
+      else encodingName = "cl100k_base";
+    }
+    const estimator = `exact tokenizer: model=${model} encoding=${encodingName}`;
+    return { count, estimator };
+  } catch (e) {
+    if (!fallbackToHeuristic) {
+      return { count: heuristic, estimator: "exact requested but tokenizer unavailable" };
+    }
+    return { count: heuristic, estimator: heuristicEstimator };
+  }
 }
 // Annotate the CommonJS export names for ESM import in node:
 0 && (module.exports = {
-  estimateTokens
+  estimateTokens,
+  estimateTokensWithMeta,
+  serializeForTokenEstimate
 });

package/dist/core/token.d.cts

@@ -1,12 +1,24 @@
+interface TokenEstimateOptions {
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
+}
+declare function serializeForTokenEstimate(value: unknown): string;
 /**
- * Estimates token count from string length.
- *
- * Approximation:
- * 1 token ≈ 4 characters (common for GPT models)
- *
- * @param text - Input string
- * @returns Estimated token count
+ * Estimates tokens using a transparent four-characters-per-token heuristic.
+ * JSON values are serialized first, matching the compact form normally sent
+ * to an API. Set exact=true to attempt a model-specific tokenizer count when
+ * the runtime has the tokenizer installed.
  */
-declare function estimateTokens(text: string): number;
+type TokenEstimateResult = {
+    count: number;
+    estimator: string;
+};
+declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
+/**
+ * Returns both a token count and an estimator string describing how the count was
+ * obtained (exact tokenizer + encoding when available, or heuristic otherwise).
+ */
+declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
 
-export { estimateTokens };
+export { type TokenEstimateOptions, type TokenEstimateResult, estimateTokens, estimateTokensWithMeta, serializeForTokenEstimate };

package/dist/core/token.d.ts

@@ -1,12 +1,24 @@
+interface TokenEstimateOptions {
+    exact?: boolean;
+    model?: string;
+    fallbackToHeuristic?: boolean;
+}
+declare function serializeForTokenEstimate(value: unknown): string;
 /**
- * Estimates token count from string length.
- *
- * Approximation:
- * 1 token ≈ 4 characters (common for GPT models)
- *
- * @param text - Input string
- * @returns Estimated token count
+ * Estimates tokens using a transparent four-characters-per-token heuristic.
+ * JSON values are serialized first, matching the compact form normally sent
+ * to an API. Set exact=true to attempt a model-specific tokenizer count when
+ * the runtime has the tokenizer installed.
  */
-declare function estimateTokens(text: string): number;
+type TokenEstimateResult = {
+    count: number;
+    estimator: string;
+};
+declare function estimateTokens(value: unknown, options?: TokenEstimateOptions): number;
+/**
+ * Returns both a token count and an estimator string describing how the count was
+ * obtained (exact tokenizer + encoding when available, or heuristic otherwise).
+ */
+declare function estimateTokensWithMeta(value: unknown, options?: TokenEstimateOptions): TokenEstimateResult;
 
-export { estimateTokens };
+export { type TokenEstimateOptions, type TokenEstimateResult, estimateTokens, estimateTokensWithMeta, serializeForTokenEstimate };

package/dist/core/token.js

@@ -1,6 +1,10 @@
 import {
-  estimateTokens
-} from "../chunk-TM6L7YF2.js";
+  estimateTokens,
+  estimateTokensWithMeta,
+  serializeForTokenEstimate
+} from "../chunk-U433WUUT.js";
 export {
-  estimateTokens
+  estimateTokens,
+  estimateTokensWithMeta,
+  serializeForTokenEstimate
 };

package/dist/core/toon.cjs

@@ -38,11 +38,11 @@ function toTOON(data, indent = 0) {
       }
       return result;
     }
-    return `${space}[${data.length}]: ${data.join(",")}`;
+    return `${space}[${data.length}]: ${data.map(formatValue).join(",")}`;
   }
   if (typeof data === "object" && data !== null) {
     let result = "";
-    for (const key in data) {
+    for (const key of Object.keys(data)) {
       const value = data[key];
       if (typeof value === "object" && value !== null) {
         result += `${space}${key}:
@@ -61,7 +61,9 @@ ${toTOON(value, indent + 1)}
 }
 function formatValue(val) {
   if (val === null || val === void 0) return "";
-  if (typeof val === "string") return val;
+  if (typeof val === "string") {
+    return val === "" || /[,\n\r]/.test(val) ? JSON.stringify(val) : val;
+  }
   return String(val);
 }
 function isPlainObject(value) {

package/dist/core/toon.js

@@ -1,6 +1,6 @@
 import {
   toTOON
-} from "../chunk-3ECVBELU.js";
+} from "../chunk-6VRLDRJK.js";
 export {
   toTOON
 };