@sohqureshi/tokenwise 1.0.0

package/dist/cli.js

@@ -0,0 +1,51 @@
+#!/usr/bin/env node
+import {
+  AIChain
+} from "./chunk-2IQAGI7Q.js";
+import "./chunk-GQ62V6ZK.js";
+import "./chunk-3ECVBELU.js";
+import "./chunk-EUFLW42L.js";
+import "./chunk-YMOSWQGE.js";
+import "./chunk-D4CFTFM3.js";
+import "./chunk-IAV75SZA.js";
+import "./chunk-TM6L7YF2.js";
+
+// src/cli.ts
+import fs from "fs";
+import path from "path";
+var args = process.argv.slice(2);
+if (args.length === 0) {
+  console.log(`
+Usage:
+  tokenwise <file> [options]
+
+Options:
+  --toon       Convert JSON to TOON format
+  --compact    Convert JSON to compact format
+  --analyze    Show token analysis
+`);
+  process.exit(0);
+}
+var filePath = path.resolve(process.cwd(), args[0]);
+if (!fs.existsSync(filePath)) {
+  console.error("\u274C File not found:", filePath);
+  process.exit(1);
+}
+var raw = fs.readFileSync(filePath, "utf-8");
+var json;
+try {
+  json = JSON.parse(raw);
+} catch (err) {
+  console.error("\u274C Invalid JSON file");
+  process.exit(1);
+}
+var tool = new AIChain(json);
+if (args.includes("--toon")) {
+  console.log(tool.toTOON().value());
+} else if (args.includes("--compact")) {
+  console.log(tool.compact().value());
+} else if (args.includes("--analyze")) {
+  console.log(tool.analyze());
+} else {
+  console.log("\u26A0\uFE0F No valid option provided. Use --toon, --compact or --analyze");
+}

package/dist/core/analyze.cjs

@@ -0,0 +1,181 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/core/analyze.ts
+var analyze_exports = {};
+__export(analyze_exports, {
+  analyze: () => analyze
+});
+module.exports = __toCommonJS(analyze_exports);
+
+// src/core/prune.ts
+function prune(obj, options = {}) {
+  const normalizedOptions = Array.isArray(options) ? { removeKeys: options } : options;
+  const {
+    removeKeys = [],
+    removeNull = true,
+    removeUndefined = true,
+    removeEmptyObjects = true,
+    removeEmptyArrays = false
+  } = normalizedOptions;
+  if (obj === null) return removeNull ? void 0 : obj;
+  if (obj === void 0) return removeUndefined ? void 0 : obj;
+  if (typeof obj !== "object") return obj;
+  if (Array.isArray(obj)) {
+    const arr = obj.map((item) => prune(item, normalizedOptions)).filter((item) => item !== void 0);
+    if (removeEmptyArrays && arr.length === 0) {
+      return void 0;
+    }
+    return arr;
+  }
+  const result = {};
+  for (const key in obj) {
+    if (removeKeys.includes(key)) continue;
+    const value = prune(obj[key], normalizedOptions);
+    if (value === void 0) continue;
+    if (removeEmptyObjects && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 0) {
+      continue;
+    }
+    result[key] = value;
+  }
+  return result;
+}
+
+// src/core/compact.ts
+function compact(obj) {
+  const pruned = prune(obj);
+  return JSON.stringify(pruned) ?? "";
+}
+
+// src/core/flatten.ts
+function flatten(obj, prefix = "", res = {}) {
+  if (obj === null || obj === void 0) return res;
+  if (typeof obj !== "object") {
+    res[prefix] = obj;
+    return res;
+  }
+  for (const key in obj) {
+    const value = obj[key];
+    const newKey = prefix ? `${prefix}.${key}` : key;
+    if (typeof value === "object" && value !== null) {
+      flatten(value, newKey, res);
+    } else {
+      res[newKey] = value;
+    }
+  }
+  return res;
+}
+
+// src/core/toon.ts
+function toTOON(data, indent = 0) {
+  const space = "  ".repeat(indent);
+  if (Array.isArray(data)) {
+    if (data.length === 0) return "[]";
+    if (data.every(isPlainObject)) {
+      const keys = Array.from(
+        new Set(data.flatMap((item) => Object.keys(item)))
+      );
+      let result = `${space}[${data.length}]{${keys.join(",")}}:
+`;
+      for (const item of data) {
+        result += space + "  " + keys.map((k) => formatValue(item[k])).join(",") + "\n";
+      }
+      return result;
+    }
+    return `${space}[${data.length}]: ${data.join(",")}`;
+  }
+  if (typeof data === "object" && data !== null) {
+    let result = "";
+    for (const key in data) {
+      const value = data[key];
+      if (typeof value === "object" && value !== null) {
+        result += `${space}${key}:
+${toTOON(value, indent + 1)}
+`;
+      } else {
+        const formatted = formatValue(value);
+        result += formatted ? `${space}${key}: ${formatted}
+` : `${space}${key}:
+`;
+      }
+    }
+    return result.trim();
+  }
+  return formatValue(data);
+}
+function formatValue(val) {
+  if (val === null || val === void 0) return "";
+  if (typeof val === "string") return val;
+  return String(val);
+}
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/core/token.ts
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+// src/core/analyze.ts
+function analyze(input, options = {}) {
+  if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
+    return {
+      originalTokens: 0,
+      optimizedTokens: 0,
+      savings: 0,
+      savingsPercent: 0,
+      optimizedData: null,
+      reductionRatio: 1
+    };
+  }
+  let originalTokens = estimateTokens(input);
+  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  let optimizedData = input;
+  if (options.prune && Array.isArray(options.prune)) {
+    optimizedData = prune(optimizedData, options.prune);
+  }
+  if (options.compact) {
+    optimizedData = compact(optimizedData);
+  }
+  if (options.flatten) {
+    optimizedData = flatten(optimizedData);
+  }
+  if (options.toTOON === true || options.toon === true) {
+    optimizedData = toTOON(optimizedData);
+  }
+  let optimizedTokens = estimateTokens(optimizedData);
+  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const savings = Math.max(0, originalTokens - optimizedTokens);
+  const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
+  const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  return {
+    originalTokens,
+    optimizedTokens,
+    savings,
+    savingsPercent,
+    optimizedData,
+    reductionRatio
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  analyze
+});

package/dist/core/analyze.d.cts

@@ -0,0 +1,13 @@
+/**
+ * Safe version of analyze() - Prevents NaN values
+ */
+declare function analyze(input: any, options?: any): {
+    originalTokens: number;
+    optimizedTokens: number;
+    savings: number;
+    savingsPercent: number;
+    optimizedData: any;
+    reductionRatio: number;
+};
+
+export { analyze };

package/dist/core/analyze.d.ts

@@ -0,0 +1,13 @@
+/**
+ * Safe version of analyze() - Prevents NaN values
+ */
+declare function analyze(input: any, options?: any): {
+    originalTokens: number;
+    optimizedTokens: number;
+    savings: number;
+    savingsPercent: number;
+    optimizedData: any;
+    reductionRatio: number;
+};
+
+export { analyze };

package/dist/core/analyze.js

@@ -0,0 +1,11 @@
+import {
+  analyze
+} from "../chunk-GQ62V6ZK.js";
+import "../chunk-3ECVBELU.js";
+import "../chunk-EUFLW42L.js";
+import "../chunk-YMOSWQGE.js";
+import "../chunk-IAV75SZA.js";
+import "../chunk-TM6L7YF2.js";
+export {
+  analyze
+};

package/dist/core/compact.cjs

@@ -0,0 +1,68 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/core/compact.ts
+var compact_exports = {};
+__export(compact_exports, {
+  compact: () => compact
+});
+module.exports = __toCommonJS(compact_exports);
+
+// src/core/prune.ts
+function prune(obj, options = {}) {
+  const normalizedOptions = Array.isArray(options) ? { removeKeys: options } : options;
+  const {
+    removeKeys = [],
+    removeNull = true,
+    removeUndefined = true,
+    removeEmptyObjects = true,
+    removeEmptyArrays = false
+  } = normalizedOptions;
+  if (obj === null) return removeNull ? void 0 : obj;
+  if (obj === void 0) return removeUndefined ? void 0 : obj;
+  if (typeof obj !== "object") return obj;
+  if (Array.isArray(obj)) {
+    const arr = obj.map((item) => prune(item, normalizedOptions)).filter((item) => item !== void 0);
+    if (removeEmptyArrays && arr.length === 0) {
+      return void 0;
+    }
+    return arr;
+  }
+  const result = {};
+  for (const key in obj) {
+    if (removeKeys.includes(key)) continue;
+    const value = prune(obj[key], normalizedOptions);
+    if (value === void 0) continue;
+    if (removeEmptyObjects && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 0) {
+      continue;
+    }
+    result[key] = value;
+  }
+  return result;
+}
+
+// src/core/compact.ts
+function compact(obj) {
+  const pruned = prune(obj);
+  return JSON.stringify(pruned) ?? "";
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  compact
+});

package/dist/core/compact.d.cts

@@ -0,0 +1,3 @@
+declare function compact(obj: any): string;
+
+export { compact };

package/dist/core/compact.d.ts

@@ -0,0 +1,3 @@
+declare function compact(obj: any): string;
+
+export { compact };

package/dist/core/compact.js

@@ -0,0 +1,7 @@
+import {
+  compact
+} from "../chunk-EUFLW42L.js";
+import "../chunk-IAV75SZA.js";
+export {
+  compact
+};

package/dist/core/flatten.cjs

@@ -0,0 +1,46 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/core/flatten.ts
+var flatten_exports = {};
+__export(flatten_exports, {
+  flatten: () => flatten
+});
+module.exports = __toCommonJS(flatten_exports);
+function flatten(obj, prefix = "", res = {}) {
+  if (obj === null || obj === void 0) return res;
+  if (typeof obj !== "object") {
+    res[prefix] = obj;
+    return res;
+  }
+  for (const key in obj) {
+    const value = obj[key];
+    const newKey = prefix ? `${prefix}.${key}` : key;
+    if (typeof value === "object" && value !== null) {
+      flatten(value, newKey, res);
+    } else {
+      res[newKey] = value;
+    }
+  }
+  return res;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  flatten
+});

package/dist/core/flatten.d.cts

@@ -0,0 +1,18 @@
+/**
+ * Flattens nested JSON into an object with dot-notation keys.
+ *
+ * Useful for:
+ * - Search indexing
+ * - AI prompt simplification
+ *
+ * @param obj - Input object
+ * @param prefix - Used internally for recursion
+ * @returns Object with dot-notation keys
+ *
+ * Example:
+ * flatten({ user: { name: "Ali" } })
+ * -> { "user.name": "Ali" }
+ */
+declare function flatten(obj: any, prefix?: string, res?: any): any;
+
+export { flatten };

package/dist/core/flatten.d.ts

@@ -0,0 +1,18 @@
+/**
+ * Flattens nested JSON into an object with dot-notation keys.
+ *
+ * Useful for:
+ * - Search indexing
+ * - AI prompt simplification
+ *
+ * @param obj - Input object
+ * @param prefix - Used internally for recursion
+ * @returns Object with dot-notation keys
+ *
+ * Example:
+ * flatten({ user: { name: "Ali" } })
+ * -> { "user.name": "Ali" }
+ */
+declare function flatten(obj: any, prefix?: string, res?: any): any;
+
+export { flatten };

package/dist/core/flatten.js

@@ -0,0 +1,6 @@
+import {
+  flatten
+} from "../chunk-YMOSWQGE.js";
+export {
+  flatten
+};

package/dist/core/natural.cjs

@@ -0,0 +1,129 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/core/natural.ts
+var natural_exports = {};
+__export(natural_exports, {
+  toNatural: () => toNatural
+});
+module.exports = __toCommonJS(natural_exports);
+function toNatural(data, depth = 0) {
+  if (data === null || data === void 0) return "nothing";
+  if (typeof data === "string") return data;
+  if (typeof data === "number") return String(data);
+  if (typeof data === "boolean") return data ? "yes" : "no";
+  if (Array.isArray(data)) {
+    if (data.length === 0) return "empty list";
+    if (data.every(isPlainObject)) {
+      return data.map((item, index) => `${index + 1}. ${stripTrailingPeriod(toNatural(item, depth + 1))}.`).join(" ");
+    }
+    const items = data.map((item) => toNatural(item, depth + 1));
+    return joinNaturalList(items);
+  }
+  if (typeof data === "object") {
+    return buildContextualStory(data, depth);
+  }
+  return String(data);
+}
+function buildContextualStory(obj, depth = 0) {
+  let name = obj.name || obj.userName || obj.user;
+  if (typeof name === "object" && name !== null && name.name) {
+    name = name.name;
+  }
+  const entries = Object.entries(obj).filter(([key]) => {
+    return !["id", "timestamp", "apiKey"].includes(key);
+  });
+  if (entries.length === 0) return "";
+  let story = name && typeof name === "string" ? `User ${name}` : "";
+  const clauses = entries.map(([key, value]) => {
+    if (key === "user" && name && typeof value === "object" && value !== null && !Array.isArray(value)) {
+      const userProps = Object.entries(value).filter(([k]) => !["id", "name", "timestamp", "apiKey"].includes(k)).map(([k, v]) => formatPropertyClause(k, v, depth + 1)).filter((c) => c !== null && c !== "").join(", ");
+      return userProps ? `(${userProps})` : null;
+    }
+    return formatPropertyClause(key, value, depth);
+  }).filter((c) => c !== null && c !== "");
+  if (clauses.length === 0) return story;
+  if (story) {
+    story += " " + clauses.join(", ");
+  } else {
+    story = clauses.join(", ");
+  }
+  return story + ".";
+}
+function formatPropertyClause(key, value, depth) {
+  const naturalKey = camelToWords(key);
+  if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+    if (["internal", "metadata", "debug"].includes(key)) {
+      return null;
+    }
+    const nested = Object.entries(value).filter(([k]) => !["id", "timestamp", "apiKey", "createdAt", "updatedAt"].includes(k)).map(([k, v]) => {
+      if (typeof v === "object" && v !== null) {
+        return formatPropertyClause(k, v, depth + 1);
+      }
+      const propValue = toNatural(v, depth + 1);
+      if (propValue === "yes" || propValue === "no") {
+        return `${camelToWords(k)} ${propValue === "yes" ? "enabled" : "disabled"}`;
+      }
+      return `${camelToWords(k)} ${propValue}`;
+    }).filter((c) => c && c.trim()).join(", ");
+    if (!nested) return null;
+    return `${naturalKey}: ${nested}`;
+  }
+  if (Array.isArray(value)) {
+    if (value.length === 0) return null;
+    if (["skills", "hobbies", "interests", "tags", "languages", "items"].includes(key)) {
+      const items2 = value.map((item) => {
+        const str = String(item);
+        return str.charAt(0).toUpperCase() + str.slice(1);
+      });
+      return `Having ${joinNaturalList(items2)}`;
+    }
+    const items = joinNaturalList(value.map((item) => toNatural(item, depth + 1)));
+    return `${naturalKey}: ${items}`;
+  }
+  const naturalValue = toNatural(value, depth + 1);
+  if (naturalValue === "yes") return `${naturalKey} enabled`;
+  if (naturalValue === "no") return `${naturalKey} disabled`;
+  if (key === "theme") return `prefers the ${naturalValue} ${key}`;
+  if (key === "age") return `age: ${naturalValue}`;
+  if (key === "email") return `email: ${naturalValue}`;
+  if (key === "city") return `address: ${naturalValue}`;
+  if (key === "debug" && naturalValue === "yes") return `debug enabled`;
+  return `${naturalKey}: ${naturalValue}`;
+}
+function camelToWords(str) {
+  return str.replace(/([A-Z])/g, " $1").toLowerCase().trim();
+}
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function joinNaturalList(items) {
+  if (items.length === 0) return "";
+  if (items.length === 1) return items[0];
+  if (items.length === 2) return `${items[0]} and ${items[1]}`;
+  const lastItem = items[items.length - 1];
+  return `${items.slice(0, -1).join(", ")} and ${lastItem}`;
+}
+function stripTrailingPeriod(value) {
+  return value.replace(/\.+$/, "");
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  toNatural
+});

package/dist/core/natural.d.cts

@@ -0,0 +1,28 @@
+/**
+ * Converts JSON into natural language story format.
+ *
+ * Goal:
+ * - Transform structured data into human-readable narratives
+ * - Identify subjects (names, users) and describe their properties
+ * - Support arrays, nested objects, and primitive types
+ * - Skip technical metadata fields for cleaner output
+ *
+ * @param data - Input JSON data
+ * @returns Natural language story string
+ *
+ * Example:
+ * Input:
+ * {
+ *   "user": {
+ *     "name": "Alice Johnson",
+ *     "email": "alice@example.com",
+ *     "skills": ["Python", "JavaScript"]
+ *   }
+ * }
+ *
+ * Output:
+ * "User Alice Johnson (email: alice@example.com, Having Python and JavaScript)."
+ */
+declare function toNatural(data: any, depth?: number): string;
+
+export { toNatural };

package/dist/core/natural.d.ts

@@ -0,0 +1,28 @@
+/**
+ * Converts JSON into natural language story format.
+ *
+ * Goal:
+ * - Transform structured data into human-readable narratives
+ * - Identify subjects (names, users) and describe their properties
+ * - Support arrays, nested objects, and primitive types
+ * - Skip technical metadata fields for cleaner output
+ *
+ * @param data - Input JSON data
+ * @returns Natural language story string
+ *
+ * Example:
+ * Input:
+ * {
+ *   "user": {
+ *     "name": "Alice Johnson",
+ *     "email": "alice@example.com",
+ *     "skills": ["Python", "JavaScript"]
+ *   }
+ * }
+ *
+ * Output:
+ * "User Alice Johnson (email: alice@example.com, Having Python and JavaScript)."
+ */
+declare function toNatural(data: any, depth?: number): string;
+
+export { toNatural };

package/dist/core/natural.js

@@ -0,0 +1,6 @@
+import {
+  toNatural
+} from "../chunk-D4CFTFM3.js";
+export {
+  toNatural
+};