@sohqureshi/tokenwise 1.0.0

package/dist/core/prune.cjs

@@ -0,0 +1,60 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/core/prune.ts
+var prune_exports = {};
+__export(prune_exports, {
+  prune: () => prune
+});
+module.exports = __toCommonJS(prune_exports);
+function prune(obj, options = {}) {
+  const normalizedOptions = Array.isArray(options) ? { removeKeys: options } : options;
+  const {
+    removeKeys = [],
+    removeNull = true,
+    removeUndefined = true,
+    removeEmptyObjects = true,
+    removeEmptyArrays = false
+  } = normalizedOptions;
+  if (obj === null) return removeNull ? void 0 : obj;
+  if (obj === void 0) return removeUndefined ? void 0 : obj;
+  if (typeof obj !== "object") return obj;
+  if (Array.isArray(obj)) {
+    const arr = obj.map((item) => prune(item, normalizedOptions)).filter((item) => item !== void 0);
+    if (removeEmptyArrays && arr.length === 0) {
+      return void 0;
+    }
+    return arr;
+  }
+  const result = {};
+  for (const key in obj) {
+    if (removeKeys.includes(key)) continue;
+    const value = prune(obj[key], normalizedOptions);
+    if (value === void 0) continue;
+    if (removeEmptyObjects && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 0) {
+      continue;
+    }
+    result[key] = value;
+  }
+  return result;
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  prune
+});

package/dist/core/prune.d.cts

@@ -0,0 +1,11 @@
+type PruneOptions = {
+    removeKeys?: string[];
+    removeNull?: boolean;
+    removeUndefined?: boolean;
+    removeEmptyObjects?: boolean;
+    removeEmptyArrays?: boolean;
+};
+type PruneInput = PruneOptions | string[];
+declare function prune(obj: any, options?: PruneInput): any;
+
+export { prune };

package/dist/core/prune.d.ts

@@ -0,0 +1,11 @@
+type PruneOptions = {
+    removeKeys?: string[];
+    removeNull?: boolean;
+    removeUndefined?: boolean;
+    removeEmptyObjects?: boolean;
+    removeEmptyArrays?: boolean;
+};
+type PruneInput = PruneOptions | string[];
+declare function prune(obj: any, options?: PruneInput): any;
+
+export { prune };

package/dist/core/prune.js

@@ -0,0 +1,6 @@
+import {
+  prune
+} from "../chunk-IAV75SZA.js";
+export {
+  prune
+};

package/dist/core/token.cjs

@@ -0,0 +1,33 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/core/token.ts
+var token_exports = {};
+__export(token_exports, {
+  estimateTokens: () => estimateTokens
+});
+module.exports = __toCommonJS(token_exports);
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  estimateTokens
+});

package/dist/core/token.d.cts

@@ -0,0 +1,12 @@
+/**
+ * Estimates token count from string length.
+ *
+ * Approximation:
+ * 1 token ≈ 4 characters (common for GPT models)
+ *
+ * @param text - Input string
+ * @returns Estimated token count
+ */
+declare function estimateTokens(text: string): number;
+
+export { estimateTokens };

package/dist/core/token.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Estimates token count from string length.
+ *
+ * Approximation:
+ * 1 token ≈ 4 characters (common for GPT models)
+ *
+ * @param text - Input string
+ * @returns Estimated token count
+ */
+declare function estimateTokens(text: string): number;
+
+export { estimateTokens };

package/dist/core/token.js

@@ -0,0 +1,6 @@
+import {
+  estimateTokens
+} from "../chunk-TM6L7YF2.js";
+export {
+  estimateTokens
+};

package/dist/core/toon.cjs

@@ -0,0 +1,73 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/core/toon.ts
+var toon_exports = {};
+__export(toon_exports, {
+  toTOON: () => toTOON
+});
+module.exports = __toCommonJS(toon_exports);
+function toTOON(data, indent = 0) {
+  const space = "  ".repeat(indent);
+  if (Array.isArray(data)) {
+    if (data.length === 0) return "[]";
+    if (data.every(isPlainObject)) {
+      const keys = Array.from(
+        new Set(data.flatMap((item) => Object.keys(item)))
+      );
+      let result = `${space}[${data.length}]{${keys.join(",")}}:
+`;
+      for (const item of data) {
+        result += space + "  " + keys.map((k) => formatValue(item[k])).join(",") + "\n";
+      }
+      return result;
+    }
+    return `${space}[${data.length}]: ${data.join(",")}`;
+  }
+  if (typeof data === "object" && data !== null) {
+    let result = "";
+    for (const key in data) {
+      const value = data[key];
+      if (typeof value === "object" && value !== null) {
+        result += `${space}${key}:
+${toTOON(value, indent + 1)}
+`;
+      } else {
+        const formatted = formatValue(value);
+        result += formatted ? `${space}${key}: ${formatted}
+` : `${space}${key}:
+`;
+      }
+    }
+    return result.trim();
+  }
+  return formatValue(data);
+}
+function formatValue(val) {
+  if (val === null || val === void 0) return "";
+  if (typeof val === "string") return val;
+  return String(val);
+}
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  toTOON
+});

package/dist/core/toon.d.cts

@@ -0,0 +1,25 @@
+/**
+ * Converts JSON into TOON-like (Token-Oriented Object Notation) format.
+ *
+ * Goal:
+ * - Remove repeated keys
+ * - Represent arrays as table-like structure
+ * - Reduce token usage for LLMs
+ *
+ * @param data - Input JSON
+ * @param indent - Internal indentation level
+ * @returns Token-efficient string format
+ *
+ * Example:
+ * Input:
+ * {
+ *   users: [{ id: 1, name: "Ali" }]
+ * }
+ *
+ * Output:
+ * users[1]{id,name}:
+ * 1,Ali
+ */
+declare function toTOON(data: any, indent?: number): string;
+
+export { toTOON };

package/dist/core/toon.d.ts

@@ -0,0 +1,25 @@
+/**
+ * Converts JSON into TOON-like (Token-Oriented Object Notation) format.
+ *
+ * Goal:
+ * - Remove repeated keys
+ * - Represent arrays as table-like structure
+ * - Reduce token usage for LLMs
+ *
+ * @param data - Input JSON
+ * @param indent - Internal indentation level
+ * @returns Token-efficient string format
+ *
+ * Example:
+ * Input:
+ * {
+ *   users: [{ id: 1, name: "Ali" }]
+ * }
+ *
+ * Output:
+ * users[1]{id,name}:
+ * 1,Ali
+ */
+declare function toTOON(data: any, indent?: number): string;
+
+export { toTOON };

package/dist/core/toon.js

@@ -0,0 +1,6 @@
+import {
+  toTOON
+} from "../chunk-3ECVBELU.js";
+export {
+  toTOON
+};

package/dist/index.cjs

@@ -0,0 +1,344 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+
+// src/index.ts
+var index_exports = {};
+__export(index_exports, {
+  AIChain: () => AIChain,
+  analyze: () => analyze,
+  compact: () => compact,
+  default: () => index_default,
+  estimateTokens: () => estimateTokens,
+  flatten: () => flatten,
+  prune: () => prune,
+  toNatural: () => toNatural,
+  toTOON: () => toTOON
+});
+module.exports = __toCommonJS(index_exports);
+
+// src/core/prune.ts
+function prune(obj, options = {}) {
+  const normalizedOptions = Array.isArray(options) ? { removeKeys: options } : options;
+  const {
+    removeKeys = [],
+    removeNull = true,
+    removeUndefined = true,
+    removeEmptyObjects = true,
+    removeEmptyArrays = false
+  } = normalizedOptions;
+  if (obj === null) return removeNull ? void 0 : obj;
+  if (obj === void 0) return removeUndefined ? void 0 : obj;
+  if (typeof obj !== "object") return obj;
+  if (Array.isArray(obj)) {
+    const arr = obj.map((item) => prune(item, normalizedOptions)).filter((item) => item !== void 0);
+    if (removeEmptyArrays && arr.length === 0) {
+      return void 0;
+    }
+    return arr;
+  }
+  const result = {};
+  for (const key in obj) {
+    if (removeKeys.includes(key)) continue;
+    const value = prune(obj[key], normalizedOptions);
+    if (value === void 0) continue;
+    if (removeEmptyObjects && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 0) {
+      continue;
+    }
+    result[key] = value;
+  }
+  return result;
+}
+
+// src/core/flatten.ts
+function flatten(obj, prefix = "", res = {}) {
+  if (obj === null || obj === void 0) return res;
+  if (typeof obj !== "object") {
+    res[prefix] = obj;
+    return res;
+  }
+  for (const key in obj) {
+    const value = obj[key];
+    const newKey = prefix ? `${prefix}.${key}` : key;
+    if (typeof value === "object" && value !== null) {
+      flatten(value, newKey, res);
+    } else {
+      res[newKey] = value;
+    }
+  }
+  return res;
+}
+
+// src/core/compact.ts
+function compact(obj) {
+  const pruned = prune(obj);
+  return JSON.stringify(pruned) ?? "";
+}
+
+// src/core/toon.ts
+function toTOON(data, indent = 0) {
+  const space = "  ".repeat(indent);
+  if (Array.isArray(data)) {
+    if (data.length === 0) return "[]";
+    if (data.every(isPlainObject)) {
+      const keys = Array.from(
+        new Set(data.flatMap((item) => Object.keys(item)))
+      );
+      let result = `${space}[${data.length}]{${keys.join(",")}}:
+`;
+      for (const item of data) {
+        result += space + "  " + keys.map((k) => formatValue(item[k])).join(",") + "\n";
+      }
+      return result;
+    }
+    return `${space}[${data.length}]: ${data.join(",")}`;
+  }
+  if (typeof data === "object" && data !== null) {
+    let result = "";
+    for (const key in data) {
+      const value = data[key];
+      if (typeof value === "object" && value !== null) {
+        result += `${space}${key}:
+${toTOON(value, indent + 1)}
+`;
+      } else {
+        const formatted = formatValue(value);
+        result += formatted ? `${space}${key}: ${formatted}
+` : `${space}${key}:
+`;
+      }
+    }
+    return result.trim();
+  }
+  return formatValue(data);
+}
+function formatValue(val) {
+  if (val === null || val === void 0) return "";
+  if (typeof val === "string") return val;
+  return String(val);
+}
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+// src/core/token.ts
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+// src/core/analyze.ts
+function analyze(input, options = {}) {
+  if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
+    return {
+      originalTokens: 0,
+      optimizedTokens: 0,
+      savings: 0,
+      savingsPercent: 0,
+      optimizedData: null,
+      reductionRatio: 1
+    };
+  }
+  let originalTokens = estimateTokens(input);
+  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  let optimizedData = input;
+  if (options.prune && Array.isArray(options.prune)) {
+    optimizedData = prune(optimizedData, options.prune);
+  }
+  if (options.compact) {
+    optimizedData = compact(optimizedData);
+  }
+  if (options.flatten) {
+    optimizedData = flatten(optimizedData);
+  }
+  if (options.toTOON === true || options.toon === true) {
+    optimizedData = toTOON(optimizedData);
+  }
+  let optimizedTokens = estimateTokens(optimizedData);
+  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const savings = Math.max(0, originalTokens - optimizedTokens);
+  const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
+  const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  return {
+    originalTokens,
+    optimizedTokens,
+    savings,
+    savingsPercent,
+    optimizedData,
+    reductionRatio
+  };
+}
+
+// src/core/natural.ts
+function toNatural(data, depth = 0) {
+  if (data === null || data === void 0) return "nothing";
+  if (typeof data === "string") return data;
+  if (typeof data === "number") return String(data);
+  if (typeof data === "boolean") return data ? "yes" : "no";
+  if (Array.isArray(data)) {
+    if (data.length === 0) return "empty list";
+    if (data.every(isPlainObject2)) {
+      return data.map((item, index) => `${index + 1}. ${stripTrailingPeriod(toNatural(item, depth + 1))}.`).join(" ");
+    }
+    const items = data.map((item) => toNatural(item, depth + 1));
+    return joinNaturalList(items);
+  }
+  if (typeof data === "object") {
+    return buildContextualStory(data, depth);
+  }
+  return String(data);
+}
+function buildContextualStory(obj, depth = 0) {
+  let name = obj.name || obj.userName || obj.user;
+  if (typeof name === "object" && name !== null && name.name) {
+    name = name.name;
+  }
+  const entries = Object.entries(obj).filter(([key]) => {
+    return !["id", "timestamp", "apiKey"].includes(key);
+  });
+  if (entries.length === 0) return "";
+  let story = name && typeof name === "string" ? `User ${name}` : "";
+  const clauses = entries.map(([key, value]) => {
+    if (key === "user" && name && typeof value === "object" && value !== null && !Array.isArray(value)) {
+      const userProps = Object.entries(value).filter(([k]) => !["id", "name", "timestamp", "apiKey"].includes(k)).map(([k, v]) => formatPropertyClause(k, v, depth + 1)).filter((c) => c !== null && c !== "").join(", ");
+      return userProps ? `(${userProps})` : null;
+    }
+    return formatPropertyClause(key, value, depth);
+  }).filter((c) => c !== null && c !== "");
+  if (clauses.length === 0) return story;
+  if (story) {
+    story += " " + clauses.join(", ");
+  } else {
+    story = clauses.join(", ");
+  }
+  return story + ".";
+}
+function formatPropertyClause(key, value, depth) {
+  const naturalKey = camelToWords(key);
+  if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+    if (["internal", "metadata", "debug"].includes(key)) {
+      return null;
+    }
+    const nested = Object.entries(value).filter(([k]) => !["id", "timestamp", "apiKey", "createdAt", "updatedAt"].includes(k)).map(([k, v]) => {
+      if (typeof v === "object" && v !== null) {
+        return formatPropertyClause(k, v, depth + 1);
+      }
+      const propValue = toNatural(v, depth + 1);
+      if (propValue === "yes" || propValue === "no") {
+        return `${camelToWords(k)} ${propValue === "yes" ? "enabled" : "disabled"}`;
+      }
+      return `${camelToWords(k)} ${propValue}`;
+    }).filter((c) => c && c.trim()).join(", ");
+    if (!nested) return null;
+    return `${naturalKey}: ${nested}`;
+  }
+  if (Array.isArray(value)) {
+    if (value.length === 0) return null;
+    if (["skills", "hobbies", "interests", "tags", "languages", "items"].includes(key)) {
+      const items2 = value.map((item) => {
+        const str = String(item);
+        return str.charAt(0).toUpperCase() + str.slice(1);
+      });
+      return `Having ${joinNaturalList(items2)}`;
+    }
+    const items = joinNaturalList(value.map((item) => toNatural(item, depth + 1)));
+    return `${naturalKey}: ${items}`;
+  }
+  const naturalValue = toNatural(value, depth + 1);
+  if (naturalValue === "yes") return `${naturalKey} enabled`;
+  if (naturalValue === "no") return `${naturalKey} disabled`;
+  if (key === "theme") return `prefers the ${naturalValue} ${key}`;
+  if (key === "age") return `age: ${naturalValue}`;
+  if (key === "email") return `email: ${naturalValue}`;
+  if (key === "city") return `address: ${naturalValue}`;
+  if (key === "debug" && naturalValue === "yes") return `debug enabled`;
+  return `${naturalKey}: ${naturalValue}`;
+}
+function camelToWords(str) {
+  return str.replace(/([A-Z])/g, " $1").toLowerCase().trim();
+}
+function isPlainObject2(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function joinNaturalList(items) {
+  if (items.length === 0) return "";
+  if (items.length === 1) return items[0];
+  if (items.length === 2) return `${items[0]} and ${items[1]}`;
+  const lastItem = items[items.length - 1];
+  return `${items.slice(0, -1).join(", ")} and ${lastItem}`;
+}
+function stripTrailingPeriod(value) {
+  return value.replace(/\.+$/, "");
+}
+
+// src/chain.ts
+var AIChain = class {
+  constructor(data) {
+    this.data = data;
+  }
+  prune(options) {
+    this.data = prune(this.data, options);
+    return this;
+  }
+  compact() {
+    this.data = compact(this.data);
+    return this;
+  }
+  flatten() {
+    this.data = flatten(this.data);
+    return this;
+  }
+  toTOON() {
+    this.data = toTOON(this.data);
+    return this;
+  }
+  toNatural() {
+    return toNatural(this.data);
+  }
+  analyze() {
+    return analyze(this.data);
+  }
+  value() {
+    return this.data;
+  }
+};
+
+// src/index.ts
+function ai(data) {
+  return new AIChain(data);
+}
+ai.prune = prune;
+ai.compact = compact;
+ai.flatten = flatten;
+ai.toTOON = toTOON;
+ai.analyze = analyze;
+ai.toNatural = toNatural;
+ai.estimateTokens = estimateTokens;
+ai.AIChain = AIChain;
+var index_default = ai;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  AIChain,
+  analyze,
+  compact,
+  estimateTokens,
+  flatten,
+  prune,
+  toNatural,
+  toTOON
+});

package/dist/index.d.cts

@@ -0,0 +1,40 @@
+import { estimateTokens } from './core/token.cjs';
+import { toNatural } from './core/natural.cjs';
+import { analyze } from './core/analyze.cjs';
+import { toTOON } from './core/toon.cjs';
+import { flatten } from './core/flatten.cjs';
+import { compact } from './core/compact.cjs';
+import { prune } from './core/prune.cjs';
+
+declare class AIChain {
+    private data;
+    constructor(data: any);
+    prune(options?: any): this;
+    compact(): this;
+    flatten(): this;
+    toTOON(): this;
+    toNatural(): string;
+    analyze(): {
+        originalTokens: number;
+        optimizedTokens: number;
+        savings: number;
+        savingsPercent: number;
+        optimizedData: any;
+        reductionRatio: number;
+    };
+    value(): any;
+}
+
+declare function ai(data: any): AIChain;
+declare namespace ai {
+    var prune: typeof prune;
+    var compact: typeof compact;
+    var flatten: typeof flatten;
+    var toTOON: typeof toTOON;
+    var analyze: typeof analyze;
+    var toNatural: typeof toNatural;
+    var estimateTokens: typeof estimateTokens;
+    var AIChain: typeof AIChain;
+}
+
+export { AIChain, analyze, compact, ai as default, estimateTokens, flatten, prune, toNatural, toTOON };