@sohqureshi/tokenwise 1.0.0

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 tokenwise
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,260 @@
+# TokenWise
+
+TokenWise is a lightweight utility for preparing JSON before sending it to AI models. It helps reduce payload noise, shrink token usage, and turn structured data into formats that are easier for LLMs to consume.
+
+---
+
+## 🧠 What is tokenwise?
+
+**tokenwise** is a lightweight utility that optimizes your data before sending it to AI models.
+
+Raw JSON is:
+
+* ❌ verbose
+* ❌ token-heavy
+* ❌ expensive for LLMs
+
+**tokenwise** helps you:
+
+* 📉 reduce token usage
+* ⚡ improve response speed
+* 💰 lower API costs
+
+---
+
+## Installation
+
+```bash
+npm install tokenwise
+```
+
+## Quick Usage
+
+```js
+import ai, { compact, flatten, toNatural, toTOON } from "tokenwise";
+
+const product = {
+  product: {
+    name: "Wireless Headphones",
+    price: 79.99
+  }
+};
+
+console.log(compact(product));
+// {"product":{"name":"Wireless Headphones","price":79.99}}
+
+console.log(flatten(product));
+// {
+//   "product.name": "Wireless Headphones",
+//   "product.price": 79.99
+// }
+
+console.log(ai(product).compact().value());
+// {"product":{"name":"Wireless Headphones","price":79.99}}
+```
+
+## Core Functions
+
+### `prune()`
+
+Removes fields you do not want to send to the model. By default it also removes `null`, `undefined`, and empty objects.
+
+```js
+import { prune } from "tokenwise";
+
+const input = {
+  user: { name: "John", age: 28 },
+  debug: true,
+  internal: { apiKey: "secret" }
+};
+
+console.log(prune(input, ["debug", "internal"]));
+// { user: { name: "John", age: 28 } }
+```
+
+### `compact()`
+
+Prunes empty/noisy values, then returns minified JSON.
+
+```js
+compact({
+  product: {
+    name: "Wireless Headphones",
+    price: 79.99
+  }
+});
+// {"product":{"name":"Wireless Headphones","price":79.99}}
+```
+
+### `flatten()`
+
+Converts nested objects and arrays into one object with dot-notation keys.
+
+```js
+flatten({
+  policy: {
+    claims: [
+      { status: "approved", amount: 1200 },
+      { status: "pending", amount: 500 }
+    ]
+  }
+});
+// {
+//   "policy.claims.0.status": "approved",
+//   "policy.claims.0.amount": 1200,
+//   "policy.claims.1.status": "pending",
+//   "policy.claims.1.amount": 500
+// }
+```
+
+### `toNatural()`
+
+Converts JSON into readable sentences or numbered pointers.
+
+```js
+toNatural([
+  {
+    user: {
+      name: "Alice Johnson",
+      email: "alice@example.com",
+      skills: ["Python", "JavaScript"]
+    }
+  }
+]);
+// 1. User Alice Johnson (email: alice@example.com, Having Python and JavaScript).
+```
+
+Medical and insurance-style data also becomes readable:
+
+```js
+toNatural({
+  policy: {
+    holderName: "Carlos Rivera",
+    policyNumber: "HLT-2048",
+    claim: {
+      status: "under review",
+      requestedAmount: 64000
+    }
+  }
+});
+// policy: holder name Carlos Rivera, policy number HLT-2048, claim: status under review, requested amount 64000.
+```
+
+### `toTOON()`
+
+Converts JSON into a compact TOON-like text format. Arrays of objects become table-style rows.
+
+```js
+toTOON({
+  users: [
+    { id: 1, name: "Ali" },
+    { id: 2, name: "John" }
+  ]
+});
+// users:
+//   [2]{id,name}:
+//     1,Ali
+//     2,John
+```
+
+If later rows contain extra keys, the schema includes them:
+
+```js
+toTOON({
+  claims: [
+    { id: "C-1", status: "approved" },
+    { id: "C-2", amount: 500 }
+  ]
+});
+// claims:
+//   [2]{id,status,amount}:
+//     C-1,approved,
+//     C-2,,500
+```
+
+## Tested Use Cases
+
+TokenWise currently has coverage for:
+
+- Product JSON minification
+- Dot-notation flattening
+- Arrays and arrays of objects
+- Medical patient and appointment data
+- Insurance policy and claim data
+- Natural-language user pointers
+- TOON table formatting
+- Null, undefined, and empty values
+
+## Why It Helps
+
+LLMs charge and reason over tokens. Sending raw JSON often includes repeated keys, unnecessary metadata, and formatting whitespace. TokenWise gives you multiple ways to reshape the same data depending on your prompt:
+
+- Use `compact()` when you need valid JSON with minimal whitespace.
+- Use `flatten()` when retrieval, search, or simple key-value context is better.
+- Use `toNatural()` when the model should read the data like human-friendly notes.
+- Use `toTOON()` when arrays of objects should be shorter than repeated JSON.
+
+## CLI
+
+```bash
+node demo.js
+```
+
+You can visualize token optimization results using planned CLI/Web visual tools.
+
+---
+
+## 🚀 Roadmap
+
+* [x] CLI support *(Coming Soon)*
+* [ ] NPM Support
+* [ ] Streaming support (GB+ data)
+* [ ] Schema-aware optimization
+* [ ] SaaS API
+
+---
+
+## 🛠 Comparison to Existing Tools *(Future Section)*
+
+Highlight where **tokenwise** stands out, offering better compacting and token estimation features compared to other libraries.
+
+---
+
+## 🤝 Contributing
+
+PRs are welcome,Feel free to open issues or submit PRs, Ideas are welcome for:
+
+- Better token compression strategies  
+- Multi-model optimization  
+- Streaming CLI support  
+
+note: Explore the `CONTRIBUTING.md` for more details for the contributons.
+
+---
+
+## 📄 License
+
+MIT License
+
+---
+
+## ❤️ Support This Project
+
+TokenWise is built to help developers reduce LLM cost and improve efficiency.
+
+If it helps you, consider supporting its development:
+
+[![Support](https://img.shields.io/badge/❤️%20Support%20TokenWise-View-black?style=for-the-badge)](https://github.com/sponsors/sohqureshi)
+
+
+![npm version](https://img.shields.io/npm/v/tokenwise)
+![downloads](https://img.shields.io/npm/dw/tokenwise)
+![license](https://img.shields.io/github/license/sohqureshi/tokenwise)
+![stars](https://img.shields.io/github/stars/sohqureshi/tokenwise?style=social)
+[![Demo](https://img.shields.io/badge/Live%20Demo-Visit-brightgreen)](https://sohqureshi.github.io/tokenwise/)
+[![Support](https://img.shields.io/badge/❤️%20Support%20Tokenwise-View-black?style=for-the-badge)](https://github.com/sponsors/sohqureshi)
+---
+
+## 💡 Vision
+
+Make AI cheaper and faster by optimizing data before it reaches the model.

package/dist/chunk-2IQAGI7Q.js

@@ -0,0 +1,54 @@
+import {
+  analyze
+} from "./chunk-GQ62V6ZK.js";
+import {
+  toTOON
+} from "./chunk-3ECVBELU.js";
+import {
+  compact
+} from "./chunk-EUFLW42L.js";
+import {
+  flatten
+} from "./chunk-YMOSWQGE.js";
+import {
+  toNatural
+} from "./chunk-D4CFTFM3.js";
+import {
+  prune
+} from "./chunk-IAV75SZA.js";
+
+// src/chain.ts
+var AIChain = class {
+  constructor(data) {
+    this.data = data;
+  }
+  prune(options) {
+    this.data = prune(this.data, options);
+    return this;
+  }
+  compact() {
+    this.data = compact(this.data);
+    return this;
+  }
+  flatten() {
+    this.data = flatten(this.data);
+    return this;
+  }
+  toTOON() {
+    this.data = toTOON(this.data);
+    return this;
+  }
+  toNatural() {
+    return toNatural(this.data);
+  }
+  analyze() {
+    return analyze(this.data);
+  }
+  value() {
+    return this.data;
+  }
+};
+
+export {
+  AIChain
+};

package/dist/chunk-33GBS5YN.js

@@ -0,0 +1,288 @@
+// src/core/prune.ts
+function prune(obj, options = {}) {
+  const normalizedOptions = Array.isArray(options) ? { removeKeys: options } : options;
+  const {
+    removeKeys = [],
+    removeNull = true,
+    removeUndefined = true,
+    removeEmptyObjects = true,
+    removeEmptyArrays = false
+  } = normalizedOptions;
+  if (obj === null) return removeNull ? void 0 : obj;
+  if (obj === void 0) return removeUndefined ? void 0 : obj;
+  if (typeof obj !== "object") return obj;
+  if (Array.isArray(obj)) {
+    const arr = obj.map((item) => prune(item, normalizedOptions)).filter((item) => item !== void 0);
+    if (removeEmptyArrays && arr.length === 0) {
+      return void 0;
+    }
+    return arr;
+  }
+  const result = {};
+  for (const key in obj) {
+    if (removeKeys.includes(key)) continue;
+    const value = prune(obj[key], normalizedOptions);
+    if (value === void 0) continue;
+    if (removeEmptyObjects && typeof value === "object" && !Array.isArray(value) && Object.keys(value).length === 0) {
+      continue;
+    }
+    result[key] = value;
+  }
+  return result;
+}
+
+// src/core/flatten.ts
+function flatten(obj, prefix = "", res = {}) {
+  if (obj === null || obj === void 0) return res;
+  if (typeof obj !== "object") {
+    res[prefix] = obj;
+    return res;
+  }
+  for (const key in obj) {
+    const value = obj[key];
+    const newKey = prefix ? `${prefix}.${key}` : key;
+    if (typeof value === "object" && value !== null) {
+      flatten(value, newKey, res);
+    } else {
+      res[newKey] = value;
+    }
+  }
+  return res;
+}
+
+// src/core/compact.ts
+function compact(obj) {
+  const pruned = prune(obj);
+  const flat = flatten(pruned);
+  return Object.entries(flat).map(([k, v]) => `${k}=${v}`).join("|");
+}
+
+// src/core/toon.ts
+function toTOON(data, indent = 0) {
+  const space = "  ".repeat(indent);
+  if (Array.isArray(data)) {
+    if (data.length === 0) return "[]";
+    if (typeof data[0] === "object") {
+      const keys = Object.keys(data[0]);
+      let result = `${space}[${data.length}]{${keys.join(",")}}:
+`;
+      for (const item of data) {
+        result += space + "  " + keys.map((k) => formatValue(item[k])).join(",") + "\n";
+      }
+      return result;
+    }
+    return `${space}[${data.length}]: ${data.join(",")}`;
+  }
+  if (typeof data === "object" && data !== null) {
+    let result = "";
+    for (const key in data) {
+      const value = data[key];
+      if (typeof value === "object") {
+        result += `${space}${key}:
+${toTOON(value, indent + 1)}
+`;
+      } else {
+        result += `${space}${key}: ${formatValue(value)}
+`;
+      }
+    }
+    return result.trim();
+  }
+  return formatValue(data);
+}
+function formatValue(val) {
+  if (val === null || val === void 0) return "";
+  if (typeof val === "string") return val;
+  return String(val);
+}
+
+// src/core/token.ts
+function estimateTokens(text) {
+  if (!text) return 0;
+  return Math.ceil(text.length / 4);
+}
+
+// src/core/analyze.ts
+function analyze(input, options = {}) {
+  if (!input || typeof input === "object" && input !== null && Object.keys(input).length === 0) {
+    return {
+      originalTokens: 0,
+      optimizedTokens: 0,
+      savings: 0,
+      savingsPercent: 0,
+      optimizedData: null,
+      reductionRatio: 1
+    };
+  }
+  let originalTokens = estimateTokens(input);
+  if (isNaN(originalTokens) || !isFinite(originalTokens)) originalTokens = 0;
+  let optimizedData = input;
+  if (options.prune && Array.isArray(options.prune)) {
+    optimizedData = prune(optimizedData, options.prune);
+  }
+  if (options.compact) {
+    optimizedData = compact(optimizedData);
+  }
+  if (options.flatten) {
+    optimizedData = flatten(optimizedData);
+  }
+  if (options.toTOON === true || options.toon === true) {
+    optimizedData = toTOON(optimizedData);
+  }
+  let optimizedTokens = estimateTokens(optimizedData);
+  if (isNaN(optimizedTokens) || !isFinite(optimizedTokens)) optimizedTokens = 0;
+  const savings = Math.max(0, originalTokens - optimizedTokens);
+  const savingsPercent = originalTokens > 0 ? Math.round(savings / originalTokens * 100) : 0;
+  const reductionRatio = originalTokens > 0 ? optimizedTokens / originalTokens : 1;
+  return {
+    originalTokens,
+    optimizedTokens,
+    savings,
+    savingsPercent,
+    optimizedData,
+    reductionRatio
+  };
+}
+
+// src/core/natural.ts
+function toNatural(data, depth = 0) {
+  if (data === null || data === void 0) return "nothing";
+  if (typeof data === "string") return data;
+  if (typeof data === "number") return String(data);
+  if (typeof data === "boolean") return data ? "yes" : "no";
+  if (Array.isArray(data)) {
+    if (data.length === 0) return "empty list";
+    if (data.every(isPlainObject)) {
+      return data.map((item, index) => `${index + 1}. ${stripTrailingPeriod(toNatural(item, depth + 1))}.`).join(" ");
+    }
+    const items = data.map((item) => toNatural(item, depth + 1));
+    return joinNaturalList(items);
+  }
+  if (typeof data === "object") {
+    return buildContextualStory(data, depth);
+  }
+  return String(data);
+}
+function buildContextualStory(obj, depth = 0) {
+  let name = obj.name || obj.userName || obj.user;
+  if (typeof name === "object" && name !== null && name.name) {
+    name = name.name;
+  }
+  const entries = Object.entries(obj).filter(([key]) => {
+    return !["id", "timestamp", "apiKey"].includes(key);
+  });
+  if (entries.length === 0) return "";
+  let story = name && typeof name === "string" ? `User ${name}` : "";
+  const clauses = entries.map(([key, value]) => {
+    if (key === "user" && name && typeof value === "object" && value !== null && !Array.isArray(value)) {
+      const userProps = Object.entries(value).filter(([k]) => !["id", "name", "timestamp", "apiKey"].includes(k)).map(([k, v]) => formatPropertyClause(k, v, depth + 1)).filter((c) => c !== null && c !== "").join(", ");
+      return userProps ? `(${userProps})` : null;
+    }
+    return formatPropertyClause(key, value, depth);
+  }).filter((c) => c !== null && c !== "");
+  if (clauses.length === 0) return story;
+  if (story) {
+    story += " " + clauses.join(", ");
+  } else {
+    story = clauses.join(", ");
+  }
+  return story + ".";
+}
+function formatPropertyClause(key, value, depth) {
+  const naturalKey = camelToWords(key);
+  if (typeof value === "object" && value !== null && !Array.isArray(value)) {
+    if (["internal", "metadata", "debug"].includes(key)) {
+      return null;
+    }
+    const nested = Object.entries(value).filter(([k]) => !["id", "timestamp", "apiKey", "createdAt", "updatedAt"].includes(k)).map(([k, v]) => {
+      if (typeof v === "object" && v !== null) {
+        return formatPropertyClause(k, v, depth + 1);
+      }
+      const propValue = toNatural(v, depth + 1);
+      if (propValue === "yes" || propValue === "no") {
+        return `${camelToWords(k)} ${propValue === "yes" ? "enabled" : "disabled"}`;
+      }
+      return `${camelToWords(k)} ${propValue}`;
+    }).filter((c) => c && c.trim()).join(", ");
+    if (!nested) return null;
+    return `${naturalKey}: ${nested}`;
+  }
+  if (Array.isArray(value)) {
+    if (value.length === 0) return null;
+    if (["skills", "hobbies", "interests", "tags", "languages", "items"].includes(key)) {
+      const items2 = value.map((item) => {
+        const str = String(item);
+        return str.charAt(0).toUpperCase() + str.slice(1);
+      });
+      return `Having ${joinNaturalList(items2)}`;
+    }
+    const items = joinNaturalList(value.map((item) => toNatural(item, depth + 1)));
+    return `${naturalKey}: ${items}`;
+  }
+  const naturalValue = toNatural(value, depth + 1);
+  if (naturalValue === "yes") return `${naturalKey} enabled`;
+  if (naturalValue === "no") return `${naturalKey} disabled`;
+  if (key === "theme") return `prefers the ${naturalValue} ${key}`;
+  if (key === "age") return `age: ${naturalValue}`;
+  if (key === "email") return `email: ${naturalValue}`;
+  if (key === "city") return `address: ${naturalValue}`;
+  if (key === "debug" && naturalValue === "yes") return `debug enabled`;
+  return `${naturalKey}: ${naturalValue}`;
+}
+function camelToWords(str) {
+  return str.replace(/([A-Z])/g, " $1").toLowerCase().trim();
+}
+function isPlainObject(value) {
+  return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function joinNaturalList(items) {
+  if (items.length === 0) return "";
+  if (items.length === 1) return items[0];
+  if (items.length === 2) return `${items[0]} and ${items[1]}`;
+  const lastItem = items[items.length - 1];
+  return `${items.slice(0, -1).join(", ")} and ${lastItem}`;
+}
+function stripTrailingPeriod(value) {
+  return value.replace(/\.+$/, "");
+}
+
+// src/chain.ts
+var AIChain = class {
+  constructor(data) {
+    this.data = data;
+  }
+  prune(options) {
+    this.data = prune(this.data, options);
+    return this;
+  }
+  compact() {
+    this.data = compact(this.data);
+    return this;
+  }
+  flatten() {
+    this.data = flatten(this.data);
+    return this;
+  }
+  toTOON() {
+    this.data = toTOON(this.data);
+    return this;
+  }
+  toNatural() {
+    return toNatural(this.data);
+  }
+  analyze() {
+    return analyze(this.data);
+  }
+  value() {
+    return this.data;
+  }
+};
+
+export {
+  prune,
+  compact,
+  toTOON,
+  analyze,
+  toNatural,
+  AIChain
+};