@webywz/context-diet 0.0.0-stage → 0.1.1

package/dist/report.d.ts

@@ -0,0 +1,24 @@
+import type { CutCounts, DietResult } from "./types.js";
+export interface JsonReport {
+    ok: true;
+    source: string;
+    input: {
+        chars: number;
+        tokens: number;
+    };
+    output: {
+        chars: number;
+        tokens: number;
+    };
+    saved: {
+        chars: number;
+        tokens: number;
+        charPercent: number;
+        tokenPercent: number;
+    };
+    cuts: CutCounts;
+    text: string;
+}
+export declare function toJsonReport(result: DietResult, source: string): JsonReport;
+export declare function formatHumanReport(result: DietResult, source: string, version: string): string;
+//# sourceMappingURL=report.d.ts.map

package/dist/report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.d.ts","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAExD,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,IAAI,CAAC;IACT,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IACzC,MAAM,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,CAAC;IAC1C,KAAK,EAAE;QACL,KAAK,EAAE,MAAM,CAAC;QACd,MAAM,EAAE,MAAM,CAAC;QACf,WAAW,EAAE,MAAM,CAAC;QACpB,YAAY,EAAE,MAAM,CAAC;KACtB,CAAC;IACF,IAAI,EAAE,SAAS,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;CACd;AAWD,wBAAgB,YAAY,CAAC,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,MAAM,GAAG,UAAU,CAiB3E;AAmBD,wBAAgB,iBAAiB,CAC/B,MAAM,EAAE,UAAU,EAClB,MAAM,EAAE,MAAM,EACd,OAAO,EAAE,MAAM,GACd,MAAM,CAcR"}

package/dist/report.js

@@ -0,0 +1,56 @@
+function pct(before, after) {
+    if (before <= 0)
+        return 0;
+    return Math.round(((before - after) / before) * 1000) / 10;
+}
+function commas(n) {
+    return n.toLocaleString("en-US");
+}
+export function toJsonReport(result, source) {
+    const savedChars = Math.max(0, result.input.chars - result.output.chars);
+    const savedTokens = Math.max(0, result.input.tokens - result.output.tokens);
+    return {
+        ok: true,
+        source,
+        input: result.input,
+        output: result.output,
+        saved: {
+            chars: savedChars,
+            tokens: savedTokens,
+            charPercent: pct(result.input.chars, result.output.chars),
+            tokenPercent: pct(result.input.tokens, result.output.tokens),
+        },
+        cuts: result.cuts,
+        text: result.text,
+    };
+}
+function cutLines(cuts) {
+    const rows = [
+        ["ansi sequences", String(cuts.ansiSequences)],
+        ["boilerplate lines", String(cuts.boilerplateLines)],
+        ["duplicate paragraphs", String(cuts.duplicateParagraphs)],
+        ["whitespace runs", String(cuts.whitespaceRuns)],
+        ["omitted heavy blobs", String(cuts.omittedHeavyBlobs)],
+        ["truncated blocks", String(cuts.truncatedBlocks)],
+        ["truncated tokens", `~${commas(cuts.truncatedTokens)}`],
+        ["llm deepen", cuts.deepened ? "yes" : "no"],
+    ];
+    const width = Math.max(...rows.map(([k]) => k.length));
+    return rows.map(([k, v]) => `  ${k.padEnd(width)}  ${v}`);
+}
+export function formatHumanReport(result, source, version) {
+    const { input, output } = result;
+    const charPct = pct(input.chars, output.chars);
+    const tokPct = pct(input.tokens, output.tokens);
+    const sign = (n) => (n > 0 ? `−${n}%` : n < 0 ? `+${Math.abs(n)}%` : "0%");
+    const lines = [
+        `context-diet ${version}`,
+        `source:   ${source}`,
+        `chars:    ${commas(input.chars)} → ${commas(output.chars)}  (${sign(charPct)})`,
+        `tokens:   ~${commas(input.tokens)} → ~${commas(output.tokens)}  (${sign(tokPct)})`,
+        `cut:`,
+        ...cutLines(result.cuts),
+    ];
+    return lines.join("\n") + "\n";
+}
+//# sourceMappingURL=report.js.map

package/dist/report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report.js","sourceRoot":"","sources":["../src/report.ts"],"names":[],"mappings":"AAiBA,SAAS,GAAG,CAAC,MAAc,EAAE,KAAa;IACxC,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,CAAC,CAAC;IAC1B,OAAO,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,GAAG,KAAK,CAAC,GAAG,MAAM,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;AAC7D,CAAC;AAED,SAAS,MAAM,CAAC,CAAS;IACvB,OAAO,CAAC,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;AACnC,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,MAAkB,EAAE,MAAc;IAC7D,MAAM,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzE,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5E,OAAO;QACL,EAAE,EAAE,IAAI;QACR,MAAM;QACN,KAAK,EAAE,MAAM,CAAC,KAAK;QACnB,MAAM,EAAE,MAAM,CAAC,MAAM;QACrB,KAAK,EAAE;YACL,KAAK,EAAE,UAAU;YACjB,MAAM,EAAE,WAAW;YACnB,WAAW,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC;YACzD,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC;SAC7D;QACD,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,IAAI,EAAE,MAAM,CAAC,IAAI;KAClB,CAAC;AACJ,CAAC;AAED,SAAS,QAAQ,CAAC,IAAe;IAC/B,MAAM,IAAI,GAA4B;QACpC,CAAC,gBAAgB,EAAE,MAAM,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;QAC9C,CAAC,mBAAmB,EAAE,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QACpD,CAAC,sBAAsB,EAAE,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC,CAAC;QAC1D,CAAC,iBAAiB,EAAE,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;QAChD,CAAC,qBAAqB,EAAE,MAAM,CAAC,IAAI,CAAC,iBAAiB,CAAC,CAAC;QACvD,CAAC,kBAAkB,EAAE,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;QAClD,CAAC,kBAAkB,EAAE,IAAI,MAAM,CAAC,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC;QACxD,CAAC,YAAY,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC;KAC7C,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACvD,OAAO,IAAI,CAAC,GAAG,CACb,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE,CACzC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,iBAAiB,CAC/B,MAAkB,EAClB,MAAc,EACd,OAAe;IAEf,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,CAAC;IACjC,MAAM,OAAO,GAAG,GAAG,CAAC,KAAK,CAAC,KAAK,EAAE,MAAM,CAAC,KAAK,CAAC,CAAC;IAC/C,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC;IAChD,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG;QACZ,gBAAgB,OAAO,EAAE;QACzB,aAAa,MAAM,EAAE;QACrB,aAAa,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,MAAM,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,GAAG;QAChF,cAAc,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,MAAM,IAAI,CAAC,MAAM,CAAC,GAAG;QACnF,MAAM;QACN,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC;KACzB,CAAC;IACF,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AACjC,CAAC"}

package/dist/tokens.d.ts

@@ -0,0 +1,6 @@
+export declare function estimateTokens(text: string): number;
+export declare function snapshot(text: string): {
+    chars: number;
+    tokens: number;
+};
+//# sourceMappingURL=tokens.d.ts.map

package/dist/tokens.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokens.d.ts","sourceRoot":"","sources":["../src/tokens.ts"],"names":[],"mappings":"AA4BA,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAoDnD;AAED,wBAAgB,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CAExE"}

package/dist/tokens.js

@@ -0,0 +1,73 @@
+/**
+ * Approximate a cl100k_base (GPT-4 family) token count without shipping a BPE
+ * table. Tuned for English prose, source code, and CJK. Typical error vs
+ * tiktoken is within ~20% on mixed technical text — enough to budget a prompt,
+ * not enough to bill on.
+ */
+const CJK_CODE_POINT = (code) => (code >= 0x4e00 && code <= 0x9fff) ||
+    (code >= 0x3400 && code <= 0x4dbf) ||
+    (code >= 0xf900 && code <= 0xfaff) ||
+    (code >= 0x3040 && code <= 0x30ff) ||
+    (code >= 0x31f0 && code <= 0x31ff) ||
+    (code >= 0xac00 && code <= 0xd7af) ||
+    (code >= 0x1100 && code <= 0x11ff);
+function isLetter(ch) {
+    const code = ch.charCodeAt(0);
+    return ((code >= 65 && code <= 90) ||
+        (code >= 97 && code <= 122) ||
+        (code >= 0xc0 && code <= 0x24f && ch.toLowerCase() !== ch.toUpperCase()));
+}
+function isDigit(ch) {
+    return ch >= "0" && ch <= "9";
+}
+export function estimateTokens(text) {
+    if (!text)
+        return 0;
+    let tokens = 0;
+    let i = 0;
+    const len = text.length;
+    while (i < len) {
+        const ch = text[i];
+        const code = text.charCodeAt(i);
+        if (ch === " " || ch === "\t" || ch === "\r") {
+            i++;
+            continue;
+        }
+        if (ch === "\n") {
+            tokens += 0.25;
+            i++;
+            continue;
+        }
+        if (CJK_CODE_POINT(code)) {
+            tokens += 1;
+            i++;
+            continue;
+        }
+        if (isLetter(ch)) {
+            let j = i + 1;
+            while (j < len && isLetter(text[j]))
+                j++;
+            const wordLen = j - i;
+            // Short function words / identifiers are usually 1 piece; longer words
+            // split roughly every 4 characters, matching cl100k's byte-pair grain.
+            tokens += wordLen <= 5 ? 1 : Math.ceil(wordLen / 4);
+            i = j;
+            continue;
+        }
+        if (isDigit(ch)) {
+            let j = i + 1;
+            while (j < len && isDigit(text[j]))
+                j++;
+            tokens += Math.max(1, Math.ceil((j - i) / 3));
+            i = j;
+            continue;
+        }
+        tokens += 1;
+        i++;
+    }
+    return Math.max(text.length === 0 ? 0 : 1, Math.round(tokens));
+}
+export function snapshot(text) {
+    return { chars: text.length, tokens: estimateTokens(text) };
+}
+//# sourceMappingURL=tokens.js.map

package/dist/tokens.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"tokens.js","sourceRoot":"","sources":["../src/tokens.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,MAAM,cAAc,GAAG,CAAC,IAAY,EAAW,EAAE,CAC/C,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;IAClC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;IAClC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;IAClC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;IAClC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;IAClC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC;IAClC,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,IAAI,MAAM,CAAC,CAAC;AAErC,SAAS,QAAQ,CAAC,EAAU;IAC1B,MAAM,IAAI,GAAG,EAAE,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;IAC9B,OAAO,CACL,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,EAAE,CAAC;QAC1B,CAAC,IAAI,IAAI,EAAE,IAAI,IAAI,IAAI,GAAG,CAAC;QAC3B,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC,WAAW,EAAE,KAAK,EAAE,CAAC,WAAW,EAAE,CAAC,CACzE,CAAC;AACJ,CAAC;AAED,SAAS,OAAO,CAAC,EAAU;IACzB,OAAO,EAAE,IAAI,GAAG,IAAI,EAAE,IAAI,GAAG,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,IAAI,CAAC,IAAI;QAAE,OAAO,CAAC,CAAC;IAEpB,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,CAAC,GAAG,CAAC,CAAC;IACV,MAAM,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC;IAExB,OAAO,CAAC,GAAG,GAAG,EAAE,CAAC;QACf,MAAM,EAAE,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;QACpB,MAAM,IAAI,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAEhC,IAAI,EAAE,KAAK,GAAG,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC7C,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAChB,MAAM,IAAI,IAAI,CAAC;YACf,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,CAAC,CAAC;YACZ,CAAC,EAAE,CAAC;YACJ,SAAS;QACX,CAAC;QAED,IAAI,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;YACjB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAAE,CAAC,EAAE,CAAC;YAC1C,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,CAAC;YACtB,uEAAuE;YACvE,uEAAuE;YACvE,MAAM,IAAI,OAAO,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC;YACpD,CAAC,GAAG,CAAC,CAAC;YACN,SAAS;QACX,CAAC;QAED,IAAI,OAAO,CAAC,EAAE,CAAC,EAAE,CAAC;YAChB,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,GAAG,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,CAAE,CAAC;gBAAE,CAAC,EAAE,CAAC;YACzC,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAC9C,CAAC,GAAG,CAAC,CAAC;YACN,SAAS;QACX,CAAC;QAED,MAAM,IAAI,CAAC,CAAC;QACZ,CAAC,EAAE,CAAC;IACN,CAAC;IAED,OAAO,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC;AACjE,CAAC;AAED,MAAM,UAAU,QAAQ,CAAC,IAAY;IACnC,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,MAAM,EAAE,cAAc,CAAC,IAAI,CAAC,EAAE,CAAC;AAC9D,CAAC"}

package/dist/truncate.d.ts

@@ -0,0 +1,4 @@
+import { type Block } from "./blocks.js";
+import type { CutCounts, KeepCategory } from "./types.js";
+export declare function truncateToBudget(blocks: Block[], maxTokens: number, keep: KeepCategory[], cuts: CutCounts): Block[];
+//# sourceMappingURL=truncate.d.ts.map

package/dist/truncate.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"truncate.d.ts","sourceRoot":"","sources":["../src/truncate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAc,KAAK,KAAK,EAAE,MAAM,aAAa,CAAC;AAErD,OAAO,KAAK,EAAE,SAAS,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AA0D1D,wBAAgB,gBAAgB,CAC9B,MAAM,EAAE,KAAK,EAAE,EACf,SAAS,EAAE,MAAM,EACjB,IAAI,EAAE,YAAY,EAAE,EACpB,IAAI,EAAE,SAAS,GACd,KAAK,EAAE,CA4ET"}

package/dist/truncate.js

@@ -0,0 +1,136 @@
+import { joinBlocks } from "./blocks.js";
+import { estimateTokens } from "./tokens.js";
+function scoreBlock(block, keep) {
+    let score = 1;
+    switch (block.type) {
+        case "heading":
+            score = 4;
+            if (keep.has("headings"))
+                score += 24;
+            break;
+        case "code":
+            score = 4;
+            if (keep.has("code"))
+                score += 24;
+            break;
+        case "list":
+            score = 3;
+            if (keep.has("lists"))
+                score += 24;
+            break;
+        case "paragraph":
+            score = 2;
+            break;
+        case "rule":
+            score = 0;
+            break;
+        case "blank":
+            score = 0;
+            break;
+        default:
+            score = 1;
+    }
+    // Prefer earlier context when scores tie.
+    return score;
+}
+function trimBlockToTokens(raw, maxTokens) {
+    if (estimateTokens(raw) <= maxTokens)
+        return raw;
+    const lines = raw.split("\n");
+    if (lines.length > 1) {
+        const kept = [];
+        for (const line of lines) {
+            const trial = [...kept, line].join("\n");
+            if (estimateTokens(trial + "\n…") > maxTokens)
+                break;
+            kept.push(line);
+        }
+        if (kept.length > 0)
+            return `${kept.join("\n")}\n…`;
+    }
+    // Character walk as a last resort.
+    let lo = 0;
+    let hi = raw.length;
+    while (lo < hi) {
+        const mid = Math.ceil((lo + hi) / 2);
+        const slice = `${raw.slice(0, mid).trimEnd()}\n…`;
+        if (estimateTokens(slice) <= maxTokens)
+            lo = mid;
+        else
+            hi = mid - 1;
+    }
+    const cut = raw.slice(0, lo).trimEnd();
+    return cut.length === 0 ? "…" : `${cut}\n…`;
+}
+export function truncateToBudget(blocks, maxTokens, keep, cuts) {
+    if (!(maxTokens > 0))
+        return blocks;
+    const keepSet = new Set(keep);
+    const tokens = blocks.map((b) => estimateTokens(b.raw));
+    let total = tokens.reduce((a, b) => a + b, 0);
+    if (total <= maxTokens)
+        return blocks;
+    const scores = blocks.map((b) => scoreBlock(b, keepSet));
+    const order = blocks
+        .map((_, i) => i)
+        .sort((a, b) => {
+        const ds = scores[a] - scores[b];
+        if (ds !== 0)
+            return ds;
+        return b - a; // drop later ties first
+    });
+    const dropped = new Set();
+    for (const i of order) {
+        if (total <= maxTokens)
+            break;
+        const block = blocks[i];
+        if (block.type === "blank")
+            continue;
+        // Never drop the last remaining non-blank block; trim it instead.
+        const remainingNonBlank = blocks.filter((b, idx) => !dropped.has(idx) && b.type !== "blank").length;
+        if (remainingNonBlank <= 1)
+            continue;
+        dropped.add(i);
+        total -= tokens[i];
+        cuts.truncatedBlocks += 1;
+        cuts.truncatedTokens += tokens[i];
+    }
+    let next = blocks.filter((_, i) => !dropped.has(i));
+    next = next.filter((b, i, arr) => {
+        if (b.type !== "blank")
+            return true;
+        if (i === 0 || i === arr.length - 1)
+            return false;
+        return arr[i - 1]?.type !== "blank";
+    });
+    total = next.reduce((a, b) => a + estimateTokens(b.raw), 0);
+    if (total <= maxTokens)
+        return next;
+    // Still over: trim the lowest-score, largest remaining block from the end.
+    const trimOrder = next
+        .map((b, i) => ({ i, b, s: scoreBlock(b, keepSet), t: estimateTokens(b.raw) }))
+        .filter((x) => x.b.type !== "blank")
+        .sort((a, b) => a.s - b.s || b.t - a.t || b.i - a.i);
+    for (const item of trimOrder) {
+        total = next.reduce((a, b) => a + estimateTokens(b.raw), 0);
+        if (total <= maxTokens)
+            break;
+        const others = total - item.t;
+        const budget = Math.max(8, maxTokens - others);
+        const trimmed = trimBlockToTokens(item.b.raw, budget);
+        if (trimmed !== item.b.raw) {
+            cuts.truncatedBlocks += 1;
+            cuts.truncatedTokens += Math.max(0, item.t - estimateTokens(trimmed));
+            next = next.map((b, i) => i === item.i ? { ...b, raw: trimmed } : b);
+        }
+    }
+    const joined = joinBlocks(next);
+    if (estimateTokens(joined) <= maxTokens)
+        return next;
+    // Absolute fallback: hard-cut the joined document.
+    const hard = trimBlockToTokens(joined, maxTokens);
+    cuts.truncatedBlocks += 1;
+    cuts.truncatedTokens += Math.max(0, estimateTokens(joined) - estimateTokens(hard));
+    return [{ type: "paragraph", raw: hard }];
+}
+//# sourceMappingURL=truncate.js.map

package/dist/truncate.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"truncate.js","sourceRoot":"","sources":["../src/truncate.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAc,MAAM,aAAa,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAG7C,SAAS,UAAU,CAAC,KAAY,EAAE,IAAuB;IACvD,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,QAAQ,KAAK,CAAC,IAAI,EAAE,CAAC;QACnB,KAAK,SAAS;YACZ,KAAK,GAAG,CAAC,CAAC;YACV,IAAI,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC;gBAAE,KAAK,IAAI,EAAE,CAAC;YACtC,MAAM;QACR,KAAK,MAAM;YACT,KAAK,GAAG,CAAC,CAAC;YACV,IAAI,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC;gBAAE,KAAK,IAAI,EAAE,CAAC;YAClC,MAAM;QACR,KAAK,MAAM;YACT,KAAK,GAAG,CAAC,CAAC;YACV,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC;gBAAE,KAAK,IAAI,EAAE,CAAC;YACnC,MAAM;QACR,KAAK,WAAW;YACd,KAAK,GAAG,CAAC,CAAC;YACV,MAAM;QACR,KAAK,MAAM;YACT,KAAK,GAAG,CAAC,CAAC;YACV,MAAM;QACR,KAAK,OAAO;YACV,KAAK,GAAG,CAAC,CAAC;YACV,MAAM;QACR;YACE,KAAK,GAAG,CAAC,CAAC;IACd,CAAC;IACD,0CAA0C;IAC1C,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAW,EAAE,SAAiB;IACvD,IAAI,cAAc,CAAC,GAAG,CAAC,IAAI,SAAS;QAAE,OAAO,GAAG,CAAC;IACjD,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC9B,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACrB,MAAM,IAAI,GAAa,EAAE,CAAC;QAC1B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,KAAK,GAAG,CAAC,GAAG,IAAI,EAAE,IAAI,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACzC,IAAI,cAAc,CAAC,KAAK,GAAG,KAAK,CAAC,GAAG,SAAS;gBAAE,MAAM;YACrD,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAClB,CAAC;QACD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;IACtD,CAAC;IACD,mCAAmC;IACnC,IAAI,EAAE,GAAG,CAAC,CAAC;IACX,IAAI,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC;IACpB,OAAO,EAAE,GAAG,EAAE,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,CAAC;QAClD,IAAI,cAAc,CAAC,KAAK,CAAC,IAAI,SAAS;YAAE,EAAE,GAAG,GAAG,CAAC;;YAC5C,EAAE,GAAG,GAAG,GAAG,CAAC,CAAC;IACpB,CAAC;IACD,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,EAAE,CAAC;IACvC,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,KAAK,CAAC;AAC9C,CAAC;AAED,MAAM,UAAU,gBAAgB,CAC9B,MAAe,EACf,SAAiB,EACjB,IAAoB,EACpB,IAAe;IAEf,IAAI,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC;QAAE,OAAO,MAAM,CAAC;IACpC,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,IAAI,CAAC,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxD,IAAI,KAAK,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC9C,IAAI,KAAK,IAAI,SAAS;QAAE,OAAO,MAAM,CAAC;IAEtC,MAAM,MAAM,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,CAAC;IACzD,MAAM,KAAK,GAAG,MAAM;SACjB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;SAChB,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;QACb,MAAM,EAAE,GAAG,MAAM,CAAC,CAAC,CAAE,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACnC,IAAI,EAAE,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QACxB,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,wBAAwB;IACxC,CAAC,CAAC,CAAC;IAEL,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,IAAI,KAAK,IAAI,SAAS;YAAE,MAAM;QAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;QACzB,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;YAAE,SAAS;QACrC,kEAAkE;QAClE,MAAM,iBAAiB,GACrB,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC,CAAC,MAAM,CAAC;QAC5E,IAAI,iBAAiB,IAAI,CAAC;YAAE,SAAS;QACrC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACf,KAAK,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC;QACpB,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;QAC1B,IAAI,CAAC,eAAe,IAAI,MAAM,CAAC,CAAC,CAAE,CAAC;IACrC,CAAC;IAED,IAAI,IAAI,GAAY,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,IAAI,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE;QAC/B,IAAI,CAAC,CAAC,IAAI,KAAK,OAAO;YAAE,OAAO,IAAI,CAAC;QACpC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAClD,OAAO,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,KAAK,OAAO,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC5D,IAAI,KAAK,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IAEpC,2EAA2E;IAC3E,MAAM,SAAS,GAAG,IAAI;SACnB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,UAAU,CAAC,CAAC,EAAE,OAAO,CAAC,EAAE,CAAC,EAAE,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;SAC9E,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,KAAK,OAAO,CAAC;SACnC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IAEvD,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,cAAc,CAAC,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;QAC5D,IAAI,KAAK,IAAI,SAAS;YAAE,MAAM;QAC9B,MAAM,MAAM,GAAG,KAAK,GAAG,IAAI,CAAC,CAAC,CAAC;QAC9B,MAAM,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,GAAG,MAAM,CAAC,CAAC;QAC/C,MAAM,OAAO,GAAG,iBAAiB,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QACtD,IAAI,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC;YAC3B,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;YAC1B,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,GAAG,CAC9B,CAAC,EACD,IAAI,CAAC,CAAC,GAAG,cAAc,CAAC,OAAO,CAAC,CACjC,CAAC;YACF,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CACvB,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAC1C,CAAC;QACJ,CAAC;IACH,CAAC;IAED,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IAChC,IAAI,cAAc,CAAC,MAAM,CAAC,IAAI,SAAS;QAAE,OAAO,IAAI,CAAC;IAErD,mDAAmD;IACnD,MAAM,IAAI,GAAG,iBAAiB,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAClD,IAAI,CAAC,eAAe,IAAI,CAAC,CAAC;IAC1B,IAAI,CAAC,eAAe,IAAI,IAAI,CAAC,GAAG,CAC9B,CAAC,EACD,cAAc,CAAC,MAAM,CAAC,GAAG,cAAc,CAAC,IAAI,CAAC,CAC9C,CAAC;IACF,OAAO,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;AAC5C,CAAC"}

package/dist/types.d.ts

@@ -0,0 +1,48 @@
+export type KeepCategory = "headings" | "code" | "lists";
+export type LlmProvider = "openai" | "anthropic";
+export interface CutCounts {
+    /** ANSI escape sequences removed. */
+    ansiSequences: number;
+    /** Repeated / chrome lines dropped (logs, banners, signatures). */
+    boilerplateLines: number;
+    /** Near-identical paragraphs dropped. */
+    duplicateParagraphs: number;
+    /** Extra blank-line runs or trailing spaces collapsed. */
+    whitespaceRuns: number;
+    /** Long base64 / data-URI blobs replaced with placeholders. */
+    omittedHeavyBlobs: number;
+    /** Blocks dropped or trimmed to fit `--max-tokens`. */
+    truncatedBlocks: number;
+    /** Estimated tokens removed by truncation (not other steps). */
+    truncatedTokens: number;
+    /** True when an optional LLM pass rewrote the text. */
+    deepened: boolean;
+}
+export interface DietOptions {
+    /** Soft cap on estimated output tokens. */
+    maxTokens?: number;
+    /** Prefer these block types when truncating to a token budget. */
+    keep?: KeepCategory[];
+    /**
+     * Optional second-pass compression via OpenAI or Anthropic.
+     * Requires a key in the environment; never implied.
+     */
+    deepen?: boolean;
+    provider?: LlmProvider;
+    /** Injected fetch (tests). Defaults to global fetch. */
+    fetch?: typeof fetch;
+    env?: NodeJS.ProcessEnv;
+}
+export interface SizeSnapshot {
+    chars: number;
+    tokens: number;
+}
+export interface DietResult {
+    text: string;
+    input: SizeSnapshot;
+    output: SizeSnapshot;
+    cuts: CutCounts;
+}
+export declare const KEEP_CATEGORIES: readonly KeepCategory[];
+export declare function emptyCuts(): CutCounts;
+//# sourceMappingURL=types.d.ts.map

package/dist/types.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAAA,MAAM,MAAM,YAAY,GAAG,UAAU,GAAG,MAAM,GAAG,OAAO,CAAC;AAEzD,MAAM,MAAM,WAAW,GAAG,QAAQ,GAAG,WAAW,CAAC;AAEjD,MAAM,WAAW,SAAS;IACxB,qCAAqC;IACrC,aAAa,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,gBAAgB,EAAE,MAAM,CAAC;IACzB,yCAAyC;IACzC,mBAAmB,EAAE,MAAM,CAAC;IAC5B,0DAA0D;IAC1D,cAAc,EAAE,MAAM,CAAC;IACvB,+DAA+D;IAC/D,iBAAiB,EAAE,MAAM,CAAC;IAC1B,uDAAuD;IACvD,eAAe,EAAE,MAAM,CAAC;IACxB,gEAAgE;IAChE,eAAe,EAAE,MAAM,CAAC;IACxB,uDAAuD;IACvD,QAAQ,EAAE,OAAO,CAAC;CACnB;AAED,MAAM,WAAW,WAAW;IAC1B,2CAA2C;IAC3C,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,kEAAkE;IAClE,IAAI,CAAC,EAAE,YAAY,EAAE,CAAC;IACtB;;;OAGG;IACH,MAAM,CAAC,EAAE,OAAO,CAAC;IACjB,QAAQ,CAAC,EAAE,WAAW,CAAC;IACvB,wDAAwD;IACxD,KAAK,CAAC,EAAE,OAAO,KAAK,CAAC;IACrB,GAAG,CAAC,EAAE,MAAM,CAAC,UAAU,CAAC;CACzB;AAED,MAAM,WAAW,YAAY;IAC3B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,UAAU;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,YAAY,CAAC;IACpB,MAAM,EAAE,YAAY,CAAC;IACrB,IAAI,EAAE,SAAS,CAAC;CACjB;AAED,eAAO,MAAM,eAAe,EAAE,SAAS,YAAY,EAIlD,CAAC;AAEF,wBAAgB,SAAS,IAAI,SAAS,CAWrC"}

package/dist/types.js

@@ -0,0 +1,18 @@
+export const KEEP_CATEGORIES = [
+    "headings",
+    "code",
+    "lists",
+];
+export function emptyCuts() {
+    return {
+        ansiSequences: 0,
+        boilerplateLines: 0,
+        duplicateParagraphs: 0,
+        whitespaceRuns: 0,
+        omittedHeavyBlobs: 0,
+        truncatedBlocks: 0,
+        truncatedTokens: 0,
+        deepened: false,
+    };
+}
+//# sourceMappingURL=types.js.map

package/dist/types.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"types.js","sourceRoot":"","sources":["../src/types.ts"],"names":[],"mappings":"AAmDA,MAAM,CAAC,MAAM,eAAe,GAA4B;IACtD,UAAU;IACV,MAAM;IACN,OAAO;CACR,CAAC;AAEF,MAAM,UAAU,SAAS;IACvB,OAAO;QACL,aAAa,EAAE,CAAC;QAChB,gBAAgB,EAAE,CAAC;QACnB,mBAAmB,EAAE,CAAC;QACtB,cAAc,EAAE,CAAC;QACjB,iBAAiB,EAAE,CAAC;QACpB,eAAe,EAAE,CAAC;QAClB,eAAe,EAAE,CAAC;QAClB,QAAQ,EAAE,KAAK;KAChB,CAAC;AACJ,CAAC"}

package/package.json

@@ -1,6 +1,60 @@
 {
   "name": "@webywz/context-diet",
-  "version": "0.0.0-stage",
-  "stub": true,
-  "description": "Temporary package placeholder for staged publishing"
-}
+  "version": "0.1.1",
+  "description": "Slim LLM context before you send it — drop noise, keep the signal, estimate tokens.",
+  "keywords": [
+    "llm",
+    "context",
+    "tokens",
+    "prompt",
+    "cli",
+    "openai",
+    "anthropic",
+    "tiktoken"
+  ],
+  "homepage": "https://github.com/webywz/context-diet#readme",
+  "bugs": {
+    "url": "https://github.com/webywz/context-diet/issues"
+  },
+  "license": "MIT",
+  "author": "webywz",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/webywz/context-diet.git"
+  },
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "import": "./dist/index.js"
+    }
+  },
+  "bin": {
+    "context-diet": "dist/cli.js"
+  },
+  "files": [
+    "dist",
+    "LICENSE",
+    "README.md"
+  ],
+  "engines": {
+    "node": ">=18"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "tsx --test test/*.test.ts",
+    "typecheck": "tsc --noEmit",
+    "dev": "tsx src/cli.ts",
+    "prepublishOnly": "npm test && npm run build && node -e \"require('fs').chmodSync('dist/cli.js',0o755)\""
+  },
+  "devDependencies": {
+    "@types/node": "^22.15.29",
+    "tsx": "^4.23.15",
+    "typescript": "^5.8.3"
+  }
+}