@shrkcrft/compress 0.1.0-alpha.16

package/dist/text/compress-markdown.js

@@ -0,0 +1,96 @@
+import { EContentType } from "../content/content-type.js";
+import { ECompressionStrategy } from "../result/compression-strategy.js";
+import { splitLines, queryTokens, queryOverlap, elide } from "./line-utils.js";
+import { finalizeLossy, passthroughResult } from "./finalize.js";
+const HEADER = /^#{1,6}\s/;
+const LIST_ITEM = /^\s*(?:[-*+]\s|\d+\.\s)/;
+const FENCE = /^\s*(?:```|~~~)/;
+const TABLE_ROW = /^\s*\|/;
+/**
+ * Markdown-aware reduction that keeps a document's SKELETON — every header, the
+ * first line of each section/paragraph, table rows, and a capped run of list
+ * items — while thinning paragraph continuations and collapsing fenced code
+ * block bodies. Structure is never dropped (headers always survive), so the
+ * outline stays navigable; the full document is recoverable via CCR.
+ *
+ * Note: this runs only when an agent explicitly compresses markdown (via
+ * `shrk compress` / `compress_context`). SharkCraft's own briefs/context are
+ * never silently passed through it.
+ */
+export function compressMarkdown(text, opts = {}) {
+    const lines = splitLines(text);
+    const minLines = opts.minLines ?? 12;
+    if (lines.length < minLines)
+        return passthroughResult(text, EContentType.Markdown);
+    const tokens = queryTokens(opts.query);
+    const maxListRun = opts.maxItems && opts.maxItems > 0 ? opts.maxItems : 8;
+    const keep = new Set();
+    let inFence = false;
+    let atParagraphStart = true; // first prose line of a paragraph is the lead
+    let listRun = 0;
+    for (let i = 0; i < lines.length; i += 1) {
+        const line = lines[i] ?? '';
+        const trimmed = line.trim();
+        if (FENCE.test(line)) {
+            keep.add(i); // keep both fences; interior is elided
+            inFence = !inFence;
+            atParagraphStart = false;
+            listRun = 0;
+            continue;
+        }
+        if (inFence) {
+            if (tokens.length > 0 && queryOverlap(line, tokens) > 0)
+                keep.add(i); // keep query-relevant code lines
+            continue;
+        }
+        if (trimmed.length === 0) {
+            atParagraphStart = true;
+            listRun = 0;
+            continue; // blank runs collapse via elide
+        }
+        if (HEADER.test(line)) {
+            keep.add(i);
+            atParagraphStart = true; // the line after a header is a section lead
+            listRun = 0;
+            continue;
+        }
+        if (TABLE_ROW.test(line)) {
+            keep.add(i); // tables are already dense structure — keep rows
+            atParagraphStart = false;
+            continue;
+        }
+        if (LIST_ITEM.test(line)) {
+            listRun += 1;
+            if (listRun <= maxListRun)
+                keep.add(i);
+            atParagraphStart = false;
+            continue;
+        }
+        // Setext header: a text line underlined by a run of `=` (h1) or `-` (h2).
+        // Keep the title AND its underline so the header survives intact.
+        const underline = (lines[i + 1] ?? '').trim();
+        if (/^=+$/.test(underline) || /^-+$/.test(underline)) {
+            keep.add(i);
+            keep.add(i + 1);
+            atParagraphStart = true; // the line after a header is a section lead
+            listRun = 0;
+            i += 1; // consume the underline (the for-loop's increment skips it)
+            continue;
+        }
+        // Prose: keep the lead line of a paragraph/section, drop continuations.
+        if (atParagraphStart || (tokens.length > 0 && queryOverlap(line, tokens) > 0)) {
+            keep.add(i);
+        }
+        atParagraphStart = false;
+        listRun = 0;
+    }
+    const body = elide(lines, keep);
+    return finalizeLossy({
+        original: text,
+        body,
+        contentType: EContentType.Markdown,
+        strategy: ECompressionStrategy.Markdown,
+        opts,
+        note: `full document: ${lines.length} lines`,
+    });
+}

package/dist/text/compress-search.d.ts

@@ -0,0 +1,11 @@
+import type { ICompressionResult } from '../result/compression-result.js';
+import type { ICompressOptions } from '../result/compress-options.js';
+/**
+ * Reduce grep / ripgrep `file:line:` output to the highest-signal matches:
+ * the first hit in every file is always kept (so no file silently vanishes),
+ * then the top matches per file by query overlap and priority keywords. Lines
+ * that aren't matches (headers, blanks) are preserved as structure. Dropped
+ * matches are elided; the full output is recoverable via CCR.
+ */
+export declare function compressSearch(text: string, opts?: ICompressOptions): ICompressionResult;
+//# sourceMappingURL=compress-search.d.ts.map

package/dist/text/compress-search.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compress-search.d.ts","sourceRoot":"","sources":["../../src/text/compress-search.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAkBtE;;;;;;GAMG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,kBAAkB,CA4D5F"}

package/dist/text/compress-search.js

@@ -0,0 +1,78 @@
+import { EContentType } from "../content/content-type.js";
+import { ECompressionStrategy } from "../result/compression-strategy.js";
+import { splitLines, elide } from "./line-utils.js";
+import { finalizeLossy, passthroughResult } from "./finalize.js";
+import { computeOptimalK } from "../table/adaptive-size.js";
+import { bm25Scores } from "../relevance/bm25.js";
+// Allow an optional Windows drive prefix (`C:`) before the path, so rg/grep
+// output captured on Windows still parses (the drive colon isn't the separator).
+const SEARCH_LINE = /^((?:[A-Za-z]:)?[^\s:]+):(\d+):(.*)$/;
+const PRIORITY_RE = /\b(?:ERROR|FAIL|TODO|FIXME|BUG|throw|panic|deprecated)\b/i;
+/**
+ * Reduce grep / ripgrep `file:line:` output to the highest-signal matches:
+ * the first hit in every file is always kept (so no file silently vanishes),
+ * then the top matches per file by query overlap and priority keywords. Lines
+ * that aren't matches (headers, blanks) are preserved as structure. Dropped
+ * matches are elided; the full output is recoverable via CCR.
+ */
+export function compressSearch(text, opts = {}) {
+    const lines = splitLines(text);
+    const matches = [];
+    for (let i = 0; i < lines.length; i += 1) {
+        const m = SEARCH_LINE.exec(lines[i] ?? '');
+        if (!m)
+            continue;
+        const file = m[1] ?? '';
+        const body = m[3] ?? '';
+        matches.push({ index: i, file, body, score: PRIORITY_RE.test(body) ? 0.5 : 0 });
+    }
+    if (matches.length < 2)
+        return passthroughResult(text, EContentType.SearchResults);
+    // P3.2: bias retained matches by BM25 relevance to the query (idf-weighted,
+    // length-normalized, ID-term boosted). No query → all zeros, so ranking falls
+    // back to the priority-keyword bonus exactly as before.
+    if (opts.query) {
+        const rel = bm25Scores(opts.query, matches.map((m) => m.body));
+        for (let k = 0; k < matches.length; k += 1)
+            matches[k].score += rel[k];
+    }
+    // P3.1: with no explicit cap, size the per-file keep from how much unique
+    // information the match bodies carry — fewer on redundant hits, up to 8 on
+    // diverse ones. An explicit `maxItems` always wins.
+    const perFile = opts.maxItems ?? computeOptimalK(matches.map((m) => m.body), { min: 2, max: 8 });
+    const byFile = new Map();
+    for (const m of matches) {
+        const list = byFile.get(m.file) ?? [];
+        list.push(m);
+        byFile.set(m.file, list);
+    }
+    const keep = new Set();
+    // Keep every non-match line (structural): headers, separators, blanks.
+    const matchIdx = new Set(matches.map((m) => m.index));
+    for (let i = 0; i < lines.length; i += 1)
+        if (!matchIdx.has(i))
+            keep.add(i);
+    for (const list of byFile.values()) {
+        // Always keep the first match in the file (so no file silently vanishes),
+        // then fill the per-file budget with the highest-scoring REMAINING matches.
+        // Excluding `first` from the ranked fill keeps the total at exactly `perFile`
+        // rather than `perFile + 1` when the first match isn't itself top-ranked.
+        const first = list[0];
+        keep.add(first.index);
+        const ranked = [...list]
+            .filter((m) => m.index !== first.index)
+            .sort((a, b) => (b.score - a.score) || (a.index - b.index))
+            .slice(0, Math.max(0, perFile - 1));
+        for (const m of ranked)
+            keep.add(m.index);
+    }
+    const body = elide(lines, keep);
+    return finalizeLossy({
+        original: text,
+        body,
+        contentType: EContentType.SearchResults,
+        strategy: ECompressionStrategy.Search,
+        opts,
+        note: `full results: ${matches.length} matches in ${byFile.size} files`,
+    });
+}

package/dist/text/finalize.d.ts

@@ -0,0 +1,21 @@
+import type { EContentType } from '../content/content-type.js';
+import { ECompressionStrategy } from '../result/compression-strategy.js';
+import type { ICompressionResult } from '../result/compression-result.js';
+import type { ICompressOptions } from '../result/compress-options.js';
+/** A no-op result: the input wasn't worth compressing. */
+export declare function passthroughResult(original: string, contentType: EContentType, note?: string): ICompressionResult;
+/**
+ * Wrap a lossy compressor's body into a result: cache the original (when a
+ * store is given) and append a CCR retrieval marker, then verify the pass
+ * actually saved tokens — if it didn't, fall back to passthrough so a
+ * compressor is never a net loss.
+ */
+export declare function finalizeLossy(params: {
+    original: string;
+    body: string;
+    contentType: EContentType;
+    strategy: ECompressionStrategy;
+    opts: ICompressOptions;
+    note: string;
+}): ICompressionResult;
+//# sourceMappingURL=finalize.d.ts.map

package/dist/text/finalize.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"finalize.d.ts","sourceRoot":"","sources":["../../src/text/finalize.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAItE,0DAA0D;AAC1D,wBAAgB,iBAAiB,CAC/B,QAAQ,EAAE,MAAM,EAChB,WAAW,EAAE,YAAY,EACzB,IAAI,SAAmC,GACtC,kBAAkB,CASpB;AAED;;;;;GAKG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE;IACpC,QAAQ,EAAE,MAAM,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,WAAW,EAAE,YAAY,CAAC;IAC1B,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,IAAI,EAAE,gBAAgB,CAAC;IACvB,IAAI,EAAE,MAAM,CAAC;CACd,GAAG,kBAAkB,CAgCrB"}

package/dist/text/finalize.js

@@ -0,0 +1,54 @@
+import { ECompressionStrategy } from "../result/compression-strategy.js";
+import { measureSavings } from "../tokens/estimate-tokens.js";
+import { formatCcrMarker } from "../ccr/ccr-marker.js";
+/** A no-op result: the input wasn't worth compressing. */
+export function passthroughResult(original, contentType, note = 'below threshold — no reduction') {
+    return {
+        compressed: original,
+        contentType,
+        strategy: ECompressionStrategy.Passthrough,
+        savings: measureSavings(original, original, contentType),
+        lossy: false,
+        note,
+    };
+}
+/**
+ * Wrap a lossy compressor's body into a result: cache the original (when a
+ * store is given) and append a CCR retrieval marker, then verify the pass
+ * actually saved tokens — if it didn't, fall back to passthrough so a
+ * compressor is never a net loss.
+ */
+export function finalizeLossy(params) {
+    const { original, body, contentType, strategy, opts, note } = params;
+    // Compare modulo `\r`: the compressors run on LF-normalized lines, so a CRLF
+    // input with NO elision yields a body that differs only by line endings —
+    // that is not a real reduction and must passthrough the original untouched.
+    const reduced = body.replace(/\r/g, '') !== original.replace(/\r/g, '');
+    if (!reduced)
+        return passthroughResult(original, contentType);
+    let compressed = body;
+    let key;
+    if (opts.store) {
+        key = opts.store.put(original);
+        // Skip the trailing marker when the body already references THIS key inline
+        // (e.g. compressLog's per-drop elision hints) — no need to repeat it. A
+        // different inline key (e.g. a diff's per-section keys) still gets the
+        // whole-blob marker appended. The marker carries only the key: the human
+        // `note` is shipped separately in the result, so repeating it on the wire
+        // would just cost tokens.
+        compressed = body.includes(`<<ccr:${key}`) ? body : `${body}\n${formatCcrMarker(key)}`;
+    }
+    const savings = measureSavings(original, compressed, contentType);
+    if (savings.after >= savings.before) {
+        return passthroughResult(original, contentType);
+    }
+    return {
+        compressed,
+        contentType,
+        strategy,
+        savings,
+        lossy: true,
+        ...(key ? { ccrKey: key } : {}),
+        note,
+    };
+}

package/dist/text/line-utils.d.ts

@@ -0,0 +1,20 @@
+/** Split into lines, tolerant of CRLF, without inventing a trailing line. */
+export declare function splitLines(text: string): string[];
+/**
+ * A normalization key for near-duplicate detection: lowercased, with numbers,
+ * hex blobs and quoted paths collapsed to placeholders and whitespace
+ * squeezed. Two log/warning lines that differ only in a counter or address
+ * share a key, so repeated noise dedupes to one representative.
+ */
+export declare function dedupeKey(line: string): string;
+/** Tokenize a query into lowercase words worth matching (length ≥ 2). */
+export declare function queryTokens(query: string | undefined): string[];
+/** How many query tokens appear in `text` (case-insensitive substring). */
+export declare function queryOverlap(text: string, tokens: readonly string[]): number;
+/**
+ * Collapse a set of kept line indices into an elided block: kept lines verbatim,
+ * each dropped run replaced by a single `… N line(s) omitted …` placeholder.
+ * Deterministic and order-preserving.
+ */
+export declare function elide(lines: readonly string[], keep: ReadonlySet<number>): string;
+//# sourceMappingURL=line-utils.d.ts.map

package/dist/text/line-utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"line-utils.d.ts","sourceRoot":"","sources":["../../src/text/line-utils.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAEjD;AAED;;;;;GAKG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAQ9C;AAED,yEAAyE;AACzE,wBAAgB,WAAW,CAAC,KAAK,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,EAAE,CAO/D;AAED,2EAA2E;AAC3E,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,CAM5E;AAED;;;;GAIG;AACH,wBAAgB,KAAK,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,MAAM,CAgBjF"}

package/dist/text/line-utils.js

@@ -0,0 +1,65 @@
+/** Split into lines, tolerant of CRLF, without inventing a trailing line. */
+export function splitLines(text) {
+    return text.replace(/\r\n/g, '\n').split('\n');
+}
+/**
+ * A normalization key for near-duplicate detection: lowercased, with numbers,
+ * hex blobs and quoted paths collapsed to placeholders and whitespace
+ * squeezed. Two log/warning lines that differ only in a counter or address
+ * share a key, so repeated noise dedupes to one representative.
+ */
+export function dedupeKey(line) {
+    return line
+        .toLowerCase()
+        .replace(/0x[0-9a-f]+/g, '<x>')
+        .replace(/\b[0-9a-f]{8,}\b/g, '<x>')
+        .replace(/\d+/g, '<n>')
+        .replace(/\s+/g, ' ')
+        .trim();
+}
+/** Tokenize a query into lowercase words worth matching (length ≥ 2). */
+export function queryTokens(query) {
+    if (!query)
+        return [];
+    const seen = new Set();
+    for (const raw of query.toLowerCase().split(/[^a-z0-9_]+/)) {
+        if (raw.length >= 2)
+            seen.add(raw);
+    }
+    return [...seen];
+}
+/** How many query tokens appear in `text` (case-insensitive substring). */
+export function queryOverlap(text, tokens) {
+    if (tokens.length === 0)
+        return 0;
+    const lower = text.toLowerCase();
+    let hits = 0;
+    for (const t of tokens)
+        if (lower.includes(t))
+            hits += 1;
+    return hits;
+}
+/**
+ * Collapse a set of kept line indices into an elided block: kept lines verbatim,
+ * each dropped run replaced by a single `… N line(s) omitted …` placeholder.
+ * Deterministic and order-preserving.
+ */
+export function elide(lines, keep) {
+    const out = [];
+    let dropped = 0;
+    for (let i = 0; i < lines.length; i += 1) {
+        if (keep.has(i)) {
+            if (dropped > 0) {
+                out.push(`… ${dropped} line${dropped === 1 ? '' : 's'} omitted`);
+                dropped = 0;
+            }
+            out.push(lines[i] ?? '');
+        }
+        else {
+            dropped += 1;
+        }
+    }
+    if (dropped > 0)
+        out.push(`… ${dropped} line${dropped === 1 ? '' : 's'} omitted`);
+    return out.join('\n');
+}

package/dist/text/lockfile-names.d.ts

@@ -0,0 +1,3 @@
+/** True when `basename` is a known dependency lockfile (case-insensitive). */
+export declare function isLockfileName(basename: string): boolean;
+//# sourceMappingURL=lockfile-names.d.ts.map

package/dist/text/lockfile-names.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"lockfile-names.d.ts","sourceRoot":"","sources":["../../src/text/lockfile-names.ts"],"names":[],"mappings":"AA8BA,8EAA8E;AAC9E,wBAAgB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAExD"}

package/dist/text/lockfile-names.js

@@ -0,0 +1,33 @@
+/**
+ * Basenames of dependency lockfiles. A lockfile diff is almost pure churn —
+ * thousands of integrity hashes and resolved versions an agent never reads —
+ * so the diff compressor elides its body to a one-line marker (recoverable via
+ * CCR). Matched by exact basename (case-insensitive), never by extension, so a
+ * hand-written `versions.lock` config isn't swept up by accident.
+ */
+const LOCKFILE_BASENAMES = new Set([
+    'package-lock.json',
+    'npm-shrinkwrap.json',
+    'yarn.lock',
+    'pnpm-lock.yaml',
+    'bun.lock',
+    'bun.lockb',
+    'packages.lock.json',
+    'cargo.lock',
+    'go.sum',
+    'composer.lock',
+    'gemfile.lock',
+    'poetry.lock',
+    'pipfile.lock',
+    'pdm.lock',
+    'gradle.lockfile',
+    'mix.lock',
+    'flake.lock',
+    'pubspec.lock',
+    'packwiz.lock',
+    'deno.lock',
+]);
+/** True when `basename` is a known dependency lockfile (case-insensitive). */
+export function isLockfileName(basename) {
+    return LOCKFILE_BASENAMES.has(basename.toLowerCase());
+}

package/dist/text/log-template.d.ts

@@ -0,0 +1,31 @@
+/**
+ * Drain-style log-template mining — a LOSSLESS pre-pass for {@link compressLog}.
+ *
+ * Repeated structured log lines (`worker-3 processing batch 17 ok` × N) carry
+ * almost no new information per line: the fixed words repeat and only a few
+ * variable tokens move. This miner tokenizes each line, replaces its variable
+ * tokens (numbers, hex, UUIDs, ISO timestamps, quoted strings) with a `{}`
+ * placeholder to form a *template*, groups consecutive lines sharing a
+ * template, and collapses each run to one template plus a compact per-column
+ * encoding of the captured variables.
+ *
+ * It is **lossless by construction**: a template is exactly the original line
+ * with its variable matches replaced by `{}`, so `template ⋈ variables` rebuilds
+ * every original line, in order — no CCR needed. {@link reconstructLogTemplates}
+ * is the inverse and is exercised by the round-trip tests.
+ *
+ * Only runs with ≥1 variable column collapse; pure-identical repeats are left
+ * for the downstream signal-selector's de-duplication so its behaviour (and the
+ * `… N omitted …` markers callers rely on) is preserved.
+ */
+export interface IMinedLog {
+    /** The transformed line list with collapsible runs replaced by blocks. */
+    lines: string[];
+    /** True when at least one run collapsed. */
+    reduced: boolean;
+}
+/** Collapse consecutive same-template runs. Lossless; reversible via {@link reconstructLogTemplates}. */
+export declare function mineLogTemplates(lines: readonly string[]): IMinedLog;
+/** Inverse of {@link mineLogTemplates}: expand every block back to its original lines. */
+export declare function reconstructLogTemplates(text: string): string;
+//# sourceMappingURL=log-template.d.ts.map

package/dist/text/log-template.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"log-template.d.ts","sourceRoot":"","sources":["../../src/text/log-template.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAiCH,MAAM,WAAW,SAAS;IACxB,0EAA0E;IAC1E,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,4CAA4C;IAC5C,OAAO,EAAE,OAAO,CAAC;CAClB;AAED,yGAAyG;AACzG,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,CAuCpE;AA4HD,0FAA0F;AAC1F,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAgC5D"}