@shrkcrft/compress 0.1.0-alpha.16

package/dist/table/sample-object-array.d.ts

@@ -0,0 +1,11 @@
+import type { ISampleOptions } from './sample-options.js';
+import type { ISampledTable } from './sampled-table.js';
+/**
+ * SmartCrusher-style lossy sampler for a homogeneous object array. Keeps
+ * representative rows — front/back anchors, query matches, numeric outliers,
+ * one per dedup class — and drops the rest, in ascending original order, with
+ * full provenance. Pure and deterministic (no RNG/clock; ties break by index).
+ * Returns `null` if the input isn't a homogeneous object array.
+ */
+export declare function sampleObjectArray(items: unknown, opts?: ISampleOptions): ISampledTable | null;
+//# sourceMappingURL=sample-object-array.d.ts.map

package/dist/table/sample-object-array.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sample-object-array.d.ts","sourceRoot":"","sources":["../../src/table/sample-object-array.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAC1D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAqCxD;;;;;;GAMG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,IAAI,GAAE,cAAmB,GAAG,aAAa,GAAG,IAAI,CA8HjG"}

package/dist/table/sample-object-array.js

@@ -0,0 +1,171 @@
+import { compactObjectArray } from "./compact-object-array.js";
+import { applyValueDictionaries } from "./apply-value-dictionaries.js";
+import { computeOptimalK } from "./adaptive-size.js";
+import { topByBm25 } from "../relevance/bm25.js";
+import { queryTokens } from "../text/line-utils.js";
+// Query matches rank ABOVE front/back anchors: when the caller asked about
+// something specific, a relevant row matters more than a positional anchor
+// under a tight cap. With no query there are no matches, so anchors lead as before.
+const PRECEDENCE = { match: 0, head: 1, tail: 2, outlier: 3 };
+/** Auto-pick the numeric column with the highest variance (ties → column name). */
+function pickOutlierField(cols, rows) {
+    let best;
+    for (let c = 0; c < cols.length; c += 1) {
+        const spec = cols[c];
+        if (spec.type !== 'int' && spec.type !== 'float')
+            continue;
+        const nums = [];
+        for (const row of rows) {
+            const v = row[c];
+            if (typeof v === 'number' && Number.isFinite(v))
+                nums.push(v);
+        }
+        if (nums.length < 2)
+            continue;
+        const mean = nums.reduce((s, v) => s + v, 0) / nums.length;
+        const variance = nums.reduce((s, v) => s + (v - mean) * (v - mean), 0) / nums.length;
+        if (!best || variance > best.variance || (variance === best.variance && spec.name < best.name)) {
+            best = { name: spec.name, variance };
+        }
+    }
+    return best?.name;
+}
+/**
+ * SmartCrusher-style lossy sampler for a homogeneous object array. Keeps
+ * representative rows — front/back anchors, query matches, numeric outliers,
+ * one per dedup class — and drops the rest, in ascending original order, with
+ * full provenance. Pure and deterministic (no RNG/clock; ties break by index).
+ * Returns `null` if the input isn't a homogeneous object array.
+ */
+export function sampleObjectArray(items, opts = {}) {
+    const table = compactObjectArray(items);
+    if (!table)
+        return null;
+    const { cols, rows, absent, originalCount } = table;
+    const anchors = opts.anchors ?? 8;
+    const outliersN = opts.outliers ?? 8;
+    const matchesN = opts.matches ?? 16;
+    // P3.1: with no explicit cap, size the keep-set from the data's information
+    // curve instead of a flat 200. Floored at the representative budget so the
+    // adaptive cap never trims the bucket reps (anchors/matches/outliers) — it
+    // only declines to over-keep on large redundant arrays. An explicit
+    // `maxItems` always wins.
+    const repBudget = anchors * 2 + matchesN + outliersN * 2;
+    const maxItems = opts.maxItems && opts.maxItems > 0
+        ? opts.maxItems
+        : computeOptimalK(rows.map((r) => JSON.stringify(r)), {
+            min: Math.min(repBudget, rows.length),
+            max: 200,
+            ...(opts.bias ? { bias: opts.bias } : {}),
+        });
+    const tokens = queryTokens(opts.query);
+    // 1. Dedup → representatives (smallest index per byte-identical row).
+    let repIdx;
+    let deduped = 0;
+    if (opts.dedup !== false) {
+        const seen = new Map();
+        for (let i = 0; i < rows.length; i += 1) {
+            const key = JSON.stringify(rows[i]);
+            if (!seen.has(key))
+                seen.set(key, i);
+            else
+                deduped += 1;
+        }
+        repIdx = [...seen.values()].sort((a, b) => a - b);
+    }
+    else {
+        repIdx = rows.map((_, i) => i);
+    }
+    // 2. Bucket selection (record first bucket that claimed each index).
+    const bucketOf = new Map();
+    const claim = (i, b) => {
+        if (!bucketOf.has(i))
+            bucketOf.set(i, b);
+    };
+    for (const i of repIdx.slice(0, anchors))
+        claim(i, 'head');
+    for (const i of repIdx.slice(Math.max(0, repIdx.length - anchors)))
+        claim(i, 'tail');
+    if (tokens.length > 0 && opts.query) {
+        // P3.2: rank query matches by BM25 (idf-weighted, length-normalized, with an
+        // exact-match boost for ID-shaped terms) instead of bare token overlap, so a
+        // uniquely-relevant row outranks one that merely repeats a common word.
+        const docs = repIdx.map((i) => JSON.stringify(rows[i]));
+        for (const localIdx of topByBm25(opts.query, docs, matchesN)) {
+            claim(repIdx[localIdx], 'match');
+        }
+    }
+    const sortField = opts.outlierField ?? pickOutlierField(cols, rows);
+    if (sortField) {
+        const col = cols.findIndex((c) => c.name === sortField);
+        if (col >= 0) {
+            const vals = repIdx
+                .map((i) => ({ i, v: rows[i]?.[col] }))
+                .filter((x) => typeof x.v === 'number' && Number.isFinite(x.v))
+                .sort((a, b) => a.v - b.v || a.i - b.i);
+            for (const { i } of vals.slice(0, outliersN))
+                claim(i, 'outlier');
+            for (const { i } of vals.slice(Math.max(0, vals.length - outliersN)))
+                claim(i, 'outlier');
+        }
+    }
+    // 3. Hard cap by precedence, always keeping the first & last representative.
+    let chosen = [...bucketOf.keys()];
+    if (chosen.length > maxItems) {
+        const firstRep = repIdx[0];
+        const lastRep = repIdx[repIdx.length - 1];
+        const forced = new Set();
+        if (firstRep !== undefined)
+            forced.add(firstRep);
+        // Only force the last endpoint if the cap has room — otherwise maxItems=1
+        // would over-keep 2 rows.
+        if (lastRep !== undefined && forced.size < maxItems)
+            forced.add(lastRep);
+        const ranked = chosen
+            .filter((i) => !forced.has(i))
+            .sort((a, b) => PRECEDENCE[bucketOf.get(a)] - PRECEDENCE[bucketOf.get(b)] || a - b);
+        const out = new Set(forced);
+        for (const i of ranked) {
+            if (out.size >= maxItems)
+                break;
+            out.add(i);
+        }
+        chosen = [...out];
+    }
+    const keptSorted = chosen.sort((a, b) => a - b);
+    // 4. Build the sampled table (ascending original order; remap absent).
+    const newIndexOf = new Map();
+    keptSorted.forEach((orig, idx) => newIndexOf.set(orig, idx));
+    const keptRows = keptSorted.map((i) => rows[i] ?? []);
+    const keptAbsent = [];
+    for (const [r, c] of absent) {
+        const nr = newIndexOf.get(r);
+        if (nr !== undefined)
+            keptAbsent.push([nr, c]);
+    }
+    const count = (b) => keptSorted.filter((i) => bucketOf.get(i) === b).length;
+    // Value-dictionary encode the kept rows (over kept indices, so dict + indices
+    // align). Decoded by the same expandColumnar deref the lossless path uses.
+    const colNames = cols.map((c) => c.name);
+    const { rows: dictRows, dict } = applyValueDictionaries(colNames, keptRows, keptAbsent);
+    return {
+        _table: {
+            cols: colNames,
+            rows: dictRows,
+            absent: keptAbsent,
+            ...(dict ? { dict } : {}),
+            n: originalCount,
+            sample: {
+                kept: keptSorted.length,
+                dropped: originalCount - keptSorted.length,
+                anchorsHead: count('head'),
+                anchorsTail: count('tail'),
+                outliers: count('outlier'),
+                matches: count('match'),
+                deduped,
+                srcRows: keptSorted,
+                ...(sortField ? { sortField } : {}),
+            },
+        },
+    };
+}

package/dist/table/sample-options.d.ts

@@ -0,0 +1,29 @@
+/**
+ * Knobs for the SmartCrusher row-sampler. All optional; defaults are
+ * deterministic. The sampler keeps representative rows (anchors / outliers /
+ * query matches / one-per-dedup-class) and drops the rest, recording why.
+ */
+export interface ISampleOptions {
+    /** Hard cap on KEPT rows. Default 200. */
+    maxItems?: number;
+    /** Rows kept from each of the front and back. Default 8. */
+    anchors?: number;
+    /** Numeric column to rank outliers on; auto-picked (highest variance) if omitted. */
+    outlierField?: string;
+    /** Extreme rows kept per tail (min side + max side). Default 8. */
+    outliers?: number;
+    /** Query whose tokens force-keep matching rows (up to `matches`). */
+    query?: string;
+    /** Max query-matched rows to force-keep. Default 16. */
+    matches?: number;
+    /** Collapse byte-identical rows to one representative (earliest). Default true. */
+    dedup?: boolean;
+    /**
+     * When no explicit `maxItems` is given, the cap is chosen adaptively from the
+     * data's information curve (P3.1). `bias` shifts that knee: keep more
+     * (`conservative`) or fewer (`aggressive`). Default `moderate`. Ignored when
+     * `maxItems` is set.
+     */
+    bias?: 'conservative' | 'moderate' | 'aggressive';
+}
+//# sourceMappingURL=sample-options.d.ts.map

package/dist/table/sample-options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sample-options.d.ts","sourceRoot":"","sources":["../../src/table/sample-options.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,WAAW,cAAc;IAC7B,0CAA0C;IAC1C,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,4DAA4D;IAC5D,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,qFAAqF;IACrF,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,mEAAmE;IACnE,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,qEAAqE;IACrE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wDAAwD;IACxD,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mFAAmF;IACnF,KAAK,CAAC,EAAE,OAAO,CAAC;IAChB;;;;;OAKG;IACH,IAAI,CAAC,EAAE,cAAc,GAAG,UAAU,GAAG,YAAY,CAAC;CACnD"}

package/dist/table/sample-options.js

@@ -0,0 +1 @@
+export {};

package/dist/table/sampled-table.d.ts

@@ -0,0 +1,33 @@
+/**
+ * A lossily-sampled columnar table. Extends the `_table` envelope (so
+ * `expandColumnar` still yields the KEPT rows verbatim) with a `sample` block
+ * recording how many rows were dropped and why. The FULL array is recovered
+ * from the CCR original, not from this structure.
+ */
+export interface ISampledTable {
+    _table: {
+        cols: string[];
+        rows: unknown[][];
+        absent: Array<[number, number]>;
+        /** Per-column value dictionaries (see {@link IColumnarTable}); decode via the same deref. */
+        dict?: Record<string, unknown[]>;
+        /** Original row count (kept + dropped) — surfaced in the sampling note. */
+        n: number;
+        sample: {
+            kept: number;
+            dropped: number;
+            anchorsHead: number;
+            anchorsTail: number;
+            outliers: number;
+            matches: number;
+            deduped: number;
+            /** Original index of each kept row, strictly ascending. */
+            srcRows: number[];
+            /** Numeric column used for outlier selection, if any. */
+            sortField?: string;
+        };
+    };
+}
+/** Type guard: a columnar table carrying sampling provenance. */
+export declare function isSampledTable(value: unknown): value is ISampledTable;
+//# sourceMappingURL=sampled-table.d.ts.map

package/dist/table/sampled-table.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"sampled-table.d.ts","sourceRoot":"","sources":["../../src/table/sampled-table.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,MAAM,WAAW,aAAa;IAC5B,MAAM,EAAE;QACN,IAAI,EAAE,MAAM,EAAE,CAAC;QACf,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC;QAClB,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QAChC,6FAA6F;QAC7F,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACjC,2EAA2E;QAC3E,CAAC,EAAE,MAAM,CAAC;QACV,MAAM,EAAE;YACN,IAAI,EAAE,MAAM,CAAC;YACb,OAAO,EAAE,MAAM,CAAC;YAChB,WAAW,EAAE,MAAM,CAAC;YACpB,WAAW,EAAE,MAAM,CAAC;YACpB,QAAQ,EAAE,MAAM,CAAC;YACjB,OAAO,EAAE,MAAM,CAAC;YAChB,OAAO,EAAE,MAAM,CAAC;YAChB,2DAA2D;YAC3D,OAAO,EAAE,MAAM,EAAE,CAAC;YAClB,yDAAyD;YACzD,SAAS,CAAC,EAAE,MAAM,CAAC;SACpB,CAAC;KACH,CAAC;CACH;AAED,iEAAiE;AACjE,wBAAgB,cAAc,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,aAAa,CAIrE"}

package/dist/table/sampled-table.js

@@ -0,0 +1,8 @@
+import { isColumnarTable } from "./columnar-json.js";
+/** Type guard: a columnar table carrying sampling provenance. */
+export function isSampledTable(value) {
+    if (!isColumnarTable(value))
+        return false;
+    const t = value._table;
+    return !!t && typeof t.sample === 'object' && t.sample !== null;
+}

package/dist/table/table-compaction.d.ts

@@ -0,0 +1,19 @@
+import type { IFieldSpec } from './field-spec.js';
+/**
+ * Lossless columnar form of a homogeneous object array. The shared schema is
+ * hoisted once into {@link cols}; each row carries only values, positionally.
+ * `absent` records `[row, col]` positions whose key was missing on the source
+ * object (distinct from a present null) so the original array reconstructs
+ * exactly.
+ */
+export interface ITableCompaction {
+    /** Hoisted column schema, in a deterministic order. */
+    cols: IFieldSpec[];
+    /** Row-major values; `rows[r][c]` aligns to `cols[c]`. */
+    rows: unknown[][];
+    /** `[row, col]` positions where the source object had no such key. */
+    absent: Array<[number, number]>;
+    /** Number of source objects (equals `rows.length`). */
+    originalCount: number;
+}
+//# sourceMappingURL=table-compaction.d.ts.map

package/dist/table/table-compaction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"table-compaction.d.ts","sourceRoot":"","sources":["../../src/table/table-compaction.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAElD;;;;;;GAMG;AACH,MAAM,WAAW,gBAAgB;IAC/B,uDAAuD;IACvD,IAAI,EAAE,UAAU,EAAE,CAAC;IACnB,0DAA0D;IAC1D,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC;IAClB,sEAAsE;IACtE,MAAM,EAAE,KAAK,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAChC,uDAAuD;IACvD,aAAa,EAAE,MAAM,CAAC;CACvB"}

package/dist/table/table-compaction.js

@@ -0,0 +1 @@
+export {};

package/dist/table/table-formats.d.ts

@@ -0,0 +1,23 @@
+import type { IColumnarTable } from './columnar-table.js';
+/**
+ * Alternative, model-read-friendly encodings of a columnar table (P4.2). The
+ * default wire shape is the columnar JSON envelope; these offer the same data
+ * as CSV or Markdown key/value blocks for shapes where a flat layout reads more
+ * accurately. Both are reversible: their inverse rebuilds the original object
+ * array exactly (cell values are JSON-encoded, so strings, numbers, booleans,
+ * null, and nested values all round-trip; an absent key stays absent).
+ *
+ * Whether any of these becomes a default awaits the P4.1 comprehension eval —
+ * until then they are opt-in. Pure and deterministic.
+ */
+type Row = Record<string, unknown>;
+/** Encode a columnar table as CSV: a header row of column names, then one row per record. */
+export declare function columnarToCsv(table: IColumnarTable): string;
+/** Inverse of {@link columnarToCsv}: rebuild the original object array. */
+export declare function csvToObjects(csv: string): Row[];
+/** Encode a columnar table as Markdown key/value blocks, one block per record. */
+export declare function columnarToMarkdownKv(table: IColumnarTable): string;
+/** Inverse of {@link columnarToMarkdownKv}: rebuild the original object array. */
+export declare function markdownKvToObjects(md: string): Row[];
+export {};
+//# sourceMappingURL=table-formats.d.ts.map

package/dist/table/table-formats.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"table-formats.d.ts","sourceRoot":"","sources":["../../src/table/table-formats.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAC;AAE1D;;;;;;;;;;GAUG;AAEH,KAAK,GAAG,GAAG,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;AA4EnC,6FAA6F;AAC7F,wBAAgB,aAAa,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM,CAc3D;AAED,2EAA2E;AAC3E,wBAAgB,YAAY,CAAC,GAAG,EAAE,MAAM,GAAG,GAAG,EAAE,CAyB/C;AAaD,kFAAkF;AAClF,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,cAAc,GAAG,MAAM,CAkBlE;AAED,kFAAkF;AAClF,wBAAgB,mBAAmB,CAAC,EAAE,EAAE,MAAM,GAAG,GAAG,EAAE,CAiDrD"}

package/dist/table/table-formats.js

@@ -0,0 +1,233 @@
+function csvEscapeField(s) {
+    return /[",\n\r]/.test(s) ? `"${s.replace(/"/g, '""')}"` : s;
+}
+/**
+ * Parse a whole CSV document into records of fields, honouring `"`-quoted
+ * fields with `""` escapes — including fields that contain commas, `\r`/`\n`,
+ * or the record separator itself. Splitting the document into lines BEFORE
+ * honouring quotes (the old approach) shattered any quoted field that carried
+ * a newline (e.g. a column name containing `\n`), so the inverse threw or
+ * dropped data. Record separators are unquoted `\n`, `\r\n`, or `\r`.
+ */
+function parseCsvRecords(csv) {
+    const records = [];
+    let row = [];
+    let field = '';
+    let inQuotes = false;
+    let pending = false; // the current record has started (content or a separator seen)
+    const endField = () => {
+        row.push(field);
+        field = '';
+        pending = true;
+    };
+    const endRecord = () => {
+        endField();
+        records.push(row);
+        row = [];
+        pending = false;
+    };
+    for (let i = 0; i < csv.length; i += 1) {
+        const ch = csv[i];
+        if (inQuotes) {
+            if (ch === '"') {
+                if (csv[i + 1] === '"') {
+                    field += '"';
+                    i += 1;
+                }
+                else
+                    inQuotes = false;
+            }
+            else
+                field += ch;
+            pending = true;
+        }
+        else if (ch === '"') {
+            inQuotes = true;
+            pending = true;
+        }
+        else if (ch === ',') {
+            endField();
+        }
+        else if (ch === '\n') {
+            endRecord();
+        }
+        else if (ch === '\r') {
+            endRecord();
+            if (csv[i + 1] === '\n')
+                i += 1;
+        }
+        else {
+            field += ch;
+            pending = true;
+        }
+    }
+    // Flush a trailing record unless the document ended exactly on a record
+    // separator (nothing pending), matching `out.join('\n')` having no trailer.
+    if (pending || field.length > 0 || row.length > 0)
+        endRecord();
+    return records;
+}
+/**
+ * Resolve a cell to its REAL value, dereferencing a dict index when the column
+ * is value-dictionary encoded. CSV/MD-KV are read-accuracy text formats, so they
+ * must show the value, not the index.
+ */
+function cellValue(table, r, c) {
+    const { cols, rows, dict } = table._table;
+    const raw = rows[r]?.[c];
+    const name = cols[c];
+    return dict && name !== undefined && Object.prototype.hasOwnProperty.call(dict, name)
+        ? dict[name][raw]
+        : raw;
+}
+/** Encode a columnar table as CSV: a header row of column names, then one row per record. */
+export function columnarToCsv(table) {
+    const { cols, rows, absent } = table._table;
+    const w = cols.length;
+    const absentSet = new Set(absent.map(([r, c]) => r * w + c));
+    const out = [cols.map(csvEscapeField).join(',')];
+    for (let r = 0; r < rows.length; r += 1) {
+        const cells = cols.map((_, c) => 
+        // Absent → a truly empty field; present → JSON-encoded value (always
+        // quoted by the escaper when it would otherwise be ambiguous).
+        absentSet.has(r * w + c) ? '' : csvEscapeField(JSON.stringify(cellValue(table, r, c))));
+        out.push(cells.join(','));
+    }
+    return out.join('\n');
+}
+/** Inverse of {@link columnarToCsv}: rebuild the original object array. */
+export function csvToObjects(csv) {
+    const records = parseCsvRecords(csv);
+    if (records.length === 0)
+        return [];
+    const header = records[0] ?? [];
+    const out = [];
+    for (let r = 1; r < records.length; r += 1) {
+        const fields = records[r] ?? [];
+        // A present value is always JSON-encoded (non-empty: even "" becomes `""`),
+        // so a truly empty parsed field unambiguously means the key was absent.
+        const obj = {};
+        for (let c = 0; c < header.length; c += 1) {
+            const name = header[c];
+            if (name === undefined)
+                continue;
+            const field = fields[c];
+            if (field === undefined || field === '')
+                continue; // absent key
+            Object.defineProperty(obj, name, {
+                value: JSON.parse(field),
+                writable: true,
+                enumerable: true,
+                configurable: true,
+            });
+        }
+        out.push(obj);
+    }
+    return out;
+}
+/**
+ * A column name is safe to write bare in a `key: value` line only when it can't
+ * collide with the `: ` separator or the line/record structure. Otherwise we
+ * JSON-encode it (a leading `"` is the decoder's signal) so the key survives the
+ * round trip — `markdownKvToObjects` splits on the FIRST `: `, which a key
+ * containing `: ` or a newline would otherwise break.
+ */
+function markdownKeyNeedsQuoting(key) {
+    return key.includes(': ') || key.includes('\n') || key.includes('\r') || key.startsWith('"');
+}
+/** Encode a columnar table as Markdown key/value blocks, one block per record. */
+export function columnarToMarkdownKv(table) {
+    const { cols, rows, absent } = table._table;
+    const w = cols.length;
+    const absentSet = new Set(absent.map(([r, c]) => r * w + c));
+    const blocks = [];
+    for (let r = 0; r < rows.length; r += 1) {
+        const present = [];
+        for (let c = 0; c < w; c += 1) {
+            if (absentSet.has(r * w + c))
+                continue;
+            const rawKey = String(cols[c]);
+            const keyText = markdownKeyNeedsQuoting(rawKey) ? JSON.stringify(rawKey) : rawKey;
+            present.push(`${keyText}: ${JSON.stringify(cellValue(table, r, c))}`);
+        }
+        // `- ` opens a record; remaining keys are indented two spaces.
+        const lines = present.map((line, idx) => (idx === 0 ? `- ${line}` : `  ${line}`));
+        blocks.push(lines.length > 0 ? lines.join('\n') : '-');
+    }
+    return blocks.join('\n');
+}
+/** Inverse of {@link columnarToMarkdownKv}: rebuild the original object array. */
+export function markdownKvToObjects(md) {
+    const out = [];
+    let cur = null;
+    const commit = () => {
+        if (cur)
+            out.push(cur);
+    };
+    for (const rawLine of md.split(/\r?\n/)) {
+        const opensRecord = rawLine.startsWith('- ');
+        const line = opensRecord ? rawLine.slice(2) : rawLine.replace(/^\s+/, '');
+        if (opensRecord) {
+            commit();
+            cur = {};
+        }
+        if (rawLine === '-') {
+            commit();
+            cur = {};
+            continue;
+        }
+        if (!cur)
+            continue;
+        let key;
+        let value;
+        if (line.startsWith('"')) {
+            // JSON-encoded key (emitted by markdownKeyNeedsQuoting): read the JSON
+            // string token, then the `: ` separator, then the JSON value.
+            const end = jsonStringEnd(line);
+            if (end < 0)
+                continue;
+            const rest = line.slice(end);
+            if (!rest.startsWith(': '))
+                continue;
+            try {
+                key = JSON.parse(line.slice(0, end));
+            }
+            catch {
+                continue;
+            }
+            value = rest.slice(2);
+        }
+        else {
+            const sep = line.indexOf(': ');
+            if (sep < 0)
+                continue;
+            key = line.slice(0, sep);
+            value = line.slice(sep + 2);
+        }
+        Object.defineProperty(cur, key, {
+            value: JSON.parse(value),
+            writable: true,
+            enumerable: true,
+            configurable: true,
+        });
+    }
+    commit();
+    return out;
+}
+/**
+ * Given a string whose char 0 is `"`, return the index just past the matching
+ * closing quote of that JSON string token (honouring `\"` escapes), or -1 if
+ * unterminated.
+ */
+function jsonStringEnd(s) {
+    for (let i = 1; i < s.length; i += 1) {
+        const ch = s[i];
+        if (ch === '\\') {
+            i += 1;
+            continue;
+        }
+        if (ch === '"')
+            return i + 1;
+    }
+    return -1;
+}

package/dist/text/compress-diff.d.ts

@@ -0,0 +1,20 @@
+import type { ICompressionResult } from '../result/compression-result.js';
+import type { ICompressOptions } from '../result/compress-options.js';
+/**
+ * Reduce a unified diff to its highest-signal lines. Two passes compose:
+ *
+ *  1. **Diff-noise offload** (this wrapper): lockfile sections
+ *     (`package-lock.json` & friends) collapse to a one-line marker, and
+ *     whitespace-only hunks (pure reindentation) collapse to a marker — the
+ *     single largest sources of useless diff tokens. Both are CCR-recoverable.
+ *  2. **Core hunk compression** ({@link compressDiffCore}): the remaining real
+ *     changes keep their changed lines plus a tight context window, capped per
+ *     file.
+ *
+ * The offload pass only engages when a `diff --git` section is actually a
+ * lockfile or contains a whitespace-only hunk; every other diff routes straight
+ * to the core compressor, byte-identical to before. Recoverable via CCR; output
+ * favours LLM readability over `git apply` fidelity.
+ */
+export declare function compressDiff(text: string, opts?: ICompressOptions): ICompressionResult;
+//# sourceMappingURL=compress-diff.d.ts.map

package/dist/text/compress-diff.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compress-diff.d.ts","sourceRoot":"","sources":["../../src/text/compress-diff.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AA0BtE;;;;;;;;;;;;;;;GAeG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,kBAAkB,CAiE1F"}