@shrkcrft/compress 0.1.0-alpha.16

package/dist/json/render-compact-json.d.ts

@@ -0,0 +1,10 @@
+/**
+ * Render a JSON value as the densest deterministic text. A homogeneous object
+ * array becomes a table block when that's shorter than minified JSON;
+ * everything else is minified JSON (whitespace stripped, structure intact).
+ * Lossless in information, though a table block is not itself re-parseable as
+ * JSON — use this only where the consumer reads text (the `compress` surface),
+ * not where a client calls `JSON.parse`.
+ */
+export declare function renderCompactJson(value: unknown): string;
+//# sourceMappingURL=render-compact-json.d.ts.map

package/dist/json/render-compact-json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"render-compact-json.d.ts","sourceRoot":"","sources":["../../src/json/render-compact-json.ts"],"names":[],"mappings":"AAGA;;;;;;;GAOG;AACH,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,CAMxD"}

package/dist/json/render-compact-json.js

@@ -0,0 +1,18 @@
+import { compactObjectArray } from "../table/compact-object-array.js";
+import { renderTable } from "../table/render-table.js";
+/**
+ * Render a JSON value as the densest deterministic text. A homogeneous object
+ * array becomes a table block when that's shorter than minified JSON;
+ * everything else is minified JSON (whitespace stripped, structure intact).
+ * Lossless in information, though a table block is not itself re-parseable as
+ * JSON — use this only where the consumer reads text (the `compress` surface),
+ * not where a client calls `JSON.parse`.
+ */
+export function renderCompactJson(value) {
+    const minified = JSON.stringify(value) ?? 'null';
+    const table = compactObjectArray(value);
+    if (!table)
+        return minified;
+    const text = renderTable(table);
+    return text.length < minified.length ? text : minified;
+}

package/dist/relevance/bm25.d.ts

@@ -0,0 +1,26 @@
+/**
+ * A small, pure BM25 relevance scorer for query-biased compression. Plain
+ * token-overlap counting ({@link queryOverlap}) is weak: it can't tell a term
+ * that appears in every row from a rare, discriminating one, and it under-
+ * weights single-term and ID/UUID exact matches. BM25 weights each query term
+ * by inverse document frequency and normalizes for row/line length, so a
+ * uniquely-relevant row outranks a row that merely repeats a common word.
+ *
+ * Deterministic: a pure function of (query, documents). No clock, no RNG, no
+ * learned state. When the query is empty every score is 0, so callers fall
+ * straight back to their no-query behaviour.
+ */
+export interface IBm25Options {
+    /** Term-frequency saturation. Default 1.2. */
+    k1?: number;
+    /** Length-normalization strength. Default 0.75. */
+    b?: number;
+}
+/** BM25 score per document for `query`. Empty query → all zeros. */
+export declare function bm25Scores(query: string, documents: readonly string[], opts?: IBm25Options): number[];
+/**
+ * Indices of the top-`k` documents by BM25 score (score > 0 only), highest
+ * first, ties broken by ascending index for determinism. Empty query → `[]`.
+ */
+export declare function topByBm25(query: string, documents: readonly string[], k: number, opts?: IBm25Options): number[];
+//# sourceMappingURL=bm25.d.ts.map

package/dist/relevance/bm25.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"bm25.d.ts","sourceRoot":"","sources":["../../src/relevance/bm25.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,MAAM,WAAW,YAAY;IAC3B,8CAA8C;IAC9C,EAAE,CAAC,EAAE,MAAM,CAAC;IACZ,mDAAmD;IACnD,CAAC,CAAC,EAAE,MAAM,CAAC;CACZ;AAuDD,oEAAoE;AACpE,wBAAgB,UAAU,CACxB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,IAAI,GAAE,YAAiB,GACtB,MAAM,EAAE,CAsCV;AAED;;;GAGG;AACH,wBAAgB,SAAS,CACvB,KAAK,EAAE,MAAM,EACb,SAAS,EAAE,SAAS,MAAM,EAAE,EAC5B,CAAC,EAAE,MAAM,EACT,IAAI,GAAE,YAAiB,GACtB,MAAM,EAAE,CAQV"}

package/dist/relevance/bm25.js

@@ -0,0 +1,115 @@
+/**
+ * A small, pure BM25 relevance scorer for query-biased compression. Plain
+ * token-overlap counting ({@link queryOverlap}) is weak: it can't tell a term
+ * that appears in every row from a rare, discriminating one, and it under-
+ * weights single-term and ID/UUID exact matches. BM25 weights each query term
+ * by inverse document frequency and normalizes for row/line length, so a
+ * uniquely-relevant row outranks a row that merely repeats a common word.
+ *
+ * Deterministic: a pure function of (query, documents). No clock, no RNG, no
+ * learned state. When the query is empty every score is 0, so callers fall
+ * straight back to their no-query behaviour.
+ */
+/** Extra idf weight for an exact ID-shaped term (UUID / long hex / email). */
+const ID_BOOST = 2.5;
+const UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+const HEX_RE = /^(?:0x)?[0-9a-f]{8,}$/i;
+const EMAIL_RE = /^[\w.+-]+@[\w-]+\.[\w.-]+$/;
+function isIdShaped(term) {
+    return UUID_RE.test(term) || HEX_RE.test(term) || EMAIL_RE.test(term);
+}
+/** Parse a query into terms, preserving ID-shaped chunks (UUIDs, emails) whole. */
+function parseQueryTerms(query) {
+    const seen = new Set();
+    const out = [];
+    for (const chunk of query.trim().split(/\s+/)) {
+        if (chunk.length === 0)
+            continue;
+        if (isIdShaped(chunk)) {
+            const t = chunk.toLowerCase();
+            if (!seen.has(t)) {
+                seen.add(t);
+                out.push({ term: t, id: true });
+            }
+            continue;
+        }
+        for (const sub of chunk.toLowerCase().split(/[^a-z0-9]+/)) {
+            if (sub.length < 1 || seen.has(sub))
+                continue;
+            seen.add(sub);
+            out.push({ term: sub, id: false });
+        }
+    }
+    return out;
+}
+function countOccurrences(haystack, needle) {
+    if (needle.length === 0)
+        return 0;
+    let count = 0;
+    let from = 0;
+    for (;;) {
+        const idx = haystack.indexOf(needle, from);
+        if (idx < 0)
+            break;
+        count += 1;
+        from = idx + needle.length;
+    }
+    return count;
+}
+/** BM25 score per document for `query`. Empty query → all zeros. */
+export function bm25Scores(query, documents, opts = {}) {
+    const n = documents.length;
+    if (n === 0)
+        return [];
+    const terms = parseQueryTerms(query);
+    if (terms.length === 0)
+        return documents.map(() => 0);
+    const k1 = opts.k1 ?? 1.2;
+    const b = opts.b ?? 0.75;
+    // Lowercase each document ONCE, then derive tokens from the lowercased form
+    // (was lowercased twice: in the tokenizer and again for ID substring matching).
+    const docLower = documents.map((d) => d.toLowerCase());
+    const docTokens = docLower.map((l) => l.split(/[^a-z0-9]+/).filter((t) => t.length > 0));
+    const dl = docTokens.map((t) => t.length);
+    const avgdl = dl.reduce((s, x) => s + x, 0) / n || 1;
+    const scores = new Array(n).fill(0);
+    // One reusable frequency buffer for all terms — every slot is overwritten each
+    // term below, so no per-term allocation (or reset) is needed.
+    const f = new Array(n).fill(0);
+    for (const { term, id } of terms) {
+        let df = 0;
+        for (let d = 0; d < n; d += 1) {
+            const count = id
+                ? countOccurrences(docLower[d], term)
+                : docTokens[d].reduce((s, t) => s + (t === term ? 1 : 0), 0);
+            f[d] = count;
+            if (count > 0)
+                df += 1;
+        }
+        if (df === 0)
+            continue;
+        let idf = Math.log(1 + (n - df + 0.5) / (df + 0.5));
+        if (id)
+            idf *= ID_BOOST;
+        for (let d = 0; d < n; d += 1) {
+            if (f[d] === 0)
+                continue;
+            const denom = f[d] + k1 * (1 - b + (b * dl[d]) / avgdl);
+            scores[d] += (idf * (f[d] * (k1 + 1))) / denom;
+        }
+    }
+    return scores;
+}
+/**
+ * Indices of the top-`k` documents by BM25 score (score > 0 only), highest
+ * first, ties broken by ascending index for determinism. Empty query → `[]`.
+ */
+export function topByBm25(query, documents, k, opts = {}) {
+    const scores = bm25Scores(query, documents, opts);
+    return scores
+        .map((score, index) => ({ score, index }))
+        .filter((x) => x.score > 0)
+        .sort((a, b) => b.score - a.score || a.index - b.index)
+        .slice(0, Math.max(0, k))
+        .map((x) => x.index);
+}

package/dist/result/compress-options.d.ts

@@ -0,0 +1,26 @@
+import type { EContentType } from '../content/content-type.js';
+import type { ICcrStore } from '../ccr/ccr-store.js';
+/**
+ * Knobs for a compression pass. All optional — the defaults produce a safe,
+ * deterministic reduction. `store` is what makes a lossy pass reversible: when
+ * present, the original is cached and a `<<ccr:…>>` marker is appended.
+ */
+export interface ICompressOptions {
+    /** Cache originals here so lossy output stays retrievable (CCR). */
+    store?: ICcrStore;
+    /** Task / query text that biases which lines or matches are kept. */
+    query?: string;
+    /** Force a content class instead of auto-detecting. */
+    contentType?: EContentType;
+    /** Soft cap on retained items/lines/matches/hunks (compressor-specific). */
+    maxItems?: number;
+    /** Below this many lines a lossy text pass returns the input untouched. */
+    minLines?: number;
+    /**
+     * Token budget for a JSON array. When set and the lossless columnar form
+     * still exceeds it, `compressJson` falls back to the lossy SmartCrusher
+     * row-sampler (kept rows + CCR original). Without it, JSON stays lossless.
+     */
+    maxTokens?: number;
+}
+//# sourceMappingURL=compress-options.d.ts.map

package/dist/result/compress-options.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compress-options.d.ts","sourceRoot":"","sources":["../../src/result/compress-options.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAErD;;;;GAIG;AACH,MAAM,WAAW,gBAAgB;IAC/B,oEAAoE;IACpE,KAAK,CAAC,EAAE,SAAS,CAAC;IAClB,qEAAqE;IACrE,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,uDAAuD;IACvD,WAAW,CAAC,EAAE,YAAY,CAAC;IAC3B,4EAA4E;IAC5E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB"}

package/dist/result/compress-options.js

@@ -0,0 +1 @@
+export {};

package/dist/result/compression-result.d.ts

@@ -0,0 +1,26 @@
+import type { EContentType } from '../content/content-type.js';
+import type { ECompressionStrategy } from './compression-strategy.js';
+import type { ITokenSavings } from '../tokens/token-savings.js';
+/**
+ * The outcome of compressing one blob. `compressed` is always safe to hand to
+ * a model as-is. When a lossy strategy dropped detail, `ccrKey` points at the
+ * cached original so the agent can call `retrieve_original` / `shrk expand`
+ * to get it back (Compress-Cache-Retrieve).
+ */
+export interface ICompressionResult {
+    /** The compressed text, ready for the model. */
+    compressed: string;
+    /** Detected (or caller-forced) content class. */
+    contentType: EContentType;
+    /** Strategy that produced `compressed`. */
+    strategy: ECompressionStrategy;
+    /** Token accounting for the pass. */
+    savings: ITokenSavings;
+    /** True when detail was dropped (and an original was cached, if a store was given). */
+    lossy: boolean;
+    /** CCR key for the cached original, when a lossy pass stored one. */
+    ccrKey?: string;
+    /** A one-line, human/agent-readable note about what happened. */
+    note: string;
+}
+//# sourceMappingURL=compression-result.d.ts.map

package/dist/result/compression-result.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compression-result.d.ts","sourceRoot":"","sources":["../../src/result/compression-result.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AAC/D,OAAO,KAAK,EAAE,oBAAoB,EAAE,MAAM,2BAA2B,CAAC;AACtE,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,4BAA4B,CAAC;AAEhE;;;;;GAKG;AACH,MAAM,WAAW,kBAAkB;IACjC,gDAAgD;IAChD,UAAU,EAAE,MAAM,CAAC;IACnB,iDAAiD;IACjD,WAAW,EAAE,YAAY,CAAC;IAC1B,2CAA2C;IAC3C,QAAQ,EAAE,oBAAoB,CAAC;IAC/B,qCAAqC;IACrC,OAAO,EAAE,aAAa,CAAC;IACvB,uFAAuF;IACvF,KAAK,EAAE,OAAO,CAAC;IACf,qEAAqE;IACrE,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;CACd"}

package/dist/result/compression-result.js

@@ -0,0 +1 @@
+export {};

package/dist/result/compression-strategy.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Which deterministic strategy produced a compression result. Surfaced so
+ * callers (and tests) can assert on *how* a payload shrank, not just that it
+ * did.
+ */
+export declare enum ECompressionStrategy {
+    /** No transform applied — output equals input (below threshold, or unknown shape). */
+    Passthrough = "passthrough",
+    /** Lossless columnar/table compaction of a homogeneous object array. */
+    Table = "table",
+    /** Log line-reduction: kept errors/warnings/summaries, dropped the rest. */
+    Log = "log",
+    /** Search-result reduction: kept the highest-signal `file:line` matches. */
+    Search = "search",
+    /** Diff reduction: capped files/hunks and trimmed surrounding context. */
+    Diff = "diff",
+    /** Generic line dedup for prose / plain text. */
+    Lines = "lines",
+    /** Code outline: kept imports / types / signatures, elided function bodies. */
+    Code = "code",
+    /** Markdown distilled to headers, section leads, list/table structure; bodies thinned. */
+    Markdown = "markdown",
+    /** Minified JSON (whitespace removed, structure preserved). */
+    MinifiedJson = "minified-json",
+    /** Lossy statistical row-sample of a huge homogeneous array (SmartCrusher). */
+    Sample = "sample",
+    /** Mixed blob segmented by type; each run compressed with its own strategy. */
+    Mixed = "mixed"
+}
+//# sourceMappingURL=compression-strategy.d.ts.map

package/dist/result/compression-strategy.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compression-strategy.d.ts","sourceRoot":"","sources":["../../src/result/compression-strategy.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,oBAAY,oBAAoB;IAC9B,sFAAsF;IACtF,WAAW,gBAAgB;IAC3B,wEAAwE;IACxE,KAAK,UAAU;IACf,4EAA4E;IAC5E,GAAG,QAAQ;IACX,4EAA4E;IAC5E,MAAM,WAAW;IACjB,0EAA0E;IAC1E,IAAI,SAAS;IACb,iDAAiD;IACjD,KAAK,UAAU;IACf,+EAA+E;IAC/E,IAAI,SAAS;IACb,0FAA0F;IAC1F,QAAQ,aAAa;IACrB,+DAA+D;IAC/D,YAAY,kBAAkB;IAC9B,+EAA+E;IAC/E,MAAM,WAAW;IACjB,+EAA+E;IAC/E,KAAK,UAAU;CAChB"}

package/dist/result/compression-strategy.js

@@ -0,0 +1,30 @@
+/**
+ * Which deterministic strategy produced a compression result. Surfaced so
+ * callers (and tests) can assert on *how* a payload shrank, not just that it
+ * did.
+ */
+export var ECompressionStrategy;
+(function (ECompressionStrategy) {
+    /** No transform applied — output equals input (below threshold, or unknown shape). */
+    ECompressionStrategy["Passthrough"] = "passthrough";
+    /** Lossless columnar/table compaction of a homogeneous object array. */
+    ECompressionStrategy["Table"] = "table";
+    /** Log line-reduction: kept errors/warnings/summaries, dropped the rest. */
+    ECompressionStrategy["Log"] = "log";
+    /** Search-result reduction: kept the highest-signal `file:line` matches. */
+    ECompressionStrategy["Search"] = "search";
+    /** Diff reduction: capped files/hunks and trimmed surrounding context. */
+    ECompressionStrategy["Diff"] = "diff";
+    /** Generic line dedup for prose / plain text. */
+    ECompressionStrategy["Lines"] = "lines";
+    /** Code outline: kept imports / types / signatures, elided function bodies. */
+    ECompressionStrategy["Code"] = "code";
+    /** Markdown distilled to headers, section leads, list/table structure; bodies thinned. */
+    ECompressionStrategy["Markdown"] = "markdown";
+    /** Minified JSON (whitespace removed, structure preserved). */
+    ECompressionStrategy["MinifiedJson"] = "minified-json";
+    /** Lossy statistical row-sample of a huge homogeneous array (SmartCrusher). */
+    ECompressionStrategy["Sample"] = "sample";
+    /** Mixed blob segmented by type; each run compressed with its own strategy. */
+    ECompressionStrategy["Mixed"] = "mixed";
+})(ECompressionStrategy || (ECompressionStrategy = {}));

package/dist/table/adaptive-size.d.ts

@@ -0,0 +1,46 @@
+/**
+ * Adaptive sample sizing — pick how many items to keep from the *shape of the
+ * information*, not a fixed K. Fixed caps keep too much on redundant data and
+ * too little on diverse data. {@link computeOptimalK} finds the knee of the
+ * unique-bigram coverage curve (where adding more items stops adding
+ * information), cross-checked against simhash near-duplicate collapse and a
+ * zlib redundancy bound.
+ *
+ * Pure and deterministic: a function of the input bytes only (no clock, no RNG,
+ * no learned state). zlib `deflate` is a fixed, deterministic transform — a
+ * lookup-driven coder, not a model — used only as a redundancy *measure*.
+ */
+export type AdaptiveBias = 'conservative' | 'moderate' | 'aggressive';
+export interface IAdaptiveOptions {
+    /** Lower bound on the result (e.g. a representative-rows floor). Default 1. */
+    min?: number;
+    /** Upper bound on the result (e.g. an existing fixed cap). Default items.length. */
+    max?: number;
+    /** Shifts the knee: keep more (conservative) or fewer (aggressive). Default 'moderate'. */
+    bias?: AdaptiveBias;
+}
+/**
+ * A 32-bit simhash over an item's token bigrams. Near-identical items map to
+ * near-identical hashes (small Hamming distance), so redundant rows don't each
+ * count as "new information".
+ */
+export declare function simhash(text: string): number;
+export declare function hammingDistance(a: number, b: number): number;
+/**
+ * Index of the knee of a monotone-nondecreasing curve via Kneedle: the point of
+ * maximum deflection from the chord between its endpoints. Returns `{ index,
+ * deflection }` where `deflection` is ~0 for a straight (no-knee) curve.
+ */
+export declare function kneedle(curve: readonly number[]): {
+    index: number;
+    deflection: number;
+};
+/** The cumulative count of distinct bigrams as items are added in order. */
+export declare function bigramCoverageCurve(items: readonly string[]): number[];
+/**
+ * Choose how many of `items` to keep. Deterministic. The result is clamped to
+ * `[min, max]` (and never exceeds `items.length`), so an explicit cap passed as
+ * `max` is always honoured.
+ */
+export declare function computeOptimalK(items: readonly string[], opts?: IAdaptiveOptions): number;
+//# sourceMappingURL=adaptive-size.d.ts.map

package/dist/table/adaptive-size.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"adaptive-size.d.ts","sourceRoot":"","sources":["../../src/table/adaptive-size.ts"],"names":[],"mappings":"AAGA;;;;;;;;;;;GAWG;AAEH,MAAM,MAAM,YAAY,GAAG,cAAc,GAAG,UAAU,GAAG,YAAY,CAAC;AAEtE,MAAM,WAAW,gBAAgB;IAC/B,+EAA+E;IAC/E,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,oFAAoF;IACpF,GAAG,CAAC,EAAE,MAAM,CAAC;IACb,2FAA2F;IAC3F,IAAI,CAAC,EAAE,YAAY,CAAC;CACrB;AA0BD;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAE5C;AAwBD,wBAAgB,eAAe,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,MAAM,GAAG,MAAM,CAQ5D;AAED;;;;GAIG;AACH,wBAAgB,OAAO,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG;IAAE,KAAK,EAAE,MAAM,CAAC;IAAC,UAAU,EAAE,MAAM,CAAA;CAAE,CAmBvF;AAED,4EAA4E;AAC5E,wBAAgB,mBAAmB,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,GAAG,MAAM,EAAE,CAEtE;AA2CD;;;;GAIG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,SAAS,MAAM,EAAE,EAAE,IAAI,GAAE,gBAAqB,GAAG,MAAM,CA+B7F"}

package/dist/table/adaptive-size.js

@@ -0,0 +1,170 @@
+import { deflateSync } from 'node:zlib';
+import { Buffer } from 'node:buffer';
+const BIAS_FACTOR = {
+    conservative: 1.5,
+    moderate: 1.0,
+    aggressive: 0.6,
+};
+/** Hamming distance below which two simhashes are "the same information". */
+const NEAR_DUP_HAMMING = 6;
+/** A coverage curve flatter than this (max knee deflection) is "diverse" → keep max. */
+const FLATNESS_EPSILON = 0.08;
+function tokenize(text) {
+    return text.toLowerCase().split(/[^a-z0-9]+/).filter((t) => t.length > 0);
+}
+/** The set of adjacent token bigrams in one item (unigrams when it has one token). */
+function bigrams(text) {
+    const toks = tokenize(text);
+    if (toks.length <= 1)
+        return new Set(toks);
+    const out = new Set();
+    for (let i = 0; i + 1 < toks.length; i += 1)
+        out.add(`${toks[i]}
${toks[i + 1]}`);
+    return out;
+}
+/**
+ * A 32-bit simhash over an item's token bigrams. Near-identical items map to
+ * near-identical hashes (small Hamming distance), so redundant rows don't each
+ * count as "new information".
+ */
+export function simhash(text) {
+    return simhashOfGrams(bigrams(text));
+}
+/** Simhash over an already-computed bigram set (so callers can reuse the set). */
+function simhashOfGrams(grams) {
+    const v = new Array(32).fill(0);
+    for (const g of grams) {
+        const h = hash32(g);
+        for (let b = 0; b < 32; b += 1)
+            v[b] += (h >>> b) & 1 ? 1 : -1;
+    }
+    let out = 0;
+    for (let b = 0; b < 32; b += 1)
+        if (v[b] > 0)
+            out |= 1 << b;
+    return out >>> 0;
+}
+function hash32(s) {
+    // FNV-1a — a fixed, deterministic non-cryptographic hash.
+    let h = 0x811c9dc5;
+    for (let i = 0; i < s.length; i += 1) {
+        h ^= s.charCodeAt(i);
+        h = Math.imul(h, 0x01000193);
+    }
+    return h >>> 0;
+}
+export function hammingDistance(a, b) {
+    let x = (a ^ b) >>> 0;
+    let count = 0;
+    while (x !== 0) {
+        x &= x - 1;
+        count += 1;
+    }
+    return count;
+}
+/**
+ * Index of the knee of a monotone-nondecreasing curve via Kneedle: the point of
+ * maximum deflection from the chord between its endpoints. Returns `{ index,
+ * deflection }` where `deflection` is ~0 for a straight (no-knee) curve.
+ */
+export function kneedle(curve) {
+    const n = curve.length;
+    if (n <= 2)
+        return { index: n - 1, deflection: 0 };
+    const first = curve[0];
+    const last = curve[n - 1];
+    const span = last - first;
+    if (span <= 0)
+        return { index: 0, deflection: 0 };
+    let bestIdx = n - 1;
+    let bestDef = 0;
+    for (let i = 0; i < n; i += 1) {
+        const xNorm = i / (n - 1);
+        const yNorm = (curve[i] - first) / span;
+        const def = yNorm - xNorm; // concave (saturating) curves bulge above the chord
+        if (def > bestDef) {
+            bestDef = def;
+            bestIdx = i;
+        }
+    }
+    return { index: bestIdx, deflection: bestDef };
+}
+/** The cumulative count of distinct bigrams as items are added in order. */
+export function bigramCoverageCurve(items) {
+    return coverageFromGrams(items.map(bigrams));
+}
+/** Cumulative distinct-bigram curve over already-computed gram sets. */
+function coverageFromGrams(gramSets) {
+    const seen = new Set();
+    const curve = [];
+    for (const grams of gramSets) {
+        for (const g of grams)
+            seen.add(g);
+        curve.push(seen.size);
+    }
+    return curve;
+}
+/** Recent-hash window for the near-dup scan, keeping it O(n) on large inputs. */
+const NEAR_DUP_WINDOW = 64;
+/**
+ * Count of items that are NOT a simhash near-duplicate of a RECENT earlier item.
+ * Bounded to a sliding window so it stays linear; near-duplicates in sampler
+ * data cluster locally, and this is only an upper-bound cross-check anyway.
+ */
+function uniqueFromGrams(gramSets) {
+    const window = [];
+    let unique = 0;
+    for (const grams of gramSets) {
+        const h = simhashOfGrams(grams);
+        if (!window.some((k) => hammingDistance(k, h) <= NEAR_DUP_HAMMING)) {
+            unique += 1;
+            window.push(h);
+            if (window.length > NEAR_DUP_WINDOW)
+                window.shift();
+        }
+    }
+    return unique;
+}
+/** zlib redundancy ratio of the joined corpus in (0, 1]; lower ⇒ more redundant. */
+function zlibRedundancy(items) {
+    const raw = items.join('\n');
+    if (raw.length === 0)
+        return 1;
+    const deflated = deflateSync(Buffer.from(raw, 'utf8')).length;
+    return Math.min(1, deflated / Buffer.byteLength(raw, 'utf8'));
+}
+/**
+ * Choose how many of `items` to keep. Deterministic. The result is clamped to
+ * `[min, max]` (and never exceeds `items.length`), so an explicit cap passed as
+ * `max` is always honoured.
+ */
+export function computeOptimalK(items, opts = {}) {
+    const n = items.length;
+    const min = Math.max(1, opts.min ?? 1);
+    const max = Math.min(opts.max ?? n, n);
+    if (n <= min)
+        return n;
+    if (max <= min)
+        return min;
+    // Tokenize + bigram each item ONCE, then reuse the sets for both the coverage
+    // curve and the simhash near-dup scan (was computed twice).
+    const gramSets = items.map(bigrams);
+    const curve = coverageFromGrams(gramSets);
+    const knee = kneedle(curve);
+    let k;
+    if (knee.deflection < FLATNESS_EPSILON) {
+        // Near-linear coverage ⇒ each item adds information ⇒ keep the most.
+        k = max;
+    }
+    else {
+        k = knee.index + 1;
+    }
+    // Bias shifts how far past the knee we keep.
+    k = Math.round(k * BIAS_FACTOR[opts.bias ?? 'moderate']);
+    // Never keep more than the number of non-near-duplicate items.
+    k = Math.min(k, uniqueFromGrams(gramSets));
+    // Very redundant corpora (low zlib ratio) pull K down toward the floor.
+    if (zlibRedundancy(items) < 0.25)
+        k = Math.min(k, Math.max(min, Math.ceil(max * 0.25)));
+    return Math.max(min, Math.min(max, k));
+}

package/dist/table/apply-value-dictionaries.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Per-column value-dictionary (enum) encoding for the columnar table — a
+ * lossless token reduction for low-cardinality columns. The columnar form
+ * already hoists the schema (column names written once), but a column like the
+ * knowledge graph's `kind`/`relation`/`source` still writes its value once per
+ * row. This pass replaces those repeats with a one-time dictionary plus a small
+ * integer index per row.
+ *
+ * Disambiguation: a cell is a dict INDEX iff its column name is a key of the
+ * returned `dict` (decided structurally, never by the cell's value/type), so a
+ * literal-number column and a dict-encoded numeric enum never collide. Absent
+ * cells get no index (they stay in `absent` and are skipped before any deref).
+ *
+ * Pure and deterministic (dict values in first-appearance order). Two guards
+ * ensure it NEVER inflates: a per-column byte check, and a table-level byte
+ * fallback that returns the dict-free rows when the dict didn't actually shrink.
+ */
+export interface IValueDictResult {
+    /** Rows with dict-encoded columns rewritten to indices (or the input rows unchanged). */
+    rows: unknown[][];
+    /** Per-column value tables; present only when at least one column was encoded. */
+    dict?: Record<string, unknown[]>;
+}
+/**
+ * Dictionary-encode every low-cardinality column that strictly shrinks. Returns
+ * the (possibly unchanged) rows and an optional `dict`. The input `rows` is
+ * never mutated — it is cloned lazily only if a column is encoded.
+ */
+export declare function applyValueDictionaries(cols: readonly string[], rows: readonly unknown[][], absent: ReadonlyArray<readonly [number, number]>): IValueDictResult;
+//# sourceMappingURL=apply-value-dictionaries.d.ts.map

package/dist/table/apply-value-dictionaries.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"apply-value-dictionaries.d.ts","sourceRoot":"","sources":["../../src/table/apply-value-dictionaries.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AAOH,MAAM,WAAW,gBAAgB;IAC/B,yFAAyF;IACzF,IAAI,EAAE,OAAO,EAAE,EAAE,CAAC;IAClB,kFAAkF;IAClF,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;CAClC;AAOD;;;;GAIG;AACH,wBAAgB,sBAAsB,CACpC,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,IAAI,EAAE,SAAS,OAAO,EAAE,EAAE,EAC1B,MAAM,EAAE,aAAa,CAAC,SAAS,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,GAC/C,gBAAgB,CAkElB"}