@shrkcrft/compress 0.1.0-alpha.16

package/dist/content/content-type.js

@@ -0,0 +1,28 @@
+/**
+ * Coarse content classes the router recognises. The class selects which
+ * deterministic compressor runs. Ordered loosely from most-specific
+ * (cheapest to over-trigger) to least.
+ */
+export var EContentType;
+(function (EContentType) {
+    /** A JSON array (top-level `[ ... ]`). The table compactor's prime target. */
+    EContentType["JsonArray"] = "json-array";
+    /** A JSON object or scalar (top-level `{ ... }` / value). */
+    EContentType["Json"] = "json";
+    /** A unified/`git` diff. */
+    EContentType["GitDiff"] = "git-diff";
+    /** grep/ripgrep `file:line:` style search output. */
+    EContentType["SearchResults"] = "search-results";
+    /** Build / test / runtime log output. */
+    EContentType["BuildLog"] = "build-log";
+    /** Source code in a recognised language. */
+    EContentType["SourceCode"] = "source-code";
+    /** Markdown prose / docs. */
+    EContentType["Markdown"] = "markdown";
+    /** YAML configuration / manifests (`key: value` mappings + `- ` lists). */
+    EContentType["Yaml"] = "yaml";
+    /** Delimiter-separated values (CSV / TSV): a stable column count per line. */
+    EContentType["Csv"] = "csv";
+    /** Anything else. */
+    EContentType["PlainText"] = "plain-text";
+})(EContentType || (EContentType = {}));

package/dist/content/detect-content-type.d.ts

@@ -0,0 +1,9 @@
+import { EContentType } from './content-type.js';
+/**
+ * Classify a blob deterministically. Order is significant: JSON is checked
+ * first (it round-trips cleanly through `JSON.parse`), then structural
+ * formats (diff/search), then heuristic ones (log/code/markdown), with
+ * plain text as the floor. Pure — same bytes in, same class out.
+ */
+export declare function detectContentType(text: string): EContentType;
+//# sourceMappingURL=detect-content-type.d.ts.map

package/dist/content/detect-content-type.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"detect-content-type.d.ts","sourceRoot":"","sources":["../../src/content/detect-content-type.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AA0FjD;;;;;GAKG;AACH,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,MAAM,GAAG,YAAY,CAkG5D"}

package/dist/content/detect-content-type.js

@@ -0,0 +1,184 @@
+import { EContentType } from "./content-type.js";
+const SEARCH_LINE = /^(?:[A-Za-z]:)?[^\s:]+:\d+:/;
+// Compiler diagnostics that aren't `path:line:` shaped: tsc / MSVC
+// `src/a.ts(10,5): error TS2322` and the `path(line):` family. These are search
+// output, not source code — routing them to SourceCode mangled them.
+const DIAGNOSTIC_LINE = /^(?:[A-Za-z]:)?[^\s:()]+\(\d+(?:,\d+)?\):\s/;
+const DIFF_HUNK = /^@@ -\d+(?:,\d+)? \+\d+(?:,\d+)? @@/;
+// YAML: `key:` / `key: value` mappings, `- ` sequence items, `---` doc markers.
+const YAML_KEY = /^\s*[\w.-]+:(?:\s|$)/;
+const YAML_LINE = /^\s*(?:[\w.-]+:(?:\s|$)|-\s|#|---\s*$|\.\.\.\s*$)/;
+// A block-introducing key (`items:` with no inline value) followed by indented
+// sequence items is unambiguously YAML — a Markdown list never has a bare
+// `word:` line introducing indented bullets. Distinguishes list-heavy YAML
+// (low key density) from a Markdown bullet list with an incidental `Note: x`.
+const YAML_BLOCK_KEY = /^\s*[\w.-]+:\s*$/;
+const YAML_INDENTED_SEQ = /^\s{2,}-\s/;
+// Log levels must appear as a LINE PREFIX (optionally after a leading
+// timestamp / bracket), not anywhere on the line — otherwise common code
+// identifiers (`const ERROR = 500`, `enum { INFO, DEBUG }`) misroute to logs.
+const LOG_MARKER = /^\s*(?:(?:\[?\d{4}-\d{2}-\d{2}[T ][\d:.,]+\]?|\S+\[\d+\]:)\s+)?\[?(?:ERROR|FATAL|FAIL(?:ED|URE)?|WARN(?:ING)?|INFO|DEBUG|NOTICE|TRACE)\b|^\S+\[\d+\]:\s|^\s*Traceback\b|^\s+at\s+\S+\s*\(|^\d{4}-\d{2}-\d{2}[T ]\d{2}:\d{2}/;
+const MARKDOWN_MARKER = /^(?:#{1,6}\s|\s*[-*]\s|\s*\d+\.\s|```|>\s|\|)/;
+// Code signals: declaration keywords + structural/statement shapes that real
+// code has but prose / markdown / INI / env / TOML / nginx do NOT (each new
+// signal measures 0.00 on those). Built as a union of annotated sources.
+const CODE_MARKER = new RegExp([
+    // declaration / punctuation (original)
+    /\b(?:function|const|let|var|class|interface|enum|import|export|def|return|public|private|func|impl|struct|package|namespace)\b/,
+    /=>/,
+    /::/,
+    /^\s*@\w+/,
+    /^[\s{}()\[\];,]+$/,
+    // typed return/param annotation: `): Foo {` / `): Foo =>` / `]: Bar =`
+    /[)\]]\s*:\s*[A-Za-z_$][\w$.<>\[\], ]*\s*(?:=>|\{|=|$)/,
+    // assignment statement terminated by `;` (rejects ==/=== via [^=]; needs trailing ;)
+    /^\s*[A-Za-z_$][\w$.[\]]*\s*(?:\+|-|\*|\/|%|\?\?|\|\||&&|<<|>>|\||&|\^)?=\s*[^=].*;\s*$/,
+    // member/method call AT LINE START: `obj.method(` (anchored so prose
+    // "the system.config()" / "noun.verb(" embedded mid-sentence doesn't match)
+    /^\s*[A-Za-z_$][\w$]*\.[A-Za-z_$][\w$]*\s*\(/,
+    // bare call statement AT LINE START ending in `;`: `doThing(args);` (anchored
+    // so log lines like "Calling fetchUser(42);" don't match)
+    /^\s*[A-Za-z_$][\w$]*\([^()]*\)\s*;\s*$/,
+    // control-flow header: `if (x) {`, `for (...)`, `while/switch/catch (...)`
+    /^\s*(?:if|for|while|switch|catch)\s*\(.*\)\s*\{?\s*$/,
+]
+    .map((r) => r.source)
+    .join('|'));
+function lineHitRatio(lines, test) {
+    if (lines.length === 0)
+        return 0;
+    const match = typeof test === 'function' ? test : (l) => test.test(l);
+    let hits = 0;
+    for (const line of lines)
+        if (match(line))
+            hits += 1;
+    return hits / lines.length;
+}
+/**
+ * Delimiter-separated values: a stable column count per line. Returns true when
+ * one of `,`/`\t`/`;` yields the SAME count (≥1) on ≥90% of the non-blank lines
+ * (≥2 of them) — a shape prose and config never have.
+ */
+function looksDelimited(nonBlank) {
+    if (nonBlank.length < 2)
+        return false;
+    // `;` is excluded: semicolon-terminated prose/code lines have a stable count
+    // of 1 and would masquerade as 2-column CSV. Real CSV/TSV uses `,` or tab.
+    for (const delim of [',', '\t']) {
+        const counts = nonBlank.map((l) => l.split(delim).length - 1);
+        const freq = new Map();
+        for (const c of counts)
+            freq.set(c, (freq.get(c) ?? 0) + 1);
+        let modal = -1;
+        let modalFreq = 0;
+        for (const [c, f] of freq) {
+            if (f > modalFreq || (f === modalFreq && c > modal)) {
+                modal = c;
+                modalFreq = f;
+            }
+        }
+        // A real CSV/TSV has the same column count (≥2 columns ⇒ ≥1 delimiter) on
+        // almost every line.
+        if (modal >= 1 && modalFreq / nonBlank.length >= 0.9)
+            return true;
+    }
+    return false;
+}
+/**
+ * Classify a blob deterministically. Order is significant: JSON is checked
+ * first (it round-trips cleanly through `JSON.parse`), then structural
+ * formats (diff/search), then heuristic ones (log/code/markdown), with
+ * plain text as the floor. Pure — same bytes in, same class out.
+ */
+export function detectContentType(text) {
+    const trimmed = text.trim();
+    if (trimmed.length === 0)
+        return EContentType.PlainText;
+    // 1. JSON — only when it actually parses, so we never mis-route prose that
+    //    merely starts with a bracket.
+    const first = trimmed[0];
+    if (first === '[' || first === '{') {
+        try {
+            const parsed = JSON.parse(trimmed);
+            if (Array.isArray(parsed))
+                return EContentType.JsonArray;
+            if (parsed !== null && typeof parsed === 'object')
+                return EContentType.Json;
+            return EContentType.Json;
+        }
+        catch {
+            // fall through — not valid JSON
+        }
+    }
+    const lines = trimmed.split('\n');
+    // 2. Git diff — a `diff --git` header or a clear hunk header set. Scan ALL
+    //    `@@` lines (a leading malformed hunk must not defeat detection).
+    if (/^diff --git /m.test(trimmed) ||
+        (lines.some((l) => DIFF_HUNK.test(l)) &&
+            /^--- /m.test(trimmed) &&
+            /^\+\+\+ /m.test(trimmed))) {
+        return EContentType.GitDiff;
+    }
+    // 3. grep / ripgrep search output (`path:line:` prefix) OR compiler
+    //    diagnostics (`path(line,col):`). Count either shape toward the ratio.
+    if (lineHitRatio(lines, (l) => SEARCH_LINE.test(l) || DIAGNOSTIC_LINE.test(l)) >= 0.6) {
+        return EContentType.SearchResults;
+    }
+    // 4. Build / test log (error / warn / timestamp markers dense enough).
+    if (lineHitRatio(lines, LOG_MARKER) >= 0.25)
+        return EContentType.BuildLog;
+    // 5. Source code — keyword / structural density over non-blank lines OUTSIDE
+    //    fenced code blocks (a markdown doc's ``` examples must not be counted as
+    //    the doc's own code). A real source file has no fences, so its basis is
+    //    unchanged. EOL punctuation alone must NOT count (prose/config ending in
+    //    `;` is not code).
+    // Only TOP-LEVEL fences (CommonMark allows ≤3 leading spaces) count — an
+    // indented backtick line shown as a prose example must not toggle the fence
+    // state and skew the balance check.
+    const fenceRe = /^ {0,3}(?:```|~~~)/;
+    // Only trust fence exclusion when fences are balanced. An odd (unterminated)
+    // count — e.g. a stray ``` inside a source file's string/comment — would
+    // otherwise flip `inFence` forever and exclude the rest of the file.
+    const fenceCount = lines.reduce((n, l) => (fenceRe.test(l) ? n + 1 : n), 0);
+    const excludeFences = fenceCount > 0 && fenceCount % 2 === 0;
+    let inFence = false;
+    const codeBasis = [];
+    for (const l of lines) {
+        if (excludeFences && fenceRe.test(l)) {
+            inFence = !inFence;
+            continue;
+        }
+        if (!inFence && l.trim().length > 0)
+            codeBasis.push(l);
+    }
+    const codeRatio = codeBasis.length > 0 ? lineHitRatio(codeBasis, CODE_MARKER) : 0;
+    if (codeRatio >= 0.45)
+        return EContentType.SourceCode;
+    const nonBlank = lines.filter((l) => l.trim().length > 0);
+    // 5b. CSV / TSV — a stable column count per line (checked before YAML/markdown
+    //     so a 2-column file isn't mistaken for `key: value` or a list).
+    if (looksDelimited(nonBlank))
+        return EContentType.Csv;
+    // 5c. YAML — ≥80% of non-blank lines are YAML-shaped AND either ≥30% are
+    //     actual `key:` mappings (mapping-heavy config) OR there's a block-key →
+    //     indented-sequence shape (list-heavy config). Both reject a plain
+    //     Markdown bullet list. Checked before markdown, which would otherwise
+    //     grab YAML's `- ` sequence items and lossily cap them.
+    if (nonBlank.length >= 2) {
+        const yamlShaped = lineHitRatio(nonBlank, YAML_LINE);
+        const keyDensity = lineHitRatio(nonBlank, YAML_KEY);
+        const blockSeq = nonBlank.some((l) => YAML_BLOCK_KEY.test(l)) && nonBlank.some((l) => YAML_INDENTED_SEQ.test(l));
+        if (yamlShaped >= 0.8 && (keyDensity >= 0.3 || blockSeq))
+            return EContentType.Yaml;
+    }
+    // 6. Markdown — a marker-dense blob, OR a prose doc with ≥2 ATX headers. The
+    //    header rule is gated so a commented script (Python/shell `# …` lines, or
+    //    a `#!`-shebang file) with low code-syntax density isn't mistaken for a doc.
+    const headerCount = lines.reduce((n, l) => (/^#{1,6}\s/.test(l) ? n + 1 : n), 0);
+    const looksLikeScript = (lines[0] ?? '').startsWith('#!');
+    if (lineHitRatio(lines, MARKDOWN_MARKER) >= 0.3 ||
+        (headerCount >= 2 && codeRatio < 0.15 && !looksLikeScript)) {
+        return EContentType.Markdown;
+    }
+    return EContentType.PlainText;
+}

package/dist/content/segment.d.ts

@@ -0,0 +1,21 @@
+import { EContentType } from './content-type.js';
+/**
+ * A typed run of a mixed blob. {@link segmentContent} splits a heterogeneous
+ * dump — prose interleaved with a JSON block and a stack trace, say — into
+ * contiguous runs so each can be compressed by its own strategy instead of
+ * forcing the whole blob through one. (P4.3)
+ */
+export interface IContentSegment {
+    type: EContentType;
+    text: string;
+}
+/**
+ * Split `text` into typed segments. Contiguous multi-line JSON blocks are
+ * isolated; the remaining lines are grouped into runs of one coarse class
+ * (blank lines extend the current run), and each run's real type is detected.
+ * A single-type blob yields exactly one segment.
+ */
+export declare function segmentContent(text: string): IContentSegment[];
+/** Content classes that have a dedicated, materially-better compressor. */
+export declare function isRichSegmentType(type: EContentType): boolean;
+//# sourceMappingURL=segment.d.ts.map

package/dist/content/segment.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"segment.d.ts","sourceRoot":"","sources":["../../src/content/segment.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,mBAAmB,CAAC;AAIjD;;;;;GAKG;AACH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,YAAY,CAAC;IACnB,IAAI,EAAE,MAAM,CAAC;CACd;AA0DD;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,eAAe,EAAE,CAoC9D;AAED,2EAA2E;AAC3E,wBAAgB,iBAAiB,CAAC,IAAI,EAAE,YAAY,GAAG,OAAO,CAS7D"}

package/dist/content/segment.js

@@ -0,0 +1,117 @@
+import { EContentType } from "./content-type.js";
+import { detectContentType } from "./detect-content-type.js";
+import { splitLines } from "../text/line-utils.js";
+/** Coarse per-line class used to group adjacent non-JSON lines. */
+function coarseClass(line) {
+    if (/^(?:diff --git |@@ |--- |\+\+\+ )/.test(line))
+        return 'diff';
+    if (/^(?:[A-Za-z]:)?[^\s:]+:\d+:/.test(line))
+        return 'search';
+    if (/^\s*\[?\d{4}-\d{2}-\d{2}[T ]/.test(line) ||
+        /^\s*\[?(?:ERROR|FATAL|FAIL(?:ED|URE)?|WARN(?:ING)?|INFO|DEBUG|NOTICE|TRACE)\b/.test(line) ||
+        /^\s+at\s+\S/.test(line) ||
+        /^\s*Traceback\b/.test(line) ||
+        /^[\w.$]*(?:Error|Exception):/.test(line)) {
+        return 'log';
+    }
+    return 'prose';
+}
+/**
+ * If a JSON value opens at `start` (a line beginning `{`/`[`), return the index
+ * of the line where it closes balanced and parses — multi-line only. Otherwise
+ * null. String/escape aware so braces inside strings don't unbalance it.
+ */
+function findJsonBlock(lines, start) {
+    const open = (lines[start] ?? '').trimStart()[0];
+    if (open !== '{' && open !== '[')
+        return null;
+    let depth = 0;
+    let inStr = false;
+    let esc = false;
+    let started = false;
+    const limit = Math.min(lines.length, start + 2000);
+    for (let j = start; j < limit; j += 1) {
+        for (const ch of lines[j] ?? '') {
+            if (inStr) {
+                if (esc)
+                    esc = false;
+                else if (ch === '\\')
+                    esc = true;
+                else if (ch === '"')
+                    inStr = false;
+                continue;
+            }
+            if (ch === '"')
+                inStr = true;
+            else if (ch === '{' || ch === '[') {
+                depth += 1;
+                started = true;
+            }
+            else if (ch === '}' || ch === ']')
+                depth -= 1;
+        }
+        if (started && depth <= 0) {
+            if (j === start)
+                return null; // single-line JSON stays inline with prose
+            try {
+                JSON.parse(lines.slice(start, j + 1).join('\n').trim());
+                return j;
+            }
+            catch {
+                return null;
+            }
+        }
+    }
+    return null;
+}
+/**
+ * Split `text` into typed segments. Contiguous multi-line JSON blocks are
+ * isolated; the remaining lines are grouped into runs of one coarse class
+ * (blank lines extend the current run), and each run's real type is detected.
+ * A single-type blob yields exactly one segment.
+ */
+export function segmentContent(text) {
+    const lines = splitLines(text);
+    const n = lines.length;
+    const cls = new Array(n).fill('');
+    let i = 0;
+    while (i < n) {
+        const trimmed = (lines[i] ?? '').trimStart();
+        if (trimmed.startsWith('{') || trimmed.startsWith('[')) {
+            const end = findJsonBlock(lines, i);
+            if (end !== null) {
+                for (let k = i; k <= end; k += 1)
+                    cls[k] = 'json';
+                i = end + 1;
+                continue;
+            }
+        }
+        cls[i] = coarseClass(lines[i] ?? '');
+        i += 1;
+    }
+    // Group consecutive same-class lines; blank lines extend the current group.
+    const groups = [];
+    for (let idx = 0; idx < n; idx += 1) {
+        const blank = (lines[idx] ?? '').trim().length === 0;
+        const last = groups[groups.length - 1];
+        if (last && (blank || cls[idx] === cls[last.start])) {
+            last.end = idx + 1;
+        }
+        else {
+            groups.push({ start: idx, end: idx + 1 });
+        }
+    }
+    return groups.map(({ start, end }) => {
+        const segText = lines.slice(start, end).join('\n');
+        return { type: detectContentType(segText), text: segText };
+    });
+}
+/** Content classes that have a dedicated, materially-better compressor. */
+export function isRichSegmentType(type) {
+    return (type === EContentType.Json ||
+        type === EContentType.JsonArray ||
+        type === EContentType.BuildLog ||
+        type === EContentType.GitDiff ||
+        type === EContentType.SearchResults ||
+        type === EContentType.SourceCode);
+}

package/dist/index.d.ts

@@ -0,0 +1,61 @@
+/**
+ * `@shrkcrft/compress` — SharkCraft's deterministic context-compression
+ * engine. Built to honour the engine's hard rule: no model inside. Every
+ * transform is a pure function of its input — content routing, lossless
+ * columnar/table compaction of object arrays, log/search/diff/line reduction,
+ * and reversible Compress-Cache-Retrieve (CCR). Used by the CLI, MCP server,
+ * and inspector to cut the tokens an agent pays for the same information.
+ */
+export { estimateTokens, measureSavings } from './tokens/estimate-tokens.js';
+export type { ITokenSavings } from './tokens/token-savings.js';
+export { EContentType } from './content/content-type.js';
+export { detectContentType } from './content/detect-content-type.js';
+export type { IContentSegment } from './content/segment.js';
+export { segmentContent, isRichSegmentType } from './content/segment.js';
+export type { ICcrEntry } from './ccr/ccr-entry.js';
+export type { ICcrStore } from './ccr/ccr-store.js';
+export { ccrKey } from './ccr/ccr-key.js';
+export { CCR_MARKER_RE, formatCcrMarker, parseCcrMarkers } from './ccr/ccr-marker.js';
+export type { ICcrMarkerRef } from './ccr/ccr-marker.js';
+export { InMemoryCcrStore } from './ccr/in-memory-ccr-store.js';
+export { FileCcrStore } from './ccr/file-ccr-store.js';
+export type { ITtlFileCcrStoreOptions } from './ccr/ttl-file-ccr-store.js';
+export { TtlFileCcrStore } from './ccr/ttl-file-ccr-store.js';
+export type { IFieldSpec } from './table/field-spec.js';
+export type { ITableCompaction } from './table/table-compaction.js';
+export { compactObjectArray } from './table/compact-object-array.js';
+export type { IColumnarTable } from './table/columnar-table.js';
+export { tableToColumnar, compactArrayToColumnar, isColumnarTable, expandColumnar, } from './table/columnar-json.js';
+export { renderTable } from './table/render-table.js';
+export { renderCompactJson } from './json/render-compact-json.js';
+export { compressJson } from './json/compress-json.js';
+export { columnarToCsv, csvToObjects, columnarToMarkdownKv, markdownKvToObjects, } from './table/table-formats.js';
+export type { IObjectMap } from './table/object-map.js';
+export { compactObjectMap, expandObjectMap, isObjectMap } from './table/object-map.js';
+export type { AdaptiveBias, IAdaptiveOptions } from './table/adaptive-size.js';
+export { computeOptimalK, simhash, hammingDistance, kneedle, bigramCoverageCurve } from './table/adaptive-size.js';
+export type { IBm25Options } from './relevance/bm25.js';
+export { bm25Scores, topByBm25 } from './relevance/bm25.js';
+export type { ISampleOptions } from './table/sample-options.js';
+export type { ISampledTable } from './table/sampled-table.js';
+export { isSampledTable } from './table/sampled-table.js';
+export { sampleObjectArray } from './table/sample-object-array.js';
+export { ECompressionStrategy } from './result/compression-strategy.js';
+export type { ICompressionResult } from './result/compression-result.js';
+export type { ICompressOptions } from './result/compress-options.js';
+export { compressLog } from './text/compress-log.js';
+export { compressSearch } from './text/compress-search.js';
+export { compressDiff } from './text/compress-diff.js';
+export { compressLines } from './text/compress-lines.js';
+export { compressMarkdown } from './text/compress-markdown.js';
+export { compressCode } from './code/compress-code.js';
+export { EVolatileKind } from './cache/volatile-kind.js';
+export type { IVolatileToken } from './cache/volatile-token.js';
+export { detectVolatileTokens } from './cache/detect-volatile-tokens.js';
+export { PLACEHOLDER_RE, formatPlaceholder } from './cache/placeholder.js';
+export type { IAlignmentBinding, IAlignmentMap } from './cache/alignment-map.js';
+export type { IAlignmentResult } from './cache/alignment-result.js';
+export { alignVolatileTokens } from './cache/align-volatile-tokens.js';
+export { restoreVolatileTokens } from './cache/restore-volatile-tokens.js';
+export { compressContent } from './compress-content.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAC;AAC7E,YAAY,EAAE,aAAa,EAAE,MAAM,2BAA2B,CAAC;AAG/D,OAAO,EAAE,YAAY,EAAE,MAAM,2BAA2B,CAAC;AACzD,OAAO,EAAE,iBAAiB,EAAE,MAAM,kCAAkC,CAAC;AACrE,YAAY,EAAE,eAAe,EAAE,MAAM,sBAAsB,CAAC;AAC5D,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,sBAAsB,CAAC;AAGzE,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,YAAY,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAAE,aAAa,EAAE,eAAe,EAAE,eAAe,EAAE,MAAM,qBAAqB,CAAC;AACtF,YAAY,EAAE,aAAa,EAAE,MAAM,qBAAqB,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAChE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,YAAY,EAAE,uBAAuB,EAAE,MAAM,6BAA6B,CAAC;AAC3E,OAAO,EAAE,eAAe,EAAE,MAAM,6BAA6B,CAAC;AAG9D,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,YAAY,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AACrE,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EACL,eAAe,EACf,sBAAsB,EACtB,eAAe,EACf,cAAc,GACf,MAAM,0BAA0B,CAAC;AAClC,OAAO,EAAE,WAAW,EAAE,MAAM,yBAAyB,CAAC;AACtD,OAAO,EAAE,iBAAiB,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGvD,OAAO,EACL,aAAa,EACb,YAAY,EACZ,oBAAoB,EACpB,mBAAmB,GACpB,MAAM,0BAA0B,CAAC;AAGlC,YAAY,EAAE,UAAU,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,gBAAgB,EAAE,eAAe,EAAE,WAAW,EAAE,MAAM,uBAAuB,CAAC;AAGvF,YAAY,EAAE,YAAY,EAAE,gBAAgB,EAAE,MAAM,0BAA0B,CAAC;AAC/E,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,eAAe,EAAE,OAAO,EAAE,mBAAmB,EAAE,MAAM,0BAA0B,CAAC;AAGnH,YAAY,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,qBAAqB,CAAC;AAG5D,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,YAAY,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,iBAAiB,EAAE,MAAM,gCAAgC,CAAC;AAGnE,OAAO,EAAE,oBAAoB,EAAE,MAAM,kCAAkC,CAAC;AACxE,YAAY,EAAE,kBAAkB,EAAE,MAAM,gCAAgC,CAAC;AACzE,YAAY,EAAE,gBAAgB,EAAE,MAAM,8BAA8B,CAAC;AAGrE,OAAO,EAAE,WAAW,EAAE,MAAM,wBAAwB,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AACvD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,OAAO,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AAG/D,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAGvD,OAAO,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACzD,YAAY,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAChE,OAAO,EAAE,oBAAoB,EAAE,MAAM,mCAAmC,CAAC;AACzE,OAAO,EAAE,cAAc,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAC3E,YAAY,EAAE,iBAAiB,EAAE,aAAa,EAAE,MAAM,0BAA0B,CAAC;AACjF,YAAY,EAAE,gBAAgB,EAAE,MAAM,6BAA6B,CAAC;AACpE,OAAO,EAAE,mBAAmB,EAAE,MAAM,kCAAkC,CAAC;AACvE,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAG3E,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC"}

package/dist/index.js

@@ -0,0 +1,49 @@
+/**
+ * `@shrkcrft/compress` — SharkCraft's deterministic context-compression
+ * engine. Built to honour the engine's hard rule: no model inside. Every
+ * transform is a pure function of its input — content routing, lossless
+ * columnar/table compaction of object arrays, log/search/diff/line reduction,
+ * and reversible Compress-Cache-Retrieve (CCR). Used by the CLI, MCP server,
+ * and inspector to cut the tokens an agent pays for the same information.
+ */
+// Tokens / accounting
+export { estimateTokens, measureSavings } from "./tokens/estimate-tokens.js";
+// Content routing
+export { EContentType } from "./content/content-type.js";
+export { detectContentType } from "./content/detect-content-type.js";
+export { segmentContent, isRichSegmentType } from "./content/segment.js";
+export { ccrKey } from "./ccr/ccr-key.js";
+export { CCR_MARKER_RE, formatCcrMarker, parseCcrMarkers } from "./ccr/ccr-marker.js";
+export { InMemoryCcrStore } from "./ccr/in-memory-ccr-store.js";
+export { FileCcrStore } from "./ccr/file-ccr-store.js";
+export { TtlFileCcrStore } from "./ccr/ttl-file-ccr-store.js";
+export { compactObjectArray } from "./table/compact-object-array.js";
+export { tableToColumnar, compactArrayToColumnar, isColumnarTable, expandColumnar, } from "./table/columnar-json.js";
+export { renderTable } from "./table/render-table.js";
+export { renderCompactJson } from "./json/render-compact-json.js";
+export { compressJson } from "./json/compress-json.js";
+// Read-accuracy table encodings (P4.2): reversible CSV / Markdown-KV views.
+export { columnarToCsv, csvToObjects, columnarToMarkdownKv, markdownKvToObjects, } from "./table/table-formats.js";
+export { compactObjectMap, expandObjectMap, isObjectMap } from "./table/object-map.js";
+export { computeOptimalK, simhash, hammingDistance, kneedle, bigramCoverageCurve } from "./table/adaptive-size.js";
+export { bm25Scores, topByBm25 } from "./relevance/bm25.js";
+export { isSampledTable } from "./table/sampled-table.js";
+export { sampleObjectArray } from "./table/sample-object-array.js";
+// Result shapes / options
+export { ECompressionStrategy } from "./result/compression-strategy.js";
+// Text compressors
+export { compressLog } from "./text/compress-log.js";
+export { compressSearch } from "./text/compress-search.js";
+export { compressDiff } from "./text/compress-diff.js";
+export { compressLines } from "./text/compress-lines.js";
+export { compressMarkdown } from "./text/compress-markdown.js";
+// Code-aware compression (outline: keep imports/types/signatures, elide bodies)
+export { compressCode } from "./code/compress-code.js";
+// Cache alignment — volatile-token detection + active reversible substitution
+export { EVolatileKind } from "./cache/volatile-kind.js";
+export { detectVolatileTokens } from "./cache/detect-volatile-tokens.js";
+export { PLACEHOLDER_RE, formatPlaceholder } from "./cache/placeholder.js";
+export { alignVolatileTokens } from "./cache/align-volatile-tokens.js";
+export { restoreVolatileTokens } from "./cache/restore-volatile-tokens.js";
+// Router
+export { compressContent } from "./compress-content.js";

package/dist/json/compress-json.d.ts

@@ -0,0 +1,18 @@
+import type { ICompressionResult } from '../result/compression-result.js';
+import type { ICompressOptions } from '../result/compress-options.js';
+/**
+ * Compress JSON losslessly. A homogeneous object array becomes a *columnar*
+ * encoding — the shared schema is hoisted once and each row carries only
+ * values — which is still valid JSON and exactly reconstructable via
+ * `expandColumnar` (absent keys, nulls and empty strings are all preserved
+ * distinctly). Anything else is minified. No detail is dropped, so no CCR
+ * marker is needed. Falls back to line dedup if the text isn't valid JSON, and
+ * passes through untouched when re-serialization would lose precision (integers
+ * beyond 2^53), so the lossless guarantee always holds.
+ *
+ * (The dense text table from `renderCompactJson` is *not* used here: it
+ * renders null / "" / absent identically, so it cannot carry the lossless
+ * guarantee this function advertises.)
+ */
+export declare function compressJson(text: string, opts?: ICompressOptions): ICompressionResult;
+//# sourceMappingURL=compress-json.d.ts.map

package/dist/json/compress-json.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"compress-json.d.ts","sourceRoot":"","sources":["../../src/json/compress-json.ts"],"names":[],"mappings":"AAEA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,iCAAiC,CAAC;AAC1E,OAAO,KAAK,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAC;AAiEtE;;;;;;;;;;;;;GAaG;AACH,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,GAAE,gBAAqB,GAAG,kBAAkB,CA6D1F"}

package/dist/json/compress-json.js

@@ -0,0 +1,139 @@
+import { EContentType } from "../content/content-type.js";
+import { ECompressionStrategy } from "../result/compression-strategy.js";
+import { estimateTokens, measureSavings } from "../tokens/estimate-tokens.js";
+import { compactArrayToColumnar } from "../table/columnar-json.js";
+import { compactObjectMap } from "../table/object-map.js";
+import { sampleObjectArray } from "../table/sample-object-array.js";
+import { compressLines } from "../text/compress-lines.js";
+import { finalizeLossy, passthroughResult } from "../text/finalize.js";
+const NUMBER_TOKEN = /-?\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/g;
+/**
+ * Blank out the contents of every JSON string (object keys AND string values),
+ * leaving structural punctuation, whitespace, and bare literals in place. After
+ * this, the only digit runs left are genuine JSON *number* literals — a digit
+ * run that lived inside a string (a record id, a git SHA, a numeric-looking
+ * code) is gone. Quote handling respects backslash escapes so `"a\""` stays a
+ * single string.
+ */
+function stripJsonStrings(text) {
+    const out = [];
+    let inString = false;
+    for (let i = 0; i < text.length; i++) {
+        const ch = text[i];
+        if (inString) {
+            if (ch === '\\') {
+                i++; // skip the escaped char too
+                continue;
+            }
+            if (ch === '"')
+                inString = false;
+            continue; // drop the string's contents
+        }
+        if (ch === '"') {
+            inString = true;
+            continue;
+        }
+        out.push(ch);
+    }
+    return out.join('');
+}
+/**
+ * True if any JSON-number *literal* in the text would NOT survive a
+ * parse→serialize round trip — it overflows to Infinity (→ `null`), or carries
+ * more than ~15 significant digits (precision loss for big integers AND
+ * decimal-split floats like `90071992547409.93`). Counting significant mantissa
+ * digits (rather than a contiguous-digit run) is what catches floats whose
+ * digits straddle the dot. Only actual number literals are inspected — digit
+ * runs inside string values / keys are stripped first, because a numeric-looking
+ * STRING round-trips verbatim and is never at risk (so a list of records with
+ * id-like string fields still compacts losslessly). Sound: when true we keep the
+ * original bytes instead of a false "lossless".
+ */
+function hasRiskyNumber(text) {
+    const scannable = stripJsonStrings(text);
+    for (const match of scannable.matchAll(NUMBER_TOKEN)) {
+        const token = match[0];
+        const n = Number(token);
+        if (!Number.isFinite(n))
+            return true; // overflow → Infinity → null
+        if (n === 0 && /[1-9]/.test(token))
+            return true; // underflow: nonzero literal → 0
+        const sig = token.replace(/[eE].*$/, '').replace(/[-.]/g, '').replace(/^0+/, '');
+        if (sig.length > 15)
+            return true;
+    }
+    return false;
+}
+/**
+ * Compress JSON losslessly. A homogeneous object array becomes a *columnar*
+ * encoding — the shared schema is hoisted once and each row carries only
+ * values — which is still valid JSON and exactly reconstructable via
+ * `expandColumnar` (absent keys, nulls and empty strings are all preserved
+ * distinctly). Anything else is minified. No detail is dropped, so no CCR
+ * marker is needed. Falls back to line dedup if the text isn't valid JSON, and
+ * passes through untouched when re-serialization would lose precision (integers
+ * beyond 2^53), so the lossless guarantee always holds.
+ *
+ * (The dense text table from `renderCompactJson` is *not* used here: it
+ * renders null / "" / absent identically, so it cannot carry the lossless
+ * guarantee this function advertises.)
+ */
+export function compressJson(text, opts = {}) {
+    let parsed;
+    try {
+        parsed = JSON.parse(text);
+    }
+    catch {
+        return compressLines(text, EContentType.PlainText, opts);
+    }
+    const forced = opts.contentType === EContentType.Json || opts.contentType === EContentType.JsonArray
+        ? opts.contentType
+        : undefined;
+    const contentType = forced ?? (Array.isArray(parsed) ? EContentType.JsonArray : EContentType.Json);
+    if (hasRiskyNumber(text)) {
+        return passthroughResult(text, contentType, 'precision-preserving passthrough');
+    }
+    const columnar = Array.isArray(parsed) ? compactArrayToColumnar(parsed) : null;
+    // P2.3: an object KEYED by id with homogeneous values hoists to a columnar
+    // `_omap` envelope — the array columnar's analogue for the common map shape.
+    const objectMap = !Array.isArray(parsed) ? compactObjectMap(parsed) : null;
+    const lossless = columnar
+        ? JSON.stringify(columnar)
+        : objectMap
+            ? JSON.stringify({ _omap: objectMap })
+            : (JSON.stringify(parsed) ?? 'null');
+    // Lossy sampler is a LAST resort: only for a homogeneous array that, even
+    // losslessly compacted, still exceeds an explicit `maxTokens` budget.
+    const budget = opts.maxTokens;
+    if (Array.isArray(parsed) && budget && budget > 0 && estimateTokens(lossless, contentType) > budget) {
+        const sampled = sampleObjectArray(parsed, {
+            ...(opts.query !== undefined ? { query: opts.query } : {}),
+            ...(opts.maxItems !== undefined ? { maxItems: opts.maxItems } : {}),
+        });
+        if (sampled) {
+            return finalizeLossy({
+                original: text,
+                body: JSON.stringify(sampled),
+                contentType,
+                strategy: ECompressionStrategy.Sample,
+                opts,
+                note: `${sampled._table.sample.dropped} of ${sampled._table.n} rows sampled`,
+            });
+        }
+    }
+    const savings = measureSavings(text, lossless, contentType);
+    if (savings.after >= savings.before)
+        return passthroughResult(text, contentType);
+    return {
+        compressed: lossless,
+        contentType,
+        strategy: columnar || objectMap ? ECompressionStrategy.Table : ECompressionStrategy.MinifiedJson,
+        savings,
+        lossy: false,
+        note: columnar
+            ? 'lossless columnar table (valid JSON; schema hoisted, keys deduped)'
+            : objectMap
+                ? 'lossless columnar object-map (valid JSON; schema hoisted, keys deduped)'
+                : 'minified JSON (whitespace removed)',
+    };
+}