@shrkcrft/compress 0.1.0-alpha.16

package/dist/text/log-template.js

@@ -0,0 +1,239 @@
+/**
+ * Drain-style log-template mining — a LOSSLESS pre-pass for {@link compressLog}.
+ *
+ * Repeated structured log lines (`worker-3 processing batch 17 ok` × N) carry
+ * almost no new information per line: the fixed words repeat and only a few
+ * variable tokens move. This miner tokenizes each line, replaces its variable
+ * tokens (numbers, hex, UUIDs, ISO timestamps, quoted strings) with a `{}`
+ * placeholder to form a *template*, groups consecutive lines sharing a
+ * template, and collapses each run to one template plus a compact per-column
+ * encoding of the captured variables.
+ *
+ * It is **lossless by construction**: a template is exactly the original line
+ * with its variable matches replaced by `{}`, so `template ⋈ variables` rebuilds
+ * every original line, in order — no CCR needed. {@link reconstructLogTemplates}
+ * is the inverse and is exercised by the round-trip tests.
+ *
+ * Only runs with ≥1 variable column collapse; pure-identical repeats are left
+ * for the downstream signal-selector's de-duplication so its behaviour (and the
+ * `… N omitted …` markers callers rely on) is preserved.
+ */
+// Block sentinels. Chosen to never occur in real logs; if the input contains
+// either, mining is skipped wholesale so a collision can't corrupt round-trip.
+const BLOCK_OPEN = '⟦';
+const BLOCK_CLOSE = '⟧';
+const PLACEHOLDER = '{}';
+/** Minimum consecutive same-template lines before a run is worth collapsing. */
+const MIN_RUN = 3;
+// Variable token classes, specific → general. Correctness of the round-trip
+// does NOT depend on this list (template+matches always rebuild the line); it
+// only governs how MUCH collapses.
+const VAR_RE = new RegExp([
+    '\\d{4}-\\d{2}-\\d{2}[T ]\\d{2}:\\d{2}:\\d{2}(?:\\.\\d+)?(?:Z|[+-]\\d{2}:?\\d{2})?', // ISO timestamp
+    '[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}', // UUID
+    '0x[0-9a-fA-F]+', // hex literal
+    '\\b[0-9a-fA-F]{12,}\\b', // long hash
+    '"(?:[^"\\\\]|\\\\.)*"', // double-quoted string
+    "'(?:[^'\\\\]|\\\\.)*'", // single-quoted string
+    '\\d+(?:\\.\\d+)?', // integer / decimal
+].join('|'), 'g');
+// High-signal lines the downstream selector keeps individually — never folded
+// into a template, so an error / crash / stack frame always stays verbatim.
+const SIGNAL_RE = /\b(?:ERROR|FATAL|FAIL(?:ED|URE)?|EXCEPTION|panic|segmentation fault|segfault|core dumped|bus error|out of memory|oom|traceback|undefined reference|undefined symbol|assertion (?:failed)|SIG(?:SEGV|ABRT|KILL|BUS|FPE|ILL))\b/i;
+const STACK_RE = /^\s+(?:at\s+\S+|File ".*", line \d+)/;
+/** Collapse consecutive same-template runs. Lossless; reversible via {@link reconstructLogTemplates}. */
+export function mineLogTemplates(lines) {
+    // A sentinel anywhere means we can't guarantee a clean round-trip — bail.
+    for (const l of lines) {
+        if (l.includes(BLOCK_OPEN) || l.includes(BLOCK_CLOSE)) {
+            return { lines: [...lines], reduced: false };
+        }
+    }
+    const templateOf = (line) => {
+        if (line.includes(PLACEHOLDER) || SIGNAL_RE.test(line) || STACK_RE.test(line))
+            return null;
+        return line.replace(VAR_RE, PLACEHOLDER);
+    };
+    const templates = lines.map(templateOf);
+    const out = [];
+    let reduced = false;
+    let i = 0;
+    while (i < lines.length) {
+        const tpl = templates[i];
+        if (tpl === null || tpl === undefined) {
+            out.push(lines[i] ?? '');
+            i += 1;
+            continue;
+        }
+        let j = i + 1;
+        while (j < lines.length && templates[j] === tpl)
+            j += 1;
+        const run = lines.slice(i, j);
+        const block = run.length >= MIN_RUN ? encodeBlock(tpl, run) : null;
+        if (block) {
+            out.push(...block);
+            reduced = true;
+        }
+        else {
+            out.push(...run);
+        }
+        i = j;
+    }
+    return { lines: out, reduced };
+}
+/**
+ * Encode one run as a template block, or null when it isn't worth it (no
+ * variable columns: pure-identical repeats are left for the selector's dedup).
+ */
+function encodeBlock(template, run) {
+    const parts = template.split(PLACEHOLDER);
+    const cols = parts.length - 1;
+    const valuesPerLine = run.map((l) => l.match(VAR_RE) ?? []);
+    // Fold constant columns into the template; collect the variable columns.
+    let folded = parts[0] ?? '';
+    const varColumns = [];
+    for (let c = 0; c < cols; c += 1) {
+        const column = valuesPerLine.map((v) => v[c] ?? '');
+        const allEqual = column.every((v) => v === column[0]);
+        if (allEqual) {
+            folded += column[0] ?? '';
+        }
+        else {
+            folded += PLACEHOLDER;
+            varColumns.push(column);
+        }
+        folded += parts[c + 1] ?? '';
+    }
+    // No moving parts → leave it for the selector's identical-line dedup.
+    if (varColumns.length === 0)
+        return null;
+    const block = [`${BLOCK_OPEN}×${run.length}${BLOCK_CLOSE} ${folded}`];
+    for (const column of varColumns) {
+        block.push(`${BLOCK_OPEN}c${BLOCK_CLOSE} ${encodeColumn(column)}`);
+    }
+    block.push(`${BLOCK_OPEN}/${BLOCK_CLOSE}`);
+    return block;
+}
+/** Encode one variable column as the tightest lossless form: seq | cyc | lit. */
+function encodeColumn(values) {
+    const seq = asArithmetic(values);
+    if (seq)
+        return `seq ${seq.start} ${seq.step}`;
+    const cyc = asCycle(values);
+    if (cyc)
+        return `cyc ${cyc.map(escapeValue).join('|')}`;
+    return `lit ${values.map(escapeValue).join('|')}`;
+}
+/** Decode a column encoding into its N values. */
+function decodeColumn(enc, n) {
+    const sp = enc.indexOf(' ');
+    const kind = sp < 0 ? enc : enc.slice(0, sp);
+    const rest = sp < 0 ? '' : enc.slice(sp + 1);
+    if (kind === 'seq') {
+        const [start, step] = rest.split(' ').map((x) => Number(x));
+        return Array.from({ length: n }, (_, k) => String((start ?? 0) + k * (step ?? 0)));
+    }
+    if (kind === 'cyc') {
+        const pat = splitEscaped(rest);
+        return Array.from({ length: n }, (_, k) => pat[k % pat.length] ?? '');
+    }
+    // lit
+    return splitEscaped(rest);
+}
+/** An integer arithmetic progression that reconstructs every value canonically, else null. */
+function asArithmetic(values) {
+    if (values.length < 2)
+        return null;
+    const nums = [];
+    for (const v of values) {
+        // Canonical non-negative integer only (no leading zeros, fits exactly).
+        if (!/^\d+$/.test(v) || (v.length > 1 && v[0] === '0') || v.length > 15)
+            return null;
+        nums.push(Number(v));
+    }
+    const step = nums[1] - nums[0];
+    for (let k = 0; k < nums.length; k += 1) {
+        if (String(nums[0] + k * step) !== values[k])
+            return null;
+    }
+    return { start: nums[0], step };
+}
+/** A short repeating pattern (period 2..8) that covers the whole column, else null. */
+function asCycle(values) {
+    const n = values.length;
+    const maxPeriod = Math.min(8, Math.floor(n / 2));
+    for (let p = 2; p <= maxPeriod; p += 1) {
+        let ok = true;
+        for (let k = 0; k < n; k += 1) {
+            if (values[k] !== values[k % p]) {
+                ok = false;
+                break;
+            }
+        }
+        if (ok)
+            return values.slice(0, p);
+    }
+    return null;
+}
+function escapeValue(s) {
+    return s.replace(/\\/g, '\\\\').replace(/\|/g, '\\|');
+}
+/** Split on unescaped `|` and unescape each field. Inverse of {@link escapeValue}. */
+function splitEscaped(s) {
+    const out = [];
+    let cur = '';
+    for (let i = 0; i < s.length; i += 1) {
+        const ch = s[i];
+        if (ch === '\\' && i + 1 < s.length) {
+            cur += s[i + 1];
+            i += 1;
+        }
+        else if (ch === '|') {
+            out.push(cur);
+            cur = '';
+        }
+        else {
+            cur += ch;
+        }
+    }
+    out.push(cur);
+    return out;
+}
+const HEADER_RE = new RegExp(`^${BLOCK_OPEN}×(\\d+)${BLOCK_CLOSE} (.*)$`);
+const COL_RE = new RegExp(`^${BLOCK_OPEN}c${BLOCK_CLOSE} (.*)$`);
+const CLOSE = `${BLOCK_OPEN}/${BLOCK_CLOSE}`;
+/** Inverse of {@link mineLogTemplates}: expand every block back to its original lines. */
+export function reconstructLogTemplates(text) {
+    const lines = text.split('\n');
+    const out = [];
+    let i = 0;
+    while (i < lines.length) {
+        const header = HEADER_RE.exec(lines[i] ?? '');
+        if (!header) {
+            out.push(lines[i] ?? '');
+            i += 1;
+            continue;
+        }
+        const n = Number(header[1]);
+        const folded = header[2] ?? '';
+        i += 1;
+        const encodings = [];
+        while (i < lines.length && lines[i] !== CLOSE) {
+            const col = COL_RE.exec(lines[i] ?? '');
+            if (col)
+                encodings.push(col[1] ?? '');
+            i += 1;
+        }
+        i += 1; // skip CLOSE
+        const columns = encodings.map((enc) => decodeColumn(enc, n));
+        const parts = folded.split(PLACEHOLDER);
+        for (let k = 0; k < n; k += 1) {
+            let line = parts[0] ?? '';
+            for (let c = 0; c < columns.length; c += 1) {
+                line += (columns[c]?.[k] ?? '') + (parts[c + 1] ?? '');
+            }
+            out.push(line);
+        }
+    }
+    return out.join('\n');
+}

package/dist/tokens/estimate-tokens.d.ts

@@ -0,0 +1,17 @@
+import type { ITokenSavings } from './token-savings.js';
+import { EContentType } from '../content/content-type.js';
+/**
+ * Approximate token count. Average English token ≈ 4 chars; denser classes use
+ * a class-specific ratio when `contentType` is supplied. Only the character
+ * term is typed — the `words * 1.3` floor is content-independent. Pure: reads
+ * only its args + a frozen table.
+ */
+export declare function estimateTokens(text: string, contentType?: EContentType): number;
+/**
+ * Measure the token delta between a before/after string (optionally typed).
+ * Clamps at zero so a compressor is never reported as a net loss. Both sides
+ * use the same `contentType`, so a uniform divisor cannot flip a real
+ * reduction into a false passthrough.
+ */
+export declare function measureSavings(before: string, after: string, contentType?: EContentType): ITokenSavings;
+//# sourceMappingURL=estimate-tokens.d.ts.map

package/dist/tokens/estimate-tokens.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"estimate-tokens.d.ts","sourceRoot":"","sources":["../../src/tokens/estimate-tokens.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,4BAA4B,CAAC;AA6B1D;;;;;GAKG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,WAAW,CAAC,EAAE,YAAY,GAAG,MAAM,CAO/E;AAED;;;;;GAKG;AACH,wBAAgB,cAAc,CAC5B,MAAM,EAAE,MAAM,EACd,KAAK,EAAE,MAAM,EACb,WAAW,CAAC,EAAE,YAAY,GACzB,aAAa,CAMf"}

package/dist/tokens/estimate-tokens.js

@@ -0,0 +1,53 @@
+import { EContentType } from "../content/content-type.js";
+/**
+ * Chars-per-token by content class. Denser content (JSON punctuation, code
+ * operators) packs more characters per BPE token than prose, so a LARGER
+ * divisor yields a LOWER (more accurate) token estimate. Prose stays at 4 —
+ * the exact legacy value — so the untyped path is byte-identical.
+ */
+const CHARS_PER_TOKEN = Object.freeze({
+    [EContentType.JsonArray]: 2.5,
+    [EContentType.Json]: 2.5,
+    [EContentType.GitDiff]: 3.0,
+    [EContentType.SearchResults]: 3.0,
+    [EContentType.BuildLog]: 3.5,
+    [EContentType.SourceCode]: 3.2,
+    [EContentType.Markdown]: 4.0,
+    [EContentType.Yaml]: 3.5,
+    [EContentType.Csv]: 3.0,
+    [EContentType.PlainText]: 4.0,
+});
+/**
+ * The legacy divisor. With no content type, `estimateTokens` reproduces the
+ * exact `max(ceil(chars/4), ceil(words*1.3))` formula bit-for-bit, keeping it
+ * in lockstep with `@shrkcrft/context`'s separate estimator. DO NOT change this
+ * default or the `words*1.3` floor without updating that peer in step.
+ */
+const DEFAULT_CHARS_PER_TOKEN = 4;
+/**
+ * Approximate token count. Average English token ≈ 4 chars; denser classes use
+ * a class-specific ratio when `contentType` is supplied. Only the character
+ * term is typed — the `words * 1.3` floor is content-independent. Pure: reads
+ * only its args + a frozen table.
+ */
+export function estimateTokens(text, contentType) {
+    if (!text)
+        return 0;
+    const divisor = contentType === undefined ? DEFAULT_CHARS_PER_TOKEN : CHARS_PER_TOKEN[contentType];
+    const chars = text.length;
+    const words = text.trim().split(/\s+/).length;
+    return Math.max(Math.ceil(chars / divisor), Math.ceil(words * 1.3));
+}
+/**
+ * Measure the token delta between a before/after string (optionally typed).
+ * Clamps at zero so a compressor is never reported as a net loss. Both sides
+ * use the same `contentType`, so a uniform divisor cannot flip a real
+ * reduction into a false passthrough.
+ */
+export function measureSavings(before, after, contentType) {
+    const b = estimateTokens(before, contentType);
+    const a = estimateTokens(after, contentType);
+    const saved = Math.max(0, b - a);
+    const ratio = b === 0 ? 0 : Math.round((saved / b) * 10000) / 10000;
+    return { before: b, after: a, saved, ratio };
+}

package/dist/tokens/token-savings.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Token-accounting summary for a compression pass. Every compressor in this
+ * package reports its effect through this shape so savings are measured, not
+ * assumed.
+ */
+export interface ITokenSavings {
+    /** Estimated tokens of the input. */
+    before: number;
+    /** Estimated tokens of the output. */
+    after: number;
+    /** `before - after` (never negative; clamped at 0). */
+    saved: number;
+    /**
+     * Fraction saved in `[0, 1]`, rounded to 4 dp. `0` when the input was
+     * empty or the output grew (a compressor must never be reported as a
+     * net loss).
+     */
+    ratio: number;
+}
+//# sourceMappingURL=token-savings.d.ts.map

package/dist/tokens/token-savings.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"token-savings.d.ts","sourceRoot":"","sources":["../../src/tokens/token-savings.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,MAAM,WAAW,aAAa;IAC5B,qCAAqC;IACrC,MAAM,EAAE,MAAM,CAAC;IACf,sCAAsC;IACtC,KAAK,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,KAAK,EAAE,MAAM,CAAC;IACd;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;CACf"}

package/dist/tokens/token-savings.js

@@ -0,0 +1 @@
+export {};

package/package.json

@@ -0,0 +1,52 @@
+{
+  "name": "@shrkcrft/compress",
+  "version": "0.1.0-alpha.16",
+  "description": "SharkCraft deterministic context-compression engine: content routing, lossless columnar/table compaction, log/search/diff line reduction, and reversible Compress-Cache-Retrieve (CCR). No model inside — every transform is a pure function of its input.",
+  "license": "MIT",
+  "author": "SharkCraft contributors",
+  "type": "module",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.d.ts",
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "bun": "./src/index.ts",
+      "import": "./dist/index.js",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md",
+    "LICENSE"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/shrkcrft/sharkcraft.git",
+    "directory": "packages/compress"
+  },
+  "homepage": "https://github.com/shrkcrft/sharkcraft",
+  "bugs": {
+    "url": "https://github.com/shrkcrft/sharkcraft/issues"
+  },
+  "keywords": [
+    "sharkcraft",
+    "compression",
+    "tokens",
+    "ccr",
+    "deterministic"
+  ],
+  "engines": {
+    "bun": ">=1.1.0",
+    "node": ">=18"
+  },
+  "scripts": {
+    "typecheck": "tsc --noEmit -p tsconfig.json"
+  },
+  "dependencies": {
+    "@shrkcrft/core": "^0.1.0-alpha.16"
+  },
+  "publishConfig": {
+    "access": "public"
+  }
+}