greprag 5.41.0 → 5.42.0

package/dist/crush/index.js

@@ -0,0 +1,48 @@
+"use strict";
+/**
+ * MIRRORED FILE — canonical source: packages/core/src/crush/index.ts.
+ * Byte-identical copy lives at packages/cli/src/crush/index.ts (the CLI is a
+ * zero-dep artifact and cannot import @greprag/core). KEEP IN SYNC —
+ * the sync guard in tests/test-crush.cjs fails the build on drift.
+ *
+ * Crush engine — pure tool-output compressors (Headroom ports).
+ * String in -> { output, stats } out. Zero deps, no I/O.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.classifyImportance = exports.containsAnyKeyword = exports.containsKeyword = exports.JSON_ERROR_KEYWORDS = exports.ERROR_KEYWORDS = exports.findKnee = exports.computeUniqueBigramCurve = exports.computeOptimalK = exports.detectStructuralOutliers = exports.detectSequentialPattern = exports.detectRareStatusValues = exports.detectErrorItems = exports.classifyArray = exports.DEFAULT_JSON_CONFIG = exports.computeKSplit = exports.crushJson = exports.DEFAULT_LOG_CONFIG = exports.normalizeForDedupe = exports.parseLogLines = exports.isSummaryLine = exports.classifyLevel = exports.detectFormat = exports.crushLog = exports.DEFAULT_SEARCH_CONFIG = exports.parseMatchLine = exports.crushSearch = exports.makePassthrough = exports.estimateTokens = void 0;
+var crush_types_1 = require("./crush-types");
+Object.defineProperty(exports, "estimateTokens", { enumerable: true, get: function () { return crush_types_1.estimateTokens; } });
+Object.defineProperty(exports, "makePassthrough", { enumerable: true, get: function () { return crush_types_1.makePassthrough; } });
+var search_compressor_1 = require("./search-compressor");
+Object.defineProperty(exports, "crushSearch", { enumerable: true, get: function () { return search_compressor_1.crushSearch; } });
+Object.defineProperty(exports, "parseMatchLine", { enumerable: true, get: function () { return search_compressor_1.parseMatchLine; } });
+Object.defineProperty(exports, "DEFAULT_SEARCH_CONFIG", { enumerable: true, get: function () { return search_compressor_1.DEFAULT_SEARCH_CONFIG; } });
+var log_compressor_1 = require("./log-compressor");
+Object.defineProperty(exports, "crushLog", { enumerable: true, get: function () { return log_compressor_1.crushLog; } });
+Object.defineProperty(exports, "detectFormat", { enumerable: true, get: function () { return log_compressor_1.detectFormat; } });
+Object.defineProperty(exports, "classifyLevel", { enumerable: true, get: function () { return log_compressor_1.classifyLevel; } });
+Object.defineProperty(exports, "isSummaryLine", { enumerable: true, get: function () { return log_compressor_1.isSummaryLine; } });
+Object.defineProperty(exports, "parseLogLines", { enumerable: true, get: function () { return log_compressor_1.parseLogLines; } });
+Object.defineProperty(exports, "normalizeForDedupe", { enumerable: true, get: function () { return log_compressor_1.normalizeForDedupe; } });
+Object.defineProperty(exports, "DEFAULT_LOG_CONFIG", { enumerable: true, get: function () { return log_compressor_1.DEFAULT_LOG_CONFIG; } });
+var json_crusher_1 = require("./json-crusher");
+Object.defineProperty(exports, "crushJson", { enumerable: true, get: function () { return json_crusher_1.crushJson; } });
+Object.defineProperty(exports, "computeKSplit", { enumerable: true, get: function () { return json_crusher_1.computeKSplit; } });
+Object.defineProperty(exports, "DEFAULT_JSON_CONFIG", { enumerable: true, get: function () { return json_crusher_1.DEFAULT_JSON_CONFIG; } });
+var json_detectors_1 = require("./json-detectors");
+Object.defineProperty(exports, "classifyArray", { enumerable: true, get: function () { return json_detectors_1.classifyArray; } });
+Object.defineProperty(exports, "detectErrorItems", { enumerable: true, get: function () { return json_detectors_1.detectErrorItems; } });
+Object.defineProperty(exports, "detectRareStatusValues", { enumerable: true, get: function () { return json_detectors_1.detectRareStatusValues; } });
+Object.defineProperty(exports, "detectSequentialPattern", { enumerable: true, get: function () { return json_detectors_1.detectSequentialPattern; } });
+Object.defineProperty(exports, "detectStructuralOutliers", { enumerable: true, get: function () { return json_detectors_1.detectStructuralOutliers; } });
+var adaptive_sizer_1 = require("./adaptive-sizer");
+Object.defineProperty(exports, "computeOptimalK", { enumerable: true, get: function () { return adaptive_sizer_1.computeOptimalK; } });
+Object.defineProperty(exports, "computeUniqueBigramCurve", { enumerable: true, get: function () { return adaptive_sizer_1.computeUniqueBigramCurve; } });
+Object.defineProperty(exports, "findKnee", { enumerable: true, get: function () { return adaptive_sizer_1.findKnee; } });
+var crush_keywords_1 = require("./crush-keywords");
+Object.defineProperty(exports, "ERROR_KEYWORDS", { enumerable: true, get: function () { return crush_keywords_1.ERROR_KEYWORDS; } });
+Object.defineProperty(exports, "JSON_ERROR_KEYWORDS", { enumerable: true, get: function () { return crush_keywords_1.JSON_ERROR_KEYWORDS; } });
+Object.defineProperty(exports, "containsKeyword", { enumerable: true, get: function () { return crush_keywords_1.containsKeyword; } });
+Object.defineProperty(exports, "containsAnyKeyword", { enumerable: true, get: function () { return crush_keywords_1.containsAnyKeyword; } });
+Object.defineProperty(exports, "classifyImportance", { enumerable: true, get: function () { return crush_keywords_1.classifyImportance; } });
+//# sourceMappingURL=index.js.map

package/dist/crush/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/crush/index.ts"],"names":[],"mappings":";AAAA;;;;;;;;GAQG;;;AAEH,6CAA0G;AAAvD,6GAAA,cAAc,OAAA;AAAE,8GAAA,eAAe,OAAA;AAClF,yDAAiH;AAAxG,gHAAA,WAAW,OAAA;AAAE,mHAAA,cAAc,OAAA;AAAE,0HAAA,qBAAqB,OAAA;AAC3D,mDAY0B;AAXxB,0GAAA,QAAQ,OAAA;AACR,8GAAA,YAAY,OAAA;AACZ,+GAAA,aAAa,OAAA;AACb,+GAAA,aAAa,OAAA;AACb,+GAAA,aAAa,OAAA;AACb,oHAAA,kBAAkB,OAAA;AAClB,oHAAA,kBAAkB,OAAA;AAMpB,+CAAkG;AAAzF,yGAAA,SAAS,OAAA;AAAE,6GAAA,aAAa,OAAA;AAAE,mHAAA,mBAAmB,OAAA;AACtD,mDAQ0B;AAPxB,+GAAA,aAAa,OAAA;AACb,kHAAA,gBAAgB,OAAA;AAChB,wHAAA,sBAAsB,OAAA;AACtB,yHAAA,uBAAuB,OAAA;AACvB,0HAAA,wBAAwB,OAAA;AAI1B,mDAAuF;AAA9E,iHAAA,eAAe,OAAA;AAAE,0HAAA,wBAAwB,OAAA;AAAE,0GAAA,QAAQ,OAAA;AAC5D,mDAM0B;AALxB,gHAAA,cAAc,OAAA;AACd,qHAAA,mBAAmB,OAAA;AACnB,iHAAA,eAAe,OAAA;AACf,oHAAA,kBAAkB,OAAA;AAClB,oHAAA,kBAAkB,OAAA"}

package/dist/crush/json-crusher.d.ts

@@ -0,0 +1,38 @@
+/**
+ * MIRRORED FILE — canonical source: packages/core/src/crush/json-crusher.ts.
+ * Byte-identical copy lives at packages/cli/src/crush/json-crusher.ts (the CLI is a
+ * zero-dep artifact and cannot import @greprag/core). KEEP IN SYNC —
+ * the sync guard in tests/test-crush.cjs fails the build on drift.
+ *
+ * JSON-array crusher — TypeScript port of the SIMPLE core of Headroom's
+ * smart_crusher: statistical sampling + error preservation. Classifies
+ * each array, keeps errors / structural outliers / rare statuses /
+ * first+last blocks, and replaces the dropped bulk with a count marker
+ * string (`"[... N items omitted ...]"`).
+ *
+ * The full Headroom orchestration (TOIN semantics, relevance planning,
+ * lossless compaction, change points) is intentionally not ported.
+ */
+import { CrushResult } from './crush-types';
+export interface JsonCrusherConfig {
+    /** Arrays smaller than this are left alone. */
+    minItemsToAnalyze: number;
+    /** Target maximum items kept per crushed array. */
+    maxItemsAfterCrush: number;
+    /** Fraction of the budget pinned to the array head. */
+    firstFraction: number;
+    /** Fraction of the budget pinned to the array tail. */
+    lastFraction: number;
+    bias: number;
+}
+export declare const DEFAULT_JSON_CONFIG: JsonCrusherConfig;
+/** First/last slot split for a keep-budget. Never exceeds kTotal. */
+export declare function computeKSplit(kTotal: number, firstFraction: number, lastFraction: number): {
+    first: number;
+    last: number;
+};
+/**
+ * Compress JSON containing large arrays. Non-JSON input, or JSON with
+ * no qualifying arrays, passes through unchanged.
+ */
+export declare function crushJson(content: string, config?: Partial<JsonCrusherConfig>): CrushResult;

package/dist/crush/json-crusher.js

@@ -0,0 +1,250 @@
+"use strict";
+/**
+ * MIRRORED FILE — canonical source: packages/core/src/crush/json-crusher.ts.
+ * Byte-identical copy lives at packages/cli/src/crush/json-crusher.ts (the CLI is a
+ * zero-dep artifact and cannot import @greprag/core). KEEP IN SYNC —
+ * the sync guard in tests/test-crush.cjs fails the build on drift.
+ *
+ * JSON-array crusher — TypeScript port of the SIMPLE core of Headroom's
+ * smart_crusher: statistical sampling + error preservation. Classifies
+ * each array, keeps errors / structural outliers / rare statuses /
+ * first+last blocks, and replaces the dropped bulk with a count marker
+ * string (`"[... N items omitted ...]"`).
+ *
+ * The full Headroom orchestration (TOIN semantics, relevance planning,
+ * lossless compaction, change points) is intentionally not ported.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.DEFAULT_JSON_CONFIG = void 0;
+exports.computeKSplit = computeKSplit;
+exports.crushJson = crushJson;
+const adaptive_sizer_1 = require("./adaptive-sizer");
+const crush_types_1 = require("./crush-types");
+const json_detectors_1 = require("./json-detectors");
+exports.DEFAULT_JSON_CONFIG = {
+    minItemsToAnalyze: 10,
+    maxItemsAfterCrush: 15,
+    firstFraction: 0.3,
+    lastFraction: 0.15,
+    bias: 1.0,
+};
+/** First/last slot split for a keep-budget. Never exceeds kTotal. */
+function computeKSplit(kTotal, firstFraction, lastFraction) {
+    if (kTotal <= 1)
+        return { first: kTotal, last: 0 };
+    const first = Math.max(1, Math.round(kTotal * firstFraction));
+    const last = Math.max(1, Math.round(kTotal * lastFraction));
+    if (first + last > kTotal)
+        return { first: Math.max(1, kTotal - 1), last: kTotal > 1 ? 1 : 0 };
+    return { first, last };
+}
+/** Find a common-ish id field whose values run sequentially, for marker enrichment. */
+function sequentialIdField(items) {
+    const first = items[0];
+    if (typeof first !== 'object' || first === null || Array.isArray(first))
+        return null;
+    for (const key of Object.keys(first)) {
+        if (!/^(id|.*_id|index|seq|n)$/i.test(key))
+            continue;
+        const values = items.map((it) => typeof it === 'object' && it !== null && !Array.isArray(it) ? it[key] ?? null : null);
+        if ((0, json_detectors_1.detectSequentialPattern)(values))
+            return key;
+    }
+    return null;
+}
+function omittedMarker(droppedCount, items, droppedIndices) {
+    const idField = sequentialIdField(items);
+    if (idField && droppedIndices.length > 0) {
+        const firstDropped = items[droppedIndices[0]];
+        const lastDropped = items[droppedIndices[droppedIndices.length - 1]];
+        const idOf = (v) => typeof v === 'object' && v !== null && !Array.isArray(v) ? v[idField] ?? null : null;
+        const lo = idOf(firstDropped);
+        const hi = idOf(lastDropped);
+        if (lo !== null && hi !== null) {
+            return `[... ${droppedCount} items omitted (${idField} ${lo}..${hi}) ...]`;
+        }
+    }
+    return `[... ${droppedCount} items omitted ...]`;
+}
+/** Crush one array of objects: errors + outliers always kept; first/last + fill to budget. */
+function crushDictArray(items, config) {
+    const n = items.length;
+    const serialized = items.map((it) => JSON.stringify(it));
+    const budget = (0, adaptive_sizer_1.computeOptimalK)(serialized, config.bias, 5, config.maxItemsAfterCrush);
+    const keep = new Set();
+    // Errors are never dropped (preservation guarantee).
+    for (const idx of (0, json_detectors_1.detectErrorItems)(items))
+        keep.add(idx);
+    // Structural outliers (rare fields + rare statuses).
+    for (const idx of (0, json_detectors_1.detectStructuralOutliers)(items))
+        keep.add(idx);
+    // First/last blocks from the remaining budget.
+    const remaining = Math.max(0, budget - keep.size);
+    if (remaining > 0) {
+        const { first, last } = computeKSplit(remaining, config.firstFraction, config.lastFraction);
+        for (let i = 0; i < Math.min(first, n); i++)
+            keep.add(i);
+        for (let i = Math.max(0, n - last); i < n; i++)
+            keep.add(i);
+        // Fill any leftover slots with an even sample of the middle.
+        let leftover = budget - keep.size;
+        if (leftover > 0) {
+            const step = Math.max(1, Math.floor(n / (leftover + 1)));
+            for (let i = step; i < n && leftover > 0; i += step) {
+                if (!keep.has(i)) {
+                    keep.add(i);
+                    leftover--;
+                }
+            }
+        }
+    }
+    if (keep.size >= n) {
+        return { items: [...items], originalCount: n, keptCount: n, crushed: false };
+    }
+    const keptIndices = [...keep].sort((a, b) => a - b);
+    const droppedIndices = [];
+    for (let i = 0; i < n; i++) {
+        if (!keep.has(i))
+            droppedIndices.push(i);
+    }
+    // Survivors in original order, with one marker at the first gap.
+    const out = [];
+    let markerPlaced = false;
+    for (let i = 0; i < n; i++) {
+        if (keep.has(i)) {
+            out.push(items[i]);
+        }
+        else if (!markerPlaced) {
+            out.push(omittedMarker(droppedIndices.length, items, droppedIndices));
+            markerPlaced = true;
+        }
+    }
+    return { items: out, originalCount: n, keptCount: keptIndices.length, crushed: true };
+}
+/** Crush a scalar (string/number) array: keep first/last + statistical outliers. */
+function crushScalarArray(items, config) {
+    const n = items.length;
+    const serialized = items.map((it) => JSON.stringify(it));
+    const budget = (0, adaptive_sizer_1.computeOptimalK)(serialized, config.bias, 5, config.maxItemsAfterCrush);
+    if (budget >= n) {
+        return { items: [...items], originalCount: n, keptCount: n, crushed: false };
+    }
+    const keep = new Set();
+    // Numeric outliers: beyond 2 standard deviations from the mean.
+    const numbers = items.every((it) => typeof it === 'number') ? items : null;
+    if (numbers) {
+        const mean = numbers.reduce((s, v) => s + v, 0) / n;
+        const variance = numbers.reduce((s, v) => s + (v - mean) ** 2, 0) / n;
+        const std = Math.sqrt(variance);
+        if (std > 0) {
+            for (let i = 0; i < n; i++) {
+                if (Math.abs(numbers[i] - mean) > 2 * std)
+                    keep.add(i);
+            }
+        }
+    }
+    // String error items.
+    if (!numbers) {
+        for (let i = 0; i < n; i++) {
+            const it = items[i];
+            if (typeof it === 'string') {
+                const lower = it.toLowerCase();
+                if (/(error|exception|failed|failure|fatal|panic|timeout)/.test(lower))
+                    keep.add(i);
+            }
+        }
+    }
+    const remaining = Math.max(0, budget - keep.size);
+    if (remaining > 0) {
+        const { first, last } = computeKSplit(remaining, config.firstFraction, config.lastFraction);
+        for (let i = 0; i < Math.min(first, n); i++)
+            keep.add(i);
+        for (let i = Math.max(0, n - last); i < n; i++)
+            keep.add(i);
+    }
+    if (keep.size >= n) {
+        return { items: [...items], originalCount: n, keptCount: n, crushed: false };
+    }
+    const out = [];
+    let dropped = 0;
+    let markerAt = -1;
+    for (let i = 0; i < n; i++) {
+        if (keep.has(i)) {
+            out.push(items[i]);
+        }
+        else {
+            dropped++;
+            if (markerAt === -1) {
+                markerAt = out.length;
+                out.push(''); // placeholder, replaced below
+            }
+        }
+    }
+    if (markerAt >= 0)
+        out[markerAt] = `[... ${dropped} items omitted ...]`;
+    return { items: out, originalCount: n, keptCount: n - dropped, crushed: true };
+}
+/** Recursively walk a JSON value, crushing qualifying arrays in place. */
+function walkAndCrush(value, config, totals) {
+    if (Array.isArray(value)) {
+        const recursed = value.map((v) => walkAndCrush(v, config, totals));
+        if (recursed.length < config.minItemsToAnalyze)
+            return recursed;
+        const kind = (0, json_detectors_1.classifyArray)(recursed);
+        let outcome = null;
+        if (kind === 'dict_array')
+            outcome = crushDictArray(recursed, config);
+        else if (kind === 'string_array' || kind === 'number_array')
+            outcome = crushScalarArray(recursed, config);
+        if (outcome && outcome.crushed) {
+            totals.original += outcome.originalCount;
+            totals.kept += outcome.keptCount;
+            totals.crushedArrays++;
+            return outcome.items;
+        }
+        totals.original += recursed.length;
+        totals.kept += recursed.length;
+        return recursed;
+    }
+    if (typeof value === 'object' && value !== null) {
+        const out = {};
+        for (const [k, v] of Object.entries(value)) {
+            out[k] = walkAndCrush(v, config, totals);
+        }
+        return out;
+    }
+    return value;
+}
+/**
+ * Compress JSON containing large arrays. Non-JSON input, or JSON with
+ * no qualifying arrays, passes through unchanged.
+ */
+function crushJson(content, config = {}) {
+    const cfg = { ...exports.DEFAULT_JSON_CONFIG, ...config };
+    let parsed;
+    try {
+        parsed = JSON.parse(content);
+    }
+    catch {
+        return (0, crush_types_1.makePassthrough)(content, 'json', 'not valid JSON');
+    }
+    const totals = { original: 0, kept: 0, crushedArrays: 0 };
+    const crushedValue = walkAndCrush(parsed, cfg, totals);
+    if (totals.crushedArrays === 0) {
+        return (0, crush_types_1.makePassthrough)(content, 'json', 'no qualifying arrays', totals.original);
+    }
+    const output = JSON.stringify(crushedValue);
+    return {
+        output,
+        stats: {
+            sourceType: 'json',
+            originalChars: content.length,
+            compressedChars: output.length,
+            compressionRatio: output.length / Math.max(content.length, 1),
+            originalUnits: totals.original,
+            keptUnits: totals.kept,
+            passthrough: false,
+        },
+    };
+}
+//# sourceMappingURL=json-crusher.js.map

package/dist/crush/json-crusher.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-crusher.js","sourceRoot":"","sources":["../../src/crush/json-crusher.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AAiCH,sCAMC;AAkMD,8BA8BC;AArQD,qDAAmD;AACnD,+CAA6D;AAC7D,qDAM0B;AAcb,QAAA,mBAAmB,GAAsB;IACpD,iBAAiB,EAAE,EAAE;IACrB,kBAAkB,EAAE,EAAE;IACtB,aAAa,EAAE,GAAG;IAClB,YAAY,EAAE,IAAI;IAClB,IAAI,EAAE,GAAG;CACV,CAAC;AAEF,qEAAqE;AACrE,SAAgB,aAAa,CAAC,MAAc,EAAE,aAAqB,EAAE,YAAoB;IACvF,IAAI,MAAM,IAAI,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IACnD,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,aAAa,CAAC,CAAC,CAAC;IAC9D,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,YAAY,CAAC,CAAC,CAAC;IAC5D,IAAI,KAAK,GAAG,IAAI,GAAG,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/F,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;AACzB,CAAC;AASD,uFAAuF;AACvF,SAAS,iBAAiB,CAAC,KAA2B;IACpD,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;IACvB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACrF,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,2BAA2B,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,SAAS;QACrD,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAC9B,OAAO,EAAE,KAAK,QAAQ,IAAI,EAAE,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,EAAiC,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CACrH,CAAC;QACF,IAAI,IAAA,wCAAuB,EAAC,MAAM,CAAC;YAAE,OAAO,GAAG,CAAC;IAClD,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,aAAa,CAAC,YAAoB,EAAE,KAA2B,EAAE,cAAiC;IACzG,MAAM,OAAO,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;IACzC,IAAI,OAAO,IAAI,cAAc,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACzC,MAAM,YAAY,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9C,MAAM,WAAW,GAAG,KAAK,CAAC,cAAc,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;QACrE,MAAM,IAAI,GAAG,CAAC,CAAY,EAAa,EAAE,CACvC,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAE,CAAgC,CAAC,OAAO,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;QACvH,MAAM,EAAE,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC;QAC9B,MAAM,EAAE,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC;QAC7B,IAAI,EAAE,KAAK,IAAI,IAAI,EAAE,KAAK,IAAI,EAAE,CAAC;YAC/B,OAAO,QAAQ,YAAY,mBAAmB,OAAO,IAAI,EAAE,KAAK,EAAE,QAAQ,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,YAAY,qBAAqB,CAAC;AACnD,CAAC;AAED,8FAA8F;AAC9F,SAAS,cAAc,CAAC,KAA2B,EAAE,MAAyB;IAC5E,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAA,gCAAe,EAAC,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAC;IAEtF,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,qDAAqD;IACrD,KAAK,MAAM,GAAG,IAAI,IAAA,iCAAgB,EAAC,KAAK,CAAC;QAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACzD,qDAAqD;IACrD,KAAK,MAAM,GAAG,IAAI,IAAA,yCAAwB,EAAC,KAAK,CAAC;QAAE,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjE,+CAA+C;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC,SAAS,EAAE,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;QAC5F,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACzD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC5D,6DAA6D;QAC7D,IAAI,QAAQ,GAAG,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC;QAClC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC;YACjB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,QAAQ,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YACzD,KAAK,IAAI,CAAC,GAAG,IAAI,EAAE,CAAC,GAAG,CAAC,IAAI,QAAQ,GAAG,CAAC,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC;gBACpD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;oBACjB,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;oBACZ,QAAQ,EAAE,CAAC;gBACb,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC/E,CAAC;IAED,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,MAAM,cAAc,GAAa,EAAE,CAAC;IACpC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,cAAc,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3C,CAAC;IAED,iEAAiE;IACjE,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAChB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;aAAM,IAAI,CAAC,YAAY,EAAE,CAAC;YACzB,GAAG,CAAC,IAAI,CAAC,aAAa,CAAC,cAAc,CAAC,MAAM,EAAE,KAAK,EAAE,cAAc,CAAC,CAAC,CAAC;YACtE,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,EAAE,WAAW,CAAC,MAAM,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AACxF,CAAC;AAED,oFAAoF;AACpF,SAAS,gBAAgB,CAAC,KAA2B,EAAE,MAAyB;IAC9E,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,MAAM,UAAU,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;IACzD,MAAM,MAAM,GAAG,IAAA,gCAAe,EAAC,UAAU,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,EAAE,MAAM,CAAC,kBAAkB,CAAC,CAAC;IACtF,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAChB,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC/E,CAAC;IAED,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,gEAAgE;IAChE,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,OAAO,EAAE,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAE,KAAkB,CAAC,CAAC,CAAC,IAAI,CAAC;IACzF,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC;QACtE,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAChC,IAAI,GAAG,GAAG,CAAC,EAAE,CAAC;YACZ,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;gBAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,GAAG;oBAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACzD,CAAC;QACH,CAAC;IACH,CAAC;IACD,sBAAsB;IACtB,IAAI,CAAC,OAAO,EAAE,CAAC;QACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YAC3B,MAAM,EAAE,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACpB,IAAI,OAAO,EAAE,KAAK,QAAQ,EAAE,CAAC;gBAC3B,MAAM,KAAK,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;gBAC/B,IAAI,sDAAsD,CAAC,IAAI,CAAC,KAAK,CAAC;oBAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YACtF,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAClB,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,GAAG,aAAa,CAAC,SAAS,EAAE,MAAM,CAAC,aAAa,EAAE,MAAM,CAAC,YAAY,CAAC,CAAC;QAC5F,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QACzD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE;YAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC9D,CAAC;IAED,IAAI,IAAI,CAAC,IAAI,IAAI,CAAC,EAAE,CAAC;QACnB,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,KAAK,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IAC/E,CAAC;IAED,MAAM,GAAG,GAAgB,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,QAAQ,GAAG,CAAC,CAAC,CAAC;IAClB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;YAChB,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC;aAAM,CAAC;YACN,OAAO,EAAE,CAAC;YACV,IAAI,QAAQ,KAAK,CAAC,CAAC,EAAE,CAAC;gBACpB,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC;gBACtB,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,8BAA8B;YAC9C,CAAC;QACH,CAAC;IACH,CAAC;IACD,IAAI,QAAQ,IAAI,CAAC;QAAE,GAAG,CAAC,QAAQ,CAAC,GAAG,QAAQ,OAAO,qBAAqB,CAAC;IACxE,OAAO,EAAE,KAAK,EAAE,GAAG,EAAE,aAAa,EAAE,CAAC,EAAE,SAAS,EAAE,CAAC,GAAG,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AACjF,CAAC;AAED,0EAA0E;AAC1E,SAAS,YAAY,CACnB,KAAgB,EAChB,MAAyB,EACzB,MAAiE;IAEjE,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;QACnE,IAAI,QAAQ,CAAC,MAAM,GAAG,MAAM,CAAC,iBAAiB;YAAE,OAAO,QAAQ,CAAC;QAChE,MAAM,IAAI,GAAG,IAAA,8BAAa,EAAC,QAAQ,CAAC,CAAC;QACrC,IAAI,OAAO,GAA6B,IAAI,CAAC;QAC7C,IAAI,IAAI,KAAK,YAAY;YAAE,OAAO,GAAG,cAAc,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;aACjE,IAAI,IAAI,KAAK,cAAc,IAAI,IAAI,KAAK,cAAc;YAAE,OAAO,GAAG,gBAAgB,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC1G,IAAI,OAAO,IAAI,OAAO,CAAC,OAAO,EAAE,CAAC;YAC/B,MAAM,CAAC,QAAQ,IAAI,OAAO,CAAC,aAAa,CAAC;YACzC,MAAM,CAAC,IAAI,IAAI,OAAO,CAAC,SAAS,CAAC;YACjC,MAAM,CAAC,aAAa,EAAE,CAAC;YACvB,OAAO,OAAO,CAAC,KAAK,CAAC;QACvB,CAAC;QACD,MAAM,CAAC,QAAQ,IAAI,QAAQ,CAAC,MAAM,CAAC;QACnC,MAAM,CAAC,IAAI,IAAI,QAAQ,CAAC,MAAM,CAAC;QAC/B,OAAO,QAAQ,CAAC;IAClB,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,EAAE,CAAC;QAChD,MAAM,GAAG,GAAiC,EAAE,CAAC;QAC7C,KAAK,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;YAC3C,GAAG,CAAC,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;GAGG;AACH,SAAgB,SAAS,CAAC,OAAe,EAAE,SAAqC,EAAE;IAChF,MAAM,GAAG,GAAsB,EAAE,GAAG,2BAAmB,EAAE,GAAG,MAAM,EAAE,CAAC;IAErE,IAAI,MAAiB,CAAC;IACtB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAc,CAAC;IAC5C,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAA,6BAAe,EAAC,OAAO,EAAE,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC5D,CAAC;IAED,MAAM,MAAM,GAAG,EAAE,QAAQ,EAAE,CAAC,EAAE,IAAI,EAAE,CAAC,EAAE,aAAa,EAAE,CAAC,EAAE,CAAC;IAC1D,MAAM,YAAY,GAAG,YAAY,CAAC,MAAM,EAAE,GAAG,EAAE,MAAM,CAAC,CAAC;IAEvD,IAAI,MAAM,CAAC,aAAa,KAAK,CAAC,EAAE,CAAC;QAC/B,OAAO,IAAA,6BAAe,EAAC,OAAO,EAAE,MAAM,EAAE,sBAAsB,EAAE,MAAM,CAAC,QAAQ,CAAC,CAAC;IACnF,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,CAAC;IAC5C,OAAO;QACL,MAAM;QACN,KAAK,EAAE;YACL,UAAU,EAAE,MAAM;YAClB,aAAa,EAAE,OAAO,CAAC,MAAM;YAC7B,eAAe,EAAE,MAAM,CAAC,MAAM;YAC9B,gBAAgB,EAAE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;YAC7D,aAAa,EAAE,MAAM,CAAC,QAAQ;YAC9B,SAAS,EAAE,MAAM,CAAC,IAAI;YACtB,WAAW,EAAE,KAAK;SACnB;KACF,CAAC;AACJ,CAAC"}

package/dist/crush/json-detectors.d.ts

@@ -0,0 +1,48 @@
+/**
+ * MIRRORED FILE — canonical source: packages/core/src/crush/json-detectors.ts.
+ * Byte-identical copy lives at packages/cli/src/crush/json-detectors.ts (the CLI is a
+ * zero-dep artifact and cannot import @greprag/core). KEEP IN SYNC —
+ * the sync guard in tests/test-crush.cjs fails the build on drift.
+ *
+ * Array classification + outlier/sequence detectors for the JSON
+ * crusher — port of the simple core of Headroom's smart_crusher
+ * (classifier.rs, outliers.rs, statistics.rs).
+ *
+ * Carries Headroom's documented fixes:
+ * - Zero-padded string IDs ("001", "002", ...) are NOT sequential
+ *   numerics — if every parseable value originated as a string, the
+ *   field is categorical and the padding matters.
+ * - Rare-status detection must not short-circuit at >10 distinct
+ *   values: cardinality cap raised to 50 with a Pareto check (smallest
+ *   K values covering >=80%; rare iff K <= 5), so datasets with dozens
+ *   of error codes keep their rare-error preservation.
+ */
+export type JsonValue = null | boolean | number | string | JsonValue[] | {
+    [key: string]: JsonValue;
+};
+export type ArrayType = 'dict_array' | 'string_array' | 'number_array' | 'bool_array' | 'nested_array' | 'mixed_array' | 'empty';
+/** Classify a JSON array by element types (walks every element). */
+export declare function classifyArray(items: readonly JsonValue[]): ArrayType;
+/**
+ * Detect whether values form a sequential numeric pattern (IDs: 1,2,3…).
+ *
+ * Zero-padding fix: string values that parse as integers are counted,
+ * but if EVERY numeric value originated as a string the field is
+ * refused as sequential (padded codes are categorical).
+ */
+export declare function detectSequentialPattern(values: readonly JsonValue[]): boolean;
+/**
+ * Detect items with rare values in status-like categorical fields
+ * (Pareto check, cardinality 2..=50). Returns item indices.
+ */
+export declare function detectRareStatusValues(items: readonly JsonValue[], commonFields: ReadonlySet<string>): number[];
+/**
+ * Detect structural outliers: items carrying a rare field (<20% of the
+ * array) plus rare-status items. Returns sorted, deduped indices.
+ */
+export declare function detectStructuralOutliers(items: readonly JsonValue[]): number[];
+/**
+ * Detect items containing error keywords — these are NEVER dropped.
+ * Scans each object item's full serialization, case-insensitively.
+ */
+export declare function detectErrorItems(items: readonly JsonValue[]): number[];

package/dist/crush/json-detectors.js

@@ -0,0 +1,211 @@
+"use strict";
+/**
+ * MIRRORED FILE — canonical source: packages/core/src/crush/json-detectors.ts.
+ * Byte-identical copy lives at packages/cli/src/crush/json-detectors.ts (the CLI is a
+ * zero-dep artifact and cannot import @greprag/core). KEEP IN SYNC —
+ * the sync guard in tests/test-crush.cjs fails the build on drift.
+ *
+ * Array classification + outlier/sequence detectors for the JSON
+ * crusher — port of the simple core of Headroom's smart_crusher
+ * (classifier.rs, outliers.rs, statistics.rs).
+ *
+ * Carries Headroom's documented fixes:
+ * - Zero-padded string IDs ("001", "002", ...) are NOT sequential
+ *   numerics — if every parseable value originated as a string, the
+ *   field is categorical and the padding matters.
+ * - Rare-status detection must not short-circuit at >10 distinct
+ *   values: cardinality cap raised to 50 with a Pareto check (smallest
+ *   K values covering >=80%; rare iff K <= 5), so datasets with dozens
+ *   of error codes keep their rare-error preservation.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.classifyArray = classifyArray;
+exports.detectSequentialPattern = detectSequentialPattern;
+exports.detectRareStatusValues = detectRareStatusValues;
+exports.detectStructuralOutliers = detectStructuralOutliers;
+exports.detectErrorItems = detectErrorItems;
+const crush_keywords_1 = require("./crush-keywords");
+/** Classify a JSON array by element types (walks every element). */
+function classifyArray(items) {
+    if (items.length === 0)
+        return 'empty';
+    let hasBool = false, hasNumber = false, hasString = false, hasObject = false, hasArray = false, hasNull = false;
+    for (const item of items) {
+        if (item === null)
+            hasNull = true;
+        else if (typeof item === 'boolean')
+            hasBool = true;
+        else if (typeof item === 'number')
+            hasNumber = true;
+        else if (typeof item === 'string')
+            hasString = true;
+        else if (Array.isArray(item))
+            hasArray = true;
+        else
+            hasObject = true;
+    }
+    if (hasBool && !hasNumber && !hasString && !hasObject && !hasArray && !hasNull)
+        return 'bool_array';
+    if (hasObject && !hasBool && !hasNumber && !hasString && !hasArray && !hasNull)
+        return 'dict_array';
+    if (hasString && !hasBool && !hasNumber && !hasObject && !hasArray && !hasNull)
+        return 'string_array';
+    if (hasNumber && !hasBool && !hasString && !hasObject && !hasArray && !hasNull)
+        return 'number_array';
+    if (hasArray && !hasBool && !hasNumber && !hasString && !hasObject && !hasNull)
+        return 'nested_array';
+    return 'mixed_array';
+}
+function isPlainObject(v) {
+    return typeof v === 'object' && v !== null && !Array.isArray(v);
+}
+function stringifyScalar(v) {
+    if (typeof v === 'string')
+        return v;
+    return JSON.stringify(v);
+}
+/**
+ * Detect whether values form a sequential numeric pattern (IDs: 1,2,3…).
+ *
+ * Zero-padding fix: string values that parse as integers are counted,
+ * but if EVERY numeric value originated as a string the field is
+ * refused as sequential (padded codes are categorical).
+ */
+function detectSequentialPattern(values) {
+    if (values.length < 5)
+        return false;
+    const nums = [];
+    let hadNonStringNumeric = false;
+    for (const v of values) {
+        if (typeof v === 'number') {
+            nums.push(v);
+            hadNonStringNumeric = true;
+        }
+        else if (typeof v === 'string' && /^[+-]?\d+$/.test(v.trim())) {
+            nums.push(parseInt(v.trim(), 10));
+            // Deliberately NOT setting hadNonStringNumeric (the fix).
+        }
+    }
+    if (nums.length < 5)
+        return false;
+    if (!hadNonStringNumeric)
+        return false;
+    const sorted = [...nums].sort((a, b) => a - b);
+    const diffs = [];
+    for (let i = 1; i < sorted.length; i++)
+        diffs.push(sorted[i] - sorted[i - 1]);
+    if (diffs.length === 0)
+        return false;
+    const avg = diffs.reduce((s, d) => s + d, 0) / diffs.length;
+    if (avg < 0.5 || avg > 2.0)
+        return false;
+    const consistent = diffs.filter((d) => d >= 0.5 && d <= 2.0).length;
+    return consistent / diffs.length > 0.8;
+}
+/**
+ * Detect items with rare values in status-like categorical fields
+ * (Pareto check, cardinality 2..=50). Returns item indices.
+ */
+function detectRareStatusValues(items, commonFields) {
+    const outliers = [];
+    const sortedFields = [...commonFields].sort();
+    for (const field of sortedFields) {
+        const values = [];
+        for (const item of items) {
+            if (isPlainObject(item) && field in item)
+                values.push(item[field]);
+        }
+        const uniqueValues = new Set();
+        for (const v of values) {
+            if (v !== null)
+                uniqueValues.add(stringifyScalar(v));
+        }
+        // Cardinality cap: 2..=50 (the >10 short-circuit fix).
+        if (uniqueValues.size < 2 || uniqueValues.size > 50)
+            continue;
+        const counts = new Map();
+        for (const v of values) {
+            const key = v === null ? '__none__' : stringifyScalar(v);
+            counts.set(key, (counts.get(key) ?? 0) + 1);
+        }
+        if (counts.size === 0)
+            continue;
+        // Pareto: smallest K values covering >=80% of items.
+        const sortedCounts = [...counts.entries()].sort((a, b) => b[1] - a[1] || (a[0] < b[0] ? -1 : 1));
+        const threshold = Math.ceil(values.length * 0.8);
+        let cumulative = 0;
+        const topK = new Set();
+        for (const [value, count] of sortedCounts) {
+            cumulative += count;
+            topK.add(value);
+            if (cumulative >= threshold)
+                break;
+        }
+        // Above K=5 the distribution is too uniform to call anything rare.
+        if (topK.size > 5)
+            continue;
+        for (let i = 0; i < items.length; i++) {
+            const item = items[i];
+            if (!isPlainObject(item) || !(field in item))
+                continue;
+            const v = item[field];
+            const key = v === null ? '__none__' : stringifyScalar(v);
+            if (!topK.has(key))
+                outliers.push(i);
+        }
+    }
+    return outliers;
+}
+/**
+ * Detect structural outliers: items carrying a rare field (<20% of the
+ * array) plus rare-status items. Returns sorted, deduped indices.
+ */
+function detectStructuralOutliers(items) {
+    if (items.length < 5)
+        return [];
+    const fieldCounts = new Map();
+    for (const item of items) {
+        if (!isPlainObject(item))
+            continue;
+        for (const key of Object.keys(item)) {
+            fieldCounts.set(key, (fieldCounts.get(key) ?? 0) + 1);
+        }
+    }
+    const n = items.length;
+    const commonFields = new Set();
+    const rareFields = new Set();
+    for (const [field, count] of fieldCounts) {
+        if (count >= n * 0.8)
+            commonFields.add(field);
+        if (count < n * 0.2)
+            rareFields.add(field);
+    }
+    const outliers = new Set();
+    for (let i = 0; i < items.length; i++) {
+        const item = items[i];
+        if (!isPlainObject(item))
+            continue;
+        if (Object.keys(item).some((k) => rareFields.has(k)))
+            outliers.add(i);
+    }
+    for (const idx of detectRareStatusValues(items, commonFields))
+        outliers.add(idx);
+    return [...outliers].sort((a, b) => a - b);
+}
+/**
+ * Detect items containing error keywords — these are NEVER dropped.
+ * Scans each object item's full serialization, case-insensitively.
+ */
+function detectErrorItems(items) {
+    const out = [];
+    for (let i = 0; i < items.length; i++) {
+        const item = items[i];
+        if (!isPlainObject(item))
+            continue;
+        const serialized = JSON.stringify(item).toLowerCase();
+        if (crush_keywords_1.JSON_ERROR_KEYWORDS.some((kw) => serialized.includes(kw)))
+            out.push(i);
+    }
+    return out;
+}
+//# sourceMappingURL=json-detectors.js.map

package/dist/crush/json-detectors.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"json-detectors.js","sourceRoot":"","sources":["../../src/crush/json-detectors.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;;;;;GAkBG;;AAgBH,sCAsBC;AAkBD,0DAwBC;AAMD,wDAiDC;AAMD,4DA4BC;AAMD,4CASC;AAtLD,qDAAuD;AAavD,oEAAoE;AACpE,SAAgB,aAAa,CAAC,KAA2B;IACvD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,OAAO,CAAC;IACvC,IAAI,OAAO,GAAG,KAAK,EACjB,SAAS,GAAG,KAAK,EACjB,SAAS,GAAG,KAAK,EACjB,SAAS,GAAG,KAAK,EACjB,QAAQ,GAAG,KAAK,EAChB,OAAO,GAAG,KAAK,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,IAAI,KAAK,IAAI;YAAE,OAAO,GAAG,IAAI,CAAC;aAC7B,IAAI,OAAO,IAAI,KAAK,SAAS;YAAE,OAAO,GAAG,IAAI,CAAC;aAC9C,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,SAAS,GAAG,IAAI,CAAC;aAC/C,IAAI,OAAO,IAAI,KAAK,QAAQ;YAAE,SAAS,GAAG,IAAI,CAAC;aAC/C,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;YAAE,QAAQ,GAAG,IAAI,CAAC;;YACzC,SAAS,GAAG,IAAI,CAAC;IACxB,CAAC;IACD,IAAI,OAAO,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,YAAY,CAAC;IACpG,IAAI,SAAS,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,YAAY,CAAC;IACpG,IAAI,SAAS,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,cAAc,CAAC;IACtG,IAAI,SAAS,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,QAAQ,IAAI,CAAC,OAAO;QAAE,OAAO,cAAc,CAAC;IACtG,IAAI,QAAQ,IAAI,CAAC,OAAO,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,IAAI,CAAC,OAAO;QAAE,OAAO,cAAc,CAAC;IACtG,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,SAAS,aAAa,CAAC,CAAY;IACjC,OAAO,OAAO,CAAC,KAAK,QAAQ,IAAI,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;AAClE,CAAC;AAED,SAAS,eAAe,CAAC,CAAY;IACnC,IAAI,OAAO,CAAC,KAAK,QAAQ;QAAE,OAAO,CAAC,CAAC;IACpC,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;AAC3B,CAAC;AAED;;;;;;GAMG;AACH,SAAgB,uBAAuB,CAAC,MAA4B;IAClE,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACpC,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,IAAI,mBAAmB,GAAG,KAAK,CAAC;IAChC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC1B,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACb,mBAAmB,GAAG,IAAI,CAAC;QAC7B,CAAC;aAAM,IAAI,OAAO,CAAC,KAAK,QAAQ,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,EAAE,CAAC;YAChE,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC;YAClC,0DAA0D;QAC5D,CAAC;IACH,CAAC;IACD,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAClC,IAAI,CAAC,mBAAmB;QAAE,OAAO,KAAK,CAAC;IAEvC,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE;QAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAC9E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IAC5D,IAAI,GAAG,GAAG,GAAG,IAAI,GAAG,GAAG,GAAG;QAAE,OAAO,KAAK,CAAC;IACzC,MAAM,UAAU,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC;IACpE,OAAO,UAAU,GAAG,KAAK,CAAC,MAAM,GAAG,GAAG,CAAC;AACzC,CAAC;AAED;;;GAGG;AACH,SAAgB,sBAAsB,CACpC,KAA2B,EAC3B,YAAiC;IAEjC,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,YAAY,GAAG,CAAC,GAAG,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;IAE9C,KAAK,MAAM,KAAK,IAAI,YAAY,EAAE,CAAC;QACjC,MAAM,MAAM,GAAgB,EAAE,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,IAAI,aAAa,CAAC,IAAI,CAAC,IAAI,KAAK,IAAI,IAAI;gBAAE,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC;QACrE,CAAC;QAED,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;QACvC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,IAAI,CAAC,KAAK,IAAI;gBAAE,YAAY,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC,CAAC;QACvD,CAAC;QACD,uDAAuD;QACvD,IAAI,YAAY,CAAC,IAAI,GAAG,CAAC,IAAI,YAAY,CAAC,IAAI,GAAG,EAAE;YAAE,SAAS;QAE9D,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;QACzC,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,GAAG,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACzD,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9C,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;YAAE,SAAS;QAEhC,qDAAqD;QACrD,MAAM,YAAY,GAAG,CAAC,GAAG,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACjG,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;QACjD,IAAI,UAAU,GAAG,CAAC,CAAC;QACnB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,YAAY,EAAE,CAAC;YAC1C,UAAU,IAAI,KAAK,CAAC;YACpB,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;YAChB,IAAI,UAAU,IAAI,SAAS;gBAAE,MAAM;QACrC,CAAC;QACD,mEAAmE;QACnE,IAAI,IAAI,CAAC,IAAI,GAAG,CAAC;YAAE,SAAS;QAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;YACtB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,IAAI,CAAC;gBAAE,SAAS;YACvD,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC;YACtB,MAAM,GAAG,GAAG,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAC;YACzD,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;gBAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;;GAGG;AACH,SAAgB,wBAAwB,CAAC,KAA2B;IAClE,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,CAAC;IAEhC,MAAM,WAAW,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC9C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;YAAE,SAAS;QACnC,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QACxD,CAAC;IACH,CAAC;IAED,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IACvB,MAAM,YAAY,GAAG,IAAI,GAAG,EAAU,CAAC;IACvC,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAC;IACrC,KAAK,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC;QACzC,IAAI,KAAK,IAAI,CAAC,GAAG,GAAG;YAAE,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC9C,IAAI,KAAK,GAAG,CAAC,GAAG,GAAG;YAAE,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IACnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;YAAE,SAAS;QACnC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;YAAE,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IACxE,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,sBAAsB,CAAC,KAAK,EAAE,YAAY,CAAC;QAAE,QAAQ,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IAEjF,OAAO,CAAC,GAAG,QAAQ,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;AAC7C,CAAC;AAED;;;GAGG;AACH,SAAgB,gBAAgB,CAAC,KAA2B;IAC1D,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC;YAAE,SAAS;QACnC,MAAM,UAAU,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;QACtD,IAAI,oCAAmB,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC7E,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}