@untangle-code/scanner 0.0.0-stage → 0.1.0

package/dist/binary-classifier.js

@@ -0,0 +1,296 @@
+/**
+ * Binary / encoding / minified-file classification from a content sample
+ * (spec.md §6.1: "binäre Dateien erkennen"; WP-05 order: "Nullbytes, Anteil
+ * nicht druckbarer Bytes, BOM-/Encoding-Erkennung" + minified heuristics).
+ *
+ * Pure functions over bytes already in memory — no I/O.
+ */
+import { DEFAULT_MINIFIED_RULE, isMinifiedShape } from "@untangle-code/domain";
+// Longest match first (utf16le/be are 2-byte prefixes of nothing else here,
+// but ordering is kept explicit and defensive).
+//
+// NOTE (checked per WP defect audit, not fixed — out of spec.md/architecture.md
+// scope): UTF-32 is not a recognized `TextEncodingGuess` and has no BOM_SPECS
+// entry. A UTF-32LE-BOM'd sample (FF FE 00 00 …) therefore still matches the
+// 2-byte utf16le prefix above and is (mis-)tagged "utf16le" with a 2-byte BOM
+// length. `classifyUtf16Body` below decodes the remaining bytes as UTF-16LE
+// in that case, which — because every UTF-32LE code unit contributes two
+// extra 0x00 bytes — reliably decodes to text containing genuine embedded
+// NUL characters, so the sample still ends up classified `isBinary: true`.
+// That is the pre-existing, undocumented-until-now behavior for UTF-32
+// input: not a regression introduced by this fix, but also not a real fix —
+// neither spec.md nor architecture.md specify UTF-32 support, so adding a
+// dedicated "utf32le"/"utf32be" encoding is a separate, out-of-scope
+// decision rather than something to guess at here.
+const BOM_SPECS = [
+    { bytes: [0xef, 0xbb, 0xbf], encoding: "utf8" },
+    { bytes: [0xff, 0xfe], encoding: "utf16le" },
+    { bytes: [0xfe, 0xff], encoding: "utf16be" },
+];
+function detectBom(sample) {
+    for (const spec of BOM_SPECS) {
+        if (sample.length < spec.bytes.length) {
+            continue;
+        }
+        let matches = true;
+        for (let i = 0; i < spec.bytes.length; i += 1) {
+            if (sample[i] !== spec.bytes[i]) {
+                matches = false;
+                break;
+            }
+        }
+        if (matches) {
+            return { hasBom: true, encoding: spec.encoding, length: spec.bytes.length };
+        }
+    }
+    return { hasBom: false, encoding: "unknown", length: 0 };
+}
+/** Control bytes that indicate non-text content; tab/LF/CR are excluded (they're common in text). */
+function isNonPrintableControlByte(byte) {
+    return (byte <= 0x08 ||
+        byte === 0x0b ||
+        byte === 0x0c ||
+        (byte >= 0x0e && byte <= 0x1f) ||
+        byte === 0x7f);
+}
+/**
+ * Classifies the body of a sample confirmed (by BOM or by the BOM-less
+ * heuristic in `detectUtf16WithoutBom`) to be UTF-16 (LE or BE).
+ *
+ * The raw "a NUL byte means binary" rule from `classifyBinarySample` cannot
+ * be applied to UTF-16 bytes directly: every Basic-Latin code unit (i.e.
+ * every plain ASCII character) carries a 0x00 byte as one half of its
+ * 16-bit unit, so a legitimate UTF-16 text file is close to 50% NUL bytes
+ * by construction. Decoding first and inspecting actual *characters*
+ * avoids that false positive while still catching real NUL characters and
+ * a high ratio of real control characters.
+ */
+function classifyUtf16Body(body, encoding, threshold, hasBom) {
+    // Non-fatal decode: a byte sequence that only coincidentally starts with a
+    // UTF-16 BOM (rare, but the classifier must stay robust for arbitrary
+    // binary input) must not throw here — invalid sequences become U+FFFD
+    // instead, which is handled like any other non-control character below.
+    const text = new TextDecoder(encoding === "utf16le" ? "utf-16le" : "utf-16be", {
+        fatal: false,
+    }).decode(body);
+    let hasNullByte = false;
+    let nonPrintable = 0;
+    let unitCount = 0;
+    for (const char of text) {
+        unitCount += 1;
+        const codePoint = char.codePointAt(0) ?? 0;
+        if (codePoint === 0) {
+            hasNullByte = true;
+            continue;
+        }
+        if (codePoint < 0x100 && isNonPrintableControlByte(codePoint)) {
+            nonPrintable += 1;
+        }
+    }
+    const nonPrintableRatio = unitCount === 0 ? 0 : nonPrintable / unitCount;
+    const isBinary = hasNullByte || nonPrintableRatio > threshold;
+    return { isBinary, hasNullByte, nonPrintableRatio, encoding, hasBom };
+}
+// --- BOM-less UTF-16 detection (defect fix, spec.md E4) --------------------
+//
+// A UTF-16 file with no BOM is legal (spec.md/architecture.md do not require
+// one, and plenty of real tooling emits BOM-less UTF-16). Without a BOM,
+// `detectBom` reports `hasBom:false` and — before this fix — the sample fell
+// straight into the raw byte-level loop below, which sees every other byte
+// as 0x00 (the high or low half of each Basic-Latin code unit) and
+// classifies the file as binary. That is a false NEGATIVE (real source text
+// silently vanishes from the index), which is worse than the false-POSITIVE
+// risk of a speculative heuristic, so this is a deliberate, narrow, and
+// documented exception to "don't guess encodings beyond spec": see
+// README.md "Entscheidung 12".
+//
+// Detection requires BOTH of the following (either alone is too weak):
+//
+//   1. Pattern: over the sample, one specific byte parity (even bytes for
+//      UTF-16BE, odd bytes for UTF-16LE) is zero at a very high, regular
+//      rate (>= UTF16_HEURISTIC_DOMINANT_NULL_RATIO), while the *opposite*
+//      parity is zero only rarely (<= UTF16_HEURISTIC_OPPOSITE_NULL_RATIO).
+//      A real binary file's NUL bytes are not reliably tied to byte parity
+//      like that; this is the "regelmäßig alternierende Nullbytes"
+//      signature of Basic-Latin text encoded as fixed-width 16-bit units.
+//   2. Validation: decoding the sample under the *candidate* encoding and
+//      counting actual code units, at least
+//      UTF16_HEURISTIC_MIN_PRINTABLE_RATIO of them must be printable
+//      (excluding NUL, other control characters, and the U+FFFD decode-
+//      error marker). This is the same "decode and inspect real
+//      characters" principle `classifyUtf16Body` already applies to
+//      BOM-confirmed UTF-16 — it is what rejects a real binary blob whose
+//      bytes happen to satisfy the parity pattern by chance.
+//
+// Both gates apply only once the sample reaches UTF16_HEURISTIC_MIN_SAMPLE_BYTES
+// (32 bytes = 16 code units) — below that, parity ratios are too noisy to
+// trust one way or the other, so short samples fall through to the existing
+// (conservative: binary-by-default) raw byte-level classification. This is
+// the documented "sehr kurze Probe" edge case; see the co-located test.
+const UTF16_HEURISTIC_MIN_SAMPLE_BYTES = 32;
+const UTF16_HEURISTIC_DOMINANT_NULL_RATIO = 0.6;
+const UTF16_HEURISTIC_OPPOSITE_NULL_RATIO = 0.1;
+const UTF16_HEURISTIC_MIN_PRINTABLE_RATIO = 0.9;
+/**
+ * Attempts to identify BOM-less UTF-16 text in `sample` via the two-gate
+ * heuristic documented above. Returns the detected encoding, or `null` when
+ * the sample is too short, does not show the alternating-NUL pattern, or
+ * fails the printability validation (including: genuinely random/binary
+ * data that happens to satisfy the parity check alone).
+ */
+function detectUtf16WithoutBom(sample) {
+    const pairCount = Math.floor(sample.length / 2);
+    if (sample.length < UTF16_HEURISTIC_MIN_SAMPLE_BYTES || pairCount === 0) {
+        return null;
+    }
+    let nullAtEven = 0; // high byte of each pair is 0x00 -> UTF-16BE ASCII signal
+    let nullAtOdd = 0; // low byte of each pair is 0x00 -> UTF-16LE ASCII signal
+    const bodyLength = pairCount * 2;
+    for (let i = 0; i < bodyLength; i += 2) {
+        if (sample[i] === 0) {
+            nullAtEven += 1;
+        }
+        if (sample[i + 1] === 0) {
+            nullAtOdd += 1;
+        }
+    }
+    const evenRatio = nullAtEven / pairCount;
+    const oddRatio = nullAtOdd / pairCount;
+    let candidate = null;
+    if (oddRatio >= UTF16_HEURISTIC_DOMINANT_NULL_RATIO &&
+        evenRatio <= UTF16_HEURISTIC_OPPOSITE_NULL_RATIO) {
+        candidate = "utf16le";
+    }
+    else if (evenRatio >= UTF16_HEURISTIC_DOMINANT_NULL_RATIO &&
+        oddRatio <= UTF16_HEURISTIC_OPPOSITE_NULL_RATIO) {
+        candidate = "utf16be";
+    }
+    if (candidate === null) {
+        return null;
+    }
+    // Gate 2: decode under the candidate encoding and require the result to be
+    // overwhelmingly printable — this is what keeps real binary data (which
+    // can coincidentally satisfy the byte-parity check above, e.g. a run of
+    // small little-endian integers) from being misread as text.
+    const body = sample.subarray(0, bodyLength);
+    const text = new TextDecoder(candidate === "utf16le" ? "utf-16le" : "utf-16be", {
+        fatal: false,
+    }).decode(body);
+    let printable = 0;
+    let total = 0;
+    for (const char of text) {
+        total += 1;
+        const codePoint = char.codePointAt(0) ?? 0;
+        if (codePoint === 0) {
+            continue;
+        }
+        if (codePoint === 0xfffd) {
+            // TextDecoder's replacement character: an invalid code unit sequence
+            // under this candidate encoding, not evidence of real text.
+            continue;
+        }
+        if (codePoint < 0x100 && isNonPrintableControlByte(codePoint)) {
+            continue;
+        }
+        printable += 1;
+    }
+    if (total === 0 || printable / total < UTF16_HEURISTIC_MIN_PRINTABLE_RATIO) {
+        return null;
+    }
+    return candidate;
+}
+/**
+ * Classifies a content sample as binary or text.
+ *
+ * A sample is binary when it contains a NUL byte or when the ratio of
+ * non-printable control bytes exceeds `nonPrintableThreshold` (default
+ * 0.3). Bytes in the 0x80–0xFF range are NOT counted as non-printable —
+ * valid UTF-8 text uses them for non-ASCII characters, and flagging them
+ * would misclassify ordinary internationalized source/comments as binary.
+ *
+ * When the sample carries a UTF-16 BOM, the body is decoded per the
+ * confirmed encoding and classified over actual characters instead
+ * (`classifyUtf16Body`) — see that function's doc comment for why the raw
+ * byte-level NUL rule does not apply there.
+ *
+ * Absent a BOM, `detectUtf16WithoutBom` still checks for a conservative,
+ * two-gate BOM-less UTF-16 signature (regular alternating NUL bytes at one
+ * parity, validated by decoding and requiring the result be overwhelmingly
+ * printable) before falling back to the raw byte-level rule — see that
+ * function's doc comment for the documented thresholds and rationale
+ * (spec.md E4: a false negative here silently drops real source text from
+ * the index).
+ */
+export function classifyBinarySample(sample, options) {
+    const threshold = options?.nonPrintableThreshold ?? 0.3;
+    const bom = detectBom(sample);
+    const body = bom.hasBom ? sample.subarray(bom.length) : sample;
+    if (bom.hasBom && (bom.encoding === "utf16le" || bom.encoding === "utf16be")) {
+        return classifyUtf16Body(body, bom.encoding, threshold, true);
+    }
+    if (!bom.hasBom) {
+        const bomlessUtf16 = detectUtf16WithoutBom(sample);
+        if (bomlessUtf16 !== null) {
+            return classifyUtf16Body(sample, bomlessUtf16, threshold, false);
+        }
+    }
+    let hasNullByte = false;
+    let nonPrintable = 0;
+    let highBit = false;
+    for (const byte of body) {
+        if (byte === 0) {
+            hasNullByte = true;
+            continue;
+        }
+        if (isNonPrintableControlByte(byte)) {
+            nonPrintable += 1;
+        }
+        else if (byte >= 0x80) {
+            highBit = true;
+        }
+    }
+    const nonPrintableRatio = body.length === 0 ? 0 : nonPrintable / body.length;
+    const isBinary = hasNullByte || nonPrintableRatio > threshold;
+    const encoding = bom.hasBom
+        ? bom.encoding
+        : isBinary
+            ? "unknown"
+            : highBit
+                ? "utf8"
+                : "ascii";
+    return { isBinary, hasNullByte, nonPrintableRatio, encoding, hasBom: bom.hasBom };
+}
+/**
+ * Heuristic minified-file detection ("sehr lange Zeilen, geringes
+ * Zeilen-zu-Byte-Verhaeltnis") from a raw content sample. Operates on
+ * newline bytes (0x0A) directly rather than decoding, so it is safe to run
+ * on the same sample used for binary classification.
+ */
+export function detectMinified(sample, heuristics, textFormat) {
+    if (sample.length === 0) {
+        return false;
+    }
+    const maxLineLengthThreshold = heuristics?.maxLineLengthThreshold ?? DEFAULT_MINIFIED_RULE.maxLineLengthThreshold;
+    const averageBytesPerLineThreshold = heuristics?.averageBytesPerLineThreshold ?? DEFAULT_MINIFIED_RULE.averageLineLengthThreshold;
+    let lineBreaks = 0;
+    let longestLine = 0;
+    let currentLine = 0;
+    for (const byte of sample) {
+        if (byte === 0x0a) {
+            lineBreaks += 1;
+            longestLine = Math.max(longestLine, currentLine);
+            currentLine = 0;
+        }
+        else {
+            currentLine += 1;
+        }
+    }
+    longestLine = Math.max(longestLine, currentLine);
+    const averageBytesPerLine = sample.length / Math.max(lineBreaks, 1);
+    // Die Entscheidung selbst liegt in `@untangle-code/domain`, damit Scanner und
+    // Chunker dieselbe Frage nicht mit verschiedenen Schwellen beantworten. Hier
+    // wird nur GEMESSEN — auf Bytes, weil dieselbe Stichprobe schon fuer die
+    // Binaerklassifikation gelesen wurde.
+    return isMinifiedShape({ maxLineLength: longestLine, averageLineLength: averageBytesPerLine }, { textFormat: textFormat ?? null }, { maxLineLengthThreshold, averageLineLengthThreshold: averageBytesPerLineThreshold });
+}
+//# sourceMappingURL=binary-classifier.js.map

package/dist/binary-classifier.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"binary-classifier.js","sourceRoot":"","sources":["../src/binary-classifier.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAU/E,4EAA4E;AAC5E,gDAAgD;AAChD,EAAE;AACF,gFAAgF;AAChF,8EAA8E;AAC9E,6EAA6E;AAC7E,8EAA8E;AAC9E,4EAA4E;AAC5E,yEAAyE;AACzE,0EAA0E;AAC1E,2EAA2E;AAC3E,uEAAuE;AACvE,4EAA4E;AAC5E,0EAA0E;AAC1E,qEAAqE;AACrE,mDAAmD;AACnD,MAAM,SAAS,GAAuB;IACpC,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE;IAC/C,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE;IAC5C,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE;CAC7C,CAAC;AAEF,SAAS,SAAS,CAAC,MAAkB;IAKnC,KAAK,MAAM,IAAI,IAAI,SAAS,EAAE,CAAC;QAC7B,IAAI,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;YACtC,SAAS;QACX,CAAC;QACD,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YAC9C,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;gBAChC,OAAO,GAAG,KAAK,CAAC;gBAChB,MAAM;YACR,CAAC;QACH,CAAC;QACD,IAAI,OAAO,EAAE,CAAC;YACZ,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9E,CAAC;IACH,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;AAC3D,CAAC;AAED,qGAAqG;AACrG,SAAS,yBAAyB,CAAC,IAAY;IAC7C,OAAO,CACL,IAAI,IAAI,IAAI;QACZ,IAAI,KAAK,IAAI;QACb,IAAI,KAAK,IAAI;QACb,CAAC,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,IAAI,CAAC;QAC9B,IAAI,KAAK,IAAI,CACd,CAAC;AACJ,CAAC;AAED;;;;;;;;;;;GAWG;AACH,SAAS,iBAAiB,CACxB,IAAgB,EAChB,QAA+B,EAC/B,SAAiB,EACjB,MAAe;IAEf,2EAA2E;IAC3E,sEAAsE;IACtE,sEAAsE;IACtE,wEAAwE;IACxE,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,EAAE;QAC7E,KAAK,EAAE,KAAK;KACb,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEhB,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,SAAS,IAAI,CAAC,CAAC;QACf,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,WAAW,GAAG,IAAI,CAAC;YACnB,SAAS;QACX,CAAC;QACD,IAAI,SAAS,GAAG,KAAK,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,YAAY,IAAI,CAAC,CAAC;QACpB,CAAC;IACH,CAAC;IAED,MAAM,iBAAiB,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,SAAS,CAAC;IACzE,MAAM,QAAQ,GAAG,WAAW,IAAI,iBAAiB,GAAG,SAAS,CAAC;IAE9D,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC;AACxE,CAAC;AAED,8EAA8E;AAC9E,EAAE;AACF,6EAA6E;AAC7E,yEAAyE;AACzE,6EAA6E;AAC7E,2EAA2E;AAC3E,mEAAmE;AACnE,4EAA4E;AAC5E,4EAA4E;AAC5E,wEAAwE;AACxE,mEAAmE;AACnE,+BAA+B;AAC/B,EAAE;AACF,uEAAuE;AACvE,EAAE;AACF,0EAA0E;AAC1E,yEAAyE;AACzE,2EAA2E;AAC3E,4EAA4E;AAC5E,2EAA2E;AAC3E,mEAAmE;AACnE,0EAA0E;AAC1E,0EAA0E;AAC1E,4CAA4C;AAC5C,qEAAqE;AACrE,wEAAwE;AACxE,gEAAgE;AAChE,oEAAoE;AACpE,0EAA0E;AAC1E,6DAA6D;AAC7D,EAAE;AACF,iFAAiF;AACjF,0EAA0E;AAC1E,4EAA4E;AAC5E,2EAA2E;AAC3E,wEAAwE;AACxE,MAAM,gCAAgC,GAAG,EAAE,CAAC;AAC5C,MAAM,mCAAmC,GAAG,GAAG,CAAC;AAChD,MAAM,mCAAmC,GAAG,GAAG,CAAC;AAChD,MAAM,mCAAmC,GAAG,GAAG,CAAC;AAEhD;;;;;;GAMG;AACH,SAAS,qBAAqB,CAAC,MAAkB;IAC/C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;IAChD,IAAI,MAAM,CAAC,MAAM,GAAG,gCAAgC,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;QACxE,OAAO,IAAI,CAAC;IACd,CAAC;IAED,IAAI,UAAU,GAAG,CAAC,CAAC,CAAC,0DAA0D;IAC9E,IAAI,SAAS,GAAG,CAAC,CAAC,CAAC,yDAAyD;IAC5E,MAAM,UAAU,GAAG,SAAS,GAAG,CAAC,CAAC;IACjC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACvC,IAAI,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YACpB,UAAU,IAAI,CAAC,CAAC;QAClB,CAAC;QACD,IAAI,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;YACxB,SAAS,IAAI,CAAC,CAAC;QACjB,CAAC;IACH,CAAC;IAED,MAAM,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;IACzC,MAAM,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;IAEvC,IAAI,SAAS,GAAiC,IAAI,CAAC;IACnD,IACE,QAAQ,IAAI,mCAAmC;QAC/C,SAAS,IAAI,mCAAmC,EAChD,CAAC;QACD,SAAS,GAAG,SAAS,CAAC;IACxB,CAAC;SAAM,IACL,SAAS,IAAI,mCAAmC;QAChD,QAAQ,IAAI,mCAAmC,EAC/C,CAAC;QACD,SAAS,GAAG,SAAS,CAAC;IACxB,CAAC;IACD,IAAI,SAAS,KAAK,IAAI,EAAE,CAAC;QACvB,OAAO,IAAI,CAAC;IACd,CAAC;IAED,2EAA2E;IAC3E,wEAAwE;IACxE,wEAAwE;IACxE,4DAA4D;IAC5D,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,IAAI,WAAW,CAAC,SAAS,KAAK,SAAS,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,UAAU,EAAE;QAC9E,KAAK,EAAE,KAAK;KACb,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAEhB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,KAAK,IAAI,CAAC,CAAC;QACX,MAAM,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;QAC3C,IAAI,SAAS,KAAK,CAAC,EAAE,CAAC;YACpB,SAAS;QACX,CAAC;QACD,IAAI,SAAS,KAAK,MAAM,EAAE,CAAC;YACzB,qEAAqE;YACrE,4DAA4D;YAC5D,SAAS;QACX,CAAC;QACD,IAAI,SAAS,GAAG,KAAK,IAAI,yBAAyB,CAAC,SAAS,CAAC,EAAE,CAAC;YAC9D,SAAS;QACX,CAAC;QACD,SAAS,IAAI,CAAC,CAAC;IACjB,CAAC;IAED,IAAI,KAAK,KAAK,CAAC,IAAI,SAAS,GAAG,KAAK,GAAG,mCAAmC,EAAE,CAAC;QAC3E,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,MAAM,UAAU,oBAAoB,CAClC,MAAkB,EAClB,OAA4C;IAE5C,MAAM,SAAS,GAAG,OAAO,EAAE,qBAAqB,IAAI,GAAG,CAAC;IACxD,MAAM,GAAG,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC;IAC9B,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IAE/D,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC,GAAG,CAAC,QAAQ,KAAK,SAAS,IAAI,GAAG,CAAC,QAAQ,KAAK,SAAS,CAAC,EAAE,CAAC;QAC7E,OAAO,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,IAAI,CAAC,CAAC;IAChE,CAAC;IAED,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC;QAChB,MAAM,YAAY,GAAG,qBAAqB,CAAC,MAAM,CAAC,CAAC;QACnD,IAAI,YAAY,KAAK,IAAI,EAAE,CAAC;YAC1B,OAAO,iBAAiB,CAAC,MAAM,EAAE,YAAY,EAAE,SAAS,EAAE,KAAK,CAAC,CAAC;QACnE,CAAC;IACH,CAAC;IAED,IAAI,WAAW,GAAG,KAAK,CAAC;IACxB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,IAAI,OAAO,GAAG,KAAK,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,IAAI,EAAE,CAAC;QACxB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;YACf,WAAW,GAAG,IAAI,CAAC;YACnB,SAAS;QACX,CAAC;QACD,IAAI,yBAAyB,CAAC,IAAI,CAAC,EAAE,CAAC;YACpC,YAAY,IAAI,CAAC,CAAC;QACpB,CAAC;aAAM,IAAI,IAAI,IAAI,IAAI,EAAE,CAAC;YACxB,OAAO,GAAG,IAAI,CAAC;QACjB,CAAC;IACH,CAAC;IAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,IAAI,CAAC,MAAM,CAAC;IAC7E,MAAM,QAAQ,GAAG,WAAW,IAAI,iBAAiB,GAAG,SAAS,CAAC;IAE9D,MAAM,QAAQ,GAAsB,GAAG,CAAC,MAAM;QAC5C,CAAC,CAAC,GAAG,CAAC,QAAQ;QACd,CAAC,CAAC,QAAQ;YACR,CAAC,CAAC,SAAS;YACX,CAAC,CAAC,OAAO;gBACP,CAAC,CAAC,MAAM;gBACR,CAAC,CAAC,OAAO,CAAC;IAEhB,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAE,iBAAiB,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC;AACpF,CAAC;AAaD;;;;;GAKG;AACH,MAAM,UAAU,cAAc,CAC5B,MAAkB,EAClB,UAA+B,EAC/B,UAAkC;IAElC,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,sBAAsB,GAC1B,UAAU,EAAE,sBAAsB,IAAI,qBAAqB,CAAC,sBAAsB,CAAC;IACrF,MAAM,4BAA4B,GAChC,UAAU,EAAE,4BAA4B,IAAI,qBAAqB,CAAC,0BAA0B,CAAC;IAE/F,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,IAAI,WAAW,GAAG,CAAC,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;YAClB,UAAU,IAAI,CAAC,CAAC;YAChB,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;YACjD,WAAW,GAAG,CAAC,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,WAAW,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IACD,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,WAAW,CAAC,CAAC;IAEjD,MAAM,mBAAmB,GAAG,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC;IACpE,8EAA8E;IAC9E,6EAA6E;IAC7E,yEAAyE;IACzE,sCAAsC;IACtC,OAAO,eAAe,CACpB,EAAE,aAAa,EAAE,WAAW,EAAE,iBAAiB,EAAE,mBAAmB,EAAE,EACtE,EAAE,UAAU,EAAE,UAAU,IAAI,IAAI,EAAE,EAClC,EAAE,sBAAsB,EAAE,0BAA0B,EAAE,4BAA4B,EAAE,CACrF,CAAC;AACJ,CAAC"}

package/dist/concurrency.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Applies `worker` to every item of `source` with at most `concurrency` in
+ * flight at once, yielding each result as it settles. Checks `signal` before
+ * pulling each new item and before yielding each result, throwing (via
+ * `AbortSignal.throwIfAborted`) as soon as an abort is observed.
+ */
+export declare function mapConcurrent<T, R>(source: AsyncIterable<T> | Iterable<T>, concurrency: number, worker: (item: T) => Promise<R>, signal?: AbortSignal): AsyncGenerator<R, void, void>;
+//# sourceMappingURL=concurrency.d.ts.map

package/dist/concurrency.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"concurrency.d.ts","sourceRoot":"","sources":["../src/concurrency.ts"],"names":[],"mappings":"AAYA;;;;;GAKG;AACH,wBAAuB,aAAa,CAAC,CAAC,EAAE,CAAC,EACvC,MAAM,EAAE,aAAa,CAAC,CAAC,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,EACtC,WAAW,EAAE,MAAM,EACnB,MAAM,EAAE,CAAC,IAAI,EAAE,CAAC,KAAK,OAAO,CAAC,CAAC,CAAC,EAC/B,MAAM,CAAC,EAAE,WAAW,GACnB,cAAc,CAAC,CAAC,EAAE,IAAI,EAAE,IAAI,CAAC,CAsD/B"}

package/dist/concurrency.js

@@ -0,0 +1,64 @@
+/**
+ * Bounded-concurrency mapping over an (async) iterable, yielding results as
+ * soon as they complete (not necessarily in source order). Backs the
+ * per-file processing stage of discovery (architecture.md §13.1/§13.2).
+ */
+import { Semaphore } from "./semaphore.js";
+/**
+ * Applies `worker` to every item of `source` with at most `concurrency` in
+ * flight at once, yielding each result as it settles. Checks `signal` before
+ * pulling each new item and before yielding each result, throwing (via
+ * `AbortSignal.throwIfAborted`) as soon as an abort is observed.
+ */
+export async function* mapConcurrent(source, concurrency, worker, signal) {
+    const semaphore = new Semaphore(concurrency);
+    const iterator = Symbol.asyncIterator in source
+        ? source[Symbol.asyncIterator]()
+        : source[Symbol.iterator]();
+    const inFlight = new Map();
+    let nextId = 0;
+    let sourceDone = false;
+    async function pullOne() {
+        if (sourceDone) {
+            return;
+        }
+        const release = await semaphore.acquire();
+        signal?.throwIfAborted();
+        const step = await iterator.next();
+        if (step.done) {
+            sourceDone = true;
+            release();
+            return;
+        }
+        const id = nextId;
+        nextId += 1;
+        const task = worker(step.value)
+            .then((value) => ({ id, value }))
+            .finally(release);
+        inFlight.set(id, task);
+    }
+    try {
+        signal?.throwIfAborted();
+        const starters = [];
+        for (let i = 0; i < concurrency; i += 1) {
+            starters.push(pullOne());
+        }
+        await Promise.all(starters);
+        while (inFlight.size > 0) {
+            signal?.throwIfAborted();
+            const settled = await Promise.race(inFlight.values());
+            inFlight.delete(settled.id);
+            if (!sourceDone) {
+                await pullOne();
+            }
+            signal?.throwIfAborted();
+            yield settled.value;
+        }
+    }
+    finally {
+        if (typeof iterator.return === "function") {
+            await iterator.return?.();
+        }
+    }
+}
+//# sourceMappingURL=concurrency.js.map

package/dist/concurrency.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"concurrency.js","sourceRoot":"","sources":["../src/concurrency.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AACH,OAAO,EAAE,SAAS,EAAE,MAAM,gBAAgB,CAAC;AAO3C;;;;;GAKG;AACH,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,aAAa,CAClC,MAAsC,EACtC,WAAmB,EACnB,MAA+B,EAC/B,MAAoB;IAEpB,MAAM,SAAS,GAAG,IAAI,SAAS,CAAC,WAAW,CAAC,CAAC;IAC7C,MAAM,QAAQ,GACZ,MAAM,CAAC,aAAa,IAAI,MAAM;QAC5B,CAAC,CAAE,MAA2B,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE;QACtD,CAAC,CAAE,MAAsB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;IAEjD,MAAM,QAAQ,GAAG,IAAI,GAAG,EAA+B,CAAC;IACxD,IAAI,MAAM,GAAG,CAAC,CAAC;IACf,IAAI,UAAU,GAAG,KAAK,CAAC;IAEvB,KAAK,UAAU,OAAO;QACpB,IAAI,UAAU,EAAE,CAAC;YACf,OAAO;QACT,CAAC;QACD,MAAM,OAAO,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,CAAC;QAC1C,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,UAAU,GAAG,IAAI,CAAC;YAClB,OAAO,EAAE,CAAC;YACV,OAAO;QACT,CAAC;QACD,MAAM,EAAE,GAAG,MAAM,CAAC;QAClB,MAAM,IAAI,CAAC,CAAC;QACZ,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;aAC5B,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,CAAC,CAAC;aAChC,OAAO,CAAC,OAAO,CAAC,CAAC;QACpB,QAAQ,CAAC,GAAG,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IACzB,CAAC;IAED,IAAI,CAAC;QACH,MAAM,EAAE,cAAc,EAAE,CAAC;QACzB,MAAM,QAAQ,GAAyB,EAAE,CAAC;QAC1C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,WAAW,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;YACxC,QAAQ,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;QAC3B,CAAC;QACD,MAAM,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QAE5B,OAAO,QAAQ,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,CAAC;YACtD,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC5B,IAAI,CAAC,UAAU,EAAE,CAAC;gBAChB,MAAM,OAAO,EAAE,CAAC;YAClB,CAAC;YACD,MAAM,EAAE,cAAc,EAAE,CAAC;YACzB,MAAM,OAAO,CAAC,KAAK,CAAC;QACtB,CAAC;IACH,CAAC;YAAS,CAAC;QACT,IAAI,OAAQ,QAA6B,CAAC,MAAM,KAAK,UAAU,EAAE,CAAC;YAChE,MAAO,QAA6B,CAAC,MAAM,EAAE,EAAE,CAAC;QAClD,CAAC;IACH,CAAC;AACH,CAAC"}

package/dist/discovery.d.ts

@@ -0,0 +1,9 @@
+import type { ScannedFileV1, ScanOptions } from "./types.js";
+/**
+ * Discovers files under `options.repoRoot`, applying ignore rules and
+ * symlink safety, and yields a `ScannedFileV1` for every file accepted for
+ * downstream analysis. Excluded/skipped paths are reported via `onSkip`,
+ * never yielded. Respects `options.signal` throughout.
+ */
+export declare function discoverFiles(options: ScanOptions): AsyncGenerator<ScannedFileV1, void, void>;
+//# sourceMappingURL=discovery.d.ts.map

package/dist/discovery.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"discovery.d.ts","sourceRoot":"","sources":["../src/discovery.ts"],"names":[],"mappings":"AAsBA,OAAO,KAAK,EAEV,aAAa,EACb,WAAW,EAGZ,MAAM,YAAY,CAAC;AA6QpB;;;;;GAKG;AACH,wBAAuB,aAAa,CAClC,OAAO,EAAE,WAAW,GACnB,cAAc,CAAC,aAAa,EAAE,IAAI,EAAE,IAAI,CAAC,CAwD3C"}

package/dist/discovery.js

@@ -0,0 +1,254 @@
+/**
+ * Recursive discovery (spec.md §6.1; architecture.md §6.4, §23.1).
+ *
+ * Walks the repository tree through `FileSystemPort` only (never
+ * `node:fs`), applies the ignore engine, resolves and guards symlinks, and
+ * hands each surviving path to bounded-concurrency classification/hashing.
+ */
+import { classifyTextFormat, assertWithinWorkspace, isUntangleError, } from "@untangle-code/domain";
+import { classifyBinarySample, detectMinified } from "./binary-classifier.js";
+import { mapConcurrent } from "./concurrency.js";
+import { createPortContentHasher } from "./hashing.js";
+import { createIgnoreEngine } from "./ignore-engine.js";
+import { detectLanguage } from "./language-classifier.js";
+import { portOptions } from "./port-options.js";
+function skip(context, relativePath, reason, detail) {
+    context.onSkip?.(detail === undefined ? { relativePath, reason } : { relativePath, reason, detail });
+}
+async function* walkDirectory(context, dir) {
+    context.signal?.throwIfAborted();
+    const entries = await context.fs.readDir(dir.absolutePath, portOptions(context.signal));
+    let pushedGitignore = false;
+    let pushedUntangleIgnore = false;
+    for (const entry of entries) {
+        if (entry.kind === "file" && entry.name === context.gitIgnoreFileName) {
+            const content = await context.fs.readTextFile(entry.path, portOptions(context.signal));
+            context.ignoreEngine.pushGitignoreLayer(dir.relativePath, content);
+            pushedGitignore = true;
+        }
+        else if (entry.kind === "file" && entry.name === context.untangleIgnoreFileName) {
+            const content = await context.fs.readTextFile(entry.path, portOptions(context.signal));
+            context.ignoreEngine.pushUntangleIgnoreLayer(dir.relativePath, content);
+            pushedUntangleIgnore = true;
+        }
+    }
+    try {
+        for (const entry of entries) {
+            context.signal?.throwIfAborted();
+            const childRelative = dir.relativePath === "" ? entry.name : `${dir.relativePath}/${entry.name}`;
+            if (entry.kind === "directory") {
+                const decision = context.ignoreEngine.decide(childRelative, true);
+                if (decision.ignored) {
+                    skip(context, childRelative, decision.reason ?? "default-exclude");
+                    continue;
+                }
+                if (context.visitedRealDirs.has(entry.path)) {
+                    skip(context, childRelative, "symlink-cycle", "directory real path already visited");
+                    continue;
+                }
+                context.visitedRealDirs.add(entry.path);
+                yield* walkDirectory(context, { relativePath: childRelative, absolutePath: entry.path });
+                continue;
+            }
+            if (entry.kind === "symlink") {
+                yield* handleSymlink(context, entry.path, childRelative);
+                continue;
+            }
+            if (entry.kind === "file") {
+                const decision = context.ignoreEngine.decide(childRelative, false);
+                if (decision.ignored) {
+                    skip(context, childRelative, decision.reason ?? "default-exclude");
+                    continue;
+                }
+                yield { relativePath: childRelative, absolutePath: entry.path };
+                continue;
+            }
+            skip(context, childRelative, "unreadable", `unsupported directory entry kind "${entry.kind}"`);
+        }
+    }
+    finally {
+        if (pushedUntangleIgnore) {
+            context.ignoreEngine.popUntangleIgnoreLayer();
+        }
+        if (pushedGitignore) {
+            context.ignoreEngine.popGitignoreLayer();
+        }
+    }
+}
+async function* handleSymlink(context, entryAbsolutePath, childRelative) {
+    let real;
+    try {
+        real = await context.fs.realPath(entryAbsolutePath, portOptions(context.signal));
+    }
+    catch {
+        skip(context, childRelative, "unreadable", "symlink target could not be resolved");
+        return;
+    }
+    try {
+        assertWithinWorkspace(real, context.roots);
+    }
+    catch (error) {
+        if (isUntangleError(error) && error.code === "PATH_OUTSIDE_WORKSPACE") {
+            skip(context, childRelative, "symlink-escape", real);
+            return;
+        }
+        throw error;
+    }
+    const targetStat = await context.fs.stat(real, portOptions(context.signal));
+    if (targetStat === null) {
+        skip(context, childRelative, "unreadable", "dangling symlink");
+        return;
+    }
+    if (targetStat.kind === "directory") {
+        const decision = context.ignoreEngine.decide(childRelative, true);
+        if (decision.ignored) {
+            skip(context, childRelative, decision.reason ?? "default-exclude");
+            return;
+        }
+        if (context.visitedRealDirs.has(real)) {
+            skip(context, childRelative, "symlink-cycle", real);
+            return;
+        }
+        context.visitedRealDirs.add(real);
+        yield* walkDirectory(context, { relativePath: childRelative, absolutePath: real });
+        return;
+    }
+    if (targetStat.kind === "file") {
+        const decision = context.ignoreEngine.decide(childRelative, false);
+        if (decision.ignored) {
+            skip(context, childRelative, decision.reason ?? "default-exclude");
+            return;
+        }
+        yield { relativePath: childRelative, absolutePath: real, stat: targetStat };
+        return;
+    }
+    skip(context, childRelative, "unreadable", `symlink target has unsupported kind "${targetStat.kind}"`);
+}
+/**
+ * Decodes a content sample as text using the *detected* encoding
+ * (WP-64 fix, spec.md E4: silent data loss is worse than a slightly slower
+ * decode). Before this fix, the sample was always decoded as UTF-8
+ * regardless of `classifyBinarySample`'s detected `encoding`, which produced
+ * mojibake for BOM-less UTF-16 files (their bytes are ~50% NUL under a
+ * UTF-8 reading) and, worse, fed that mojibake into `detectLanguage`'s
+ * shebang sniff for extensionless scripts — silently misclassifying a
+ * legitimate `#!/usr/bin/env python3` UTF-16 script as unknown/no language.
+ *
+ * UTF-16 (LE/BE, whether BOM-confirmed or detected BOM-less by
+ * `detectUtf16WithoutBom`) decodes under its own `TextDecoder` label so the
+ * BOM — if present — is consumed correctly. Every other guess (`utf8`,
+ * `ascii`, `unknown`) falls back to UTF-8 non-fatal decoding, which is a
+ * safe superset of ASCII and the correct default per spec.md.
+ */
+function decodeSampleText(sample, encoding) {
+    try {
+        if (encoding === "utf16le" || encoding === "utf16be") {
+            return new TextDecoder(encoding === "utf16le" ? "utf-16le" : "utf-16be", {
+                fatal: false,
+            }).decode(sample);
+        }
+        return new TextDecoder("utf-8", { fatal: false }).decode(sample);
+    }
+    catch {
+        return "";
+    }
+}
+async function processCandidate(config, candidate) {
+    const stat = candidate.stat ?? (await config.fs.stat(candidate.absolutePath, portOptions(config.signal)));
+    if (stat === null) {
+        config.onSkip?.({
+            relativePath: candidate.relativePath,
+            reason: "unreadable",
+            detail: "vanished before stat",
+        });
+        return null;
+    }
+    if (stat.sizeBytes > config.maxFileSizeBytes) {
+        config.onSkip?.({ relativePath: candidate.relativePath, reason: "size-limit" });
+        return null;
+    }
+    const bytes = await config.fs.readFile(candidate.absolutePath, portOptions(config.signal));
+    const sample = bytes.subarray(0, Math.min(bytes.length, config.sampleBytes));
+    const binary = classifyBinarySample(sample, {
+        nonPrintableThreshold: config.nonPrintableThreshold,
+    });
+    if (binary.isBinary) {
+        config.onSkip?.({ relativePath: candidate.relativePath, reason: "binary" });
+        return null;
+    }
+    const sampleText = decodeSampleText(sample, binary.encoding);
+    const language = detectLanguage(candidate.relativePath, sampleText);
+    // Die Textklasse steht eine Zeile weiter oben schon fest und kostet nichts:
+    // sie ist rein pfadbasiert. Ohne sie entschied die Einzelzeilenregel auch
+    // ueber Prosa — und ob sie zuschlug, haing daran, ob die erste lange Zeile
+    // zufaellig in die 8-KB-Stichprobe fiel.
+    const isMinified = detectMinified(sample, undefined, classifyTextFormat(candidate.relativePath));
+    const contentHash = await config.hasher.hashFile(candidate.absolutePath, stat.sizeBytes, portOptions(config.signal));
+    return {
+        relativePath: candidate.relativePath,
+        absolutePath: candidate.absolutePath,
+        sizeBytes: stat.sizeBytes,
+        mtimeEpochMs: stat.mtimeEpochMs,
+        contentHash,
+        languageId: language.languageId,
+        languageDetectedBy: language.detectedBy,
+        isMinified,
+        encoding: binary.encoding,
+        hasBom: binary.hasBom,
+    };
+}
+const DEFAULT_MAX_FILE_SIZE_BYTES = 5 * 1024 * 1024;
+const DEFAULT_SAMPLE_BYTES = 8192;
+const DEFAULT_CONCURRENCY = 8;
+const DEFAULT_NON_PRINTABLE_THRESHOLD = 0.3;
+/**
+ * Discovers files under `options.repoRoot`, applying ignore rules and
+ * symlink safety, and yields a `ScannedFileV1` for every file accepted for
+ * downstream analysis. Excluded/skipped paths are reported via `onSkip`,
+ * never yielded. Respects `options.signal` throughout.
+ */
+export async function* discoverFiles(options) {
+    const maxFileSizeBytes = options.maxFileSizeBytes ?? DEFAULT_MAX_FILE_SIZE_BYTES;
+    const sampleBytes = options.sampleBytes ?? DEFAULT_SAMPLE_BYTES;
+    const concurrency = options.concurrency ?? DEFAULT_CONCURRENCY;
+    const nonPrintableThreshold = options.nonPrintableThreshold ?? DEFAULT_NON_PRINTABLE_THRESHOLD;
+    const gitIgnoreFileName = options.ignore?.gitIgnoreFileName ?? ".gitignore";
+    const untangleIgnoreFileName = options.ignore?.untangleIgnoreFileName ?? ".untangleignore";
+    options.signal?.throwIfAborted();
+    // Step 2 (architecture.md §23.1): lexical check before any I/O.
+    assertWithinWorkspace(options.repoRoot, options.roots);
+    // Step 1+3: canonically resolve the roots themselves too, not just the
+    // repo root and symlink targets encountered later — a root path can
+    // itself contain an unresolved symlink component (e.g. macOS's
+    // `/var` -> `/private/var`), and comparing a realpath-resolved candidate
+    // against a *literal* root would then spuriously look like an escape.
+    const resolvedRoots = await Promise.all(options.roots.map((root) => options.fs.realPath(root, portOptions(options.signal))));
+    const rootReal = await options.fs.realPath(options.repoRoot, portOptions(options.signal));
+    assertWithinWorkspace(rootReal, resolvedRoots);
+    const context = {
+        fs: options.fs,
+        ignoreEngine: createIgnoreEngine(options.ignore),
+        roots: resolvedRoots,
+        visitedRealDirs: new Set([rootReal]),
+        gitIgnoreFileName,
+        untangleIgnoreFileName,
+        onSkip: options.onSkip,
+        signal: options.signal,
+    };
+    const candidates = walkDirectory(context, { relativePath: "", absolutePath: rootReal });
+    const processConfig = {
+        fs: options.fs,
+        maxFileSizeBytes,
+        sampleBytes,
+        nonPrintableThreshold,
+        hasher: options.hasher ?? createPortContentHasher(options.fs),
+        onSkip: options.onSkip,
+        signal: options.signal,
+    };
+    for await (const result of mapConcurrent(candidates, concurrency, (candidate) => processCandidate(processConfig, candidate), options.signal)) {
+        if (result !== null) {
+            yield result;
+        }
+    }
+}
+//# sourceMappingURL=discovery.js.map