@vibe-agent-toolkit/utils 0.2.0-rc.2 → 0.2.0-rc.4

package/dist/text-content.js

@@ -0,0 +1,348 @@
+/**
+ * **The one way VAT turns file bytes into text.**
+ *
+ * `local/no-raw-text-decode` enforces that — but only where it is registered,
+ * which today is `packages/utils/src` and `packages/resources/src`. Those two
+ * own the seam and every corpus-document read; the rest of the repo is a
+ * migration ledger, not a covered claim, and `eslint.config.js` carries it in
+ * cost order. Do not read this docstring as "nothing else in the repo decodes".
+ *
+ * ## Where this lives, and why it is a `utils` primitive
+ *
+ * Bytes-to-text is a pure function of its argument. It knows nothing about
+ * content keys, parse caches or the projection — `readContentWithKey` in
+ * `@vibe-agent-toolkit/resources` *composes* this with a raw-bytes content key,
+ * and the key is the projection concept, not the decode.
+ *
+ * It has to live here for a harder reason than tidiness. `resources` depends on
+ * `utils` and `utils` must never depend on `resources`, so a seam in `resources`
+ * with a lint rule shipped from `utils` would flag `utils`' own reads
+ * (`gitignore-checker.ts` reads an adopter's `.gitignore`; `project-utils.ts`
+ * reads an adopter's `package.json`) while giving them no legal way to comply.
+ * The rule would then be widened with exemptions until it meant nothing. Placing
+ * the primitive at the bottom of the arrow is what makes the guardrail
+ * enforceable.
+ *
+ * `./text` is also deliberately **pure** — no `node:*` import at all — so bytes
+ * from a git blob, an HTTP body or a zip entry decode through the same function
+ * as bytes from disk. The file-reading half is in `./fs` (`readTextContent`),
+ * where everything that touches `node:fs` lives.
+ *
+ * ## The defect this exists for
+ *
+ * `readContentWithKey` (in `@vibe-agent-toolkit/resources`) used to call
+ * `bytes.toString('utf-8')` unconditionally. Measured end to end, on a real
+ * `working-tree-encoding=UTF-16` checkout (see `resources`'
+ * `test/system/git-hostile-config.system.test.ts`):
+ *
+ * ```text
+ * bytes on disk   40 B, BOM fe ff, UTF-16BE
+ * decoded         "��# \0D\0o\0c\0…"   NUL-interleaved mojibake
+ * looksBinary     true  — the decoded string carries NULs
+ * projection      BLOB_NOT_TEXT, no blob row, 0 sections, 0 references
+ * the same doc    1 heading, 1 link, from its UTF-8 bytes
+ * ```
+ *
+ * So VAT could not read a UTF-16 document *at all*, and the reason it matters is
+ * not exotic: **PowerShell 5.1's `Out-File` and `>` write UTF-16LE by default**,
+ * so a Windows-authored document lands squarely in that hole.
+ *
+ * ## What is a fact here, and what is an assumption
+ *
+ * The distinction is carried in the result ({@link TextProvenance.encodingSource})
+ * rather than left to prose, because the two are not the same kind of claim:
+ *
+ * | input | encoding | encodingSource | why |
+ * |---|---|---|---|
+ * | leading `ef bb bf` | `utf-8` | `bom` | the bytes say so |
+ * | leading `ff fe` (not `ff fe 00 00`) | `utf-16le` | `bom` | the bytes say so |
+ * | leading `fe ff` | `utf-16be` | `bom` | the bytes say so |
+ * | leading `ff fe 00 00` | `utf-32le` | `bom` | the bytes say so |
+ * | leading `00 00 fe ff` | `utf-32be` | `bom` | the bytes say so |
+ * | anything else | `utf-8` | `assumed` | the defensible default |
+ *
+ * ## And what proves the assumption WRONG
+ *
+ * `encoding` says what was guessed; {@link TextProvenance.replacementCharacters}
+ * is what the guess cost. A malformed sequence decodes to U+FFFD instead of
+ * throwing, so a mis-decoded document arrives as a well-formed JS string full of
+ * garbage — and a byte-level BPE tokenizer has no out-of-vocabulary concept, so
+ * it embeds and indexes that garbage without erroring anywhere. Counting the
+ * substitutions is what turns "we assumed UTF-8" into "we assumed UTF-8 and were
+ * demonstrably wrong 3,200 times in this file".
+ *
+ * It is counted **without paying for it on the clean path**. Every decode runs
+ * first through a `fatal: true` decoder, which throws on the first malformed
+ * sequence rather than substituting; a file that decodes cleanly — nearly every
+ * file — costs exactly one decode and no scan at all. Only a file that actually
+ * threw is decoded a second time in substituting mode and scanned for U+FFFD, so
+ * the O(n) scan is charged entirely to broken input.
+ *
+ * That ordering also buys a correctness property a scan alone cannot have: a
+ * document that *legitimately contains* U+FFFD is valid input, so the fatal
+ * decoder does not throw and it is reported as **0** replacements rather than
+ * accused of a bad decode. A bare scan would count its own content against it.
+ *
+ * 🪤 **The UTF-32LE BOM starts with the UTF-16LE BOM.** `ff fe 00 00` matches
+ * `ff fe`, so a table tested shortest-first decodes every UTF-32LE document as
+ * NUL-interleaved UTF-16 — the same bug this module exists to fix, one encoding
+ * further down. {@link BOMS} is therefore ordered longest-first and the test
+ * suite pins that ordering directly.
+ *
+ * ## The two limitations, recorded rather than guessed around
+ *
+ * - **BOM-less UTF-16 is not detected.** It is undecidable from bytes alone: the
+ *   same byte string is a legal, different UTF-8 document. A NUL-density
+ *   heuristic would decide it *usually* correctly and silently wrongly the rest
+ *   of the time, and "silently wrongly" is the failure class this whole module
+ *   is a reaction to. BOM-less input is UTF-8.
+ * - **Latin charsets are not detected either, and there is no windows-1252
+ *   fallback.** "These bytes are not valid UTF-8" is a fact; "therefore they are
+ *   latin-1" is a guess, and it is equally consistent with a UTF-8 document
+ *   carrying one corrupt byte. Malformed input gets U+FFFD. In practice the
+ *   Latin family costs little: every ASCII byte string is valid UTF-8 and
+ *   decodes correctly, so only high bytes are affected.
+ *
+ * Both are pinned as tests that state what is given up, so adding a heuristic
+ * later has to edit an assertion rather than quietly widen a claim.
+ *
+ * ## Why `TextDecoder` for three encodings and hand-rolled code for two
+ *
+ * `TextDecoder` implements the WHATWG Encoding Standard, which **deliberately
+ * omits UTF-32** — no engine offers it. Node's `Buffer` is narrower still:
+ * `utf8`, `utf16le`/`ucs2`, `latin1`, `ascii` and the binary-to-text codecs, with
+ * **no UTF-16BE at all** (the encoding a round trip through git's
+ * `working-tree-encoding=UTF-16` actually produces). So `TextDecoder` carries
+ * utf-8/utf-16le/utf-16be, and UTF-32 is converted here.
+ *
+ * Refusing UTF-32 loudly was the alternative, and it was rejected for one
+ * reason: the BOM has to be *recognised* regardless (see the trap above), so the
+ * choice was never "detect it or not" — only "having detected it, decode it or
+ * throw". Decoding is ~20 lines and leaves no hole.
+ *
+ * ## The BOM is stripped, and that is load-bearing in both directions
+ *
+ * A surviving leading U+FEFF stops `# Heading` from parsing as a heading. And
+ * every offset downstream is a **character** offset over this decoded string —
+ * `lineStartOffsets` in `resources`' `projection/blob-sections.ts` derives them
+ * from `content.split('\n')`, and `parseMarkdownContent` takes remark's own
+ * character positions — so stripping shifts all of them consistently. What must
+ * NOT shift is the content key, which is computed over the raw bytes; see
+ * `resources`' `content-key.ts`.
+ */
+/**
+ * Byte-order marks, **longest first**.
+ *
+ * The ordering is the correctness property, not a formatting choice — see the
+ * module docstring's trap. Frozen so a caller cannot reorder it in place.
+ */
+const BOMS = Object.freeze([
+    { bytes: [0xff, 0xfe, 0x00, 0x00], encoding: 'utf-32le' },
+    { bytes: [0x00, 0x00, 0xfe, 0xff], encoding: 'utf-32be' },
+    { bytes: [0xef, 0xbb, 0xbf], encoding: 'utf-8' },
+    { bytes: [0xff, 0xfe], encoding: 'utf-16le' },
+    { bytes: [0xfe, 0xff], encoding: 'utf-16be' },
+]);
+/**
+ * One `TextDecoder` per encoding, built once.
+ *
+ * Constructing a decoder per call is measurable on a corpus of thousands of
+ * files, and these are stateless when `stream: false` (the default for
+ * `decode()`), so one instance is safe to share.
+ *
+ * `ignoreBOM: true` on purpose: this module has already removed the BOM by the
+ * time a decoder sees the bytes, and leaving the option at its default would put
+ * a second, silent BOM-stripping step behind the deliberate one — so a document
+ * whose *content* legitimately begins with U+FEFF would lose a character that
+ * nothing here decided to remove.
+ */
+const DECODERS = new Map([
+    // No `eslint-disable` needed: this file is the `exemptFiles` entry for
+    // `local/no-raw-text-decode` in the repo's `eslint.config.js`. If the rule
+    // starts firing here, that entry has drifted from this path.
+    ['utf-8', new TextDecoder('utf-8', { ignoreBOM: true })],
+    ['utf-16le', new TextDecoder('utf-16le', { ignoreBOM: true })],
+    ['utf-16be', new TextDecoder('utf-16be', { ignoreBOM: true })],
+]);
+/**
+ * The same three encodings in **fatal** mode — throw rather than substitute.
+ *
+ * This is the whole cost model for {@link TextProvenance.replacementCharacters}.
+ * A substituting decoder cannot tell a caller whether it substituted, so the only
+ * other way to know is to scan every decoded string for U+FFFD — an O(n) pass
+ * over every file in a corpus to learn that almost none of them needed it. A
+ * fatal decoder answers the same question by *not throwing*, at no extra cost on
+ * the clean path, and the expensive route is taken only where there is genuinely
+ * something to count.
+ *
+ * Same `ignoreBOM: true` as their substituting twins, for the same reason: the
+ * BOM is already gone by the time either sees the bytes.
+ */
+const FATAL_DECODERS = new Map([
+    ['utf-8', new TextDecoder('utf-8', { ignoreBOM: true, fatal: true })],
+    ['utf-16le', new TextDecoder('utf-16le', { ignoreBOM: true, fatal: true })],
+    ['utf-16be', new TextDecoder('utf-16be', { ignoreBOM: true, fatal: true })],
+]);
+/** How many code points to spread into one `String.fromCodePoint` call. */
+const CODE_POINT_CHUNK = 4096;
+/** The last Unicode code point. */
+const MAX_CODE_POINT = 0x10_ff_ff;
+/** First and last UTF-16 surrogate — never a scalar value on their own. */
+const SURROGATE_FIRST = 0xd8_00;
+const SURROGATE_LAST = 0xdf_ff;
+/** U+FFFD REPLACEMENT CHARACTER, what a malformed unit becomes. */
+const REPLACEMENT_CODE_POINT = 0xff_fd;
+/** Does `bytes` begin with `prefix`? */
+function startsWith(bytes, prefix) {
+    if (bytes.length < prefix.length)
+        return false;
+    return prefix.every((byte, index) => bytes[index] === byte);
+}
+/**
+ * The encoding a BOM announces, and how many bytes it occupies.
+ *
+ * @param bytes - Raw bytes, from the start of the file
+ * @returns The BOM's encoding and length, or `null` when there is no BOM
+ */
+function bomAt(bytes) {
+    for (const bom of BOMS) {
+        if (startsWith(bytes, bom.bytes)) {
+            return { encoding: bom.encoding, length: bom.bytes.length };
+        }
+    }
+    return null;
+}
+/**
+ * Is `codePoint` something `String.fromCodePoint` will accept?
+ *
+ * A lone surrogate and anything past U+10FFFF both throw, and a corrupt document
+ * must not surface as an exception out of a decoder.
+ */
+function isScalarValue(codePoint) {
+    if (codePoint > MAX_CODE_POINT)
+        return false;
+    return codePoint < SURROGATE_FIRST || codePoint > SURROGATE_LAST;
+}
+/**
+ * Build a string from code points, in chunks.
+ *
+ * `String.fromCodePoint(...points)` blows the engine's argument limit somewhere
+ * in the tens of thousands, which a real document reaches — so the spread is
+ * bounded rather than whole-array.
+ */
+function fromCodePoints(points) {
+    if (points.length <= CODE_POINT_CHUNK)
+        return String.fromCodePoint(...points);
+    const parts = [];
+    for (let start = 0; start < points.length; start += CODE_POINT_CHUNK) {
+        parts.push(String.fromCodePoint(...points.slice(start, start + CODE_POINT_CHUNK)));
+    }
+    return parts.join('');
+}
+/**
+ * Decode UTF-32, by hand, because no engine does.
+ *
+ * Every 4-byte unit that is not a Unicode scalar value becomes U+FFFD, as does a
+ * trailing run of 1–3 bytes that cannot form a unit. That mirrors what
+ * `TextDecoder` does with malformed input, so the two paths fail the same way.
+ *
+ * There is no fatal-first pass here and none is needed: this loop *decides* each
+ * substitution, so it can count them as it makes them — free, and exact. The
+ * fatal-decoder trick exists only because `TextDecoder` refuses to say.
+ *
+ * @param bytes - The content bytes, BOM already removed
+ * @param littleEndian - Byte order the BOM announced
+ * @returns The decoded string and how many units were replaced
+ */
+function decodeUtf32(bytes, littleEndian) {
+    const view = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
+    const whole = bytes.byteLength - (bytes.byteLength % 4);
+    const points = [];
+    let replacementCharacters = 0;
+    for (let offset = 0; offset < whole; offset += 4) {
+        const codePoint = view.getUint32(offset, littleEndian);
+        if (isScalarValue(codePoint)) {
+            points.push(codePoint);
+        }
+        else {
+            points.push(REPLACEMENT_CODE_POINT);
+            replacementCharacters += 1;
+        }
+    }
+    if (whole !== bytes.byteLength) {
+        points.push(REPLACEMENT_CODE_POINT);
+        replacementCharacters += 1;
+    }
+    return { text: fromCodePoints(points), replacementCharacters };
+}
+/**
+ * Count the U+FFFD in a string that a fatal decode already refused.
+ *
+ * Only ever called on input known to be malformed, which is what keeps the O(n)
+ * scan off the common path. A scan rather than a regex or a split: it allocates
+ * nothing on a string that may be megabytes long. U+FFFD is a BMP character, so
+ * the comparison is exact whether the index lands on it or on a surrogate half.
+ *
+ * @param text - The substituting decoder's output
+ * @returns How many replacement characters it contains
+ */
+function countReplacementCharacters(text) {
+    let count = 0;
+    for (let index = 0; index < text.length; index += 1) {
+        if (text.codePointAt(index) === REPLACEMENT_CODE_POINT)
+            count += 1;
+    }
+    return count;
+}
+/**
+ * **The canonical content-decoding seam.** Turn file bytes into the text a
+ * parser should see.
+ *
+ * Takes bytes rather than a path so that a caller which must also key, hash or
+ * measure the raw bytes reads the file exactly once — `readContentWithKey` in
+ * `@vibe-agent-toolkit/resources` is that caller, and its key must stay over the
+ * raw byte preimage whatever this function decides the characters are. For the
+ * ordinary "read a file, give me its text" case use `readTextContent` from
+ * `@vibe-agent-toolkit/utils/fs`.
+ *
+ * @param bytes - The exact bytes read from disk
+ * @returns The decoded text, the encoding used, whether that was a fact, and how
+ *   many characters the decode had to replace
+ *
+ * @example
+ * ```typescript
+ * const bytes = await readFile(path);
+ * const { text, encoding, encodingSource, replacementCharacters } = decodeTextContent(bytes);
+ * // UTF-16BE file: encoding 'utf-16be', encodingSource 'bom', 0 replacements,
+ * // text with no BOM and no NULs
+ * ```
+ */
+export function decodeTextContent(bytes) {
+    const bom = bomAt(bytes);
+    const encoding = bom?.encoding ?? 'utf-8';
+    const body = bom === null ? bytes : bytes.subarray(bom.length);
+    const encodingSource = bom === null ? 'assumed' : 'bom';
+    if (encoding === 'utf-32le' || encoding === 'utf-32be') {
+        return { ...decodeUtf32(body, encoding === 'utf-32le'), encoding, encodingSource };
+    }
+    const decoder = DECODERS.get(encoding);
+    const fatalDecoder = FATAL_DECODERS.get(encoding);
+    if (decoder === undefined || fatalDecoder === undefined) {
+        // Unreachable: BOMS, DECODERS and FATAL_DECODERS cover the same five
+        // encodings between them. Thrown rather than defaulted, because a silent fall
+        // back to UTF-8 here would reproduce the exact defect this module was written
+        // to remove.
+        throw new Error(`no decoder for encoding "${encoding}"`);
+    }
+    try {
+        // The clean path, and the only one nearly every file takes: one decode, no
+        // scan. A throw here is the ONLY evidence that a substitution happened —
+        // see FATAL_DECODERS.
+        return { text: fatalDecoder.decode(body), encoding, encodingSource, replacementCharacters: 0 };
+    }
+    catch {
+        const text = decoder.decode(body);
+        return { text, encoding, encodingSource, replacementCharacters: countReplacementCharacters(text) };
+    }
+}
+//# sourceMappingURL=text-content.js.map

package/dist/text-content.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-content.js","sourceRoot":"","sources":["../src/text-content.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAmIG;AA8CH;;;;;GAKG;AACH,MAAM,IAAI,GACR,MAAM,CAAC,MAAM,CAAC;IACZ,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE;IACzD,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE;IACzD,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,OAAO,EAAE;IAChD,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE;IAC7C,EAAE,KAAK,EAAE,CAAC,IAAI,EAAE,IAAI,CAAC,EAAE,QAAQ,EAAE,UAAU,EAAE;CAC9C,CAAC,CAAC;AAEL;;;;;;;;;;;;GAYG;AACH,MAAM,QAAQ,GAAgE,IAAI,GAAG,CAAC;IACpF,uEAAuE;IACvE,2EAA2E;IAC3E,6DAA6D;IAC7D,CAAC,OAAO,EAAE,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACxD,CAAC,UAAU,EAAE,IAAI,WAAW,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IAC9D,CAAC,UAAU,EAAE,IAAI,WAAW,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;CAC/D,CAAC,CAAC;AAEH;;;;;;;;;;;;;GAaG;AACH,MAAM,cAAc,GAAgE,IAAI,GAAG,CAAC;IAC1F,CAAC,OAAO,EAAE,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IACrE,CAAC,UAAU,EAAE,IAAI,WAAW,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;IAC3E,CAAC,UAAU,EAAE,IAAI,WAAW,CAAC,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;CAC5E,CAAC,CAAC;AAEH,2EAA2E;AAC3E,MAAM,gBAAgB,GAAG,IAAI,CAAC;AAE9B,mCAAmC;AACnC,MAAM,cAAc,GAAG,UAAU,CAAC;AAElC,2EAA2E;AAC3E,MAAM,eAAe,GAAG,OAAO,CAAC;AAChC,MAAM,cAAc,GAAG,OAAO,CAAC;AAE/B,mEAAmE;AACnE,MAAM,sBAAsB,GAAG,OAAO,CAAC;AAEvC,wCAAwC;AACxC,SAAS,UAAU,CAAC,KAAiB,EAAE,MAAyB;IAC9D,IAAI,KAAK,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM;QAAE,OAAO,KAAK,CAAC;IAC/C,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED;;;;;GAKG;AACH,SAAS,KAAK,CAAC,KAAiB;IAC9B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,UAAU,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YACjC,OAAO,EAAE,QAAQ,EAAE,GAAG,CAAC,QAAQ,EAAE,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;QAC9D,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,SAAiB;IACtC,IAAI,SAAS,GAAG,cAAc;QAAE,OAAO,KAAK,CAAC;IAC7C,OAAO,SAAS,GAAG,eAAe,IAAI,SAAS,GAAG,cAAc,CAAC;AACnE,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,MAAyB;IAC/C,IAAI,MAAM,CAAC,MAAM,IAAI,gBAAgB;QAAE,OAAO,MAAM,CAAC,aAAa,CAAC,GAAG,MAAM,CAAC,CAAC;IAC9E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,IAAI,gBAAgB,EAAE,CAAC;QACrE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,GAAG,gBAAgB,CAAC,CAAC,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,WAAW,CAClB,KAAiB,EACjB,YAAqB;IAErB,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,KAAK,CAAC,MAAM,EAAE,KAAK,CAAC,UAAU,EAAE,KAAK,CAAC,UAAU,CAAC,CAAC;IAC5E,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,GAAG,CAAC,KAAK,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC;IACxD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,qBAAqB,GAAG,CAAC,CAAC;IAC9B,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,GAAG,KAAK,EAAE,MAAM,IAAI,CAAC,EAAE,CAAC;QACjD,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,YAAY,CAAC,CAAC;QACvD,IAAI,aAAa,CAAC,SAAS,CAAC,EAAE,CAAC;YAC7B,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACzB,CAAC;aAAM,CAAC;YACN,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;YACpC,qBAAqB,IAAI,CAAC,CAAC;QAC7B,CAAC;IACH,CAAC;IACD,IAAI,KAAK,KAAK,KAAK,CAAC,UAAU,EAAE,CAAC;QAC/B,MAAM,CAAC,IAAI,CAAC,sBAAsB,CAAC,CAAC;QACpC,qBAAqB,IAAI,CAAC,CAAC;IAC7B,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,cAAc,CAAC,MAAM,CAAC,EAAE,qBAAqB,EAAE,CAAC;AACjE,CAAC;AAED;;;;;;;;;;GAUG;AACH,SAAS,0BAA0B,CAAC,IAAY;IAC9C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,KAAK,IAAI,CAAC,EAAE,CAAC;QACpD,IAAI,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,KAAK,sBAAsB;YAAE,KAAK,IAAI,CAAC,CAAC;IACrE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;;;;;;;;;;;;;;GAsBG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAiB;IACjD,MAAM,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC;IACzB,MAAM,QAAQ,GAAG,GAAG,EAAE,QAAQ,IAAI,OAAO,CAAC;IAC1C,MAAM,IAAI,GAAG,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAC/D,MAAM,cAAc,GAAmB,GAAG,KAAK,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;IAExE,IAAI,QAAQ,KAAK,UAAU,IAAI,QAAQ,KAAK,UAAU,EAAE,CAAC;QACvD,OAAO,EAAE,GAAG,WAAW,CAAC,IAAI,EAAE,QAAQ,KAAK,UAAU,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC;IACrF,CAAC;IACD,MAAM,OAAO,GAAG,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACvC,MAAM,YAAY,GAAG,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IAClD,IAAI,OAAO,KAAK,SAAS,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;QACxD,qEAAqE;QACrE,8EAA8E;QAC9E,8EAA8E;QAC9E,aAAa;QACb,MAAM,IAAI,KAAK,CAAC,4BAA4B,QAAQ,GAAG,CAAC,CAAC;IAC3D,CAAC;IAED,IAAI,CAAC;QACH,2EAA2E;QAC3E,yEAAyE;QACzE,sBAAsB;QACtB,OAAO,EAAE,IAAI,EAAE,YAAY,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,cAAc,EAAE,qBAAqB,EAAE,CAAC,EAAE,CAAC;IACjG,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAClC,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,cAAc,EAAE,qBAAqB,EAAE,0BAA0B,CAAC,IAAI,CAAC,EAAE,CAAC;IACrG,CAAC;AACH,CAAC"}

package/dist/text-file.d.ts

@@ -0,0 +1,45 @@
+/**
+ * Read a file and decode it through the one content-decoding seam.
+ *
+ * The replacement for `readFile(path, 'utf-8')`, which decodes inside `fs` with
+ * no byte-order-mark handling and no way to express UTF-16BE at all. The
+ * *decision* about what the bytes say lives in `text-content.ts`, which is pure
+ * and reaches no `node:*` module; this file is only the two lines that get the
+ * bytes off disk, and it lives on the `./fs` entry with everything else here
+ * that touches the filesystem.
+ *
+ * Two functions rather than one, because the callers genuinely differ: an
+ * enumeration or parse lane is asynchronous throughout, while a manifest or
+ * config probe on a startup path is not, and handing the latter a Promise makes
+ * it worse rather than more consistent.
+ *
+ * A caller that needs the RAW bytes as well as the text — to hash them, key
+ * them, or report `stat().size` — must not use these: read the bytes once itself
+ * and call `decodeTextContent` on them, so the digest and the characters come
+ * from the same read. `readContentWithKey` in `@vibe-agent-toolkit/resources` is
+ * that caller and is shaped exactly that way.
+ */
+import { type DecodedText } from './text-content.js';
+/**
+ * Read a file and decode it through {@link decodeTextContent}.
+ *
+ * @param filePath - Path to read
+ * @returns The decoded text, the encoding used, whether that was a fact, and the replacement-character count
+ * @throws Whatever `readFile` throws — callers decide whether that is fatal
+ *
+ * @example
+ * ```typescript
+ * const { text, encoding } = await readTextContent(docPath);
+ * // A PowerShell-written document: encoding 'utf-16le', text with no BOM
+ * ```
+ */
+export declare function readTextContent(filePath: string): Promise<DecodedText>;
+/**
+ * {@link readTextContent}, synchronously.
+ *
+ * @param filePath - Path to read
+ * @returns The decoded text, the encoding used, whether that was a fact, and the replacement-character count
+ * @throws Whatever `readFileSync` throws
+ */
+export declare function readTextContentSync(filePath: string): DecodedText;
+//# sourceMappingURL=text-file.d.ts.map

package/dist/text-file.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-file.d.ts","sourceRoot":"","sources":["../src/text-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAKH,OAAO,EAAqB,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAExE;;;;;;;;;;;;GAYG;AACH,wBAAsB,eAAe,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,WAAW,CAAC,CAG5E;AAED;;;;;;GAMG;AACH,wBAAgB,mBAAmB,CAAC,QAAQ,EAAE,MAAM,GAAG,WAAW,CAGjE"}

package/dist/text-file.js

@@ -0,0 +1,53 @@
+/**
+ * Read a file and decode it through the one content-decoding seam.
+ *
+ * The replacement for `readFile(path, 'utf-8')`, which decodes inside `fs` with
+ * no byte-order-mark handling and no way to express UTF-16BE at all. The
+ * *decision* about what the bytes say lives in `text-content.ts`, which is pure
+ * and reaches no `node:*` module; this file is only the two lines that get the
+ * bytes off disk, and it lives on the `./fs` entry with everything else here
+ * that touches the filesystem.
+ *
+ * Two functions rather than one, because the callers genuinely differ: an
+ * enumeration or parse lane is asynchronous throughout, while a manifest or
+ * config probe on a startup path is not, and handing the latter a Promise makes
+ * it worse rather than more consistent.
+ *
+ * A caller that needs the RAW bytes as well as the text — to hash them, key
+ * them, or report `stat().size` — must not use these: read the bytes once itself
+ * and call `decodeTextContent` on them, so the digest and the characters come
+ * from the same read. `readContentWithKey` in `@vibe-agent-toolkit/resources` is
+ * that caller and is shaped exactly that way.
+ */
+import { readFileSync } from 'node:fs';
+import { readFile } from 'node:fs/promises';
+import { decodeTextContent } from './text-content.js';
+/**
+ * Read a file and decode it through {@link decodeTextContent}.
+ *
+ * @param filePath - Path to read
+ * @returns The decoded text, the encoding used, whether that was a fact, and the replacement-character count
+ * @throws Whatever `readFile` throws — callers decide whether that is fatal
+ *
+ * @example
+ * ```typescript
+ * const { text, encoding } = await readTextContent(docPath);
+ * // A PowerShell-written document: encoding 'utf-16le', text with no BOM
+ * ```
+ */
+export async function readTextContent(filePath) {
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- caller-supplied path, same trust level as the parsers this feeds
+    return decodeTextContent(await readFile(filePath));
+}
+/**
+ * {@link readTextContent}, synchronously.
+ *
+ * @param filePath - Path to read
+ * @returns The decoded text, the encoding used, whether that was a fact, and the replacement-character count
+ * @throws Whatever `readFileSync` throws
+ */
+export function readTextContentSync(filePath) {
+    // eslint-disable-next-line security/detect-non-literal-fs-filename -- caller-supplied path, same trust level as the parsers this feeds
+    return decodeTextContent(readFileSync(filePath));
+}
+//# sourceMappingURL=text-file.js.map

package/dist/text-file.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text-file.js","sourceRoot":"","sources":["../src/text-file.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;GAoBG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,SAAS,CAAC;AACvC,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAE5C,OAAO,EAAE,iBAAiB,EAAoB,MAAM,mBAAmB,CAAC;AAExE;;;;;;;;;;;;GAYG;AACH,MAAM,CAAC,KAAK,UAAU,eAAe,CAAC,QAAgB;IACpD,uIAAuI;IACvI,OAAO,iBAAiB,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC;AACrD,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,mBAAmB,CAAC,QAAgB;IAClD,uIAAuI;IACvI,OAAO,iBAAiB,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;AACnD,CAAC"}

package/dist/text.d.ts

@@ -0,0 +1,16 @@
+/**
+ * @vibe-agent-toolkit/utils/text
+ *
+ * **The one way bytes become text.** Encoding detection from a byte-order mark,
+ * BOM stripping, and a stated default when there is no BOM — see
+ * `text-content.ts` for what is a fact, what is an assumption, and the two
+ * limitations that are deliberately not guessed around.
+ *
+ * Pure: this entry reaches no `node:*` builtin and no third-party package, so
+ * bytes from a git blob, an HTTP response or a zip entry decode through exactly
+ * the same function as bytes from disk. For the read-a-file case, import
+ * `readTextContent` from `@vibe-agent-toolkit/utils/fs`, which is this plus a
+ * `readFile`.
+ */
+export { decodeTextContent, type DecodedText, type EncodingSource, type TextEncoding, type TextProvenance, } from './text-content.js';
+//# sourceMappingURL=text.d.ts.map

package/dist/text.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.d.ts","sourceRoot":"","sources":["../src/text.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EACL,iBAAiB,EACjB,KAAK,WAAW,EAChB,KAAK,cAAc,EACnB,KAAK,YAAY,EACjB,KAAK,cAAc,GACpB,MAAM,mBAAmB,CAAC"}

package/dist/text.js

@@ -0,0 +1,16 @@
+/**
+ * @vibe-agent-toolkit/utils/text
+ *
+ * **The one way bytes become text.** Encoding detection from a byte-order mark,
+ * BOM stripping, and a stated default when there is no BOM — see
+ * `text-content.ts` for what is a fact, what is an assumption, and the two
+ * limitations that are deliberately not guessed around.
+ *
+ * Pure: this entry reaches no `node:*` builtin and no third-party package, so
+ * bytes from a git blob, an HTTP response or a zip entry decode through exactly
+ * the same function as bytes from disk. For the read-a-file case, import
+ * `readTextContent` from `@vibe-agent-toolkit/utils/fs`, which is this plus a
+ * `readFile`.
+ */
+export { decodeTextContent, } from './text-content.js';
+//# sourceMappingURL=text.js.map

package/dist/text.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"text.js","sourceRoot":"","sources":["../src/text.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAEH,OAAO,EACL,iBAAiB,GAKlB,MAAM,mBAAmB,CAAC"}

package/dist/timing-dump.d.ts

@@ -0,0 +1,136 @@
+/**
+ * The on-disk plumbing every VAT timing seam shares.
+ *
+ * Two seams write per-process JSON dumps to an operator-named directory:
+ * `@vibe-agent-toolkit/resources`' `parse-timing.ts` (which pass inside a parser
+ * owns the time) and this package's `crawl-timing.ts` (which contributor or
+ * crawler owns the time). They sit in different packages because `crawl-timing`
+ * has to bracket `GitTracker`, which is here — so this module lives at the lower
+ * of the two and is exported for the higher one. What they
+ * MEASURE could not be less alike — one axis is a closed enum of parser passes,
+ * the other is an open set of contributor ids — but every property that makes
+ * the *file* trustworthy is identical between them, and the lab's
+ * `harness/dumps.ts` already says so from the reading end:
+ *
+ * 1. A directory that cannot be created is reported the moment the seam turns
+ *    on, while there is still a run to abandon — never at exit, where a failure
+ *    costs the whole measurement.
+ * 2. A dump failure is written to **stderr and never thrown**. These run from
+ *    an `exit` listener, where a throw changes the process's exit behaviour, and
+ *    they must never touch stdout, which carries vat's report.
+ * 3. A pid can file more than one dump, and the name is CLAIMED rather than
+ *    checked. `vat validate` spawns the vat binary once per phase and pids are
+ *    reused, so `<stem>-<pid>.json` genuinely collides; worse, worker threads
+ *    SHARE their parent's pid, so a thread pool's whole cohort competes for one
+ *    `<stem>-<pid>` sequence at the same instant. The name gains a counter
+ *    rather than overwriting, and the counter is settled by
+ *    {@link EXCLUSIVE_CREATE} so the OS — not a check-then-write gap — decides
+ *    who won it.
+ * 4. The process's own wall and CPU time is read ONCE, at dump time. It is a
+ *    lifetime figure for the process and never a duration of the measured work;
+ *    its value is the RATIO, which tells a reader whether the wall-timed
+ *    brackets above it were competing with a loaded machine.
+ *
+ *    ⚠️ **A reader must never SUM these across dumps.** Point 3 means one
+ *    command routinely files several, and a vat command's phase processes
+ *    overlap in time — so summing their lifetimes counts the same wall clock
+ *    more than once and produces a "total" longer than the command took. The
+ *    figure is per process, and the only honest aggregate over several is the
+ *    per-process ratio read one dump at a time. The `crawl` facet keeps one
+ *    record per dump and publishes no total for exactly this reason; `parse`
+ *    still sums, which is review finding F2 (2026-08-14) and is annotated at
+ *    `facets/parse/dump.ts`. This list is where both seams learn what makes a
+ *    dump trustworthy, so the hazard belongs here rather than only beside the
+ *    consumer that already fixed it.
+ *
+ * Writing that twice would give two seams two chances to diverge on the one
+ * thing a reader has to be able to trust identically. What each seam keeps for
+ * itself is its accumulator shape, its dump body and the noun it is called by.
+ */
+/**
+ * Process-level wall and CPU time, read ONCE when a dump is written.
+ *
+ * All three are lifetime figures for the whole process, not for the measured
+ * work: the point of carrying them is the *ratio*. CPU well below wall means the
+ * process was waiting rather than computing, and every wall-timed bracket in the
+ * dump carries that waiting inside it. CPU above wall is normal and not an
+ * error — `process.cpuUsage()` sums every thread, including libuv's pool.
+ */
+export interface TimingProcess {
+    /** Wall clock since this process started. */
+    wallMs: number;
+    /** User CPU consumed by the process, across all its threads. */
+    cpuUserMs: number;
+    /** System CPU consumed by the process, across all its threads. */
+    cpuSystemMs: number;
+}
+/**
+ * Ceiling on the pid-collision search. A directory holding this many dumps for
+ * one pid is a runaway, not a collision, and giving up on a reported line is a
+ * better outcome than spinning.
+ *
+ * Exhausting it does NOT fall back to overwriting the last slot. That was the
+ * old behaviour, and it destroyed a dump for the same reason the check-then-
+ * write gap did — silently, and precisely when the directory was busiest.
+ */
+export declare const MAX_DUMP_COLLISIONS = 1000;
+/**
+ * Reduce a raw env value to a directory or `null`.
+ *
+ * An empty-string value counts as absent: `VAT_PARSE_TIMING=` in a shell profile
+ * is a variable somebody meant to unset, not a request to dump into the process's
+ * working directory.
+ *
+ * @param raw - The env var's value, if set
+ * @returns The dump directory, or `null` when the seam is off
+ */
+export declare function normalizeTimingDirectory(raw: string | undefined): string | null;
+/**
+ * Report a dump problem on stderr.
+ *
+ * Never throws and never touches stdout: vat's stdout carries a YAML report, and
+ * an exit handler that threw would change the process's exit behaviour.
+ *
+ * @param noun - What the seam is called, so a reader knows which instrument failed
+ * @param target - Path the failure concerns
+ * @param error - Whatever was caught
+ */
+export declare function reportTimingDumpFailure(noun: string, target: string, error: unknown): void;
+/**
+ * Create the dump directory, swallowing failure.
+ *
+ * Done once when the seam turns on rather than at exit, so a bad path is
+ * reported while there is still a run to abandon — and so the exit handler does
+ * the minimum possible work.
+ *
+ * @param noun - What the seam is called, for the failure line
+ * @param directory - Directory dumps will be written to
+ */
+export declare function ensureTimingDirectory(noun: string, directory: string): void;
+/**
+ * Read this process's lifetime wall and CPU time.
+ *
+ * Called ONCE per dump — two syscalls for a whole run, which is why the process
+ * level can afford a CPU reading that a per-bracket level cannot. Deliberately
+ * not an accumulator and deliberately not reset: it describes the process, not
+ * the measurement window.
+ *
+ * @returns Wall clock and CPU since process start, in milliseconds
+ */
+export declare function readTimingProcess(): TimingProcess;
+/**
+ * Write one seam's dump, if the seam is on.
+ *
+ * The body is built lazily, inside this call, so a disabled seam never pays to
+ * snapshot accumulators nobody will read — and a `build` that throws is reported
+ * like any other failure, because this runs from an `exit` listener where a
+ * throw would change the process's exit behaviour.
+ *
+ * @param noun - What the seam is called, for any failure line
+ * @param directory - Where to write, or `null` when the seam is off
+ * @param basename - Basename stem for the file
+ * @param build - Produces the dump body
+ * @returns The path written, or `null` when the seam is off or nothing was written
+ */
+export declare function writeTimingDump(noun: string, directory: string | null, basename: string, build: () => unknown): string | null;
+//# sourceMappingURL=timing-dump.d.ts.map

package/dist/timing-dump.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"timing-dump.d.ts","sourceRoot":"","sources":["../src/timing-dump.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgDG;AAOH;;;;;;;;GAQG;AACH,MAAM,WAAW,aAAa;IAC5B,6CAA6C;IAC7C,MAAM,EAAE,MAAM,CAAC;IACf,gEAAgE;IAChE,SAAS,EAAE,MAAM,CAAC;IAClB,kEAAkE;IAClE,WAAW,EAAE,MAAM,CAAC;CACrB;AAQD;;;;;;;;GAQG;AACH,eAAO,MAAM,mBAAmB,OAAO,CAAC;AA6BxC;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,GAAG,MAAM,GAAG,IAAI,CAE/E;AAED;;;;;;;;;GASG;AACH,wBAAgB,uBAAuB,CAAC,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI,CAG1F;AAED;;;;;;;;;GASG;AACH,wBAAgB,qBAAqB,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAM3E;AAED;;;;;;;;;GASG;AACH,wBAAgB,iBAAiB,IAAI,aAAa,CAOjD;AA0DD;;;;;;;;;;;;;GAaG;AACH,wBAAgB,eAAe,CAC7B,IAAI,EAAE,MAAM,EACZ,SAAS,EAAE,MAAM,GAAG,IAAI,EACxB,QAAQ,EAAE,MAAM,EAChB,KAAK,EAAE,MAAM,OAAO,GACnB,MAAM,GAAG,IAAI,CAoBf"}