@tikoci/centrs 0.1.3 → 0.1.6

package/src/explain/coordinates.ts

@@ -0,0 +1,488 @@
+/**
+ * Coordinate contract for `explain` (centrs canonicalizer).
+ *
+ * Ratified by the phase-0 lab, question Q15 (#185) and contracted in
+ * `commands/explain/README.md` (the byte-count-preserving coordinate rule).
+ * This is the production promotion of the lab's reference mapper; the six
+ * invariants it satisfies are pinned as anchor/property tests in
+ * `test/unit/explain-coordinates.test.ts` against
+ * `test/fixtures/explain/coordinates.json`.
+ *
+ * THREE COORDINATE SPACES
+ *
+ *   original    the caller's JS string. UTF-16 code units. Addressable as
+ *               (line, utf16Col) — the LSP position space.
+ *   originalU8  the UTF-8 encoding of `original`. Byte offsets. This is what a
+ *               byte-oriented device sees on the wire (RouterOS `highlight`
+ *               emits one token per INPUT BYTE).
+ *   analyzed    originalU8 with every non-ASCII byte replaced by the SUB
+ *               placeholder (0x1A), SAME LENGTH as originalU8. Pure ASCII, so a
+ *               JS string built from it has index === byte offset. This is the
+ *               surface the offline parser runs on and the space in which probe
+ *               offsets (`--cursor <byte>`, highlight spans) are contracted.
+ *
+ * WHY BYTE-COUNT-PRESERVING NORMALIZATION (the load-bearing choice)
+ *
+ *   A non-ASCII character is replaced by AS MANY SUB bytes as its UTF-8 length,
+ *   never collapsed to one. That makes an `analyzed` byte offset === the
+ *   `originalU8` byte offset identically, so a device highlight span (byte
+ *   offsets over the UTF-8 it received) and an offline span (analyzed byte
+ *   offsets) share ONE coordinate system with no fixup. SUB (U+001A) is chosen
+ *   because it is ASCII, is not whitespace, and is none of the RouterOS
+ *   delimiters (`; \n { } [ ] ( ) " \ # $ = /`), so it stays glued inside its
+ *   token and cannot forge a keyword, menu name, or separator.
+ *
+ * The production parser consumes the `analyzed` STRING (pure ASCII ⇒ its JS
+ * index IS the byte offset); this module converts those byte offsets back to
+ * the original (line, utf16Col) for LSP consumers. centrs does NOT NFC-fold:
+ * a combining mark is its own code point, its own run, and its own column.
+ */
+
+import type { Defect } from "./defects.ts";
+
+/** SUB (U+001A) — the byte-count-preserving stand-in for a non-ASCII byte. */
+export const SUB = 0x1a;
+
+/** U+FFFD REPLACEMENT CHARACTER — where a lone surrogate normalizes to. */
+const REPLACEMENT = 0xfffd;
+
+/** BYTE ORDER MARK (U+FEFF) — a defect only in leading position; see `coordinateDefects`. */
+const BOM = 0xfeff;
+
+/** A 0-based (line, column) position in `original`; column is a UTF-16 count. */
+export interface Position {
+	line: number;
+	col: number;
+}
+
+/** A half-open (line, col) range in `original` — the LSP range space. */
+export interface Range {
+	start: Position;
+	end: Position;
+}
+
+/** One code point of `original`, located in every coordinate space. */
+export interface CharRun {
+	/** Unicode scalar value; a lone surrogate is recorded as U+FFFD. */
+	codePoint: number;
+	/** Offset in `original` (UTF-16 code units). */
+	utf16Start: number;
+	/** 1 (BMP) or 2 (astral surrogate pair) UTF-16 units. */
+	utf16Len: number;
+	/** Offset in `analyzed`/`originalU8` (bytes). */
+	byteStart: number;
+	/** UTF-8 byte length: 1..4 (a lone surrogate → 3 via U+FFFD). */
+	byteLen: number;
+	/** Whether this run is a single ASCII byte (analyzed byte === original byte). */
+	ascii: boolean;
+	/** 0-based line. */
+	line: number;
+	/** 0-based UTF-16 column within the line. */
+	col: number;
+}
+
+/** The full coordinate analysis of one input string. */
+export interface CoordinateAnalysis {
+	original: string;
+	/** Pure-ASCII surface; `analyzed.length === originalU8.length`. */
+	analyzed: Uint8Array;
+	originalU8: Uint8Array;
+	/**
+	 * One run per code point.
+	 *
+	 * On the {@link CoordinateAnalysis.ascii} path this array is built ON FIRST
+	 * READ, not by `analyzeCoordinates`: every caller that only needs an offset
+	 * or a position can answer from arithmetic there, and materializing one
+	 * object per character was the single largest allocation in the analyzer
+	 * (#322). Reading it is always sound — the contract is the array, and the
+	 * lazy path builds the same one the eager path would have.
+	 */
+	runs: CharRun[];
+	/** Byte offset of each line's first character (index === 0-based line). */
+	lineStarts: number[];
+	/**
+	 * True when every code point of `original` is ASCII, so `analyzed`,
+	 * `originalU8` and `original` agree byte for byte and a byte offset IS a
+	 * UTF-16 offset.
+	 *
+	 * This is the property the three coordinate spaces were defined to make
+	 * checkable, and it is what lets a caller skip the mapping rather than
+	 * re-derive an identity one. It is a FACT about the input, never a mode: the
+	 * non-ASCII path below is unchanged and stays the only reading of a
+	 * non-ASCII document.
+	 */
+	ascii: boolean;
+}
+
+/** UTF-8 byte length of a Unicode scalar value. */
+function utf8Len(cp: number): number {
+	if (cp < 0x80) return 1;
+	if (cp < 0x800) return 2;
+	if (cp < 0x10000) return 3;
+	return 4;
+}
+
+/** Encode a Unicode scalar value to its UTF-8 bytes. */
+function utf8Encode(cp: number): number[] {
+	if (cp < 0x80) return [cp];
+	if (cp < 0x800) return [0xc0 | (cp >> 6), 0x80 | (cp & 0x3f)];
+	if (cp < 0x10000)
+		return [0xe0 | (cp >> 12), 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f)];
+	return [
+		0xf0 | (cp >> 18),
+		0x80 | ((cp >> 12) & 0x3f),
+		0x80 | ((cp >> 6) & 0x3f),
+		0x80 | (cp & 0x3f),
+	];
+}
+
+/**
+ * Any code point outside ASCII.
+ *
+ * `u`-mode, so the test is over CODE POINTS: an astral character is one match
+ * rather than two halves, and a lone surrogate — which is not a scalar value
+ * and therefore not ASCII either — matches as itself. That is the same
+ * population the walk below normalizes, so the two cannot disagree about what
+ * "all ASCII" means.
+ */
+const NON_ASCII = /[^\p{ASCII}]/u;
+
+/**
+ * Build the coordinate analysis of `original`: its UTF-8 encoding, the
+ * byte-count-preserving `analyzed` surface, and one {@link CharRun} per code
+ * point locating it in all three spaces.
+ *
+ * An ALL-ASCII input takes the identity path. Nothing about the contract
+ * changes there — the three spaces coincide, which is what the
+ * byte-count-preserving rule above was chosen to guarantee — so the mapping is
+ * arithmetic and the per-character {@link CharRun} array is deferred to its
+ * first reader. The analyzer re-derived that map for every nested block body it
+ * descended into, and `pathresolve`'s own `locate` discards the coordinate
+ * facts of a body anyway (they are properties of the root input): on a 60 KiB
+ * document nested 64 deep it decoded 7.9 MiB and allocated a run object per
+ * byte of it, a quarter of total analysis time (#322).
+ */
+export function analyzeCoordinates(original: string): CoordinateAnalysis {
+	if (!NON_ASCII.test(original)) return asciiCoordinates(original);
+	const runs: CharRun[] = [];
+	const u8: number[] = [];
+	const analyzed: number[] = [];
+	const lineStarts: number[] = [0];
+	let line = 0;
+	let col = 0;
+	let utf16Pos = 0;
+
+	// `for..of` yields code points and handles surrogate pairs; we track the
+	// UTF-16 offset/len separately so an astral char advances col by 2 (its
+	// UTF-16 width), which is what an LSP position expects.
+	for (const ch of original) {
+		let cp = ch.codePointAt(0) as number;
+		const utf16Len = ch.length; // 1 or 2
+		// A lone surrogate survives in a JS string as a single UTF-16 unit whose
+		// code point is in [0xD800,0xDFFF]; it is not a valid scalar → U+FFFD.
+		if (cp >= 0xd800 && cp <= 0xdfff) cp = REPLACEMENT;
+		const ascii = cp < 0x80;
+		const byteStart = u8.length;
+
+		for (const b of utf8Encode(cp)) {
+			u8.push(b);
+			analyzed.push(ascii ? b : SUB);
+		}
+
+		runs.push({
+			codePoint: cp,
+			utf16Start: utf16Pos,
+			utf16Len,
+			byteStart,
+			byteLen: utf8Len(cp),
+			ascii,
+			line,
+			col,
+		});
+
+		utf16Pos += utf16Len;
+		// `\n` (0x0A) ends a line. A lone `\r` is an ordinary column, so a `\r\n`
+		// sequence advances exactly one line (on the `\n`). Tabs are one column,
+		// never expanded — column counts UTF-16 units, not visual width.
+		if (cp === 0x0a) {
+			line++;
+			col = 0;
+			lineStarts.push(u8.length);
+		} else {
+			col += utf16Len;
+		}
+	}
+
+	return {
+		original,
+		analyzed: Uint8Array.from(analyzed),
+		originalU8: Uint8Array.from(u8),
+		runs,
+		lineStarts,
+		ascii: false,
+	};
+}
+
+/**
+ * The identity analysis of an all-ASCII input.
+ *
+ * One byte per code unit, so `analyzed`, `originalU8` and `original` are the
+ * same sequence and every offset is the same number in all three spaces. Only
+ * `lineStarts` needs a scan; `runs` is built on demand by {@link asciiRuns}
+ * because most callers never ask for it.
+ */
+function asciiCoordinates(original: string): CoordinateAnalysis {
+	const len = original.length;
+	const u8 = new Uint8Array(len);
+	const lineStarts = [0];
+	for (let i = 0; i < len; i++) {
+		const code = original.charCodeAt(i);
+		u8[i] = code;
+		if (code === 0x0a) lineStarts.push(i + 1);
+	}
+	let runs: CharRun[] | undefined;
+	return {
+		original,
+		// `analyzed` and `originalU8` are equal by definition here, but they stay
+		// separate arrays: they are separately typed, separately indexed surfaces
+		// and a shared buffer would let a write through one be read through the
+		// other.
+		analyzed: u8,
+		originalU8: u8.slice(),
+		get runs(): CharRun[] {
+			runs ??= asciiRuns(original, lineStarts);
+			return runs;
+		},
+		lineStarts,
+		ascii: true,
+	};
+}
+
+/** The per-character runs of an all-ASCII input, built on first read. */
+function asciiRuns(original: string, lineStarts: readonly number[]): CharRun[] {
+	const runs: CharRun[] = new Array(original.length);
+	let line = 0;
+	for (let i = 0; i < original.length; i++) {
+		const codePoint = original.charCodeAt(i);
+		runs[i] = {
+			codePoint,
+			utf16Start: i,
+			utf16Len: 1,
+			byteStart: i,
+			byteLen: 1,
+			ascii: true,
+			line,
+			col: i - (lineStarts[line] as number),
+		};
+		if (codePoint === 0x0a) line++;
+	}
+	return runs;
+}
+
+/** 0-based line containing `byte` — the last line start at or before it. */
+function lineAtByte(lineStarts: readonly number[], byte: number): number {
+	let lo = 0;
+	let hi = lineStarts.length - 1;
+	while (lo < hi) {
+		const mid = (lo + hi + 1) >> 1;
+		if ((lineStarts[mid] as number) <= byte) lo = mid;
+		else hi = mid - 1;
+	}
+	return lo;
+}
+
+/**
+ * The {@link CharRun} whose byte range contains `byte`. Throws if `byte` is not
+ * inside any run (i.e. `byte < 0` or `byte >= analyzed.length`).
+ */
+export function runAtByte(a: CoordinateAnalysis, byte: number): CharRun {
+	const len = a.analyzed.length;
+	if (!Number.isInteger(byte) || byte < 0 || byte >= len)
+		throw new Error(
+			`runAtByte: ${byte} is not a valid interior byte offset; expected an integer in [0, ${len})`,
+		);
+	// On the identity path the run IS the byte, so answer it rather than build
+	// (and then binary-search) an array of one object per character.
+	if (a.ascii) {
+		const line = lineAtByte(a.lineStarts, byte);
+		return {
+			codePoint: a.original.charCodeAt(byte),
+			utf16Start: byte,
+			utf16Len: 1,
+			byteStart: byte,
+			byteLen: 1,
+			ascii: true,
+			line,
+			col: byte - (a.lineStarts[line] as number),
+		};
+	}
+	let lo = 0;
+	let hi = a.runs.length - 1;
+	while (lo <= hi) {
+		const mid = (lo + hi) >> 1;
+		const r = a.runs[mid] as CharRun;
+		if (byte < r.byteStart) hi = mid - 1;
+		else if (byte >= r.byteStart + r.byteLen) lo = mid + 1;
+		else return r;
+	}
+	// Unreachable: a valid interior byte always lies inside exactly one run.
+	throw new Error(`runAtByte: no run contains byte ${byte} in [0, ${len})`);
+}
+
+/**
+ * Map an `analyzed` byte offset to an original (line, col) position. A byte
+ * landing INSIDE a multi-byte char snaps to that char's boundary (V6).
+ * `byte === analyzed.length` is the end-of-input cursor and maps to the
+ * position just past the last character.
+ */
+export function byteToPosition(a: CoordinateAnalysis, byte: number): Position {
+	const len = a.analyzed.length;
+	if (!Number.isInteger(byte) || byte < 0 || byte > len)
+		throw new Error(
+			`byteToPosition: ${byte} is not a valid cursor offset; expected an integer in [0, ${len}]`,
+		);
+	if (byte === len) {
+		// The identity path reads the same answer off `lineStarts`: a trailing
+		// newline has already pushed the line that the cursor sits at column 0 of.
+		if (a.ascii) {
+			const line = a.lineStarts.length - 1;
+			return { line, col: len - (a.lineStarts[line] as number) };
+		}
+		const last = a.runs.at(-1);
+		if (!last) return { line: 0, col: 0 };
+		if (last.codePoint === 0x0a) return { line: last.line + 1, col: 0 };
+		return { line: last.line, col: last.col + last.utf16Len };
+	}
+	const run = runAtByte(a, byte);
+	return { line: run.line, col: run.col };
+}
+
+/**
+ * Map a half-open analyzed-byte span to an original (line, col) range — the
+ * conversion an LSP consumer needs for every defect region and every token span.
+ *
+ * Deliberately built on {@link byteToPosition} in both directions rather than on
+ * {@link positionToByte}: that one is an O(n) linear scan over `runs`, so using
+ * it here would make converting a document's worth of spans quadratic.
+ * `byteToPosition` binary-searches. Both endpoints snap to a character boundary
+ * (V6), and `end === analyzed.length` is legal, so a span reaching end-of-input
+ * converts without a special case.
+ */
+export function byteSpanToRange(
+	a: CoordinateAnalysis,
+	start: number,
+	end: number,
+): Range {
+	if (end < start)
+		throw new Error(
+			`byteSpanToRange: end ${end} precedes start ${start}; spans are half-open and forward`,
+		);
+	return { start: byteToPosition(a, start), end: byteToPosition(a, end) };
+}
+
+/**
+ * The defect regions that fall out of the coordinate pass itself: a leading BOM,
+ * and every run of bytes the ASCII normalization stood in for.
+ *
+ * Both classes are {@link isPositionalFact}s, never errors — see that predicate
+ * for why. They come from `runs` rather than from a second scan because
+ * `analyzeCoordinates` has already decided, per code point, whether it is ASCII
+ * (`CharRun.ascii`); re-deriving that would be a chance to disagree with it.
+ *
+ * Non-ASCII runs are COALESCED: a CJK comment or an emoji-bearing value is one
+ * region, not one per code point. Contiguity is measured in bytes, so adjacent
+ * non-ASCII characters merge even across a surrogate pair.
+ *
+ * Only a U+FEFF in LEADING position is a `bom`. The same code point later in the
+ * document is a zero-width no-break space — ordinary content — and is reported
+ * as part of its `non-ascii` run like any other character.
+ */
+export function coordinateDefects(a: CoordinateAnalysis): Defect[] {
+	// Both classes are non-ASCII code points by definition (U+FEFF included), so
+	// an all-ASCII input has neither and the runs never need building.
+	if (a.ascii) return [];
+	const defects: Defect[] = [];
+	const first = a.runs[0];
+	const hasBom = first !== undefined && first.codePoint === BOM;
+	if (hasBom && first !== undefined)
+		defects.push({
+			code: "bom",
+			start: first.byteStart,
+			end: first.byteStart + first.byteLen,
+		});
+
+	let runStart = -1;
+	let runEnd = -1;
+	for (let i = hasBom ? 1 : 0; i < a.runs.length; i++) {
+		const run = a.runs[i] as CharRun;
+		if (run.ascii) {
+			if (runStart >= 0) {
+				defects.push({ code: "non-ascii", start: runStart, end: runEnd });
+				runStart = -1;
+			}
+			continue;
+		}
+		if (runStart < 0) runStart = run.byteStart;
+		runEnd = run.byteStart + run.byteLen;
+	}
+	if (runStart >= 0)
+		defects.push({ code: "non-ascii", start: runStart, end: runEnd });
+
+	return defects;
+}
+
+/**
+ * Map an original (line, col) position to an `analyzed` byte offset. `col` may
+ * equal the line's UTF-16 length (the position just past the last character /
+ * at the newline). Throws if the position addresses no boundary.
+ */
+export function positionToByte(
+	a: CoordinateAnalysis,
+	line: number,
+	col: number,
+): number {
+	if (!Number.isInteger(line) || !Number.isInteger(col) || line < 0 || col < 0)
+		throw new Error(
+			`positionToByte: (line ${line}, col ${col}) is not a valid position; expected non-negative integers`,
+		);
+	// On the identity path a position is arithmetic: the column IS the byte
+	// offset from the line start.
+	//
+	// The bound is the byte one PAST the line's last character, and on a line
+	// that ends in a newline that last character is the newline — so the bound
+	// is the next line's start, not one before it. That is the boundary the scan
+	// below reaches through `lastOnLine`, whose last run on the line is the
+	// newline itself; getting it wrong refused (line 0, col 2) of `a\n\nb`,
+	// which the scan answers with byte 2.
+	if (a.ascii) {
+		if (line >= a.lineStarts.length)
+			throw new Error(
+				`positionToByte: no character boundary at (line ${line}, col ${col})`,
+			);
+		const start = a.lineStarts[line] as number;
+		const end = (a.lineStarts[line + 1] ?? a.original.length) as number;
+		if (start + col > end)
+			throw new Error(
+				`positionToByte: no character boundary at (line ${line}, col ${col})`,
+			);
+		return start + col;
+	}
+	// Single pass: return an exact (line, col) hit immediately; otherwise keep
+	// the last run seen on the target line to resolve a just-past-the-end column.
+	let lastOnLine: CharRun | undefined;
+	for (const run of a.runs) {
+		if (run.line !== line) continue;
+		if (run.col === col) return run.byteStart;
+		lastOnLine = run;
+	}
+	// col past the last char of the line → byte after that line's last char.
+	if (lastOnLine && col === lastOnLine.col + lastOnLine.utf16Len)
+		return lastOnLine.byteStart + lastOnLine.byteLen;
+	// Empty line (only as the final position after a trailing newline) or col 0
+	// of a line with no chars → that line's start byte.
+	if (col === 0 && line < a.lineStarts.length)
+		return a.lineStarts[line] as number;
+	throw new Error(
+		`positionToByte: no character boundary at (line ${line}, col ${col})`,
+	);
+}

package/src/explain/defects.ts

@@ -0,0 +1,218 @@
+/**
+ * Defect regions for `explain` (centrs canonicalizer).
+ *
+ * Ratified by the phase-0 lab, question Q14 (#185), whose coordinate half
+ * `commands/explain/README.md` states as "**Malformed input carries a defect
+ * *region*** — each diagnostic points at the byte span of its defect". This
+ * module is that span, and since #202 it is the ONLY structural-surprise channel
+ * an analyzer carries.
+ *
+ * WHY ONE CHANNEL
+ *
+ *   It was two for exactly one issue. Every analyzer used to carry a flat
+ *   `notes: string[]` beside this one, and #192 added the regions additively so
+ *   that any movement in a `notes` value during that change was a real bug rather
+ *   than refactor noise. The convergence is this: a note answered "did anything
+ *   go wrong", a defect answers that AND "where", so the note channel was
+ *   strictly weaker and is gone. The two questions a caller actually asks are
+ *   {@link hasStructuralDefect} (the abstention gate the notes used to serve) and
+ *   the regions themselves.
+ *
+ *   The swap was measured, not assumed: over the frozen 913-document corpus and
+ *   19,972 targeted mutations (delimiter, quote, escape and non-ASCII injections
+ *   that made the gate fire on 6,908 of them), `notes.length > 0` and
+ *   `hasStructuralDefect(defects)` disagreed on ZERO inputs in all five analyzers,
+ *   and no `containsWrite` tristate moved.
+ *
+ * COORDINATES
+ *
+ *   `start`/`end` are ANALYZED-byte offsets, half-open, in exactly the space
+ *   `coordinates.ts` contracts. Because that normalization is byte-count
+ *   preserving, a defect region is directly comparable to a device `highlight`
+ *   span with no fixup. Every region is non-empty: a defect that is "at" one
+ *   byte spans that byte (`[i, i+1)`), never `[i, i)`.
+ *
+ * WHY REGIONS ARE NOT MERELY THE NOTE STRINGS PARSED
+ *
+ *   Two real losses in the retired note channel are what this module fixed, and
+ *   neither was recoverable by parsing `over-depth:<n>` back out of a string. Both are
+ *   reproduced in `test/unit/explain-defects.test.ts`:
+ *
+ *   - `pathresolve` collected notes into a `Set`, so several `over-depth` events
+ *     at different offsets collapsed into ONE entry and every offset was lost.
+ *     Two independent 260-deep block chains in one document produced a single
+ *     `"over-depth"` note; they are two distinct defects.
+ *   - `pathresolve` emitted a BARE `over-depth` with no offset at all, while
+ *     `segment` and `symbols` emitted `over-depth:<byte>` — the same class in two
+ *     shapes, only one of them locatable.
+ *
+ *   A third suspected loss did NOT survive checking, and is recorded so it is
+ *   not re-derived: segmenting a block body and discarding its findings LOOKS
+ *   like it would hide a defect inside `do={…}`. It does not. The top-level
+ *   segmenter scans the whole byte range including block bodies, and
+ *   `scopeBlocks` only returns depth-0 braces, so the outer scan is a strict
+ *   superset — six probe shapes (unterminated strings, stray closes and
+ *   unclosed delimiters inside bodies, nested two deep) surfaced no body-only
+ *   defect. Body defects propagate and `ScopeBlock.start` exists because that is
+ *   what lets a defect the WALKER itself raises (its block-descent
+ *   `over-depth`) carry a document-space region — not because either recovers a
+ *   hidden class.
+ *
+ * CLASSES DELIBERATELY NOT DETECTED
+ *
+ *   `truncation` and `stray mid-token delimiter` appeared in the phase-0 draft
+ *   of the detectable set and are deferred. **The decision and its reasoning
+ *   live in `commands/explain/README.md` (the executable spec), not here** —
+ *   this vocabulary follows that list rather than narrowing it. In short:
+ *   truncation is indistinguishable from a finished token without a schema (it
+ *   is a `--complete` continuation-state question against a live target), and
+ *   the stray-delimiter class needs a probe matrix before a detector is worth
+ *   having — the #201 round is the standing evidence that a lexical rule
+ *   curated from intuition produces defects the corpus cannot reach.
+ *
+ *   Re-opening either means grounding it first and amending the README's list;
+ *   `test/unit/explain-defects.test.ts` asserts they stay undetected so the two
+ *   cannot silently drift apart.
+ */
+
+/**
+ * One structural defect, located in analyzed-byte space.
+ *
+ * The `code` vocabulary is the one the retired note channel used, so a fixture
+ * or transcript from before #202 is still comparable entry for entry.
+ */
+export interface Defect {
+	code: DefectCode;
+	/** analyzed-byte offset, inclusive. */
+	start: number;
+	/** analyzed-byte offset, exclusive. Always `> start`. */
+	end: number;
+	/**
+	 * The delimiter or byte that raised it, when the class has one — `"}"` for
+	 * `unbalanced-close`, the opener for `unclosed`. Never a sentence: prose
+	 * belongs in a rendered diagnostic, not in the structural record.
+	 */
+	detail?: string;
+}
+
+export type DefectCode =
+	| "over-depth"
+	| "bad-escape"
+	| "bad-string-escape"
+	| "bad-sigil"
+	| "invalid-hash"
+	| "invalid-apostrophe"
+	| "unterminated-string"
+	| "unclosed"
+	| "unbalanced-close"
+	| "bom"
+	| "non-ascii";
+
+/**
+ * `bom` and `non-ascii` are POSITIONAL FACTS, not errors.
+ *
+ * This predicate exists so a renderer cannot give them error severity by
+ * accident, and it is a guard rather than a comment because the failure it
+ * prevents is silent and user-facing: `/system identity set name="router-🚀"`
+ * (`commands/explain/examples.md` example 22) is a perfectly legal command whose
+ * value is non-ASCII by design. Failing it under `--fail-on error` would be a
+ * false positive on correct input, and RouterOS comments and string values carry
+ * non-ASCII routinely.
+ *
+ * What these two regions actually record is where the byte-count-preserving
+ * normalization stood in for bytes the analyzer cannot read — i.e. the spans a
+ * consumer needs in order to map back to the original text. That is a coordinate
+ * fact. The other structural codes mark input the analyzer could not structurally
+ * resolve, which is a different thing.
+ *
+ * {@link hasStructuralDefect} is where that distinction is enforced, so a UTF-8
+ * comment can never flip a document's write tristate to `unknown`.
+ */
+export function isPositionalFact(code: DefectCode): boolean {
+	return code === "bom" || code === "non-ascii";
+}
+
+/**
+ * Does this list carry a defect an analyzer must ABSTAIN on?
+ *
+ * The abstention gate, and the reason {@link isPositionalFact} is a function
+ * rather than a comment: a defect list is not uniformly "bad input". Nine codes
+ * mark structure the analyzer could not resolve — part of the document was never
+ * walked, so no verdict over it is safe. The other two are coordinate facts about
+ * bytes that were read perfectly well. Gating on `defects.length > 0` would
+ * abstain on `/system identity set name="router-🚀"`, a legal command, and on the
+ * ~12% of the phase-0 corpus that carries non-ASCII.
+ */
+export function hasStructuralDefect(defects: readonly Defect[]): boolean {
+	return defects.some((d) => !isPositionalFact(d.code));
+}
+
+/** A defect spanning exactly the byte at `at`. */
+export function defectAt(
+	code: DefectCode,
+	at: number,
+	detail?: string,
+): Defect {
+	return detail === undefined
+		? { code, start: at, end: at + 1 }
+		: { code, start: at, end: at + 1, detail };
+}
+
+/**
+ * Shift every defect by `base` — the move that carries a block body's
+ * body-relative regions into document space.
+ *
+ * Always returns a new ARRAY, so a caller may accumulate into the result without
+ * touching the input; at `base === 0` the elements themselves are shared, which
+ * is safe because a Defect is never mutated after construction.
+ */
+export function rebaseDefects(
+	defects: readonly Defect[],
+	base: number,
+): Defect[] {
+	if (base === 0) return [...defects];
+	return defects.map((d) =>
+		d.detail === undefined
+			? { code: d.code, start: d.start + base, end: d.end + base }
+			: {
+					code: d.code,
+					start: d.start + base,
+					end: d.end + base,
+					detail: d.detail,
+				},
+	);
+}
+
+/**
+ * Concatenate defect lists, dropping exact duplicates.
+ *
+ * Identity is `code + start + end + detail` — the WHOLE region, never the code
+ * alone. That distinction is the entire point: the retired note channel
+ * de-duplicated by stringified note, so two `over-depth` events at bytes 40 and
+ * 900 became one `"over-depth"` entry and the second offset was gone. Here they
+ * are two defects.
+ *
+ * Order is stable (first occurrence wins) so a caller can report defects in
+ * discovery order without sorting; callers that want source order should sort by
+ * `start` themselves, since discovery order differs between the statement walk
+ * and the bracket walk.
+ *
+ * The key is JSON rather than a joined string so that an ABSENT `detail` and an
+ * empty-string one stay distinguishable, and so the separator cannot be a raw
+ * control byte sitting invisibly in this file.
+ */
+export function mergeDefects(
+	...lists: readonly (readonly Defect[])[]
+): Defect[] {
+	const seen = new Set<string>();
+	const out: Defect[] = [];
+	for (const list of lists) {
+		for (const d of list) {
+			const key = JSON.stringify([d.code, d.start, d.end, d.detail]);
+			if (seen.has(key)) continue;
+			seen.add(key);
+			out.push(d);
+		}
+	}
+	return out;
+}