@tikoci/centrs 0.1.2 → 0.1.4

package/src/explain/coordinates.ts

@@ -0,0 +1,329 @@
+/**
+ * Coordinate contract for `explain` (centrs canonicalizer).
+ *
+ * Ratified by the phase-0 lab, question Q15 (#185) and contracted in
+ * `commands/explain/README.md` (the byte-count-preserving coordinate rule).
+ * This is the production promotion of the lab's reference mapper; the six
+ * invariants it satisfies are pinned as anchor/property tests in
+ * `test/unit/explain-coordinates.test.ts` against
+ * `test/fixtures/explain/coordinates.json`.
+ *
+ * THREE COORDINATE SPACES
+ *
+ *   original    the caller's JS string. UTF-16 code units. Addressable as
+ *               (line, utf16Col) — the LSP position space.
+ *   originalU8  the UTF-8 encoding of `original`. Byte offsets. This is what a
+ *               byte-oriented device sees on the wire (RouterOS `highlight`
+ *               emits one token per INPUT BYTE).
+ *   analyzed    originalU8 with every non-ASCII byte replaced by the SUB
+ *               placeholder (0x1A), SAME LENGTH as originalU8. Pure ASCII, so a
+ *               JS string built from it has index === byte offset. This is the
+ *               surface the offline parser runs on and the space in which probe
+ *               offsets (`--cursor <byte>`, highlight spans) are contracted.
+ *
+ * WHY BYTE-COUNT-PRESERVING NORMALIZATION (the load-bearing choice)
+ *
+ *   A non-ASCII character is replaced by AS MANY SUB bytes as its UTF-8 length,
+ *   never collapsed to one. That makes an `analyzed` byte offset === the
+ *   `originalU8` byte offset identically, so a device highlight span (byte
+ *   offsets over the UTF-8 it received) and an offline span (analyzed byte
+ *   offsets) share ONE coordinate system with no fixup. SUB (U+001A) is chosen
+ *   because it is ASCII, is not whitespace, and is none of the RouterOS
+ *   delimiters (`; \n { } [ ] ( ) " \ # $ = /`), so it stays glued inside its
+ *   token and cannot forge a keyword, menu name, or separator.
+ *
+ * The production parser consumes the `analyzed` STRING (pure ASCII ⇒ its JS
+ * index IS the byte offset); this module converts those byte offsets back to
+ * the original (line, utf16Col) for LSP consumers. centrs does NOT NFC-fold:
+ * a combining mark is its own code point, its own run, and its own column.
+ */
+
+import type { Defect } from "./defects.ts";
+
+/** SUB (U+001A) — the byte-count-preserving stand-in for a non-ASCII byte. */
+export const SUB = 0x1a;
+
+/** U+FFFD REPLACEMENT CHARACTER — where a lone surrogate normalizes to. */
+const REPLACEMENT = 0xfffd;
+
+/** BYTE ORDER MARK (U+FEFF) — a defect only in leading position; see `coordinateDefects`. */
+const BOM = 0xfeff;
+
+/** A 0-based (line, column) position in `original`; column is a UTF-16 count. */
+export interface Position {
+	line: number;
+	col: number;
+}
+
+/** A half-open (line, col) range in `original` — the LSP range space. */
+export interface Range {
+	start: Position;
+	end: Position;
+}
+
+/** One code point of `original`, located in every coordinate space. */
+export interface CharRun {
+	/** Unicode scalar value; a lone surrogate is recorded as U+FFFD. */
+	codePoint: number;
+	/** Offset in `original` (UTF-16 code units). */
+	utf16Start: number;
+	/** 1 (BMP) or 2 (astral surrogate pair) UTF-16 units. */
+	utf16Len: number;
+	/** Offset in `analyzed`/`originalU8` (bytes). */
+	byteStart: number;
+	/** UTF-8 byte length: 1..4 (a lone surrogate → 3 via U+FFFD). */
+	byteLen: number;
+	/** Whether this run is a single ASCII byte (analyzed byte === original byte). */
+	ascii: boolean;
+	/** 0-based line. */
+	line: number;
+	/** 0-based UTF-16 column within the line. */
+	col: number;
+}
+
+/** The full coordinate analysis of one input string. */
+export interface CoordinateAnalysis {
+	original: string;
+	/** Pure-ASCII surface; `analyzed.length === originalU8.length`. */
+	analyzed: Uint8Array;
+	originalU8: Uint8Array;
+	runs: CharRun[];
+	/** Byte offset of each line's first character (index === 0-based line). */
+	lineStarts: number[];
+}
+
+/** UTF-8 byte length of a Unicode scalar value. */
+function utf8Len(cp: number): number {
+	if (cp < 0x80) return 1;
+	if (cp < 0x800) return 2;
+	if (cp < 0x10000) return 3;
+	return 4;
+}
+
+/** Encode a Unicode scalar value to its UTF-8 bytes. */
+function utf8Encode(cp: number): number[] {
+	if (cp < 0x80) return [cp];
+	if (cp < 0x800) return [0xc0 | (cp >> 6), 0x80 | (cp & 0x3f)];
+	if (cp < 0x10000)
+		return [0xe0 | (cp >> 12), 0x80 | ((cp >> 6) & 0x3f), 0x80 | (cp & 0x3f)];
+	return [
+		0xf0 | (cp >> 18),
+		0x80 | ((cp >> 12) & 0x3f),
+		0x80 | ((cp >> 6) & 0x3f),
+		0x80 | (cp & 0x3f),
+	];
+}
+
+/**
+ * Build the coordinate analysis of `original`: its UTF-8 encoding, the
+ * byte-count-preserving `analyzed` surface, and one {@link CharRun} per code
+ * point locating it in all three spaces.
+ */
+export function analyzeCoordinates(original: string): CoordinateAnalysis {
+	const runs: CharRun[] = [];
+	const u8: number[] = [];
+	const analyzed: number[] = [];
+	const lineStarts: number[] = [0];
+	let line = 0;
+	let col = 0;
+	let utf16Pos = 0;
+
+	// `for..of` yields code points and handles surrogate pairs; we track the
+	// UTF-16 offset/len separately so an astral char advances col by 2 (its
+	// UTF-16 width), which is what an LSP position expects.
+	for (const ch of original) {
+		let cp = ch.codePointAt(0) as number;
+		const utf16Len = ch.length; // 1 or 2
+		// A lone surrogate survives in a JS string as a single UTF-16 unit whose
+		// code point is in [0xD800,0xDFFF]; it is not a valid scalar → U+FFFD.
+		if (cp >= 0xd800 && cp <= 0xdfff) cp = REPLACEMENT;
+		const ascii = cp < 0x80;
+		const byteStart = u8.length;
+
+		for (const b of utf8Encode(cp)) {
+			u8.push(b);
+			analyzed.push(ascii ? b : SUB);
+		}
+
+		runs.push({
+			codePoint: cp,
+			utf16Start: utf16Pos,
+			utf16Len,
+			byteStart,
+			byteLen: utf8Len(cp),
+			ascii,
+			line,
+			col,
+		});
+
+		utf16Pos += utf16Len;
+		// `\n` (0x0A) ends a line. A lone `\r` is an ordinary column, so a `\r\n`
+		// sequence advances exactly one line (on the `\n`). Tabs are one column,
+		// never expanded — column counts UTF-16 units, not visual width.
+		if (cp === 0x0a) {
+			line++;
+			col = 0;
+			lineStarts.push(u8.length);
+		} else {
+			col += utf16Len;
+		}
+	}
+
+	return {
+		original,
+		analyzed: Uint8Array.from(analyzed),
+		originalU8: Uint8Array.from(u8),
+		runs,
+		lineStarts,
+	};
+}
+
+/**
+ * The {@link CharRun} whose byte range contains `byte`. Throws if `byte` is not
+ * inside any run (i.e. `byte < 0` or `byte >= analyzed.length`).
+ */
+export function runAtByte(a: CoordinateAnalysis, byte: number): CharRun {
+	const len = a.analyzed.length;
+	if (!Number.isInteger(byte) || byte < 0 || byte >= len)
+		throw new Error(
+			`runAtByte: ${byte} is not a valid interior byte offset; expected an integer in [0, ${len})`,
+		);
+	let lo = 0;
+	let hi = a.runs.length - 1;
+	while (lo <= hi) {
+		const mid = (lo + hi) >> 1;
+		const r = a.runs[mid] as CharRun;
+		if (byte < r.byteStart) hi = mid - 1;
+		else if (byte >= r.byteStart + r.byteLen) lo = mid + 1;
+		else return r;
+	}
+	// Unreachable: a valid interior byte always lies inside exactly one run.
+	throw new Error(`runAtByte: no run contains byte ${byte} in [0, ${len})`);
+}
+
+/**
+ * Map an `analyzed` byte offset to an original (line, col) position. A byte
+ * landing INSIDE a multi-byte char snaps to that char's boundary (V6).
+ * `byte === analyzed.length` is the end-of-input cursor and maps to the
+ * position just past the last character.
+ */
+export function byteToPosition(a: CoordinateAnalysis, byte: number): Position {
+	const len = a.analyzed.length;
+	if (!Number.isInteger(byte) || byte < 0 || byte > len)
+		throw new Error(
+			`byteToPosition: ${byte} is not a valid cursor offset; expected an integer in [0, ${len}]`,
+		);
+	if (byte === len) {
+		const last = a.runs.at(-1);
+		if (!last) return { line: 0, col: 0 };
+		if (last.codePoint === 0x0a) return { line: last.line + 1, col: 0 };
+		return { line: last.line, col: last.col + last.utf16Len };
+	}
+	const run = runAtByte(a, byte);
+	return { line: run.line, col: run.col };
+}
+
+/**
+ * Map a half-open analyzed-byte span to an original (line, col) range — the
+ * conversion an LSP consumer needs for every defect region and every token span.
+ *
+ * Deliberately built on {@link byteToPosition} in both directions rather than on
+ * {@link positionToByte}: that one is an O(n) linear scan over `runs`, so using
+ * it here would make converting a document's worth of spans quadratic.
+ * `byteToPosition` binary-searches. Both endpoints snap to a character boundary
+ * (V6), and `end === analyzed.length` is legal, so a span reaching end-of-input
+ * converts without a special case.
+ */
+export function byteSpanToRange(
+	a: CoordinateAnalysis,
+	start: number,
+	end: number,
+): Range {
+	if (end < start)
+		throw new Error(
+			`byteSpanToRange: end ${end} precedes start ${start}; spans are half-open and forward`,
+		);
+	return { start: byteToPosition(a, start), end: byteToPosition(a, end) };
+}
+
+/**
+ * The defect regions that fall out of the coordinate pass itself: a leading BOM,
+ * and every run of bytes the ASCII normalization stood in for.
+ *
+ * Both classes are {@link isPositionalFact}s, never errors — see that predicate
+ * for why. They come from `runs` rather than from a second scan because
+ * `analyzeCoordinates` has already decided, per code point, whether it is ASCII
+ * (`CharRun.ascii`); re-deriving that would be a chance to disagree with it.
+ *
+ * Non-ASCII runs are COALESCED: a CJK comment or an emoji-bearing value is one
+ * region, not one per code point. Contiguity is measured in bytes, so adjacent
+ * non-ASCII characters merge even across a surrogate pair.
+ *
+ * Only a U+FEFF in LEADING position is a `bom`. The same code point later in the
+ * document is a zero-width no-break space — ordinary content — and is reported
+ * as part of its `non-ascii` run like any other character.
+ */
+export function coordinateDefects(a: CoordinateAnalysis): Defect[] {
+	const defects: Defect[] = [];
+	const first = a.runs[0];
+	const hasBom = first !== undefined && first.codePoint === BOM;
+	if (hasBom && first !== undefined)
+		defects.push({
+			code: "bom",
+			start: first.byteStart,
+			end: first.byteStart + first.byteLen,
+		});
+
+	let runStart = -1;
+	let runEnd = -1;
+	for (let i = hasBom ? 1 : 0; i < a.runs.length; i++) {
+		const run = a.runs[i] as CharRun;
+		if (run.ascii) {
+			if (runStart >= 0) {
+				defects.push({ code: "non-ascii", start: runStart, end: runEnd });
+				runStart = -1;
+			}
+			continue;
+		}
+		if (runStart < 0) runStart = run.byteStart;
+		runEnd = run.byteStart + run.byteLen;
+	}
+	if (runStart >= 0)
+		defects.push({ code: "non-ascii", start: runStart, end: runEnd });
+
+	return defects;
+}
+
+/**
+ * Map an original (line, col) position to an `analyzed` byte offset. `col` may
+ * equal the line's UTF-16 length (the position just past the last character /
+ * at the newline). Throws if the position addresses no boundary.
+ */
+export function positionToByte(
+	a: CoordinateAnalysis,
+	line: number,
+	col: number,
+): number {
+	if (!Number.isInteger(line) || !Number.isInteger(col) || line < 0 || col < 0)
+		throw new Error(
+			`positionToByte: (line ${line}, col ${col}) is not a valid position; expected non-negative integers`,
+		);
+	// Single pass: return an exact (line, col) hit immediately; otherwise keep
+	// the last run seen on the target line to resolve a just-past-the-end column.
+	let lastOnLine: CharRun | undefined;
+	for (const run of a.runs) {
+		if (run.line !== line) continue;
+		if (run.col === col) return run.byteStart;
+		lastOnLine = run;
+	}
+	// col past the last char of the line → byte after that line's last char.
+	if (lastOnLine && col === lastOnLine.col + lastOnLine.utf16Len)
+		return lastOnLine.byteStart + lastOnLine.byteLen;
+	// Empty line (only as the final position after a trailing newline) or col 0
+	// of a line with no chars → that line's start byte.
+	if (col === 0 && line < a.lineStarts.length)
+		return a.lineStarts[line] as number;
+	throw new Error(
+		`positionToByte: no character boundary at (line ${line}, col ${col})`,
+	);
+}

package/src/explain/defects.ts

@@ -0,0 +1,217 @@
+/**
+ * Defect regions for `explain` (centrs canonicalizer).
+ *
+ * Ratified by the phase-0 lab, question Q14 (#185), whose coordinate half
+ * `commands/explain/README.md` states as "**Malformed input carries a defect
+ * *region*** — each diagnostic points at the byte span of its defect". This
+ * module is that span, and since #202 it is the ONLY structural-surprise channel
+ * an analyzer carries.
+ *
+ * WHY ONE CHANNEL
+ *
+ *   It was two for exactly one issue. Every analyzer used to carry a flat
+ *   `notes: string[]` beside this one, and #192 added the regions additively so
+ *   that any movement in a `notes` value during that change was a real bug rather
+ *   than refactor noise. The convergence is this: a note answered "did anything
+ *   go wrong", a defect answers that AND "where", so the note channel was
+ *   strictly weaker and is gone. The two questions a caller actually asks are
+ *   {@link hasStructuralDefect} (the abstention gate the notes used to serve) and
+ *   the regions themselves.
+ *
+ *   The swap was measured, not assumed: over the frozen 913-document corpus and
+ *   19,972 targeted mutations (delimiter, quote, escape and non-ASCII injections
+ *   that made the gate fire on 6,908 of them), `notes.length > 0` and
+ *   `hasStructuralDefect(defects)` disagreed on ZERO inputs in all five analyzers,
+ *   and no `containsWrite` tristate moved.
+ *
+ * COORDINATES
+ *
+ *   `start`/`end` are ANALYZED-byte offsets, half-open, in exactly the space
+ *   `coordinates.ts` contracts. Because that normalization is byte-count
+ *   preserving, a defect region is directly comparable to a device `highlight`
+ *   span with no fixup. Every region is non-empty: a defect that is "at" one
+ *   byte spans that byte (`[i, i+1)`), never `[i, i)`.
+ *
+ * WHY REGIONS ARE NOT MERELY THE NOTE STRINGS PARSED
+ *
+ *   Two real losses in the retired note channel are what this module fixed, and
+ *   neither was recoverable by parsing `over-depth:<n>` back out of a string. Both are
+ *   reproduced in `test/unit/explain-defects.test.ts`:
+ *
+ *   - `pathresolve` collected notes into a `Set`, so several `over-depth` events
+ *     at different offsets collapsed into ONE entry and every offset was lost.
+ *     Two independent 260-deep block chains in one document produced a single
+ *     `"over-depth"` note; they are two distinct defects.
+ *   - `pathresolve` emitted a BARE `over-depth` with no offset at all, while
+ *     `segment` and `symbols` emitted `over-depth:<byte>` — the same class in two
+ *     shapes, only one of them locatable.
+ *
+ *   A third suspected loss did NOT survive checking, and is recorded so it is
+ *   not re-derived: segmenting a block body and discarding its findings LOOKS
+ *   like it would hide a defect inside `do={…}`. It does not. The top-level
+ *   segmenter scans the whole byte range including block bodies, and
+ *   `scopeBlocks` only returns depth-0 braces, so the outer scan is a strict
+ *   superset — six probe shapes (unterminated strings, stray closes and
+ *   unclosed delimiters inside bodies, nested two deep) surfaced no body-only
+ *   defect. Body defects propagate and `ScopeBlock.start` exists because that is
+ *   what lets a defect the WALKER itself raises (its block-descent
+ *   `over-depth`) carry a document-space region — not because either recovers a
+ *   hidden class.
+ *
+ * CLASSES DELIBERATELY NOT DETECTED
+ *
+ *   `truncation` and `stray mid-token delimiter` appeared in the phase-0 draft
+ *   of the detectable set and are deferred. **The decision and its reasoning
+ *   live in `commands/explain/README.md` (the executable spec), not here** —
+ *   this vocabulary follows that list rather than narrowing it. In short:
+ *   truncation is indistinguishable from a finished token without a schema (it
+ *   is a `--complete` continuation-state question against a live target), and
+ *   the stray-delimiter class needs a probe matrix before a detector is worth
+ *   having — the #201 round is the standing evidence that a lexical rule
+ *   curated from intuition produces defects the corpus cannot reach.
+ *
+ *   Re-opening either means grounding it first and amending the README's list;
+ *   `test/unit/explain-defects.test.ts` asserts they stay undetected so the two
+ *   cannot silently drift apart.
+ */
+
+/**
+ * One structural defect, located in analyzed-byte space.
+ *
+ * The `code` vocabulary is the one the retired note channel used, so a fixture
+ * or transcript from before #202 is still comparable entry for entry.
+ */
+export interface Defect {
+	code: DefectCode;
+	/** analyzed-byte offset, inclusive. */
+	start: number;
+	/** analyzed-byte offset, exclusive. Always `> start`. */
+	end: number;
+	/**
+	 * The delimiter or byte that raised it, when the class has one — `"}"` for
+	 * `unbalanced-close`, the opener for `unclosed`. Never a sentence: prose
+	 * belongs in a rendered diagnostic, not in the structural record.
+	 */
+	detail?: string;
+}
+
+export type DefectCode =
+	| "over-depth"
+	| "bad-escape"
+	| "bad-string-escape"
+	| "bad-sigil"
+	| "invalid-hash"
+	| "unterminated-string"
+	| "unclosed"
+	| "unbalanced-close"
+	| "bom"
+	| "non-ascii";
+
+/**
+ * `bom` and `non-ascii` are POSITIONAL FACTS, not errors.
+ *
+ * This predicate exists so a renderer cannot give them error severity by
+ * accident, and it is a guard rather than a comment because the failure it
+ * prevents is silent and user-facing: `/system identity set name="router-🚀"`
+ * (`commands/explain/examples.md` example 22) is a perfectly legal command whose
+ * value is non-ASCII by design. Failing it under `--fail-on error` would be a
+ * false positive on correct input, and RouterOS comments and string values carry
+ * non-ASCII routinely.
+ *
+ * What these two regions actually record is where the byte-count-preserving
+ * normalization stood in for bytes the analyzer cannot read — i.e. the spans a
+ * consumer needs in order to map back to the original text. That is a coordinate
+ * fact. The other structural codes mark input the analyzer could not structurally
+ * resolve, which is a different thing.
+ *
+ * {@link hasStructuralDefect} is where that distinction is enforced, so a UTF-8
+ * comment can never flip a document's write tristate to `unknown`.
+ */
+export function isPositionalFact(code: DefectCode): boolean {
+	return code === "bom" || code === "non-ascii";
+}
+
+/**
+ * Does this list carry a defect an analyzer must ABSTAIN on?
+ *
+ * The abstention gate, and the reason {@link isPositionalFact} is a function
+ * rather than a comment: a defect list is not uniformly "bad input". Seven codes
+ * mark structure the analyzer could not resolve — part of the document was never
+ * walked, so no verdict over it is safe. The other two are coordinate facts about
+ * bytes that were read perfectly well. Gating on `defects.length > 0` would
+ * abstain on `/system identity set name="router-🚀"`, a legal command, and on the
+ * ~12% of the phase-0 corpus that carries non-ASCII.
+ */
+export function hasStructuralDefect(defects: readonly Defect[]): boolean {
+	return defects.some((d) => !isPositionalFact(d.code));
+}
+
+/** A defect spanning exactly the byte at `at`. */
+export function defectAt(
+	code: DefectCode,
+	at: number,
+	detail?: string,
+): Defect {
+	return detail === undefined
+		? { code, start: at, end: at + 1 }
+		: { code, start: at, end: at + 1, detail };
+}
+
+/**
+ * Shift every defect by `base` — the move that carries a block body's
+ * body-relative regions into document space.
+ *
+ * Always returns a new ARRAY, so a caller may accumulate into the result without
+ * touching the input; at `base === 0` the elements themselves are shared, which
+ * is safe because a Defect is never mutated after construction.
+ */
+export function rebaseDefects(
+	defects: readonly Defect[],
+	base: number,
+): Defect[] {
+	if (base === 0) return [...defects];
+	return defects.map((d) =>
+		d.detail === undefined
+			? { code: d.code, start: d.start + base, end: d.end + base }
+			: {
+					code: d.code,
+					start: d.start + base,
+					end: d.end + base,
+					detail: d.detail,
+				},
+	);
+}
+
+/**
+ * Concatenate defect lists, dropping exact duplicates.
+ *
+ * Identity is `code + start + end + detail` — the WHOLE region, never the code
+ * alone. That distinction is the entire point: the retired note channel
+ * de-duplicated by stringified note, so two `over-depth` events at bytes 40 and
+ * 900 became one `"over-depth"` entry and the second offset was gone. Here they
+ * are two defects.
+ *
+ * Order is stable (first occurrence wins) so a caller can report defects in
+ * discovery order without sorting; callers that want source order should sort by
+ * `start` themselves, since discovery order differs between the statement walk
+ * and the bracket walk.
+ *
+ * The key is JSON rather than a joined string so that an ABSENT `detail` and an
+ * empty-string one stay distinguishable, and so the separator cannot be a raw
+ * control byte sitting invisibly in this file.
+ */
+export function mergeDefects(
+	...lists: readonly (readonly Defect[])[]
+): Defect[] {
+	const seen = new Set<string>();
+	const out: Defect[] = [];
+	for (const list of lists) {
+		for (const d of list) {
+			const key = JSON.stringify([d.code, d.start, d.end, d.detail]);
+			if (seen.has(key)) continue;
+			seen.add(key);
+			out.push(d);
+		}
+	}
+	return out;
+}

package/src/explain/is-known-menu.ts

@@ -0,0 +1,17 @@
+import { lookupPath } from "./catalog.ts";
+import { isMenuPath } from "./menus.ts";
+
+/**
+ * Whether `segments` name a known RouterOS menu or settings path.
+ *
+ * Union of the two shipped structure tables (#235): `MENU_PATHS` (#207,
+ * `isMenuPath`) and `PATH_CATALOG` menu/settings rows (#228, `lookupPath`).
+ * Presence is load-bearing, absence abstains — the caller must not read
+ * absence as evidence of a command.
+ */
+export function isKnownMenuPath(segments: readonly string[]): boolean {
+	const entry = lookupPath(segments);
+	return (
+		isMenuPath(segments) || entry?.kind === "menu" || entry?.kind === "settings"
+	);
+}