@tikoci/centrs 0.1.3 → 0.1.6

package/src/explain/segment.ts

@@ -0,0 +1,1165 @@
+/**
+ * Statement segmenter for `explain` (centrs canonicalizer).
+ *
+ * Ratified by the phase-0 lab, question Q1 (#185) and promoted from the
+ * throwaway probe `.scratch/explain-lab-segmenter.ts`. This is the first walker
+ * stage: a schema-free, device-free splitter of multi-statement RouterOS input
+ * into top-level statements, matching `(evl …)` sibling adjacency in `:parse`
+ * IL. It never throws; structural surprises are reported as located
+ * {@link Defect}s.
+ *
+ * Segmentation runs on the byte-count-preserving `analyzed` surface from
+ * `./coordinates.ts`, so a segment's `start`/`end` are **analyzed-byte offsets**
+ * (half-open, aligned with the coordinate contract and device highlight spans),
+ * while `text` is the **original** human-readable statement, recovered through
+ * the coordinate mapping. Because non-ASCII bytes are `SUB` in `analyzed` and
+ * `SUB` is none of the RouterOS delimiters, prose or names cannot corrupt the
+ * delimiter stack.
+ *
+ * The boundary rules (grounded from the documented scripting semantics before
+ * any scoring, so the disagreement bins meant something):
+ *   H1  Statements separate on `;` and newline, at nesting depth 0 only.
+ *   H2  Depth counts `{}`, `[]`, `()` uniformly; a newline inside any of them
+ *       continues the statement.
+ *   H3  `"` opens a string; `\` escapes the next char inside it; a string
+ *       suppresses every separator and depth character. RouterOS has no
+ *       single-quote string form, so `'` is an ordinary character. A string is
+ *       not opaque to the NEXT quote, though: a `$[…]`/`$(…)` substitution
+ *       inside it is code that may carry strings of its own, so where a string
+ *       ends is decided by the one shared `scanQuotedString` (#199).
+ *   H4  `#` starts a comment in statement-leading position (start of input, or
+ *       the first non-space after a separator in a statement-bearing context)
+ *       — and, per H5 below, at the immediate start of a line that a
+ *       continuation carried into. A scope/stored-script brace starts such a
+ *       context; an array brace or parenthesized expression does not, and an
+ *       unquoted `#` there is a hard device error (#245). A real comment runs to
+ *       end of line and produces no statement, so its apostrophes and braces
+ *       cannot leak into the delimiter stack.
+ *   H5  A backslash at end of line is a continuation: it does not separate. Its
+ *       reach is wider than one line, and was measured on CHR 7.23.3 (#215):
+ *         - blank and whitespace-only lines right after the `\` are part of the
+ *           continuation (the device classes the whole run `escaped`);
+ *         - a `#` at the IMMEDIATE start of such a line is a comment, and the
+ *           comment line does not end the pending statement — `:local \` +
+ *           newline + `# c` + newline + `foo 1` still declares `foo`;
+ *         - after a continuation COMMENT line the reach is spent: the next
+ *           blank line ENDS the statement (where directly after the `\` it
+ *           would not), while ordinary content simply continues it;
+ *         - a continuation never moves statement-leading position. `do={\` +
+ *           newline + `:local x 1` still reads `:local` as the head, and an
+ *           indented `#` is a comment there (lead) but content mid-statement.
+ *   H6  Empty statements (`;;`, blank lines) produce nothing.
+ *   H7  A statement that is only a `{…}` group (optionally behind a bare menu
+ *       path) is a CONTAINER: IL flattens its children into siblings, so its
+ *       children are segmented in its place.
+ *
+ * The scan is a single left-to-right pass (Q17, #190): one index-based walker
+ * with an explicit `frames` stack does H7 flattening inline — no per-container
+ * recursion and no re-scan of container bodies. Each open container is a frame
+ * whose inner statements accumulate in a buffer; on its closing `}` the frame
+ * either COMMITS (its buffered statements, behind an optional menu-prefix
+ * segment, replace the container) or is DISCARDED (an empty body, trailing
+ * content after the `}`, or an unclosed brace — the parent statement absorbs the
+ * whole `{…}` verbatim). This reproduces the recursive `containerOpen` +
+ * body re-scan contract the earlier version pinned, at O(n) time / O(depth)
+ * space with no stack-overflow surface.
+ *
+ * H7 stays depth-bounded even so. `explain` accepts untrusted editor/MCP input,
+ * so `MAX_CONTAINER_DEPTH` container frames is a hard cap: a `{` that would open
+ * a deeper container is treated as an opaque group instead, the innermost
+ * remainder stays one segment, and an `over-depth` defect is raised at that
+ * offset. With recursion gone the cap is now a resource guard (bounding the
+ * `frames` array and abstention granularity), not the stack-safety boundary it
+ * once was.
+ */
+
+import {
+	analyzeCoordinates,
+	type CoordinateAnalysis,
+	coordinateDefects,
+	runAtByte,
+} from "./coordinates.ts";
+import { type Defect, defectAt, mergeDefects } from "./defects.ts";
+import {
+	MAX_STRING_FRAME_DEPTH,
+	scanQuotedString as scanQuotedStringShared,
+} from "./quoted-string.ts";
+import {
+	braceStartsStatements,
+	canRetainStatementIndex,
+	hashStartsHardError,
+	isIndexedStatementStart,
+	matchingBraceEnd,
+	scopeNameFromMasked,
+} from "./scope-brace.ts";
+
+/** One top-level statement located in analyzed-byte space. */
+export interface Segment {
+	/** analyzed-byte offset, inclusive. */
+	start: number;
+	/** analyzed-byte offset, exclusive. */
+	end: number;
+	/** the original (human-readable) statement text, trimmed. */
+	text: string;
+	/** the separator that ENDED this statement, for the boundary taxonomy. */
+	terminator: ";" | "newline" | "eof";
+	/**
+	 * true when the statement *looks* like pure menu navigation: a `/`-rooted
+	 * path of path-shaped tokens with no `=`, quotes, brackets, or `$`. This is
+	 * a SHAPE hint, not a verb claim — offline (schema-free) cannot tell a
+	 * trailing verb (`/ip address print`) from a deeper menu (`/ip address`), so
+	 * both are `menuOnly: true`. Q4/Q6 decide what it means. See `isMenuOnly`.
+	 */
+	menuOnly: boolean;
+}
+
+/** The full segmentation of one input string. */
+export interface SegmentResult {
+	segments: Segment[];
+	/** comment spans (analyzed-byte offsets), dropped from the statement stream. */
+	comments: { start: number; end: number }[];
+	/**
+	 * The input was pure ASCII, so an analyzed-byte offset into it is also a JS
+	 * string index — the test a caller needs before it may treat the spans above
+	 * as offsets into `original` itself. Published from the coordinate pass this
+	 * result already ran rather than re-derived, so the two cannot disagree.
+	 */
+	ascii: boolean;
+	/**
+	 * Unbalanced delimiters and other structural surprises, each located; never
+	 * a throw. Includes the two coordinate-pass classes (`bom`, `non-ascii`).
+	 */
+	defects: Defect[];
+}
+
+const isSpace = (c: string): boolean => c === " " || c === "\t" || c === "\r";
+
+/** Where a double-quoted string ends, and whether it was closed at all. */
+export interface QuotedStringScan {
+	/** index just past the closing `"`, or `text.length` when unterminated. */
+	end: number;
+	closed: boolean;
+}
+
+/**
+ * Find the end of the double-quoted string that opens at `open` (H3).
+ *
+ * The one shared string skip for every structural scan in `explain`. A string is
+ * NOT opaque up to the next `"`: a `$[…]` or `$(…)` substitution inside it is
+ * CODE, and that code may open strings of its own —
+ * `:local a "$[[:parse "(\"x\")"]]"` is ONE string on the device (CHR 7.23.2
+ * `/console/inspect request=highlight` classes the following `#` line `comment`
+ * and both `$a` occurrences `variable-local`). A scanner that stops at the first
+ * nested `"` flips its quote phase and reads every later comment as string
+ * content; on the frozen 913-script corpus that hid 2,184 device-`comment` bytes
+ * across 7 files, and cost `symbols.ts` 3.1 points of precision before it grew
+ * the frame model this function now shares (#198/#199).
+ *
+ * Frames mirror `symbols.ts`: a `"` frame is string phase (`\` escapes the next
+ * character, `$[`/`$(` push a code frame), anything else is code phase (`"`
+ * opens a nested string, brackets nest, a mismatched close is content rather
+ * than a close — the same line `segment.ts` and `symbols.ts` take elsewhere).
+ * Iterative, so deeply nested input cannot overflow the stack. Escapes and
+ * comments inside the substitution are NOT interpreted here: the whole string,
+ * substitutions included, stays opaque to the caller — only its END moves.
+ *
+ * Frame depth is capped for the same reason `MAX_CONTAINER_DEPTH` is: `explain`
+ * accepts untrusted editor/MCP input. Past `MAX_STRING_FRAME_DEPTH` the scan
+ * stops and reports the string UNCLOSED — fail-closed, the direction every
+ * caller already handles (`unterminated-string` → `structuralDefect` →
+ * `unresolved`). Only an input that opens that many frames without closing them
+ * can reach it, which is malformed by construction; sequential substitutions
+ * (`"$[a]$[b]…"`) pop and never accumulate.
+ */
+export function scanQuotedString(
+	text: string,
+	open: number,
+	limit: number = text.length,
+): QuotedStringScan {
+	return scanQuotedStringWithin(text, open, limit, MAX_STRING_FRAME_DEPTH);
+}
+
+function scanQuotedStringWithin(
+	text: string,
+	open: number,
+	limit: number,
+	frameBudget: number,
+): QuotedStringScan {
+	return scanQuotedStringShared(
+		text,
+		open,
+		limit,
+		(source, bracket, end, frameDepth) =>
+			scanStringInterpolationWithin(
+				source,
+				bracket,
+				end,
+				frameBudget - frameDepth,
+			).end,
+		frameBudget,
+	);
+}
+
+/**
+ * How far a H5 `\<newline>` continuation still reaches (#215, CHR 7.23.3).
+ *
+ *   "none"    — no continuation pending.
+ *   "escape"  — inside the `\<newline>` run itself. Blank and whitespace-only
+ *               lines stay inside it (the device classes them `escaped`).
+ *   "comment" — the run was ended by a continuation COMMENT line, which does
+ *               not end the pending statement but does spend the run's reach:
+ *               a following blank line now terminates.
+ *
+ * `Continuation` and the `lineStart` flag beside it are the vocabulary the
+ * three parallel walkers (`maskComments`, `scanAscii`, `symbols.ts`) share, so
+ * a change to the rule can be made in the same shape in each. #199 was two of
+ * them drifting apart; this state is where the third one can.
+ */
+export type Continuation = "none" | "escape" | "comment";
+
+/**
+ * Blank out RouterOS comments so a later delimiter re-scan cannot be fooled by
+ * `#` text (a `}` or `[` inside a comment is not a real delimiter). A `#` starts
+ * a comment in statement-leading position (H4) or at the immediate start of a
+ * line a continuation carried into (H5). Statement-leading is contextual:
+ * scope/stored-script braces admit it, array/group expressions do not. Any
+ * intervening space or tab makes a continuation-line `#` content unless the
+ * surrounding statement context is still empty. Comments are opaque inside
+ * strings. Comment characters become spaces; length and every non-comment
+ * offset are preserved, so indices stay valid against the original and callers
+ * can slice the original for content. Idempotent.
+ */
+export function maskComments(original: string): string {
+	if (recentMaskInput === original && recentMaskOutput !== undefined)
+		return recentMaskOutput;
+	const cached = maskCache.get(original);
+	if (cached !== undefined) {
+		// Refresh recency — see `scope-brace.ts` → `statementIndex` for why an
+		// insertion-ordered eviction loses exactly the entry worth keeping.
+		maskCache.delete(original);
+		maskCache.set(original, cached);
+		adoptMask(original, cached);
+		return cached;
+	}
+	const masked = maskUncached(original);
+	if (cacheMask(original, masked)) adoptMask(original, masked);
+	return masked;
+}
+
+const MASK_CACHE_LIMIT = 4;
+const MASK_CACHE_BYTE_LIMIT = 8 * 1024 * 1024;
+const maskCache = new Map<string, string>();
+let maskCacheBytes = 0;
+// Same shape, and the same reason, as `scope-brace.ts`'s boundary cache: one
+// slot for the hot object, a small bounded map behind it, and adoption on a hit
+// so a content-equal twin is compared once rather than on every call.
+let recentMaskInput: string | undefined;
+let recentMaskOutput: string | undefined;
+
+/** Adopt `original` as the single-slot key (see `scope-brace.ts` → `adoptBoundaryText`). */
+function adoptMask(original: string, masked: string): void {
+	recentMaskInput = original;
+	recentMaskOutput = masked;
+}
+
+function maskCacheEntryBytes(original: string, masked: string): number {
+	// Comment-free input is its own mask, so that entry holds one string.
+	return (original.length + (masked === original ? 0 : masked.length)) * 2;
+}
+
+function cacheMask(original: string, masked: string): boolean {
+	const entryBytes = maskCacheEntryBytes(original, masked);
+	if (entryBytes > MASK_CACHE_BYTE_LIMIT) return false;
+	while (
+		maskCache.size >= MASK_CACHE_LIMIT ||
+		maskCacheBytes + entryBytes > MASK_CACHE_BYTE_LIMIT
+	) {
+		const oldest = maskCache.entries().next().value;
+		if (oldest === undefined) break;
+		maskCache.delete(oldest[0]);
+		maskCacheBytes -= maskCacheEntryBytes(oldest[0], oldest[1]);
+		if (recentMaskInput === oldest[0]) {
+			recentMaskInput = undefined;
+			recentMaskOutput = undefined;
+		}
+	}
+	maskCache.set(original, masked);
+	maskCacheBytes += entryBytes;
+	return true;
+}
+
+/**
+ * A region of one document, carried with that document's comment mask.
+ *
+ * Every reader below the statement walk asks its questions of a statement's
+ * `text` or of a `do={…}` body — and a statement's text CONTAINS its whole
+ * nested subtree, so each enclosing level re-derived the mask, and the
+ * statement index behind it, over bytes an inner level had already read. That
+ * is the O(bytes x depth) term: a 60 KiB document nested 64 deep masked 27.7 MB
+ * (#322).
+ *
+ * A range replaces the substring. `text` and `masked` are always the WHOLE
+ * document; `start` and `end` say which part of it this reader may look at. The
+ * mask is then derived once per document and every nested reading is an offset
+ * into it.
+ *
+ * Soundness is a property of where the ranges come from, not of this type: a
+ * comment mask is decided by the statement context a byte sits in, and a
+ * statement boundary and a scope body both begin one — statement-leading, no
+ * continuation pending, an enclosing context that admits statements. So the
+ * mask of such a region is the document's mask restricted to it. Readers still
+ * have to be told the floor (see `scope-brace.ts` → `scopeNameFromMasked`) so
+ * that lookback stops where a fresh call on the region alone would stop.
+ */
+export interface MaskedRange {
+	/** The whole document. Offsets below index into this, never into a slice. */
+	readonly text: string;
+	/** `maskComments(text)`. */
+	readonly masked: string;
+	readonly start: number;
+	readonly end: number;
+	/** This range's string indices are analyzed-byte offsets. */
+	readonly coordinateIdentity?: true;
+}
+
+/** The range covering all of `text`, masking it once. */
+export function documentRange(
+	text: string,
+	coordinateIdentity = false,
+): MaskedRange {
+	return {
+		text,
+		masked: maskComments(text),
+		start: 0,
+		end: text.length,
+		...(coordinateIdentity ? { coordinateIdentity: true as const } : {}),
+	};
+}
+
+/**
+ * Locate one `$[…]` body's closing bracket using its own comment context.
+ *
+ * The document mask deliberately leaves an enclosing string opaque. Once `$[`
+ * enters code, statement-leading comments are real again, so brackets inside
+ * them cannot close or extend the substitution. The returned range starts at
+ * `open` and retains that local mask for the readers of the body.
+ */
+export function scanStringInterpolation(
+	text: string,
+	open: number,
+	limit: number = text.length,
+): { end: number; range: MaskedRange } {
+	return scanStringInterpolationWithin(
+		text,
+		open,
+		limit,
+		MAX_STRING_FRAME_DEPTH - 1,
+	);
+}
+
+function scanStringInterpolationWithin(
+	text: string,
+	open: number,
+	limit: number,
+	frameBudget: number,
+): { end: number; range: MaskedRange } {
+	const local = text.slice(open, limit);
+	// Masking must find each code-level string boundary comment-aware. Reuse
+	// those answers below instead of recursively scanning the same strings again.
+	const quotedEnds = new Map<number, number>();
+	const range: MaskedRange = {
+		text: local,
+		masked: maskUncached(local, frameBudget, quotedEnds),
+		start: 0,
+		end: local.length,
+	};
+	if (frameBudget < 1) return { end: limit, range };
+	const frames: string[] = ["["];
+	for (let i = 1; i < range.end; i++) {
+		const c = range.masked[i];
+		if (c === '"') {
+			const quotedEnd = quotedEnds.get(i);
+			if (quotedEnd === undefined) return { end: limit, range };
+			i = quotedEnd - 1;
+			continue;
+		}
+		if (c === "[" || c === "(" || c === "{") {
+			if (frames.length >= frameBudget) return { end: limit, range };
+			frames.push(c);
+			continue;
+		}
+		if (c !== "]" && c !== ")" && c !== "}") continue;
+		const want = c === "]" ? "[" : c === ")" ? "(" : "{";
+		if (frames.at(-1) !== want) continue;
+		frames.pop();
+		if (frames.length === 0) return { end: open + i, range };
+	}
+	return { end: limit, range };
+}
+
+/**
+ * The sub-range `[start, end)` of `parent`, in `parent`'s own coordinates.
+ *
+ * Offsets must be JS string indices — for a statement span that means the
+ * document was ASCII ({@link SegmentResult.ascii}), which is the only case in
+ * which an analyzed-byte offset is one.
+ */
+export function nestedRange(
+	parent: MaskedRange,
+	start: number,
+	end: number,
+): MaskedRange {
+	return {
+		text: parent.text,
+		masked: parent.masked,
+		start: parent.start + start,
+		end: parent.start + end,
+		...(parent.coordinateIdentity ? { coordinateIdentity: true as const } : {}),
+	};
+}
+
+/**
+ * The range `[start, end)` of the same document, in ABSOLUTE offsets.
+ *
+ * The counterpart to {@link nestedRange}, for a reader that is already working
+ * in the document's own offsets rather than in a parent's.
+ */
+export function rangeAt(
+	parent: MaskedRange,
+	start: number,
+	end: number,
+): MaskedRange {
+	return {
+		text: parent.text,
+		masked: parent.masked,
+		start,
+		end,
+		...(parent.coordinateIdentity ? { coordinateIdentity: true as const } : {}),
+	};
+}
+
+/** The comment mask of what this range covers. */
+export function rangeMask(range: MaskedRange): string {
+	return range.masked.slice(range.start, range.end);
+}
+
+/**
+ * {@link rangeMask}, tolerating no range — for readers that take one OPTIONALLY
+ * and fall back to masking their own text: `maskOf(range) ?? maskComments(text)`.
+ */
+export function maskOf(range: MaskedRange | undefined): string | undefined {
+	return range === undefined ? undefined : rangeMask(range);
+}
+
+/**
+ * The mask itself. Pure, so the memo above cannot change an answer — only how
+ * often it is computed.
+ *
+ * The walk asks for the mask of the SAME statement several times over: the
+ * structural-defect scan, the scope-block scan and the bracket scan each take
+ * one, and the bare-directive test takes another over the trimmed spelling.
+ * Each of those re-ran this scan AND, through `braceStartsStatements`, the
+ * statement index behind it — and a statement's text carries its whole nested
+ * subtree, so the repetition was paid once per enclosing level (#322).
+ */
+function maskUncached(
+	original: string,
+	frameBudget: number = MAX_STRING_FRAME_DEPTH,
+	quotedEnds?: Map<number, number>,
+): string {
+	// Masked output is built from whole slices, and only once a comment is
+	// actually found: comment-free input is returned as the SAME string object.
+	// `split("")` allocated one string per character of every call — 16 walks
+	// over a 64 KiB document made a million of them — where a document's comment
+	// runs are a handful of slices (#313).
+	let pieces: string[] | undefined;
+	let copied = 0;
+	const contexts: { char: "{" | "[" | "("; statements: boolean }[] = [];
+	let atLead = true;
+	let cont: Continuation = "none";
+	let contLineStart = false;
+	for (let i = 0; i < original.length; i++) {
+		const c = original[i];
+		const inCont = cont !== "none";
+		const contComment = c === "#" && inCont && contLineStart;
+		if (inCont && !contComment) {
+			if (c === " " || c === "\t" || c === "\r") {
+				contLineStart = false;
+			} else if (c === "\n" && cont === "escape") {
+				contLineStart = true;
+				continue; // a blank line inside the `\` run is still the run
+			} else {
+				// Real content, or the blank line that spends a comment run's reach.
+				cont = "none";
+				contLineStart = false;
+			}
+		}
+		if (c === '"') {
+			atLead = false;
+			const end = scanQuotedStringWithin(
+				original,
+				i,
+				original.length,
+				frameBudget - contexts.length,
+			).end;
+			quotedEnds?.set(i, end);
+			i = end - 1;
+			continue; // i rests on the closing quote (or past end)
+		}
+		// H4 is unchanged and H5 only ADDS the immediate-line-start case: an
+		// indented `#` is a comment when the statement is still empty (`do={\` +
+		// newline + `  # c`) and content when it is not (`:put a\` + newline +
+		// `  # x`, which the device classes `error`) — exactly what `atLead` says.
+		if (c === "#" && (atLead || contComment)) {
+			const commentStart = i;
+			while (i < original.length && original[i] !== "\n") i++;
+			if (pieces === undefined) pieces = [];
+			pieces.push(
+				original.slice(copied, commentStart),
+				" ".repeat(i - commentStart),
+			);
+			copied = i;
+			if (contComment) {
+				// The comment line does not end the statement: step over its newline
+				// (the loop's i++) with the continuation still pending.
+				cont = "comment";
+				contLineStart = true;
+				continue;
+			}
+			i--; // let the loop advance onto the newline (or end)
+			continue;
+		}
+		if (
+			c === "\\" &&
+			(original[i + 1] === "\n" ||
+				(original[i + 1] === "\r" && original[i + 2] === "\n"))
+		) {
+			// H5 leaves `atLead` alone: `do={\` + newline + `:local x 1` still reads
+			// `:local` as the statement head on the device.
+			cont = "escape";
+			contLineStart = true;
+			i += original[i + 1] === "\r" ? 2 : 1;
+			continue;
+		}
+		if (c === "{" || c === "[" || c === "(") {
+			const enclosing = contexts.at(-1)?.statements ?? true;
+			const statements =
+				c === "[" ||
+				(c === "{" && enclosing && braceStartsStatements(original, i));
+			contexts.push({ char: c, statements });
+			// A bracket is a nested statement context too: `[# c\n:put 1]` starts
+			// with a real comment, while `[:put #value]` consumes the lead on `:put`
+			// and keeps the hash as that command's value.
+			atLead = statements;
+		} else if (c === "}" || c === "]" || c === ")") {
+			const want = c === "}" ? "{" : c === "]" ? "[" : "(";
+			if (contexts[contexts.length - 1]?.char === want) contexts.pop();
+			atLead = false;
+		} else if (c === ";" || c === "\n") {
+			atLead = contexts.at(-1)?.statements ?? true;
+		} else if (c !== " " && c !== "\t" && c !== "\r") atLead = false;
+	}
+	if (pieces === undefined) return original;
+	pieces.push(original.slice(copied));
+	return pieces.join("");
+}
+
+/**
+ * Container-frame depth cap.
+ *
+ * Not a RouterOS grammar limit — an implementation guard derived from the Q17
+ * stress finding (the lab walker overflowed the JS stack around 32k nested
+ * containers). The single-pass scanner no longer recurses, so this is now a
+ * resource guard rather than a stack-safety boundary: 256 preserves far more
+ * nesting than normal scripts use while bounding the `frames` array. A `{` that
+ * would open frame 257 is kept as an opaque group and the analysis abstains with
+ * an `over-depth:<analyzed-byte-offset>` note.
+ */
+const MAX_CONTAINER_DEPTH = 256;
+
+/**
+ * Segment `original` into top-level statements. Spans are analyzed-byte offsets
+ * (see module header); `text` is the original substring for each span.
+ *
+ * A statement-bearing `range` may anchor the same `original` text in its
+ * enclosing document. On ASCII input, structural lookback can reuse that
+ * document's statement index instead of indexing every nested body (#322).
+ * The scan still reads the original bytes, including comments: replacing them
+ * with the mask would lose comment spans and H5 continuation state. Non-ASCII
+ * input, or a body opener skipped by the raw document index (a quote inside an
+ * earlier comment), keeps the standalone analyzed-byte scan.
+ */
+export function segmentStatements(
+	original: string,
+	range?: MaskedRange,
+): SegmentResult {
+	return segmentStatementsInternal(original, range, false);
+}
+
+/** Segment one known scope body while treating its child scopes as opaque. */
+export function segmentScopeBody(
+	original: string,
+	range: MaskedRange,
+): SegmentResult {
+	return segmentStatementsInternal(
+		original,
+		range,
+		range.coordinateIdentity === true,
+	);
+}
+
+function segmentStatementsInternal(
+	original: string,
+	range: MaskedRange | undefined,
+	opaqueChildScopes: boolean,
+): SegmentResult {
+	const identity =
+		range?.coordinateIdentity === true &&
+		range.end - range.start === original.length;
+	const analysis = identity ? undefined : analyzeCoordinates(original);
+	// `analyzed` is pure ASCII, so a string built from it has index === byte.
+	// When the INPUT was already ASCII the two are the same sequence, so the
+	// original is that string and the decode is skipped (#322).
+	const ascii =
+		analysis === undefined || analysis.ascii
+			? original
+			: new TextDecoder().decode(analysis.analyzed);
+	const anchored =
+		(analysis === undefined || analysis.ascii) &&
+		range !== undefined &&
+		canRetainStatementIndex(range.text) &&
+		isIndexedStatementStart(range.text, range.start)
+			? range
+			: undefined;
+	const raw = scanAscii(ascii, anchored, opaqueChildScopes);
+	// Recover the human-readable `text` for each segment from the original.
+	const segments = raw.segments.map((s) => ({
+		...s,
+		text:
+			analysis === undefined
+				? original.slice(s.start, s.end)
+				: originalSlice(analysis, s.start, s.end),
+	}));
+	return {
+		segments,
+		comments: raw.comments,
+		ascii: analysis?.ascii ?? true,
+		// Coordinate-pass regions first: they are facts about the input as
+		// received, and they exist even when the scan finds nothing structural.
+		defects: mergeDefects(
+			analysis === undefined ? [] : coordinateDefects(analysis),
+			raw.defects,
+		),
+	};
+}
+
+/** Original substring for an analyzed-byte span (boundaries are char-aligned). */
+function originalSlice(
+	a: CoordinateAnalysis,
+	start: number,
+	end: number,
+): string {
+	return a.original.slice(utf16At(a, start), utf16At(a, end));
+}
+
+/** UTF-16 offset in the original for an analyzed-byte boundary. */
+function utf16At(a: CoordinateAnalysis, byte: number): number {
+	if (byte >= a.analyzed.length) return a.original.length;
+	// Identity on an all-ASCII input: one byte per code unit in both spaces.
+	return a.ascii ? byte : runAtByte(a, byte).utf16Start;
+}
+
+/** A located statement before its `text` is recovered from the original. */
+type RawSegment = Omit<Segment, "text">;
+
+/**
+ * One nesting level of the single-pass walker. The top level is `frames[0]`
+ * (`container: false`); each open CONTAINER `{…}` pushes a frame. Statements
+ * accumulate in the active (innermost) frame's `buffer`; a container's buffer is
+ * spliced into its parent on COMMIT and dropped on DISCARD.
+ */
+interface Frame {
+	/** in-progress statement start in analyzed-byte space, or -1 (H6). */
+	stmtStart: number;
+	/** H4 — still before the first real token of the current statement? */
+	atLead: boolean;
+	/** H5 — how far the pending `\<newline>` continuation still reaches. */
+	cont: Continuation;
+	/** H5 — at the immediate start of a line the continuation carried into? */
+	contLineStart: boolean;
+	/**
+	 * A mismatched close occurred in this statement/container. A container with
+	 * a known structural defect must DISCARD rather than promote its buffered
+	 * children as confident sibling statements.
+	 */
+	structurallyInvalid: boolean;
+	/** Start/end of the prefix range already classified for H7. */
+	prefixScanStart: number;
+	prefixScanEnd: number;
+	/** Incremental equivalent of trimming and checking the H7 prefix. */
+	prefixSawNonSpace: boolean;
+	prefixValid: boolean;
+	/** statements collected here; the flattened output for a committed container. */
+	buffer: RawSegment[];
+	/** true for a container frame; false for the synthetic top level. */
+	container: boolean;
+	/** analyzed-byte index of the `{` that opened this container. */
+	openIndex: number;
+	/**
+	 * Start of the statement this container belongs to — the menu prefix before
+	 * the `{`. On DISCARD the parent statement resumes from here, so the whole
+	 * `{…}` is absorbed verbatim, matching the old `containerOpen` = -1 path.
+	 */
+	prefixStart: number;
+}
+
+/**
+ * Single left-to-right walk of a pure-ASCII string (byte === string index),
+ * flattening H7 containers inline. Returns spans only; `segmentStatements`
+ * recovers each `text` from the original afterward.
+ */
+function scanAscii(
+	ascii: string,
+	range?: MaskedRange,
+	opaqueChildScopes = false,
+): {
+	segments: RawSegment[];
+	comments: { start: number; end: number }[];
+	defects: Defect[];
+} {
+	// Local indices remain local in the result; only index-backed questions use
+	// document coordinates. The floor prevents an enclosing head from becoming
+	// the head of the body being segmented.
+	const indexed = range?.text ?? ascii;
+	const structural = range?.masked ?? ascii;
+	const floor = range?.start ?? 0;
+	const comments: { start: number; end: number }[] = [];
+	const defects: Defect[] = [];
+	const overDepth: number[] = [];
+	let hardHashSeen = false;
+	let hardApostropheSeen = false;
+	// H2 — every open bracket, for balance. Each frame carries WHERE it opened so
+	// an `unclosed` defect can point at the opener rather than at the end of
+	// input.
+	const delimStack: { char: string; at: number; statements: boolean }[] = [];
+	const top: Frame = {
+		stmtStart: -1,
+		atLead: true,
+		cont: "none",
+		contLineStart: false,
+		structurallyInvalid: false,
+		prefixScanStart: -1,
+		prefixScanEnd: -1,
+		prefixSawNonSpace: false,
+		prefixValid: true,
+		buffer: [],
+		container: false,
+		openIndex: -1,
+		prefixStart: -1,
+	};
+	const frames: Frame[] = [top];
+	const cur = (): Frame => frames[frames.length - 1] as Frame;
+
+	// At container level when every open bracket is a container `{` — i.e. no
+	// value bracket (`[ ( ` or a non-container `{`) is open. Only then do
+	// separators split and only then can a `{` open a new container. (Container
+	// frames only open at this level, so they always sit at the bottom of
+	// `delimStack`; plain brackets sit above.)
+	const atContainerLevel = (): boolean =>
+		delimStack.length === frames.length - 1;
+
+	const ensureStmt = (f: Frame, i: number): void => {
+		if (f.stmtStart < 0) f.stmtStart = i;
+	};
+
+	const resetPrefixScan = (f: Frame): void => {
+		f.prefixScanStart = -1;
+		f.prefixScanEnd = -1;
+		f.prefixSawNonSpace = false;
+		f.prefixValid = true;
+	};
+
+	const makeSegment = (
+		start: number,
+		end: number,
+		terminator: Segment["terminator"],
+	): RawSegment | null => {
+		const raw = ascii.slice(start, end);
+		const trimmed = raw.trim();
+		if (trimmed.length === 0) return null; // H6
+		const lead = raw.length - raw.trimStart().length;
+		const s = start + lead;
+		return {
+			start: s,
+			end: s + trimmed.length,
+			terminator,
+			menuOnly: isMenuOnly(trimmed),
+		};
+	};
+
+	const flush = (
+		f: Frame,
+		end: number,
+		terminator: Segment["terminator"],
+	): void => {
+		if (f.stmtStart < 0) return;
+		const s = makeSegment(f.stmtStart, end, terminator);
+		f.stmtStart = -1;
+		resetPrefixScan(f);
+		if (s) f.buffer.push(s);
+		// The synthetic top frame can recover at a real statement boundary. A
+		// container stays poisoned until it closes so a defect in one buffered
+		// child cannot make later children look independently trustworthy.
+		if (!f.container) f.structurallyInvalid = false;
+	};
+
+	// A container prefix is empty or a bare `/`-menu path with no `=[($"` — the
+	// same test the old `containerOpen` applied to the group's prefix.
+	const validPrefix = (f: Frame, start: number, end: number): boolean => {
+		if (f.prefixScanStart !== start || f.prefixScanEnd > end) {
+			resetPrefixScan(f);
+			f.prefixScanStart = start;
+			f.prefixScanEnd = start;
+		}
+		if (!f.prefixValid) {
+			f.prefixScanEnd = end;
+			return false;
+		}
+		for (let i = f.prefixScanEnd; i < end; i++) {
+			const c = ascii[i] as string;
+			if (!f.prefixSawNonSpace) {
+				if (c.trim().length === 0) continue;
+				f.prefixSawNonSpace = true;
+				if (c !== "/") {
+					f.prefixValid = false;
+					break;
+				}
+			}
+			if (/[=[($"]/.test(c)) {
+				f.prefixValid = false;
+				break;
+			}
+		}
+		f.prefixScanEnd = end;
+		return f.prefixValid;
+	};
+
+	const openContainer = (i: number): void => {
+		const parent = cur();
+		// prefixStart = the parent statement's start, or the `{` itself when the
+		// container has no prefix (so a DISCARD absorbs from the `{`).
+		const prefixStart = parent.stmtStart >= 0 ? parent.stmtStart : i;
+		delimStack.push({ char: "{", at: i, statements: true });
+		frames.push({
+			stmtStart: -1,
+			atLead: true,
+			cont: "none",
+			contLineStart: false,
+			structurallyInvalid: parent.structurallyInvalid,
+			prefixScanStart: -1,
+			prefixScanEnd: -1,
+			prefixSawNonSpace: false,
+			prefixValid: true,
+			buffer: [],
+			container: true,
+			openIndex: i,
+			prefixStart,
+		});
+	};
+
+	// A container's `}` ends the parent statement (so the group is a real trailing
+	// container) when the next non-blank char is a separator, end of input, or the
+	// `}` that closes an enclosing container. Any other trailing content means the
+	// `}` was mid-statement — the old `containerOpen` = -1 case.
+	const trailingEndsStatement = (closeIndex: number): boolean => {
+		let j = closeIndex + 1;
+		while (j < ascii.length && isSpace(ascii[j] as string)) j++;
+		if (j >= ascii.length) return true;
+		const n = ascii[j];
+		if (n === ";" || n === "\n") return true;
+		if (n === "}") return cur().container; // closes the parent container
+		return false;
+	};
+
+	const closeContainer = (i: number): void => {
+		const frame = frames.pop() as Frame;
+		// Flush the body's last statement. Its terminator is "eof" — the end of
+		// the container body, matching the old per-body scan; the `}` itself is
+		// not a statement terminator token (only `;`/newline/actual EOF are).
+		flush(frame, i, "eof");
+		const parent = cur();
+		if (
+			!frame.structurallyInvalid &&
+			frame.buffer.length > 0 &&
+			trailingEndsStatement(i)
+		) {
+			// COMMIT: menu-prefix segment (if any), then the buffered body, replace
+			// the container in the parent stream; the parent statement is consumed.
+			const prefix = makeSegment(frame.prefixStart, frame.openIndex, "newline");
+			if (prefix) parent.buffer.push(prefix);
+			for (const s of frame.buffer) parent.buffer.push(s);
+			parent.stmtStart = -1;
+			resetPrefixScan(parent);
+		} else {
+			// DISCARD: empty body or trailing content — the parent statement absorbs
+			// the whole `{…}` verbatim by resuming from the container's prefix.
+			parent.stmtStart = frame.prefixStart;
+			parent.structurallyInvalid ||= frame.structurallyInvalid;
+		}
+		parent.atLead = false;
+	};
+
+	let i = 0;
+	while (i < ascii.length) {
+		const c = ascii[i] as string;
+		const f = cur();
+
+		// H4/H5 — a comment is statement-leading, or at the immediate start of a
+		// line a continuation carried into. Both are decided from state as it
+		// stands BEFORE the continuation bookkeeping below, so an indented `#`
+		// mid-continuation stays content.
+		const inCont = f.cont !== "none";
+		const contComment = c === "#" && inCont && f.contLineStart;
+		if (c === "#" && (f.atLead || contComment)) {
+			const nl = ascii.indexOf("\n", i);
+			const end = nl === -1 ? ascii.length : nl;
+			comments.push({ start: i, end });
+			if (contComment) {
+				// The comment line does not end the statement — step over its newline
+				// rather than letting the separator branch flush there.
+				f.cont = "comment";
+				f.contLineStart = true;
+				i = nl === -1 ? end : end + 1;
+			} else {
+				i = end;
+			}
+			continue;
+		}
+
+		if (inCont) {
+			if (c === " " || c === "\t" || c === "\r") {
+				f.contLineStart = false;
+			} else if (c === "\n" && f.cont === "escape") {
+				// A blank line inside the `\` run is still the run: no flush.
+				f.contLineStart = true;
+				i++;
+				continue;
+			} else {
+				// Real content, or the blank line that spends a comment run's reach
+				// and so falls through to the separator branch below.
+				f.cont = "none";
+				f.contLineStart = false;
+			}
+		}
+
+		// #245 — once the device reaches an unquoted hash that cannot be a
+		// statement-leading comment or value, highlight marks that byte `error` and
+		// stops. Record only that first hard hash, then keep scanning solely to
+		// recover balanced spans without pretending later hashes were classified.
+		if (
+			c === "#" &&
+			!hardHashSeen &&
+			hashStartsHardError(
+				indexed,
+				floor + i,
+				delimStack.at(-1)?.statements === false,
+				floor,
+			)
+		) {
+			defects.push(defectAt("invalid-hash", i, "#"));
+			hardHashSeen = true;
+			f.structurallyInvalid = true;
+			ensureStmt(f, i);
+			f.atLead = false;
+			i++;
+			continue;
+		}
+
+		// #355 — `'` is NOT a RouterOS token: not a string delimiter, and not a
+		// legal bare byte. `:put [:parse]` rejects `name=don't`, `name=don't'`,
+		// `name='abc'` and `:put 'abc'` byte-identically on 7.21.5 / 7.23.5 /
+		// 7.24.2 / 7.25beta3, so this is a lexical rule and not a quote-balance
+		// one — the BALANCED forms are rejected too, and only `"` opens a string.
+		//
+		// Only the FIRST apostrophe is recorded, for the same reason as the hard
+		// hash above and grounded the same way: the device reports one column and
+		// it is always the first apostrophe (`name=don't'` reports column 30, the
+		// one at `don|'t`, not the trailing one), because classification stops
+		// there. A second occurrence is not an independent finding.
+		//
+		// Apostrophes inside a `"`-string or a comment never reach this branch:
+		// both runs are consumed whole above (H3 and H4/H5), which is why
+		// `comment="it's here"` stays legal with no special case here.
+		if (c === "'" && !hardApostropheSeen) {
+			defects.push(defectAt("invalid-apostrophe", i, "'"));
+			hardApostropheSeen = true;
+			f.structurallyInvalid = true;
+			ensureStmt(f, i);
+			f.atLead = false;
+			i++;
+			continue;
+		}
+
+		// H3 — string.
+		if (c === '"') {
+			ensureStmt(f, i);
+			f.atLead = false;
+			const str = scanQuotedString(ascii, i);
+			if (!str.closed) {
+				// The region is the whole unterminated run — from the opening quote
+				// to where the scan gave up — not just the quote. A consumer
+				// highlighting the defect wants the text that is swallowed by it.
+				defects.push({ code: "unterminated-string", start: i, end: str.end });
+			}
+			i = str.end;
+			continue;
+		}
+
+		// H5 — line continuation.
+		if (
+			c === "\\" &&
+			(ascii[i + 1] === "\n" ||
+				(ascii[i + 1] === "\r" && ascii[i + 2] === "\n"))
+		) {
+			// `atLead` deliberately survives: `do={\` + newline + `:local x 1` still
+			// reads `:local` as the statement head on the device (#215).
+			ensureStmt(f, i);
+			f.cont = "escape";
+			f.contLineStart = true;
+			i += ascii[i + 1] === "\r" ? 3 : 2;
+			continue;
+		}
+
+		// H2/H7 — opens. A `{` at container level with a valid prefix opens a
+		// container frame (or, past the cap, abstains); everything else is a plain
+		// group that only tracks balance and suppresses separators.
+		if (c === "{" || c === "[" || c === "(") {
+			if (
+				c === "{" &&
+				atContainerLevel() &&
+				validPrefix(f, f.stmtStart >= 0 ? f.stmtStart : i, i)
+			) {
+				if (frames.length - 1 < MAX_CONTAINER_DEPTH) {
+					openContainer(i);
+				} else {
+					overDepth.push(i);
+					ensureStmt(f, i);
+					delimStack.push({ char: "{", at: i, statements: true });
+					f.atLead = true;
+				}
+			} else {
+				ensureStmt(f, i);
+				const enclosing = delimStack.at(-1)?.statements ?? true;
+				if (
+					opaqueChildScopes &&
+					c === "{" &&
+					atContainerLevel() &&
+					scopeNameFromMasked(structural, floor + i, floor) !== null
+				) {
+					const end = matchingBraceEnd(structural, floor + i);
+					if (end !== undefined && end < floor + ascii.length) {
+						f.atLead = false;
+						i = end - floor + 1;
+						continue;
+					}
+				}
+				const statements =
+					c === "[" ||
+					(c === "{" &&
+						enclosing &&
+						braceStartsStatements(indexed, floor + i, floor));
+				delimStack.push({ char: c, at: i, statements });
+				f.atLead = statements;
+			}
+			i++;
+			continue;
+		}
+
+		// H2 — closes.
+		if (c === "}" || c === "]" || c === ")") {
+			const want = c === "}" ? "{" : c === "]" ? "[" : "(";
+			if (delimStack[delimStack.length - 1]?.char === want) {
+				const isContainerClose =
+					c === "}" &&
+					frames.length > 1 &&
+					delimStack.length === frames.length - 1;
+				delimStack.pop();
+				if (isContainerClose) {
+					closeContainer(i);
+				} else {
+					ensureStmt(f, i);
+					f.atLead = false;
+				}
+			} else {
+				defects.push(defectAt("unbalanced-close", i, c));
+				ensureStmt(f, i);
+				f.atLead = false;
+				f.structurallyInvalid = true;
+			}
+			i++;
+			continue;
+		}
+
+		// H1 — separators. They END a statement only at container level, but
+		// restore statement-leading position only in a statement-bearing context.
+		// Array and parenthesized expression members do not admit bare `#` comments.
+		if (c === ";" || c === "\n") {
+			if (atContainerLevel()) flush(f, i, c === ";" ? ";" : "newline");
+			f.atLead = delimStack.at(-1)?.statements ?? true;
+			i++;
+			continue;
+		}
+
+		if (isSpace(c)) {
+			i++;
+			continue;
+		}
+
+		ensureStmt(f, i);
+		f.atLead = false;
+		i++;
+	}
+
+	// End of input: any still-open container never closed, so it never flattens —
+	// unwind each as a DISCARD (outermost open container's prefix wins), then flush
+	// the surviving top-level statement.
+	while (frames.length > 1) {
+		const frame = frames.pop() as Frame;
+		const parent = cur();
+		parent.stmtStart = frame.prefixStart;
+		parent.atLead = false;
+	}
+	flush(top, ascii.length, "eof");
+	if (delimStack.length > 0) {
+		// One defect per still-open delimiter, each pointing at its OPENER —
+		// "something is unclosed" is not actionable, "the `{` at byte 41 is
+		// unclosed" is.
+		for (const d of delimStack)
+			defects.push(defectAt("unclosed", d.at, d.char));
+	}
+
+	return {
+		segments: top.buffer,
+		comments,
+		defects: [
+			...defects,
+			// No `detail`: `symbols.ts` and `pathresolve.ts` emit this class without
+			// one, and `mergeDefects` keys on it, so a `"{"` here would make the same
+			// event fail to de-duplicate across analyzers.
+			...overDepth.map((offset) => defectAt("over-depth", offset)),
+		],
+	};
+}
+
+/**
+ * A menu-navigation SHAPE: a `/`-rooted path of path-shaped tokens (letters,
+ * digits, space, and `/_.-`) with no `=`, quotes, brackets, or `$` — e.g.
+ * `/ip address`, `/interface`, `/`. It is deliberately shape-only: offline
+ * cannot tell a trailing verb (`/ip address print`) from a deeper menu without
+ * a schema, so `print` still reads as menu-shaped here. A HINT for Q4/Q6, not a
+ * claim that no verb is present.
+ */
+function isMenuOnly(trimmed: string): boolean {
+	if (!trimmed.startsWith("/")) return false;
+	return /^\/[A-Za-z0-9 /_.-]*$/.test(trimmed);
+}