@tikoci/centrs 0.1.4 → 0.1.7

package/src/explain/segment.ts

@@ -80,8 +80,18 @@ import {
 	runAtByte,
 } from "./coordinates.ts";
 import { type Defect, defectAt, mergeDefects } from "./defects.ts";
-import { scanQuotedString as scanQuotedStringShared } from "./quoted-string.ts";
-import { braceStartsStatements, hashStartsHardError } from "./scope-brace.ts";
+import {
+	MAX_STRING_FRAME_DEPTH,
+	scanQuotedString as scanQuotedStringShared,
+} from "./quoted-string.ts";
+import {
+	braceStartsStatements,
+	canRetainStatementIndex,
+	hashStartsHardError,
+	isIndexedStatementStart,
+	matchingBraceEnd,
+	scopeNameFromMasked,
+} from "./scope-brace.ts";
 
 /** One top-level statement located in analyzed-byte space. */
 export interface Segment {
@@ -108,6 +118,13 @@ export interface SegmentResult {
 	segments: Segment[];
 	/** comment spans (analyzed-byte offsets), dropped from the statement stream. */
 	comments: { start: number; end: number }[];
+	/**
+	 * The input was pure ASCII, so an analyzed-byte offset into it is also a JS
+	 * string index — the test a caller needs before it may treat the spans above
+	 * as offsets into `original` itself. Published from the coordinate pass this
+	 * result already ran rather than re-derived, so the two cannot disagree.
+	 */
+	ascii: boolean;
 	/**
 	 * Unbalanced delimiters and other structural surprises, each located; never
 	 * a throw. Includes the two coordinate-pass classes (`bom`, `non-ascii`).
@@ -154,8 +171,33 @@ export interface QuotedStringScan {
  * can reach it, which is malformed by construction; sequential substitutions
  * (`"$[a]$[b]…"`) pop and never accumulate.
  */
-export function scanQuotedString(text: string, open: number): QuotedStringScan {
-	return scanQuotedStringShared(text, open);
+export function scanQuotedString(
+	text: string,
+	open: number,
+	limit: number = text.length,
+): QuotedStringScan {
+	return scanQuotedStringWithin(text, open, limit, MAX_STRING_FRAME_DEPTH);
+}
+
+function scanQuotedStringWithin(
+	text: string,
+	open: number,
+	limit: number,
+	frameBudget: number,
+): QuotedStringScan {
+	return scanQuotedStringShared(
+		text,
+		open,
+		limit,
+		(source, bracket, end, frameDepth) =>
+			scanStringInterpolationWithin(
+				source,
+				bracket,
+				end,
+				frameBudget - frameDepth,
+			).end,
+		frameBudget,
+	);
 }
 
 /**
@@ -188,7 +230,250 @@ export type Continuation = "none" | "escape" | "comment";
  * can slice the original for content. Idempotent.
  */
 export function maskComments(original: string): string {
-	const out = original.split("");
+	if (recentMaskInput === original && recentMaskOutput !== undefined)
+		return recentMaskOutput;
+	const cached = maskCache.get(original);
+	if (cached !== undefined) {
+		// Refresh recency — see `scope-brace.ts` → `statementIndex` for why an
+		// insertion-ordered eviction loses exactly the entry worth keeping.
+		maskCache.delete(original);
+		maskCache.set(original, cached);
+		adoptMask(original, cached);
+		return cached;
+	}
+	const masked = maskUncached(original);
+	if (cacheMask(original, masked)) adoptMask(original, masked);
+	return masked;
+}
+
+const MASK_CACHE_LIMIT = 4;
+const MASK_CACHE_BYTE_LIMIT = 8 * 1024 * 1024;
+const maskCache = new Map<string, string>();
+let maskCacheBytes = 0;
+// Same shape, and the same reason, as `scope-brace.ts`'s boundary cache: one
+// slot for the hot object, a small bounded map behind it, and adoption on a hit
+// so a content-equal twin is compared once rather than on every call.
+let recentMaskInput: string | undefined;
+let recentMaskOutput: string | undefined;
+
+/** Adopt `original` as the single-slot key (see `scope-brace.ts` → `adoptBoundaryText`). */
+function adoptMask(original: string, masked: string): void {
+	recentMaskInput = original;
+	recentMaskOutput = masked;
+}
+
+function maskCacheEntryBytes(original: string, masked: string): number {
+	// Comment-free input is its own mask, so that entry holds one string.
+	return (original.length + (masked === original ? 0 : masked.length)) * 2;
+}
+
+function cacheMask(original: string, masked: string): boolean {
+	const entryBytes = maskCacheEntryBytes(original, masked);
+	if (entryBytes > MASK_CACHE_BYTE_LIMIT) return false;
+	while (
+		maskCache.size >= MASK_CACHE_LIMIT ||
+		maskCacheBytes + entryBytes > MASK_CACHE_BYTE_LIMIT
+	) {
+		const oldest = maskCache.entries().next().value;
+		if (oldest === undefined) break;
+		maskCache.delete(oldest[0]);
+		maskCacheBytes -= maskCacheEntryBytes(oldest[0], oldest[1]);
+		if (recentMaskInput === oldest[0]) {
+			recentMaskInput = undefined;
+			recentMaskOutput = undefined;
+		}
+	}
+	maskCache.set(original, masked);
+	maskCacheBytes += entryBytes;
+	return true;
+}
+
+/**
+ * A region of one document, carried with that document's comment mask.
+ *
+ * Every reader below the statement walk asks its questions of a statement's
+ * `text` or of a `do={…}` body — and a statement's text CONTAINS its whole
+ * nested subtree, so each enclosing level re-derived the mask, and the
+ * statement index behind it, over bytes an inner level had already read. That
+ * is the O(bytes x depth) term: a 60 KiB document nested 64 deep masked 27.7 MB
+ * (#322).
+ *
+ * A range replaces the substring. `text` and `masked` are always the WHOLE
+ * document; `start` and `end` say which part of it this reader may look at. The
+ * mask is then derived once per document and every nested reading is an offset
+ * into it.
+ *
+ * Soundness is a property of where the ranges come from, not of this type: a
+ * comment mask is decided by the statement context a byte sits in, and a
+ * statement boundary and a scope body both begin one — statement-leading, no
+ * continuation pending, an enclosing context that admits statements. So the
+ * mask of such a region is the document's mask restricted to it. Readers still
+ * have to be told the floor (see `scope-brace.ts` → `scopeNameFromMasked`) so
+ * that lookback stops where a fresh call on the region alone would stop.
+ */
+export interface MaskedRange {
+	/** The whole document. Offsets below index into this, never into a slice. */
+	readonly text: string;
+	/** `maskComments(text)`. */
+	readonly masked: string;
+	readonly start: number;
+	readonly end: number;
+	/** This range's string indices are analyzed-byte offsets. */
+	readonly coordinateIdentity?: true;
+}
+
+/** The range covering all of `text`, masking it once. */
+export function documentRange(
+	text: string,
+	coordinateIdentity = false,
+): MaskedRange {
+	return {
+		text,
+		masked: maskComments(text),
+		start: 0,
+		end: text.length,
+		...(coordinateIdentity ? { coordinateIdentity: true as const } : {}),
+	};
+}
+
+/**
+ * Locate one `$[…]` body's closing bracket using its own comment context.
+ *
+ * The document mask deliberately leaves an enclosing string opaque. Once `$[`
+ * enters code, statement-leading comments are real again, so brackets inside
+ * them cannot close or extend the substitution. The returned range starts at
+ * `open` and retains that local mask for the readers of the body.
+ */
+export function scanStringInterpolation(
+	text: string,
+	open: number,
+	limit: number = text.length,
+): { end: number; range: MaskedRange } {
+	return scanStringInterpolationWithin(
+		text,
+		open,
+		limit,
+		MAX_STRING_FRAME_DEPTH - 1,
+	);
+}
+
+function scanStringInterpolationWithin(
+	text: string,
+	open: number,
+	limit: number,
+	frameBudget: number,
+): { end: number; range: MaskedRange } {
+	const local = text.slice(open, limit);
+	// Masking must find each code-level string boundary comment-aware. Reuse
+	// those answers below instead of recursively scanning the same strings again.
+	const quotedEnds = new Map<number, number>();
+	const range: MaskedRange = {
+		text: local,
+		masked: maskUncached(local, frameBudget, quotedEnds),
+		start: 0,
+		end: local.length,
+	};
+	if (frameBudget < 1) return { end: limit, range };
+	const frames: string[] = ["["];
+	for (let i = 1; i < range.end; i++) {
+		const c = range.masked[i];
+		if (c === '"') {
+			const quotedEnd = quotedEnds.get(i);
+			if (quotedEnd === undefined) return { end: limit, range };
+			i = quotedEnd - 1;
+			continue;
+		}
+		if (c === "[" || c === "(" || c === "{") {
+			if (frames.length >= frameBudget) return { end: limit, range };
+			frames.push(c);
+			continue;
+		}
+		if (c !== "]" && c !== ")" && c !== "}") continue;
+		const want = c === "]" ? "[" : c === ")" ? "(" : "{";
+		if (frames.at(-1) !== want) continue;
+		frames.pop();
+		if (frames.length === 0) return { end: open + i, range };
+	}
+	return { end: limit, range };
+}
+
+/**
+ * The sub-range `[start, end)` of `parent`, in `parent`'s own coordinates.
+ *
+ * Offsets must be JS string indices — for a statement span that means the
+ * document was ASCII ({@link SegmentResult.ascii}), which is the only case in
+ * which an analyzed-byte offset is one.
+ */
+export function nestedRange(
+	parent: MaskedRange,
+	start: number,
+	end: number,
+): MaskedRange {
+	return {
+		text: parent.text,
+		masked: parent.masked,
+		start: parent.start + start,
+		end: parent.start + end,
+		...(parent.coordinateIdentity ? { coordinateIdentity: true as const } : {}),
+	};
+}
+
+/**
+ * The range `[start, end)` of the same document, in ABSOLUTE offsets.
+ *
+ * The counterpart to {@link nestedRange}, for a reader that is already working
+ * in the document's own offsets rather than in a parent's.
+ */
+export function rangeAt(
+	parent: MaskedRange,
+	start: number,
+	end: number,
+): MaskedRange {
+	return {
+		text: parent.text,
+		masked: parent.masked,
+		start,
+		end,
+		...(parent.coordinateIdentity ? { coordinateIdentity: true as const } : {}),
+	};
+}
+
+/** The comment mask of what this range covers. */
+export function rangeMask(range: MaskedRange): string {
+	return range.masked.slice(range.start, range.end);
+}
+
+/**
+ * {@link rangeMask}, tolerating no range — for readers that take one OPTIONALLY
+ * and fall back to masking their own text: `maskOf(range) ?? maskComments(text)`.
+ */
+export function maskOf(range: MaskedRange | undefined): string | undefined {
+	return range === undefined ? undefined : rangeMask(range);
+}
+
+/**
+ * The mask itself. Pure, so the memo above cannot change an answer — only how
+ * often it is computed.
+ *
+ * The walk asks for the mask of the SAME statement several times over: the
+ * structural-defect scan, the scope-block scan and the bracket scan each take
+ * one, and the bare-directive test takes another over the trimmed spelling.
+ * Each of those re-ran this scan AND, through `braceStartsStatements`, the
+ * statement index behind it — and a statement's text carries its whole nested
+ * subtree, so the repetition was paid once per enclosing level (#322).
+ */
+function maskUncached(
+	original: string,
+	frameBudget: number = MAX_STRING_FRAME_DEPTH,
+	quotedEnds?: Map<number, number>,
+): string {
+	// Masked output is built from whole slices, and only once a comment is
+	// actually found: comment-free input is returned as the SAME string object.
+	// `split("")` allocated one string per character of every call — 16 walks
+	// over a 64 KiB document made a million of them — where a document's comment
+	// runs are a handful of slices (#313).
+	let pieces: string[] | undefined;
+	let copied = 0;
 	const contexts: { char: "{" | "[" | "("; statements: boolean }[] = [];
 	let atLead = true;
 	let cont: Continuation = "none";
@@ -211,7 +496,14 @@ export function maskComments(original: string): string {
 		}
 		if (c === '"') {
 			atLead = false;
-			i = scanQuotedString(original, i).end - 1;
+			const end = scanQuotedStringWithin(
+				original,
+				i,
+				original.length,
+				frameBudget - contexts.length,
+			).end;
+			quotedEnds?.set(i, end);
+			i = end - 1;
 			continue; // i rests on the closing quote (or past end)
 		}
 		// H4 is unchanged and H5 only ADDS the immediate-line-start case: an
@@ -219,10 +511,14 @@ export function maskComments(original: string): string {
 		// newline + `  # c`) and content when it is not (`:put a\` + newline +
 		// `  # x`, which the device classes `error`) — exactly what `atLead` says.
 		if (c === "#" && (atLead || contComment)) {
-			while (i < original.length && original[i] !== "\n") {
-				out[i] = " ";
-				i++;
-			}
+			const commentStart = i;
+			while (i < original.length && original[i] !== "\n") i++;
+			if (pieces === undefined) pieces = [];
+			pieces.push(
+				original.slice(copied, commentStart),
+				" ".repeat(i - commentStart),
+			);
+			copied = i;
 			if (contComment) {
 				// The comment line does not end the statement: step over its newline
 				// (the loop's i++) with the continuation still pending.
@@ -263,7 +559,9 @@ export function maskComments(original: string): string {
 			atLead = contexts.at(-1)?.statements ?? true;
 		} else if (c !== " " && c !== "\t" && c !== "\r") atLead = false;
 	}
-	return out.join("");
+	if (pieces === undefined) return original;
+	pieces.push(original.slice(copied));
+	return pieces.join("");
 }
 
 /**
@@ -282,23 +580,76 @@ const MAX_CONTAINER_DEPTH = 256;
 /**
  * Segment `original` into top-level statements. Spans are analyzed-byte offsets
  * (see module header); `text` is the original substring for each span.
+ *
+ * A statement-bearing `range` may anchor the same `original` text in its
+ * enclosing document. On ASCII input, structural lookback can reuse that
+ * document's statement index instead of indexing every nested body (#322).
+ * The scan still reads the original bytes, including comments: replacing them
+ * with the mask would lose comment spans and H5 continuation state. Non-ASCII
+ * input, or a body opener skipped by the raw document index (a quote inside an
+ * earlier comment), keeps the standalone analyzed-byte scan.
  */
-export function segmentStatements(original: string): SegmentResult {
-	const analysis = analyzeCoordinates(original);
+export function segmentStatements(
+	original: string,
+	range?: MaskedRange,
+): SegmentResult {
+	return segmentStatementsInternal(original, range, false);
+}
+
+/** Segment one known scope body while treating its child scopes as opaque. */
+export function segmentScopeBody(
+	original: string,
+	range: MaskedRange,
+): SegmentResult {
+	return segmentStatementsInternal(
+		original,
+		range,
+		range.coordinateIdentity === true,
+	);
+}
+
+function segmentStatementsInternal(
+	original: string,
+	range: MaskedRange | undefined,
+	opaqueChildScopes: boolean,
+): SegmentResult {
+	const identity =
+		range?.coordinateIdentity === true &&
+		range.end - range.start === original.length;
+	const analysis = identity ? undefined : analyzeCoordinates(original);
 	// `analyzed` is pure ASCII, so a string built from it has index === byte.
-	const ascii = new TextDecoder().decode(analysis.analyzed);
-	const raw = scanAscii(ascii);
+	// When the INPUT was already ASCII the two are the same sequence, so the
+	// original is that string and the decode is skipped (#322).
+	const ascii =
+		analysis === undefined || analysis.ascii
+			? original
+			: new TextDecoder().decode(analysis.analyzed);
+	const anchored =
+		(analysis === undefined || analysis.ascii) &&
+		range !== undefined &&
+		canRetainStatementIndex(range.text) &&
+		isIndexedStatementStart(range.text, range.start)
+			? range
+			: undefined;
+	const raw = scanAscii(ascii, anchored, opaqueChildScopes);
 	// Recover the human-readable `text` for each segment from the original.
 	const segments = raw.segments.map((s) => ({
 		...s,
-		text: originalSlice(analysis, s.start, s.end),
+		text:
+			analysis === undefined
+				? original.slice(s.start, s.end)
+				: originalSlice(analysis, s.start, s.end),
 	}));
 	return {
 		segments,
 		comments: raw.comments,
+		ascii: analysis?.ascii ?? true,
 		// Coordinate-pass regions first: they are facts about the input as
 		// received, and they exist even when the scan finds nothing structural.
-		defects: mergeDefects(coordinateDefects(analysis), raw.defects),
+		defects: mergeDefects(
+			analysis === undefined ? [] : coordinateDefects(analysis),
+			raw.defects,
+		),
 	};
 }
 
@@ -314,7 +665,8 @@ function originalSlice(
 /** UTF-16 offset in the original for an analyzed-byte boundary. */
 function utf16At(a: CoordinateAnalysis, byte: number): number {
 	if (byte >= a.analyzed.length) return a.original.length;
-	return runAtByte(a, byte).utf16Start;
+	// Identity on an all-ASCII input: one byte per code unit in both spaces.
+	return a.ascii ? byte : runAtByte(a, byte).utf16Start;
 }
 
 /** A located statement before its `text` is recovered from the original. */
@@ -366,15 +718,26 @@ interface Frame {
  * flattening H7 containers inline. Returns spans only; `segmentStatements`
  * recovers each `text` from the original afterward.
  */
-function scanAscii(ascii: string): {
+function scanAscii(
+	ascii: string,
+	range?: MaskedRange,
+	opaqueChildScopes = false,
+): {
 	segments: RawSegment[];
 	comments: { start: number; end: number }[];
 	defects: Defect[];
 } {
+	// Local indices remain local in the result; only index-backed questions use
+	// document coordinates. The floor prevents an enclosing head from becoming
+	// the head of the body being segmented.
+	const indexed = range?.text ?? ascii;
+	const structural = range?.masked ?? ascii;
+	const floor = range?.start ?? 0;
 	const comments: { start: number; end: number }[] = [];
 	const defects: Defect[] = [];
 	const overDepth: number[] = [];
 	let hardHashSeen = false;
+	let hardApostropheSeen = false;
 	// H2 — every open bracket, for balance. Each frame carries WHERE it opened so
 	// an `unclosed` defect can point at the opener rather than at the end of
 	// input.
@@ -596,7 +959,12 @@ function scanAscii(ascii: string): {
 		if (
 			c === "#" &&
 			!hardHashSeen &&
-			hashStartsHardError(ascii, i, delimStack.at(-1)?.statements === false)
+			hashStartsHardError(
+				indexed,
+				floor + i,
+				delimStack.at(-1)?.statements === false,
+				floor,
+			)
 		) {
 			defects.push(defectAt("invalid-hash", i, "#"));
 			hardHashSeen = true;
@@ -607,6 +975,31 @@ function scanAscii(ascii: string): {
 			continue;
 		}
 
+		// #355 — `'` is NOT a RouterOS token: not a string delimiter, and not a
+		// legal bare byte. `:put [:parse]` rejects `name=don't`, `name=don't'`,
+		// `name='abc'` and `:put 'abc'` byte-identically on 7.21.5 / 7.23.5 /
+		// 7.24.2 / 7.25beta3, so this is a lexical rule and not a quote-balance
+		// one — the BALANCED forms are rejected too, and only `"` opens a string.
+		//
+		// Only the FIRST apostrophe is recorded, for the same reason as the hard
+		// hash above and grounded the same way: the device reports one column and
+		// it is always the first apostrophe (`name=don't'` reports column 30, the
+		// one at `don|'t`, not the trailing one), because classification stops
+		// there. A second occurrence is not an independent finding.
+		//
+		// Apostrophes inside a `"`-string or a comment never reach this branch:
+		// both runs are consumed whole above (H3 and H4/H5), which is why
+		// `comment="it's here"` stays legal with no special case here.
+		if (c === "'" && !hardApostropheSeen) {
+			defects.push(defectAt("invalid-apostrophe", i, "'"));
+			hardApostropheSeen = true;
+			f.structurallyInvalid = true;
+			ensureStmt(f, i);
+			f.atLead = false;
+			i++;
+			continue;
+		}
+
 		// H3 — string.
 		if (c === '"') {
 			ensureStmt(f, i);
@@ -657,9 +1050,24 @@ function scanAscii(ascii: string): {
 			} else {
 				ensureStmt(f, i);
 				const enclosing = delimStack.at(-1)?.statements ?? true;
+				if (
+					opaqueChildScopes &&
+					c === "{" &&
+					atContainerLevel() &&
+					scopeNameFromMasked(structural, floor + i, floor) !== null
+				) {
+					const end = matchingBraceEnd(structural, floor + i);
+					if (end !== undefined && end < floor + ascii.length) {
+						f.atLead = false;
+						i = end - floor + 1;
+						continue;
+					}
+				}
 				const statements =
 					c === "[" ||
-					(c === "{" && enclosing && braceStartsStatements(ascii, i));
+					(c === "{" &&
+						enclosing &&
+						braceStartsStatements(indexed, floor + i, floor));
 				delimStack.push({ char: c, at: i, statements });
 				f.atLead = statements;
 			}