@tikoci/centrs 0.1.3 → 0.1.6

package/src/explain/separator.ts

@@ -0,0 +1,187 @@
+/**
+ * The missing statement separator between two command-shaped runs (#311).
+ *
+ * RouterOS accepts slash-shaped operands after a verb — `/file remove
+ * /flash/skins/foo.html` lowers cleanly to `numbers=/flash/skins/foo.html` — so
+ * "a `/` after the verb" is NOT the signal, and a rule built on it would reject
+ * valid input. What the device rejects is narrower: a known **menu** path in
+ * operand position, followed by a console **verb**.
+ *
+ * ## What the device actually does (CHR 7.24.2, `:parse` + `highlight`)
+ *
+ * #311 reported the error byte as the `=` of the trailing attribute
+ * (`expected end of command (line 1 column 55)` for `/ip/address add
+ * interface=ether1 /ip/route add gateway=192.168.88.1`). Probing the shape
+ * around that row shows the `=` is an ARTEFACT of that particular head:
+ * `add` abbreviates `/ip/address/add`'s own `address=` argument, so the console
+ * reads `add gateway` as `address=gateway` and only complains at the `=` after
+ * it. Vary the head so the second verb abbreviates nothing and the byte moves —
+ * `/system/note set note=hello /ip/route add gateway=1.2.3.4` is not a hard
+ * reject at all; `:parse` returns IL carrying `bad parameter /ip/route
+ * (line 1 column 38)`, naming the second PATH. The same input also proves the
+ * head's own argument list is not safe to keep: `… /ip/route add placeholder` lowers
+ * to `address=placeholder;;interface=ether1`, a phantom attribute the source never
+ * wrote, and whether that happens depends on per-menu argument names — schema
+ * knowledge offline does not ship.
+ *
+ * So the grounded invariant is: **RouterOS never accepts a known menu path
+ * followed by a verb in operand position.** Across every probed row it answers
+ * either `bad parameter <that path>` or `expected end of command`; none is
+ * clean. It is also the shape with an obvious fix, which is what makes it worth
+ * a diagnostic rather than an abstention.
+ *
+ * ## What this deliberately does not decide
+ *
+ * A bare operand the head does not accept (`… placeholder`, `/file remove a b`) also
+ * draws `bad parameter`, and so does a menu path with no verb after it
+ * (`… /ip/route`). Those are the generic "this menu takes no such positional"
+ * complaint — deciding them needs the per-menu positional list, which
+ * `catalog.ts` deliberately is not, and a separator is not necessarily the fix.
+ * They abstain.
+ *
+ * The rule reads only what the argument lexer already DECIDED. It never
+ * re-scans bytes, so a `[…]` substitution (a located token the lexer declined
+ * to decode, hence no `value` and no operand), a `{…}` body (a separate
+ * flattened statement), a quoted operand and a `/` inside a VALUE are all
+ * outside its input by construction rather than by a guard it could get wrong.
+ * `bareOperand` is that construction: `value` absent is the lexer saying it
+ * decoded nothing, whatever the token's bytes spell.
+ *
+ * ## The reach it has, stated
+ *
+ * The input is `lexArgumentTokens`, the skip-tolerant reading (#316), never the
+ * strict `lexArguments`. The strict lexer discards every token it decoded as
+ * soon as it declines one, so a statement holding a variable — `… gateway=$g` —
+ * used to carry no token list for this rule at all and the run in it was missed
+ * while its literal twin was found. The tolerant reading locates the
+ * undecodable token and keeps walking, so the run is found on either side of
+ * it; that the strict reading still refuses the statement is correct and
+ * separate, since nothing here makes `$g` renderable.
+ *
+ * What remains out of reach is an unterminated string and an unbalanced
+ * delimiter, where no walk can find a boundary to resume from. That is still a
+ * reach limit rather than a judgement: nothing here decides those shapes are
+ * acceptable.
+ */
+
+import type { Argument } from "./args.ts";
+import { isKnownMenuPath } from "./is-known-menu.ts";
+import { VERBS } from "./verbs.ts";
+
+/** A second command-shaped run where a `;` or newline should have been. */
+export interface MissingSeparator {
+	/** The whole second run — first byte of its path through the end of its verb. */
+	span: { start: number; end: number };
+	/** Where a `;` or newline belongs: the first byte of the second path. */
+	insertAt: number;
+	/** The second run's menu path, slash-spelled and leading-slashed. */
+	path: string;
+	/** The verb token that closed the second run. */
+	verb: string;
+}
+
+// Both patterns below admit a segment that STARTS WITH A DIGIT. `/interface/6to4`
+// is the shipped tables' one such name — a menu in both `MENU_PATHS` and
+// `PATH_CATALOG` — and requiring a leading letter made every spelling of it miss
+// the rule. These are only a cheap shape filter; presence in the structure table
+// is the actual gate, so a wider first character cannot admit a path the table
+// does not already list.
+
+/** A bare path segment run, `route` or `firewall/filter`, with no leading slash. */
+const RELATIVE_SEGMENTS =
+	/^[A-Za-z0-9][A-Za-z0-9-]*(?:\/[A-Za-z0-9][A-Za-z0-9-]*)*$/;
+
+/** An absolute menu path spelled with slashes, `/ip` or `/ip/firewall/filter`. */
+const ABSOLUTE_SEGMENTS =
+	/^\/[A-Za-z0-9][A-Za-z0-9-]*(?:\/[A-Za-z0-9][A-Za-z0-9-]*)*$/;
+
+/**
+ * A positional whose source text IS its value — so it carries no quotes, no
+ * substitution and no escape. `Argument.value` is already quote-stripped, which
+ * makes the comparison the whole quoting check; `value` absent means the lexer
+ * declined to decode the token and there is nothing to match against.
+ */
+function bareOperand(token: Argument | undefined): string | undefined {
+	if (token === undefined || token.kind !== "positional") return undefined;
+	if (token.value === undefined || token.value !== token.text) return undefined;
+	return token.value;
+}
+
+/**
+ * Every complete second command-shaped run in one statement's argument tokens.
+ *
+ * Coordinate-space agnostic: the spans come back in whatever space `tokens`
+ * were lexed in, so the caller passes document-rebased tokens and gets document
+ * offsets.
+ */
+export function findMissingSeparators(
+	tokens: readonly Argument[],
+): MissingSeparator[] {
+	const out: MissingSeparator[] = [];
+	let i = 0;
+	while (i < tokens.length) {
+		const run = separatorAt(tokens, i);
+		if (run === null) {
+			i++;
+			continue;
+		}
+		out.push(run.separator);
+		i = run.next;
+	}
+	return out;
+}
+
+/** One run starting at `i`, or `null` when the tokens there are not that shape. */
+function separatorAt(
+	tokens: readonly Argument[],
+	i: number,
+): { separator: MissingSeparator; next: number } | null {
+	const head = bareOperand(tokens[i]);
+	if (head === undefined || !ABSOLUTE_SEGMENTS.test(head)) return null;
+	const run = longestKnownMenuRun(tokens, i, head);
+	if (run === null) return null;
+	const verb = bareOperand(tokens[run.last + 1]);
+	if (verb === undefined || !VERBS.has(verb)) return null;
+	const pathStart = tokens[i]?.span.start;
+	const verbEnd = tokens[run.last + 1]?.span.end;
+	if (pathStart === undefined || verbEnd === undefined) return null;
+	return {
+		separator: {
+			span: { start: pathStart, end: verbEnd },
+			insertAt: pathStart,
+			path: `/${run.segments.join("/")}`,
+			verb,
+		},
+		next: run.last + 2,
+	};
+}
+
+/**
+ * The longest run of operands from `i` that spells a known menu path.
+ *
+ * A menu path may be written with slashes, spaces, or a mix (Q3 rule R4), so
+ * `/ip/route`, `/ip route` and `/ip route` split across operands are the same
+ * path and all of them have to be reachable — the device rejects the spaced
+ * spelling identically. Longest wins: `/ip route` is a menu and so is `/ip`, and
+ * only the longer one leaves `add` in verb position. `null` when no prefix is a
+ * known menu — absence in the structure table abstains and never asserts a
+ * command (`is-known-menu.ts`).
+ */
+function longestKnownMenuRun(
+	tokens: readonly Argument[],
+	i: number,
+	head: string,
+): { last: number; segments: string[] } | null {
+	const segments = head.slice(1).split("/");
+	const runs: { last: number; segments: string[] }[] = [];
+	if (isKnownMenuPath(segments))
+		runs.push({ last: i, segments: [...segments] });
+	for (let j = i + 1; j < tokens.length; j++) {
+		const next = bareOperand(tokens[j]);
+		if (next === undefined || !RELATIVE_SEGMENTS.test(next)) break;
+		segments.push(...next.split("/"));
+		if (isKnownMenuPath(segments))
+			runs.push({ last: j, segments: [...segments] });
+	}
+	return runs.at(-1) ?? null;
+}

package/src/explain/split-owner.ts

@@ -0,0 +1,194 @@
+/**
+ * "Which statement owns this span?" as an index rather than a scan (#317).
+ *
+ * `symbol-values.ts` asks this question once per symbol, once per value and
+ * once per scope brace. Answering it by scanning every statement split makes
+ * the product quadratic: on 62 KiB of `:local v<i> <i>; :put $v<i>;` the scan
+ * ran 6,144 queries over 4,096 splits — 25.2 M visits, 42% of total analysis
+ * time — and the visit count quadrupled for every doubling of the input.
+ *
+ * ## The answer that has to be preserved, exactly
+ *
+ * The **shortest ADDRESSABLE split span containing `[start, end)`**, and on a
+ * length tie the **earliest split index**. Addressable means the span slices
+ * back to the split's own text: `pathresolve.ts` can hand back a *widened
+ * fallback* span, and a span that is not this statement's own bytes cannot
+ * prove statement ownership (the same gate `valuesOf`/`argumentsOf` apply to
+ * their own reads).
+ * Shortest — not first, not outermost — is what keeps `:local f do={:put 1}`
+ * from claiming the body's `1` for `f`.
+ *
+ * ## Why a parent chain is enough
+ *
+ * Statement spans NEST: a parent statement's span contains its scope bodies'
+ * statements. If the addressable spans are a laminar family — every pair
+ * either nested or disjoint, never crossing — then the spans containing a
+ * given point form a chain, so the answer is reachable by walking up from the
+ * deepest span that starts at or before `start`.
+ *
+ * Laminarity is **measured, not assumed**. Over the pinned 948-script corpus
+ * (`corpus-pin.json`), all 18,487 addressable spans of 18,648 splits are
+ * laminar: zero crossings, zero duplicate `(start, end)` pairs, maximum
+ * nesting depth 11. That is one genre of input (see #203 on the corpus bias),
+ * so it is a measurement, not a proof — {@link buildSplitOwnerIndex} therefore
+ * DETECTS a crossing while it builds and falls back to the exhaustive scan for
+ * that document. The fallback is the old code, so a crossing costs speed and
+ * never correctness, and `test/unit/explain-split-owner.test.ts` drives both
+ * paths against a brute-force oracle on randomly generated span families.
+ */
+
+import type { DocumentVerbSplit } from "./verbsplit.ts";
+
+/** A prepared owner lookup over one document's splits. */
+export interface SplitOwnerIndex {
+	/**
+	 * The owning split index for `[start, end)`, or `undefined` when no
+	 * addressable span contains it.
+	 */
+	ownerOf(start: number, end: number): number | undefined;
+	/**
+	 * Were the addressable spans laminar? `false` means this index answers by
+	 * exhaustive scan. Exposed so the unit test can prove it drove both paths,
+	 * not for callers to branch on.
+	 */
+	readonly laminar: boolean;
+}
+
+interface Entry {
+	start: number;
+	end: number;
+	/** Index into the original `splits` array. */
+	split: number;
+}
+
+/** The addressable spans, in split order. */
+function addressableEntries(
+	analyzed: string,
+	splits: readonly DocumentVerbSplit[],
+): Entry[] {
+	const entries: Entry[] = [];
+	for (let i = 0; i < splits.length; i++) {
+		const split = splits[i] as DocumentVerbSplit;
+		const { start, end } = split.span;
+		// A widened fallback span cannot prove statement ownership: it is not
+		// this statement's own bytes.
+		if (analyzed.slice(start, end) !== split.text) continue;
+		entries.push({ start, end, split: i });
+	}
+	return entries;
+}
+
+/**
+ * The exhaustive scan, over the addressable entries rather than over `splits`.
+ * Same answer as the pre-#317 loop: the filter it applied per candidate is the
+ * one `addressableEntries` applied once, and `length < ownerLength` is strict,
+ * so ascending split order still resolves a length tie to the earliest index.
+ */
+function scanOwner(
+	entries: readonly Entry[],
+	start: number,
+	end: number,
+): number | undefined {
+	let owner: number | undefined;
+	let ownerLength = Number.POSITIVE_INFINITY;
+	for (const entry of entries) {
+		if (entry.start > start || end > entry.end) continue;
+		const length = entry.end - entry.start;
+		if (length < ownerLength) {
+			owner = entry.split;
+			ownerLength = length;
+		}
+	}
+	return owner;
+}
+
+/**
+ * Prepare the owner lookup for one document. Build once per analysis and reuse
+ * it for every query: building it per query would be the scan it replaces.
+ */
+export function buildSplitOwnerIndex(
+	analyzed: string,
+	splits: readonly DocumentVerbSplit[],
+): SplitOwnerIndex {
+	const entries = addressableEntries(analyzed, splits);
+	// Parents first: ascending start, and on equal starts the longer span
+	// (the container) before the shorter one. The last entry with
+	// `start <= query.start` is then the deepest candidate, which is what the
+	// binary search below relies on.
+	const ordered = [...entries].sort(
+		(a, b) => a.start - b.start || b.end - a.end || a.split - b.split,
+	);
+	// Two splits can claim exactly the same span; the sort has made them
+	// adjacent, and dropping all but the first keeps the earliest index. It
+	// also leaves the nesting STRICT, which is what lets the walk stop at the
+	// first long-enough span: after this, two distinct entries of equal length
+	// cannot both contain one query, because in a laminar family nested plus
+	// equal length means identical.
+	const sorted = ordered.filter((entry, i) => {
+		const previous = ordered[i - 1];
+		return (
+			previous === undefined ||
+			previous.start !== entry.start ||
+			previous.end !== entry.end
+		);
+	});
+	const parent = new Int32Array(sorted.length).fill(-1);
+	const stack: number[] = [];
+	let laminar = true;
+	for (let i = 0; i < sorted.length; i++) {
+		const cur = sorted[i] as Entry;
+		while (stack.length > 0) {
+			const top = sorted[stack[stack.length - 1] as number] as Entry;
+			if (top.end <= cur.start) stack.pop();
+			else break;
+		}
+		const topIndex = stack[stack.length - 1];
+		if (topIndex !== undefined) {
+			const top = sorted[topIndex] as Entry;
+			// Overlapping but not containing: the family is not laminar, so the
+			// containing spans of a point are not a chain and the walk below
+			// could miss the real owner.
+			if (top.end < cur.end) laminar = false;
+			parent[i] = topIndex;
+		}
+		stack.push(i);
+	}
+
+	if (!laminar)
+		return {
+			laminar,
+			ownerOf: (start, end) => scanOwner(entries, start, end),
+		};
+
+	return {
+		laminar,
+		ownerOf(start, end) {
+			// A ZERO-WIDTH query is outside the nesting model and takes the scan.
+			// Containment of `[p, p)` is satisfied by a span that merely TOUCHES
+			// `p`, so `[a, p)` and `[p, b)` both "contain" it while being disjoint
+			// — the containing spans stop being a chain and the walk below can
+			// reach only one of them. No real query is zero-width (a symbol, a
+			// value anchor and a scope brace all span at least one byte; measured
+			// 0 of 19,292 symbol occurrences over the pinned corpus), so this
+			// costs nothing in practice and keeps the answer exact if that ever
+			// stops being true.
+			if (start >= end) return scanOwner(entries, start, end);
+			// Last entry whose span starts at or before `start`.
+			let lo = 0;
+			let hi = sorted.length;
+			while (lo < hi) {
+				const mid = (lo + hi) >> 1;
+				if ((sorted[mid] as Entry).start <= start) lo = mid + 1;
+				else hi = mid;
+			}
+			// Ends grow as the walk rises, so the first span long enough to
+			// cover `end` is the shortest one that does. Entries that stop
+			// before `start` are simply skipped by the same test.
+			for (let i = lo - 1; i >= 0; i = parent[i] as number) {
+				const entry = sorted[i] as Entry;
+				if (entry.end >= end) return entry.split;
+			}
+			return undefined;
+		},
+	};
+}

package/src/explain/string-tokens.ts

@@ -0,0 +1,77 @@
+/**
+ * B3 string fill — claims quoted-string spans on the residual.
+ *
+ * Every RouterOS double-quoted string is a lexical unit the device highlights
+ * (even when its value is not a literal, the delimiters are). `data.values`
+ * already claims argument value strings (quotes included), so this fill runs
+ * **after** `value` and sees only the residual: quoted declaration names
+ * (`:global "set-dns"`), quoted variable references (`$"set-dns"`), and
+ * string content inside substitutions that value anchoring did not locate.
+ *
+ * All quoted runs are produced by the single shared scanner
+ * `src/explain/quoted-string.ts` (`scanQuotedString`), the same primitive the
+ * segmenter and symbol resolver use, so boundaries cannot drift.
+ *
+ * One `string` class per quoted run, delimiters included, regardless of whether
+ * the string is a literal value, a name, or code. The delimiters are part of
+ * the run because they are its own first and last byte, so nothing is gained by
+ * naming them separately (#264 B5).
+ *
+ * A run is **not** necessarily one token, and escapes are why. The `escaped`
+ * fill (`src/explain/escape-tokens.ts`) claims valid escapes two stages earlier,
+ * so what reaches here is the run minus its escapes and this fill emits a
+ * fragment either side of each one — `"a\nb"` is `string("a) escaped(\n)
+ * string(b")`. Splitting the run that way is not new machinery: `clipToResidual`
+ * already fragmented these runs around `variable-*` spans. #264 made that split
+ * because a valid escape is recoverable from no other published surface, where a
+ * delimiter is recoverable from the run's own first and last byte — which is why
+ * the escapes are named and the delimiters are not.
+ */
+
+import type { ExplainToken } from "../explain.ts";
+import { scanQuotedString } from "./quoted-string.ts";
+import { clipToResidual } from "./token-ranges.ts";
+
+/** Quoted-string spans on the residual. */
+export function stringSpans(
+	analyzed: string,
+	residual: readonly { start: number; end: number }[],
+): ExplainToken[] {
+	const len = analyzed.length;
+	if (len === 0 || residual.length === 0) return [];
+	const raw: { start: number; end: number }[] = [];
+	for (let i = 0; i < analyzed.length; i++) {
+		if (analyzed[i] !== '"') continue;
+		// Earlier fills own semantic regions. In particular, a quote inside a
+		// comment must not open a string that paints later statements when that
+		// comment has no matching quote. A real residual string always has its
+		// opening delimiter on the residual; already-owned strings need no fill.
+		if (clipToResidual(i, i + 1, residual).length === 0) continue;
+		const scan = scanQuotedString(analyzed, i);
+		if (!scan.closed) {
+			// Unterminated strings are already a defect; claim from opener to
+			// end so the residual does not stay gap-filled as unclassified
+			// while the diagnostic says the same bytes are defective.
+			raw.push({ start: i, end: analyzed.length });
+			break;
+		}
+		raw.push({ start: i, end: scan.end });
+		i = scan.end - 1;
+	}
+	if (raw.length === 0) return [];
+	const out: ExplainToken[] = [];
+	for (const { start, end } of raw) {
+		if (!Number.isInteger(start) || !Number.isInteger(end)) continue;
+		if (start < 0 || end <= start || end > len) continue;
+		for (const r of clipToResidual(start, end, residual)) {
+			out.push({
+				start: r.start,
+				end: r.end,
+				class: "string" as const,
+				ev: "e13",
+			});
+		}
+	}
+	out.sort((a, b) => a.start - b.start || a.end - b.end);
+	return out;
+}