@tikoci/centrs 0.1.3 → 0.1.6

package/src/explain/escape-tokens.ts

@@ -0,0 +1,77 @@
+/**
+ * Escape fill — claims valid string-internal escape sequences on the residual.
+ *
+ * The one split #264 B5's rule admits and PR #330 did not make. B5's rule is
+ * that a merged class is split only where a consumer needs the distinction AND
+ * cannot recover it by joining `tokens[]` to another published surface on byte
+ * offsets. A valid escape passes both halves: only INVALID escapes surface
+ * elsewhere (as `bad-string-escape` diagnostics), so a consumer holding the
+ * token stream has no published surface to join against to find the valid ones,
+ * and the device does name them — `escaped` is one of the 19 classes the Q13
+ * capture observed.
+ *
+ * Unlike `arg-sep` this needs a real walk rather than a relabel of one
+ * analyzer's own output, and it takes bytes out of TWO fills at once rather
+ * than one. In byte terms it is still a retag: measured over the corpus
+ * `escaped` claims 19,960 bytes, `string` gives up 17,672 and `value` 2,288,
+ * and classified coverage is identical to the last decimal. Nothing that was
+ * `unclassified` becomes claimed — an escape only ever lives inside a run some
+ * fill already held.
+ *
+ * The walk is not this module's. `src/explain/quoted-string.ts` already owned
+ * the escape grammar for `bad-string-escape`, and `walkStringEscapes` now
+ * returns both halves of that single reading, so an `escaped` token and a
+ * `bad-string-escape` diagnostic cannot disagree about which bytes are an
+ * escape. This module only clips that list onto the residual.
+ *
+ * ## Fill order: before `value`, after `arg`
+ *
+ * An escape is interior to a quoted run, so it must claim before the two fills
+ * that would otherwise swallow it whole (`value`, then `string`) — after them
+ * it could claim nothing. It runs after `arg` because an argument NAME run is
+ * the analyzer's own located token and nothing inside it is a string interior;
+ * leaving `arg` ahead keeps the located reading winning over a byte scan, which
+ * is the same precedence every earlier fill was given (#290 design decision 1).
+ * Splitting a `value` or `string` run into fragments around an escape is the
+ * mechanism those fills already have — `~1,720` leaf value bytes were already
+ * being fragmented by `variable-*` spans before this fill existed.
+ */
+
+import type { ExplainToken } from "../explain.ts";
+import type { StringEscapeSpan } from "./quoted-string.ts";
+import { clipToResidual } from "./token-ranges.ts";
+
+/**
+ * Valid string-internal escape spans on the residual.
+ *
+ * `analyzed` is the ASCII-normalized document text (length checks only);
+ * `residual` is the gap set left by earlier fills (sorted, no overlaps);
+ * `escapes` are the valid escapes from `walkStringEscapes`, already in
+ * analyzed-byte space. Every emitted span's bytes are fully inside `residual`,
+ * sorted by `start`, non-overlapping, and carry `class: "escaped"`.
+ */
+export function escapeSpans(
+	analyzed: string,
+	residual: readonly { start: number; end: number }[],
+	escapes: readonly StringEscapeSpan[],
+): ExplainToken[] {
+	const len = analyzed.length;
+	if (len === 0 || residual.length === 0 || escapes.length === 0) return [];
+
+	const out: ExplainToken[] = [];
+	for (const { start, end } of escapes) {
+		if (!Number.isInteger(start) || !Number.isInteger(end)) continue;
+		if (start < 0 || end <= start || end > len) continue;
+		for (const r of clipToResidual(start, end, residual)) {
+			out.push({
+				start: r.start,
+				end: r.end,
+				class: "escaped" as const,
+				ev: "e16",
+			});
+		}
+	}
+
+	out.sort((a, b) => a.start - b.start || a.end - b.end);
+	return out;
+}

package/src/explain/highlight-projection.ts

@@ -0,0 +1,308 @@
+/**
+ * The `centrs → highlight` projection (#264 B4, measured by #263).
+ *
+ * `data.tokens[]` is a centrs-owned vocabulary; RouterOS
+ * `/console/inspect request=highlight` is a *different* vocabulary produced by
+ * a *different* kind of analyzer. This module is the declared bridge between
+ * them, and it exists so the agreement report
+ * (`scripts/explain-highlight-agreement.ts`) can say what offline analysis got
+ * right against device truth instead of eyeballing two streams side by side.
+ *
+ * ## The projection is declared, never fitted
+ *
+ * Every entry below is justified by what the centrs class was *defined* to
+ * mean — `commands/explain/README.md` → the token-census block and
+ * `src/explain.ts`'s `ExplainTokenClass` doc — not by which device class it
+ * happened to land on most often. A projection tuned to the measurement would
+ * make the measurement circular: agreement would only ever say "the map was
+ * fitted", which is not a fact about the parser. So a centrs class whose
+ * *intent* has no single device counterpart projects to `null` and the report
+ * counts those bytes as `unprojected`, reporting what the device said about
+ * them as its own table. That table is the evidence #264 read to decide
+ * which provisional merges deserved to be split — two did, `arg-sep` (B5) and
+ * `escaped` — and `commands/explain/README.md` -> *The vocabulary, and what
+ * earns a class* records why the rest stay merged.
+ *
+ * ## Why the device stream is not one oracle but four
+ *
+ * Scoring a byte requires knowing what kind of answer the device gave, and
+ * `highlight` mixes four:
+ *
+ * - **syntax** — a class an offline analyzer can in principle decide from the
+ *   source text alone (`cmd`, `dir`, `arg`, `comment`, the `variable-*`
+ *   family, `escaped`, `syntax-meta`).
+ * - **non-syntax** — `obj-inactive`, `obj-dynamic`, `obj-disabled`,
+ *   `variable-undefined`, `syntax-obsolete`. The source text alone does not
+ *   decide these, so they are reported as their own bucket and never as a
+ *   wrong answer. Note what abstains: centrs has no class for most of them
+ *   (`ExplainSymbolClass` excludes `undefined`), but where it *does* claim the
+ *   byte — an `arg` the device calls `obj-inactive` — the token stands and it
+ *   is the SCORER that declines to grade it. Saying *why* each one is out of
+ *   reach is a separate question, and it is deliberately **not** answered by
+ *   the class name — see `applicabilityOf` below.
+ * - **silence** — `none`. The device assigned no class. It is the device's
+ *   abstention, not a class named "none", so agreeing with it is not an
+ *   agreement (probes: "an abstention is not a disagreement").
+ * - **parser-stop** — `error`. Measured over the committed slice this byte
+ *   appears **at most once per script, always exactly one byte wide**, and on
+ *   long-term 7.23.5 every byte after it is `none`: the device stops
+ *   classifying rather than recovering. Most of the slice's bytes sit in that
+ *   tail, so a percentage that ignores it measures the device giving up.
+ *   Stable 7.24.2 and development 7.25beta3 do classify again after the `error`
+ *   in one run, which is why the report gives that a bucket of its own instead
+ *   of asserting the rule (the counts live in the generated README block, not
+ *   here, so they cannot go stale). The rule and its exception swapped builds
+ *   with the #342 channel realignment; the shape of the claim did not.
+ *
+ * ## Upstream drift
+ *
+ * `DEVICE_CLASS_KIND` lists the 19 classes the phase-0 capture observed on
+ * 7.23.2 / 7.24rc2 — and the #342 recapture found the SAME 19 on 7.24.2,
+ * 7.23.5 and 7.25beta3, so the vocabulary has held across five builds and two
+ * capture rounds. A class outside it is `"unknown"` — reported, never
+ * silently folded into one of the four kinds — because a new upstream class is
+ * exactly the drift the centrs-owned vocabulary exists to absorb, and guessing
+ * its kind would hide it.
+ */
+
+import type { ExplainTokenClass } from "../explain.ts";
+
+/**
+ * What kind of answer a device highlight class is, from an offline analyzer's
+ * point of view. See the module header for why the distinction is load-bearing.
+ */
+export type DeviceClassKind =
+	| "syntax"
+	| "non-syntax"
+	| "silence"
+	| "parser-stop"
+	| "unknown";
+
+/**
+ * The 19 classes the Q13 capture observed, by kind.
+ *
+ * Source: `test/fixtures/explain/highlight-streams.slice.json` →
+ * `selection.classesInCapture`, which the slice's own integrity test pins
+ * against the stream data. Keep the two in step: the agreement report fails
+ * loudly when the slice carries a class this table does not name.
+ */
+export const DEVICE_CLASS_KIND: Readonly<Record<string, DeviceClassKind>> = {
+	arg: "syntax",
+	"arg-dot": "syntax",
+	"arg-scope": "syntax",
+	cmd: "syntax",
+	comment: "syntax",
+	dir: "syntax",
+	escaped: "syntax",
+	"syntax-meta": "syntax",
+	"variable-auto": "syntax",
+	"variable-global": "syntax",
+	"variable-local": "syntax",
+	"variable-parameter": "syntax",
+	"obj-disabled": "non-syntax",
+	"obj-dynamic": "non-syntax",
+	"obj-inactive": "non-syntax",
+	"syntax-obsolete": "non-syntax",
+	"variable-undefined": "non-syntax",
+	none: "silence",
+	error: "parser-stop",
+};
+
+/** The kind of a device class; `"unknown"` for anything the capture never saw. */
+export function deviceClassKind(deviceClass: string): DeviceClassKind {
+	return DEVICE_CLASS_KIND[deviceClass] ?? "unknown";
+}
+
+/**
+ * One projection entry: the device classes a centrs class predicts, plus the
+ * reason it predicts them. `accepts: null` is a deliberate abstention.
+ */
+export interface ProjectionEntry {
+	/** Device classes this centrs class claims, or `null` to abstain. */
+	readonly accepts: readonly string[] | null;
+	/** Why — in terms of what the centrs class was defined to mean. */
+	readonly because: string;
+}
+
+/**
+ * centrs token class → the device highlight classes it predicts.
+ *
+ * Total over `ExplainTokenClass`: every class has an entry, and an entry is
+ * either a non-empty accept set or an explicit abstention.
+ */
+export const HIGHLIGHT_PROJECTION: Readonly<
+	Record<ExplainTokenClass, ProjectionEntry>
+> = {
+	comment: {
+		accepts: ["comment"],
+		because:
+			"name-coincident: the class was taken from the device vocabulary and means the same run of source",
+	},
+	"variable-local": {
+		accepts: ["variable-local"],
+		because: "name-coincident (the Q13 symbol classes are the device's own)",
+	},
+	"variable-global": {
+		accepts: ["variable-global"],
+		because: "name-coincident (the Q13 symbol classes are the device's own)",
+	},
+	"variable-auto": {
+		accepts: ["variable-auto"],
+		because: "name-coincident (the Q13 symbol classes are the device's own)",
+	},
+	"variable-parameter": {
+		accepts: ["variable-parameter"],
+		because: "name-coincident (the Q13 symbol classes are the device's own)",
+	},
+	dir: {
+		accepts: ["dir"],
+		because:
+			"name-coincident: resolved menu bytes, leading `/` or `:` included, which is the run the device also calls `dir`",
+	},
+	cmd: {
+		accepts: ["cmd"],
+		because: "name-coincident: the resolved command-name run",
+	},
+	arg: {
+		accepts: ["arg", "arg-dot", "arg-scope"],
+		because:
+			"argument-name bytes; the device splits dotted spellings (`.id`, `security.authentication-types`) into `arg-dot` / `arg-scope` and centrs deliberately does not, so all three are the same claim. The `=` is no longer part of this class — #264 B5 gave it `arg-sep`, which is why this entry now predicts only name classes",
+	},
+	"arg-sep": {
+		accepts: ["syntax-meta"],
+		because:
+			"the `=` binding one argument name to its value is structure punctuation, not part of the name, and `syntax-meta` is the device's only class for structure. Declared from what the class means, not fitted: before #264 B5 split it out these bytes were the whole of the `arg` -> `syntax-meta` disagreement, which is the measurement that made the split worth its name (the count is in the generated agreement block, so it cannot go stale here)",
+	},
+	operator: {
+		accepts: ["syntax-meta"],
+		because:
+			"`syntax-meta` is the device's residual structure class and the only place an operator byte can land (#255): the device merges adjacent structure runs, so it names no operator boundary, but it does decide structure-versus-word",
+	},
+	brace: {
+		accepts: ["syntax-meta"],
+		because:
+			"scope and array delimiters are structure punctuation, and `syntax-meta` is the device's only class for it",
+	},
+	string: {
+		accepts: null,
+		because:
+			"the token spans the quoted run MINUS its escapes, which the device still splits two ways — delimiters `syntax-meta` and the interior it declines to classify at all. Naming the escapes (#264) removed the third way and is why this entry's residue is now delimiters, not delimiters plus escapes; the run still has no single device class, so the abstention stands",
+	},
+	escaped: {
+		accepts: ["escaped"],
+		because:
+			"name-coincident: the class means one valid string-internal escape sequence, which is the run the device also calls `escaped`. Declared from meaning before it was measured — `escaped` was already in `DEVICE_CLASS_KIND` as a syntax class, and the agreement report's `unprojected` table is what showed the bytes were being merged away rather than what chose the target",
+	},
+	value: {
+		accepts: null,
+		because:
+			"the device has no value class. It classifies an argument's value only when the value is itself something else (a substitution, an object name, an escape — the last of which #264 split out), so a `value` token has no device counterpart to be right or wrong about",
+	},
+	unclassified: {
+		accepts: null,
+		because: "centrs's own abstention — no analyzer claimed the byte",
+	},
+};
+
+/** The device classes a centrs class predicts, or `null` where it abstains. */
+export function projectTokenClass(
+	tokenClass: ExplainTokenClass,
+): readonly string[] | null {
+	return HIGHLIGHT_PROJECTION[tokenClass].accepts;
+}
+
+/**
+ * Why a device answer is or is not within reach of offline analysis (#263).
+ *
+ * This is a **second axis**, orthogonal to whether a byte agreed. Agreement
+ * asks "did the two answers match"; applicability asks "what kind of fact was
+ * the device deciding, and could source text alone have decided it". A byte
+ * has one of each.
+ *
+ * The categories are #263's, and the load-bearing instruction there is that
+ * they are assigned **from context, never by assigning a whole highlight class
+ * to one bucket**. So the input is the *pair* — what centrs read the byte as
+ * and what the device called it — plus whether the captured versions even
+ * agreed on the device's answer.
+ */
+export type Applicability =
+	/** Syntax and structure justify the conclusion; offline analysis can reach it. */
+	| "offline-decidable"
+	/** Needs a per-path field/argument schema offline centrs deliberately does not ship. */
+	| "schema-dependent"
+	/** Needs runtime objects or session state outside the input. */
+	| "state-dependent"
+	/**
+	 * Deciding needs version context: either the captured RouterOS versions
+	 * disagree about this byte, so there is no single device answer, or the
+	 * device's answer is itself a statement about the running version
+	 * (`syntax-obsolete`) even where the captures agree.
+	 */
+	| "version-dependent"
+	/** No supported basis for a category. Explicit, never a silent default. */
+	| "uncategorized"
+	/** The device gave no answer here (`none`, or the post-`error` tail). */
+	| "no-device-answer";
+
+/**
+ * Categorize one byte's device answer.
+ *
+ * `versionsAgree` is whether every captured version gave this byte the same
+ * class. It is checked **first**, ahead even of the device's own silence: when
+ * the captures disagree there is no single device answer to be schema- or
+ * state-dependent about, and the disagreement is itself the fact worth
+ * reporting. That ordering matters in practice — a byte one build classifies
+ * and the next leaves `none` is a version difference, not an absence, and
+ * checking silence first would file it as "the device said nothing" on the
+ * strength of whichever build happens to be the base.
+ */
+export function applicabilityOf(
+	centrsClass: string,
+	deviceClass: string,
+	versionsAgree: boolean,
+): Applicability {
+	if (!versionsAgree) return "version-dependent";
+	const kind = deviceClassKind(deviceClass);
+	if (kind === "silence" || kind === "parser-stop") return "no-device-answer";
+	if (kind === "syntax") return "offline-decidable";
+	// `syntax-obsolete` is the running version's deprecation table talking: the
+	// same bytes are unremarkable on an older build. Measured on the slice it
+	// lands on the whitespace where modern syntax wants `=` (`} else {`,
+	// `:foreach conn in $conns`).
+	if (deviceClass === "syntax-obsolete") return "version-dependent";
+	if (deviceClass === "obj-dynamic" || deviceClass === "obj-disabled")
+		return "state-dependent";
+	if (deviceClass === "obj-inactive") {
+		// Same class, two different reasons — which is exactly why the class name
+		// cannot decide the category. On a name centrs read as an argument, the
+		// device is saying the menu's schema has no such field; on a menu or
+		// command name it is saying the package or menu is not present on this
+		// box.
+		//
+		// `arg-sep` rides with `arg` because it is the same fact about the same
+		// attribute: the device marking an attribute inactive marks the `=` that
+		// binds it too, and what it is saying there is still "this menu has no
+		// such field". Splitting the class (#264 B5) did not split the reason.
+		if (centrsClass === "arg" || centrsClass === "arg-sep")
+			return "schema-dependent";
+		if (centrsClass === "dir" || centrsClass === "cmd")
+			return "state-dependent";
+		return "uncategorized";
+	}
+	// `variable-undefined` is NOT blanket state-dependent (#263, explicitly). A
+	// bare word the device resolved to an undefined reference may be an external
+	// global, a query/filter field name, or a lexical spelling — three different
+	// categories — and nothing committed here separates them. Equally, an absent
+	// binding in this document is not evidence of a runtime undefined variable.
+	// Leaving it explicit is the point; folding it into a category would be the
+	// unsupported claim.
+	//
+	// This branch reaches the same answer as the fallthrough below on purpose:
+	// the requirement belongs at the point of decision, where the next reader
+	// making a rule for this class will be standing, not only in the prose. It
+	// is pinned by a test.
+	if (deviceClass === "variable-undefined") return "uncategorized";
+	// Any other non-syntax class, and any class this build has never seen: no
+	// supported basis for a category, said out loud rather than guessed.
+	return "uncategorized";
+}

package/src/explain/is-known-menu.ts

@@ -0,0 +1,17 @@
+import { lookupPath } from "./catalog.ts";
+import { isMenuPath } from "./menus.ts";
+
+/**
+ * Whether `segments` name a known RouterOS menu or settings path.
+ *
+ * Union of the two shipped structure tables (#235): `MENU_PATHS` (#207,
+ * `isMenuPath`) and `PATH_CATALOG` menu/settings rows (#228, `lookupPath`).
+ * Presence is load-bearing, absence abstains — the caller must not read
+ * absence as evidence of a command.
+ */
+export function isKnownMenuPath(segments: readonly string[]): boolean {
+	const entry = lookupPath(segments);
+	return (
+		isMenuPath(segments) || entry?.kind === "menu" || entry?.kind === "settings"
+	);
+}