@dforge-core/metadata 0.0.20 → 0.0.23

package/src/dsl/index.ts

@@ -0,0 +1,36 @@
+// @dforge-core/metadata/dsl — the action-DSL analyzer.
+//
+// A separate entry point, not part of the root barrel: this is a lexer, a
+// parser and a rule engine, while the root export is registries and types that
+// the web app and the VS Code webview pull into a browser bundle. Importing
+// `@dforge-core/metadata` must not drag a tokenizer in behind it.
+//
+// Consumers: the language server (diagnostics, completion, hover,
+// go-to-definition) and the module validator that runs before a pack.
+
+export { tokenize, stringValue } from "./lexer";
+export type { Token, TokenKind } from "./lexer";
+
+export { parseDsl, BLOCK_KINDS } from "./parse";
+export type {
+	BlockKind,
+	CallRef,
+	DslBlock,
+	DslDocument,
+	DslParam,
+	FieldRef,
+	GlobalRef,
+	Span,
+} from "./parse";
+
+export { BUILTINS, BUILTIN_BY_NAME, BUILTIN_VALUES } from "./builtins";
+export type { Builtin } from "./builtins";
+
+export { checkDsl } from "./check";
+export type {
+	ColumnLookup,
+	DslContext,
+	DslIssue,
+	DslSeverity,
+	EntityShape,
+} from "./types";

package/src/dsl/lexer.ts

@@ -0,0 +1,364 @@
+// Hand-rolled lexer for the dForge action DSL.
+//
+// Deliberately not the tree-sitter grammar: that one exists to be compiled into
+// Zed, and pulling a native parser (or a wasm runtime) into an npx-launched
+// language server — or into a validator that runs before a pack — costs far
+// more than it buys for the handful of constructs the rules need.
+
+export type TokenKind =
+	| "ident"
+	| "number"
+	| "string"
+	| "template"
+	| "regex"
+	| "punct"
+	| "comment";
+
+export interface Token {
+	kind: TokenKind;
+	/** Source text, verbatim — quotes included for strings. */
+	text: string;
+	start: number;
+	end: number;
+	line: number;
+	character: number;
+	/** True when a line break sits between this token and the previous one. */
+	startsLine: boolean;
+}
+
+const PUNCT3 = ["===", "!=="];
+const PUNCT2 = [
+	"==", "!=", "<=", ">=", "&&", "||", "++", "--",
+	"+=", "-=", "*=", "/=", "%=", "${", "=>",
+];
+
+/**
+ * Template literals and the `${…}` holes in them. Inside a template we are
+ * reading literal text; inside a hole we are reading code again, and the brace
+ * count says which `}` closes it.
+ */
+type Frame =
+	| { kind: "template"; chunkStart: number }
+	| { kind: "interp"; braces: number };
+
+/**
+ * Keywords whose `(…)` holds a condition rather than a parameter list. What
+ * follows that `)` is a statement — so a regex may open there, and the brace
+ * after it is not a function body.
+ */
+export const CONTROL_KEYWORDS = new Set([
+	"if",
+	"while",
+	"for",
+	"switch",
+	"catch",
+	"with",
+]);
+
+export function tokenize(text: string): Token[] {
+	const tokens: Token[] = [];
+	let i = 0;
+	let line = 0;
+	let lineStart = 0;
+	let pendingLineBreak = true;
+	const frames: Frame[] = [];
+	// Whether each open paren belongs to a control condition, so that the `)`
+	// closing it is known to be followed by a statement.
+	const parenIsControl: boolean[] = [];
+	let closedControlParen = false;
+	// Whether each open brace began a statement block rather than an object,
+	// so that the `}` closing it is known to leave a statement position.
+	const braceIsBlock: boolean[] = [];
+	let closedBlockBrace = false;
+
+	const push = (kind: TokenKind, start: number, end: number) => {
+		tokens.push({
+			kind,
+			text: text.slice(start, end),
+			start,
+			end,
+			line,
+			character: start - lineStart,
+			startsLine: pendingLineBreak,
+		});
+		pendingLineBreak = false;
+	};
+
+	while (i < text.length) {
+		const frame = frames[frames.length - 1];
+
+		// ── literal text inside a template ──────────────────────────────
+		// Emitted as one token per chunk, so the analyzer sees the code in
+		// `${…}` but never the prose around it. The `${` closes the chunk
+		// before it and the `}` opens the chunk after, which keeps the brace
+		// count of anything reading these tokens balanced.
+		if (frame?.kind === "template") {
+			const startLine = line;
+			const startLineStart = lineStart;
+			while (i < text.length) {
+				const ch = text[i]!;
+				if (ch === "\\") {
+					i += 2;
+					continue;
+				}
+				if (ch === "`" || (ch === "$" && text[i + 1] === "{")) break;
+				if (ch === "\n") {
+					line++;
+					lineStart = i + 1;
+				}
+				i++;
+			}
+			if (text[i] === "`") {
+				i++;
+				frames.pop();
+			} else if (i < text.length) {
+				i += 2;
+				frames.push({ kind: "interp", braces: 0 });
+			} else {
+				frames.pop(); // unterminated
+			}
+			const savedLine = line;
+			const savedLineStart = lineStart;
+			line = startLine;
+			lineStart = startLineStart;
+			push("template", frame.chunkStart, i);
+			line = savedLine;
+			lineStart = savedLineStart;
+			continue;
+		}
+
+		const c = text[i]!;
+
+		// A leading BOM is not a column. The compiler strips it before anchoring
+		// block headers at `^execute:`, so the first real token must still land
+		// at character 0 here or every block-scoped rule stands down.
+		if (c === "\uFEFF" && i === 0) {
+			i++;
+			lineStart = i;
+			continue;
+		}
+
+		if (c === "\n") {
+			i++;
+			line++;
+			lineStart = i;
+			pendingLineBreak = true;
+			continue;
+		}
+		if (c === " " || c === "\t" || c === "\r") {
+			i++;
+			continue;
+		}
+
+		if (c === "/" && text[i + 1] === "/") {
+			const start = i;
+			while (i < text.length && text[i] !== "\n") i++;
+			push("comment", start, i);
+			continue;
+		}
+		if (c === "/" && text[i + 1] === "*") {
+			const start = i;
+			const startLine = line;
+			const startLineStart = lineStart;
+			i += 2;
+			while (i < text.length && !(text[i] === "*" && text[i + 1] === "/")) {
+				if (text[i] === "\n") {
+					line++;
+					lineStart = i + 1;
+				}
+				i++;
+			}
+			i = Math.min(i + 2, text.length);
+			// Report the comment at its opening position, not its end.
+			const savedLine = line;
+			const savedLineStart = lineStart;
+			line = startLine;
+			lineStart = startLineStart;
+			push("comment", start, i);
+			line = savedLine;
+			lineStart = savedLineStart;
+			continue;
+		}
+
+		if (c === "'" || c === '"') {
+			const start = i;
+			const quote = c;
+			i++;
+			while (i < text.length && text[i] !== quote && text[i] !== "\n") {
+				if (text[i] === "\\") i++;
+				i++;
+			}
+			if (text[i] === quote) i++;
+			push("string", start, i);
+			continue;
+		}
+
+		if (c === "`") {
+			frames.push({ kind: "template", chunkStart: i });
+			i++;
+			continue;
+		}
+
+		if (
+			c === "/" &&
+			regexAllowedAfter(
+				tokens[tokens.length - 1],
+				closedControlParen,
+				closedBlockBrace,
+			)
+		) {
+			const end = scanRegex(text, i);
+			if (end > 0) {
+				push("regex", i, end);
+				i = end;
+				continue;
+			}
+		}
+
+		if (/[0-9]/.test(c)) {
+			const start = i;
+			while (i < text.length && /[0-9._eE]/.test(text[i]!)) i++;
+			push("number", start, i);
+			continue;
+		}
+
+		if (/[A-Za-z_$]/.test(c)) {
+			const start = i;
+			while (i < text.length && /[A-Za-z0-9_$]/.test(text[i]!)) i++;
+			push("ident", start, i);
+			continue;
+		}
+
+		if (c === "(") {
+			const prev = tokens[tokens.length - 1];
+			parenIsControl.push(
+				prev?.kind === "ident" && CONTROL_KEYWORDS.has(prev.text),
+			);
+		} else if (c === ")") {
+			closedControlParen = parenIsControl.pop() ?? false;
+		}
+
+		if (frame?.kind === "interp") {
+			if (c === "}" && frame.braces === 0) {
+				frames.pop();
+				// The `}` opens the next literal chunk rather than becoming a
+				// token of its own — see the chunk comment above.
+				(frames[frames.length - 1] as { chunkStart: number }).chunkStart = i;
+				i++;
+				continue;
+			}
+			if (c === "{") frame.braces++;
+			else if (c === "}") frame.braces--;
+		}
+
+		if (c === "{") {
+			braceIsBlock.push(isBlockBrace(tokens[tokens.length - 1]));
+		} else if (c === "}") {
+			closedBlockBrace = braceIsBlock.pop() ?? false;
+		}
+
+		const three = text.slice(i, i + 3);
+		if (PUNCT3.includes(three)) {
+			push("punct", i, i + 3);
+			i += 3;
+			continue;
+		}
+		const two = text.slice(i, i + 2);
+		if (PUNCT2.includes(two)) {
+			push("punct", i, i + 2);
+			i += 2;
+			continue;
+		}
+		push("punct", i, i + 1);
+		i++;
+	}
+
+	return tokens;
+}
+
+/**
+ * Whether a `/` here opens a regex literal rather than dividing. The usual
+ * rule: a regex may start where an expression may start, so it follows an
+ * operator or a keyword but not a value.
+ *
+ * A `)` depends on what it closed: after a control condition a statement
+ * follows, so `if (x) /re/.test(s)` is a regex, while `(a + b) / 2` divides.
+ * A template chunk likewise depends on which end it is — one that stops at
+ * `${` opens an expression, one that ran to the closing backtick is a value.
+ *
+ * A `}` likewise depends on what it closed: ending a statement block leaves a
+ * statement position, so `if (x) {}` may be followed by a regex, while an
+ * object literal is a value and divides.
+ */
+function regexAllowedAfter(
+	prev: Token | undefined,
+	closedControlParen: boolean,
+	closedBlockBrace: boolean,
+): boolean {
+	if (!prev) return true;
+	if (prev.kind === "number" || prev.kind === "string") return false;
+	if (prev.kind === "template") return prev.text.endsWith("${");
+	if (prev.kind === "ident") return REGEX_PRECEDING_KEYWORDS.has(prev.text);
+	if (prev.kind === "regex") return false;
+	if (prev.kind === "comment") return true;
+	if (prev.text === ")") return closedControlParen;
+	if (prev.text === "}") return closedBlockBrace;
+	return !REGEX_BLOCKING_PUNCT.has(prev.text);
+}
+
+/**
+ * Whether the `{` after `prev` opens a statement block rather than an object
+ * literal. A block follows a condition, another block, a statement end or one
+ * of the keywords that take a bare body; anything else — an `=`, a `(`, a `,`,
+ * a `=>` — is expression position, where the brace is a value.
+ */
+function isBlockBrace(prev: Token | undefined): boolean {
+	if (!prev) return true;
+	if (prev.kind === "ident") return BLOCK_PRECEDING_KEYWORDS.has(prev.text);
+	if (prev.kind !== "punct") return false;
+	return prev.text === ")" || prev.text === "}" || prev.text === "{" || prev.text === ";";
+}
+
+const BLOCK_PRECEDING_KEYWORDS = new Set(["else", "do", "try", "finally"]);
+
+const REGEX_PRECEDING_KEYWORDS = new Set([
+	"return", "typeof", "instanceof", "in", "of", "new", "delete", "void",
+	"case", "do", "else", "throw", "yield", "await",
+]);
+
+const REGEX_BLOCKING_PUNCT = new Set(["]", "++", "--"]);
+
+/**
+ * End offset of the regex literal starting at `start`, or -1 if it is not one.
+ * A `/` inside a character class does not close the literal, and a literal
+ * never spans a line — hitting one means this was a division after all.
+ */
+function scanRegex(text: string, start: number): number {
+	let i = start + 1;
+	let inClass = false;
+	// An empty pattern would be the `//` comment the caller already handled.
+	if (text[i] === "/" || text[i] === "*") return -1;
+	for (; i < text.length; i++) {
+		const c = text[i]!;
+		if (c === "\\") {
+			i++;
+			continue;
+		}
+		if (c === "\n") return -1;
+		if (c === "[") inClass = true;
+		else if (c === "]") inClass = false;
+		else if (c === "/" && !inClass) {
+			i++;
+			while (i < text.length && /[a-z]/.test(text[i]!)) i++;
+			return i;
+		}
+	}
+	return -1;
+}
+
+/** Strip the surrounding quotes from a string token's text. */
+export function stringValue(token: Token): string {
+	if (token.kind !== "string") return token.text;
+	const body = token.text.slice(1, token.text.endsWith(token.text[0]!) && token.text.length > 1 ? -1 : undefined);
+	return body.replace(/\\(.)/g, "$1");
+}