@effected/yaml 0.1.0

package/internal/cst-parser.js

@@ -0,0 +1,716 @@
+import { lexAll } from "./lexer.js";
+
+//#region src/internal/cst-parser.ts
+/**
+* Hardening: cap CST nesting so hostile inputs cannot overflow the stack
+* while the tree is being BUILT. Set slightly above the composer's
+* MAX_NESTING_DEPTH (256) so the composer's own guard — which reports the
+* user-facing NestingDepthExceeded diagnostic — always fires first when the
+* capped tree is composed.
+*/
+const MAX_CST_DEPTH = 264;
+/**
+* When the depth budget is exhausted, consume the current token and return
+* a flat error node instead of recursing. Returns null when within budget.
+*/
+function guardDepth(state) {
+	if (state.depth < MAX_CST_DEPTH) return null;
+	const token = advance(state);
+	return {
+		type: "error",
+		source: token?.value ?? "",
+		offset: token?.offset ?? state.text.length,
+		length: token?.length ?? 0
+	};
+}
+function atEnd(state) {
+	return state.pos >= state.tokens.length;
+}
+function peek(state) {
+	return state.tokens[state.pos];
+}
+function advance(state) {
+	const token = state.tokens[state.pos];
+	state.pos++;
+	return token;
+}
+/** Check if the current token is a trivia token (whitespace, newline, comment). */
+function isTrivia(token) {
+	return token.kind === "whitespace" || token.kind === "newline" || token.kind === "comment";
+}
+/** Check if the current token starts a new document boundary. */
+function isDocumentBoundary(token) {
+	return token.kind === "document-start" || token.kind === "document-end";
+}
+/** Build a CstNode from collected children, computing source from the text. */
+function makeContainerNode(type, children, text) {
+	if (children.length === 0) return {
+		type,
+		source: "",
+		offset: 0,
+		length: 0,
+		children
+	};
+	const first = children[0];
+	const last = children[children.length - 1];
+	const offset = first.offset;
+	const end = last.offset + last.length;
+	return {
+		type,
+		source: text.slice(offset, end),
+		offset,
+		length: end - offset,
+		children
+	};
+}
+/**
+* Create a leaf CstNode from a token, using the raw source text.
+*
+* We slice from the original text rather than using `token.value` because
+* the lexer decodes certain tokens (e.g. quoted scalars have quotes stripped
+* and escape sequences resolved in `value`), but the CST must preserve
+* the raw source text exactly as written.
+*/
+function makeLeafNode(type, token, text) {
+	return {
+		type,
+		source: text.slice(token.offset, token.offset + token.length),
+		offset: token.offset,
+		length: token.length
+	};
+}
+/**
+* Consume trivia tokens (whitespace, newline, comment) and return them as CST nodes.
+*/
+function consumeTrivia(state) {
+	const nodes = [];
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token || !isTrivia(token)) break;
+		advance(state);
+		if (token.kind === "comment") nodes.push(makeLeafNode("comment", token, state.text));
+		else if (token.kind === "newline") nodes.push(makeLeafNode("newline", token, state.text));
+		else nodes.push(makeLeafNode("whitespace", token, state.text));
+	}
+	return nodes;
+}
+/**
+* Consume a single trivia-or-content token and return it as a CST node.
+* Used for tokens that don't form higher-level structures.
+*/
+function consumeLeafToken(state) {
+	const token = peek(state);
+	if (!token) return void 0;
+	advance(state);
+	switch (token.kind) {
+		case "whitespace": return makeLeafNode("whitespace", token, state.text);
+		case "newline": return makeLeafNode("newline", token, state.text);
+		case "comment": return makeLeafNode("comment", token, state.text);
+		case "scalar": return makeLeafNode("flow-scalar", token, state.text);
+		case "anchor": return makeLeafNode("anchor", token, state.text);
+		case "alias": return makeLeafNode("alias", token, state.text);
+		case "tag": return makeLeafNode("tag", token, state.text);
+		case "directive": return makeLeafNode("directive", token, state.text);
+		case "flow-separator": return makeLeafNode("whitespace", token, state.text);
+		case "block-map-value":
+		case "block-map-key":
+		case "block-seq-entry": return makeLeafNode("whitespace", token, state.text);
+		case "document-start":
+		case "document-end": return makeLeafNode("whitespace", token, state.text);
+		case "flow-map-start":
+		case "flow-map-end":
+		case "flow-seq-start":
+		case "flow-seq-end": return makeLeafNode("whitespace", token, state.text);
+		case "block-map-start":
+		case "block-seq-start": return makeLeafNode("whitespace", token, state.text);
+		case "byte-order-mark": return makeLeafNode("whitespace", token, state.text);
+		case "error": return makeLeafNode("error", token, state.text);
+		default: return makeLeafNode("error", token, state.text);
+	}
+}
+/**
+* Parse a flow mapping (curly braces).
+*/
+function parseFlowMapping(state) {
+	const guarded = guardDepth(state);
+	if (guarded !== null) return guarded;
+	state.depth++;
+	try {
+		return parseFlowMappingInner(state);
+	} finally {
+		state.depth--;
+	}
+}
+function parseFlowMappingInner(state) {
+	const children = [];
+	const open = advance(state);
+	if (open) children.push(makeLeafNode("whitespace", open, state.text));
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token) break;
+		if (token.kind === "flow-map-end") {
+			const close = advance(state);
+			if (close) children.push(makeLeafNode("whitespace", close, state.text));
+			break;
+		}
+		if (token.kind === "flow-map-start") children.push(parseFlowMapping(state));
+		else if (token.kind === "flow-seq-start") children.push(parseFlowSequence(state));
+		else {
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+		}
+	}
+	return makeContainerNode("flow-map", children, state.text);
+}
+/**
+* Parse a flow sequence: [ ... ]
+*/
+function parseFlowSequence(state) {
+	const guarded = guardDepth(state);
+	if (guarded !== null) return guarded;
+	state.depth++;
+	try {
+		return parseFlowSequenceInner(state);
+	} finally {
+		state.depth--;
+	}
+}
+function parseFlowSequenceInner(state) {
+	const children = [];
+	const open = advance(state);
+	if (open) children.push(makeLeafNode("whitespace", open, state.text));
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token) break;
+		if (token.kind === "flow-seq-end") {
+			const close = advance(state);
+			if (close) children.push(makeLeafNode("whitespace", close, state.text));
+			break;
+		}
+		if (token.kind === "flow-map-start") children.push(parseFlowMapping(state));
+		else if (token.kind === "flow-seq-start") children.push(parseFlowSequence(state));
+		else {
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+		}
+	}
+	return makeContainerNode("flow-seq", children, state.text);
+}
+/**
+* Parse a block scalar token. The lexer already handles the block scalar
+* content (literal `|` or folded `\>`), so we just need to wrap it.
+*/
+function parseBlockScalar(state) {
+	const token = advance(state);
+	if (!token) return {
+		type: "block-scalar",
+		source: "",
+		offset: 0,
+		length: 0
+	};
+	return {
+		type: "block-scalar",
+		source: state.text.slice(token.offset, token.offset + token.length),
+		offset: token.offset,
+		length: token.length
+	};
+}
+/**
+* Check if the current position has a block scalar indicator in the original text.
+*/
+function isBlockScalarToken(state) {
+	const token = peek(state);
+	if (token?.kind !== "scalar") return false;
+	const ch = state.text[token.offset];
+	return ch === "|" || ch === ">";
+}
+/**
+* Check if the last non-trivia child in a CST node list is a value separator
+* (`:`) with no subsequent value content. This indicates the next block
+* structure should be consumed as the mapping value, not ejected as a sibling.
+*/
+function lastNonTriviaIsValueSep(children) {
+	for (let i = children.length - 1; i >= 0; i--) {
+		const c = children[i];
+		if (!c) continue;
+		if (c.type === "whitespace" && c.source === ":") return true;
+		if (c.type === "newline" || c.type === "comment") continue;
+		if (c.type === "whitespace") continue;
+		if (c.type === "anchor" || c.type === "tag") continue;
+		return false;
+	}
+	return false;
+}
+/**
+* Find the column of the first block-seq-entry in the token stream,
+* skipping the zero-width block-seq-start marker and any trivia.
+* Falls back to `fallback` if no entry is found.
+*/
+function findFirstSeqEntryColumn(state, fallback) {
+	for (let i = state.pos; i < state.tokens.length; i++) {
+		const t = state.tokens[i];
+		if (!t) break;
+		if (t.kind === "block-seq-start") continue;
+		if (isTrivia(t)) continue;
+		if (t.kind === "block-seq-entry") return t.column;
+		break;
+	}
+	return fallback;
+}
+/**
+* Parse a block mapping at the given indentation level.
+*/
+function parseBlockMapping(state, indent) {
+	const guarded = guardDepth(state);
+	if (guarded !== null) return guarded;
+	state.depth++;
+	try {
+		return parseBlockMappingInner(state, indent);
+	} finally {
+		state.depth--;
+	}
+}
+function parseBlockMappingInner(state, indent) {
+	const children = [];
+	let sawExplicitKey = false;
+	if (peek(state)?.kind === "block-map-start") advance(state);
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token) break;
+		if (isDocumentBoundary(token)) break;
+		if (token.kind === "block-seq-start" && token.column <= indent && children.length > 0) {
+			if (!lastNonTriviaIsValueSep(children) && !sawExplicitKey) break;
+		}
+		if (isTrivia(token)) {
+			children.push(...consumeTrivia(state));
+			continue;
+		}
+		if (token.kind === "block-map-key") {
+			sawExplicitKey = true;
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+			continue;
+		}
+		if (token.kind === "block-map-value") {
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+			children.push(...parseBlockValue(state, indent, sawExplicitKey));
+			sawExplicitKey = false;
+			continue;
+		}
+		if (token.kind === "scalar" || token.kind === "anchor" || token.kind === "alias" || token.kind === "tag") {
+			if (token.column < indent && children.length > 0) break;
+			if (isBlockScalarToken(state)) {
+				children.push(parseBlockScalar(state));
+				continue;
+			}
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+			continue;
+		}
+		if (token.kind === "block-map-start") {
+			if (token.column > indent) children.push(parseBlockMapping(state, token.column));
+			else if (token.column === indent) advance(state);
+			else break;
+			continue;
+		}
+		if (token.kind === "block-seq-start") {
+			const seqIndent = sawExplicitKey ? findFirstSeqEntryColumn(state, token.column) : token.column;
+			children.push(parseBlockSequence(state, seqIndent));
+			continue;
+		}
+		if (token.kind === "block-seq-entry") {
+			if (token.column === indent && lastNonTriviaIsValueSep(children)) {
+				const seqChildren = [];
+				while (!atEnd(state)) {
+					const seqToken = peek(state);
+					if (!seqToken) break;
+					if (seqToken.kind === "block-seq-entry" && seqToken.column === indent) {
+						seqChildren.push(...consumeTrivia(state));
+						const entry = consumeLeafToken(state);
+						if (entry) seqChildren.push(entry);
+						seqChildren.push(...parseSequenceEntryContent(state, indent));
+					} else if (isTrivia(seqToken)) seqChildren.push(...consumeTrivia(state));
+					else break;
+				}
+				if (seqChildren.length > 0) children.push(makeContainerNode("block-seq", seqChildren, state.text));
+				continue;
+			}
+			if (token.column <= indent) break;
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+			continue;
+		}
+		if (token.kind === "flow-map-start") {
+			children.push(parseFlowMapping(state));
+			continue;
+		}
+		if (token.kind === "flow-seq-start") {
+			children.push(parseFlowSequence(state));
+			continue;
+		}
+		const leaf = consumeLeafToken(state);
+		if (leaf) children.push(leaf);
+	}
+	return makeContainerNode("block-map", children, state.text);
+}
+/**
+* Parse the value part after a ":" in a block mapping.
+*/
+function parseBlockValue(state, _parentIndent, explicitKey = false) {
+	const nodes = [];
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token) break;
+		if (token.kind === "whitespace") {
+			nodes.push(...consumeTrivia(state));
+			continue;
+		}
+		if (token.kind === "newline") break;
+		if (token.kind === "comment") break;
+		if (token.kind === "flow-map-start") {
+			nodes.push(parseFlowMapping(state));
+			break;
+		}
+		if (token.kind === "flow-seq-start") {
+			nodes.push(parseFlowSequence(state));
+			break;
+		}
+		if (isBlockScalarToken(state)) {
+			nodes.push(parseBlockScalar(state));
+			break;
+		}
+		if (explicitKey && (token.kind === "block-seq-start" || token.kind === "block-seq-entry")) {
+			const seqIndent = findFirstSeqEntryColumn(state, token.column);
+			nodes.push(parseBlockSequence(state, seqIndent));
+			break;
+		}
+		if (token.kind === "scalar" || token.kind === "anchor" || token.kind === "alias" || token.kind === "tag") {
+			const leaf = consumeLeafToken(state);
+			if (leaf) nodes.push(leaf);
+			continue;
+		}
+		break;
+	}
+	return nodes;
+}
+/**
+* Parse a block sequence at the given indentation level.
+*/
+function parseBlockSequence(state, indent) {
+	const guarded = guardDepth(state);
+	if (guarded !== null) return guarded;
+	state.depth++;
+	try {
+		return parseBlockSequenceInner(state, indent);
+	} finally {
+		state.depth--;
+	}
+}
+function parseBlockSequenceInner(state, indent) {
+	const children = [];
+	if (peek(state)?.kind === "block-seq-start") advance(state);
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token) break;
+		if (isDocumentBoundary(token)) break;
+		if (isTrivia(token)) {
+			children.push(...consumeTrivia(state));
+			continue;
+		}
+		if (token.kind === "block-seq-entry") {
+			if (token.column < indent) break;
+			if (token.column > indent) break;
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+			children.push(...parseSequenceEntryContent(state, indent));
+			continue;
+		}
+		if (token.kind === "block-map-start" && token.column > indent) {
+			children.push(parseBlockMapping(state, token.column));
+			continue;
+		}
+		if (token.kind === "block-seq-start") {
+			if (token.column > indent) children.push(parseBlockSequence(state, token.column));
+			else advance(state);
+			continue;
+		}
+		break;
+	}
+	return makeContainerNode("block-seq", children, state.text);
+}
+/**
+* Look ahead (without consuming) to see if a block-map-value token exists
+* before the next newline / document boundary / sequence entry at this indent.
+*/
+function hasImplicitMapAhead(state, seqIndent) {
+	let flowDepth = 0;
+	for (let i = state.pos; i < state.tokens.length; i++) {
+		const t = state.tokens[i];
+		if (!t) break;
+		if (t.kind === "flow-map-start" || t.kind === "flow-seq-start") {
+			flowDepth++;
+			continue;
+		}
+		if (t.kind === "flow-map-end" || t.kind === "flow-seq-end") {
+			flowDepth--;
+			continue;
+		}
+		if (flowDepth > 0) continue;
+		if (t.kind === "newline") return false;
+		if (isDocumentBoundary(t)) return false;
+		if (t.kind === "block-seq-entry" && t.column <= seqIndent) return false;
+		if (t.kind === "block-map-value") return true;
+	}
+	return false;
+}
+/**
+* Parse the content of a sequence entry (after the `-`).
+*/
+function parseSequenceEntryContent(state, seqIndent) {
+	const nodes = [];
+	const nextToken = findNextNonTrivia(state);
+	if (nextToken && nextToken.kind === "block-seq-entry" && nextToken.column > seqIndent) {} else if (hasImplicitMapAhead(state, seqIndent)) {
+		nodes.push(parseImplicitBlockMapping(state, seqIndent));
+		return nodes;
+	}
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token) break;
+		if (isDocumentBoundary(token)) break;
+		if (token.kind === "block-seq-entry" && token.column <= seqIndent) break;
+		if (token.kind === "block-seq-entry" && token.column > seqIndent) {
+			nodes.push(parseBlockSequence(state, token.column));
+			continue;
+		}
+		if (isTrivia(token)) {
+			nodes.push(...consumeTrivia(state));
+			continue;
+		}
+		if (token.kind === "block-map-start") {
+			nodes.push(parseBlockMapping(state, token.column));
+			continue;
+		}
+		if (token.kind === "block-seq-start") {
+			if (token.column <= seqIndent) break;
+			nodes.push(parseBlockSequence(state, token.column));
+			continue;
+		}
+		if (token.kind === "flow-map-start") {
+			nodes.push(parseFlowMapping(state));
+			continue;
+		}
+		if (token.kind === "flow-seq-start") {
+			nodes.push(parseFlowSequence(state));
+			continue;
+		}
+		if (isBlockScalarToken(state)) {
+			nodes.push(parseBlockScalar(state));
+			continue;
+		}
+		if (token.kind === "scalar" || token.kind === "block-map-value" || token.kind === "block-map-key" || token.kind === "anchor" || token.kind === "alias" || token.kind === "tag") {
+			if (token.column <= seqIndent) break;
+			const leaf = consumeLeafToken(state);
+			if (leaf) nodes.push(leaf);
+			continue;
+		}
+		break;
+	}
+	return nodes;
+}
+/**
+* Parse an implicit block mapping (no block-map-start token from the lexer).
+* This occurs inside sequence entries like `- a: 1`.
+*/
+function parseImplicitBlockMapping(state, seqIndent) {
+	const guarded = guardDepth(state);
+	if (guarded !== null) return guarded;
+	state.depth++;
+	try {
+		return parseImplicitBlockMappingInner(state, seqIndent);
+	} finally {
+		state.depth--;
+	}
+}
+function parseImplicitBlockMappingInner(state, seqIndent) {
+	const children = [];
+	let entryIndent = -1;
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token) break;
+		if (isDocumentBoundary(token)) break;
+		if (token.kind === "block-seq-entry" && token.column <= seqIndent) break;
+		if (isTrivia(token)) {
+			children.push(...consumeTrivia(state));
+			continue;
+		}
+		if (token.kind === "block-map-value") {
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+			children.push(...parseBlockValue(state, seqIndent));
+			continue;
+		}
+		if (token.kind === "scalar" || token.kind === "anchor" || token.kind === "alias" || token.kind === "tag") {
+			if (isBlockScalarToken(state)) {
+				children.push(parseBlockScalar(state));
+				continue;
+			}
+			if (entryIndent >= 0 && token.column <= seqIndent) break;
+			if (entryIndent < 0) entryIndent = token.column;
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+			continue;
+		}
+		if (token.kind === "block-map-start") {
+			if (entryIndent >= 0 && token.column <= entryIndent) {
+				advance(state);
+				continue;
+			}
+			children.push(parseBlockMapping(state, token.column));
+			continue;
+		}
+		if (token.kind === "block-seq-start") {
+			if (token.column <= seqIndent && children.length > 0) break;
+			children.push(parseBlockSequence(state, token.column));
+			continue;
+		}
+		if (token.kind === "flow-map-start") {
+			children.push(parseFlowMapping(state));
+			continue;
+		}
+		if (token.kind === "flow-seq-start") {
+			children.push(parseFlowSequence(state));
+			continue;
+		}
+		const leaf = consumeLeafToken(state);
+		if (leaf) children.push(leaf);
+	}
+	return makeContainerNode("block-map", children, state.text);
+}
+/**
+* Parse a single document from the token stream.
+*/
+function parseDocument(state) {
+	const children = [];
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token) break;
+		if (token.kind === "directive") {
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+			continue;
+		}
+		if (isTrivia(token) && token.kind !== "comment") {
+			children.push(...consumeTrivia(state));
+			continue;
+		}
+		if (token.kind === "comment") {
+			const nextNonTrivia = findNextNonTrivia(state);
+			if (nextNonTrivia?.kind === "directive" || nextNonTrivia?.kind === "document-start") {
+				children.push(...consumeTrivia(state));
+				continue;
+			}
+			break;
+		}
+		break;
+	}
+	if (!atEnd(state)) {
+		if (peek(state)?.kind === "document-start") {
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+		}
+	}
+	while (!atEnd(state)) {
+		const token = peek(state);
+		if (!token) break;
+		if (token.kind === "document-start") break;
+		if (token.kind === "document-end") {
+			const leaf = consumeLeafToken(state);
+			if (leaf) children.push(leaf);
+			while (!atEnd(state)) {
+				const t = peek(state);
+				if (!t) break;
+				if (t.kind === "newline" || t.kind === "whitespace" || t.kind === "comment") children.push(...consumeTrivia(state));
+				else break;
+			}
+			break;
+		}
+		if (isTrivia(token)) {
+			children.push(...consumeTrivia(state));
+			continue;
+		}
+		if (token.kind === "block-map-start") {
+			children.push(parseBlockMapping(state, token.column));
+			continue;
+		}
+		if (token.kind === "block-seq-start") {
+			children.push(parseBlockSequence(state, token.column));
+			continue;
+		}
+		if (token.kind === "flow-map-start") {
+			children.push(parseFlowMapping(state));
+			continue;
+		}
+		if (token.kind === "flow-seq-start") {
+			children.push(parseFlowSequence(state));
+			continue;
+		}
+		if (isBlockScalarToken(state)) {
+			children.push(parseBlockScalar(state));
+			continue;
+		}
+		const leaf = consumeLeafToken(state);
+		if (leaf) children.push(leaf);
+	}
+	return makeContainerNode("document", children, state.text);
+}
+/**
+* Find the next non-trivia token without consuming.
+*/
+function findNextNonTrivia(state) {
+	for (let i = state.pos; i < state.tokens.length; i++) {
+		const t = state.tokens[i];
+		if (t && !isTrivia(t)) return t;
+	}
+}
+/**
+* Parse all documents from the token stream.
+*/
+function parseDocuments(tokens, text) {
+	const state = {
+		tokens,
+		text,
+		pos: 0,
+		depth: 0
+	};
+	const documents = [];
+	if (atEnd(state)) return [{
+		type: "document",
+		source: "",
+		offset: 0,
+		length: 0,
+		children: []
+	}];
+	while (!atEnd(state)) {
+		const before = state.pos;
+		const doc = parseDocument(state);
+		const hasDocStart = doc.children?.some((c) => c.type === "whitespace" && c.source === "---");
+		const hasContent = doc.children?.some((c) => c.type !== "whitespace" && c.type !== "newline" && c.type !== "comment" && c.type !== "error");
+		if (hasDocStart || hasContent || documents.length === 0) documents.push(doc);
+		if (state.pos === before && !atEnd(state)) state.pos++;
+	}
+	return documents;
+}
+/**
+* Parse YAML source text and collect all CST document nodes into an array.
+*
+* Each CST node preserves every character of the original input, including
+* whitespace, comments, and structural indicators. No value interpretation
+* occurs at this stage — `true` is still the string `"true"`.
+*/
+function parseCSTAll(text) {
+	return parseDocuments(lexAll(text), text);
+}
+
+//#endregion
+export { parseCSTAll };