@effected/yaml 0.1.0

package/internal/lexer.js

@@ -0,0 +1,959 @@
+//#region src/internal/lexer.ts
+/**
+* Create a new YAML scanner for the given source text.
+*
+* Returns a stateful, pull-based scanner. Call {@link YamlScanner.scan} to
+* advance to the next token, then use `getToken*` methods to inspect it.
+*/
+function createScanner(text) {
+	let pos = 0;
+	let line = 0;
+	let col = 0;
+	/**
+	* The indentation level of the current line. Set to `col` when we encounter
+	* the first non-whitespace character after a newline (or at start of input).
+	*/
+	let lineIndent = 0;
+	/** Whether `lineIndent` has been locked (set) for the current line. */
+	let lineIndentLocked = false;
+	/** Flow nesting depth (positive means we are inside flow context). */
+	let flowDepth = 0;
+	/** Whether we've emitted block-map-start / block-seq-start for the current indent. */
+	const blockStarted = /* @__PURE__ */ new Map();
+	/** Set when the block scalar scanner produces an empty value (contentIndent === 0). */
+	let afterEmptyBlockScalar = false;
+	/** Set when the previous token was a quoted scalar (single or double quoted). */
+	let afterQuotedScalar = false;
+	let afterFlowCollectionEnd = false;
+	/** Buffer of tokens to emit before scanning the next real token. */
+	const pending = [];
+	/** Mutable holder for the most recently produced token, set by the public {@link scan} method. */
+	const state = { currentToken: null };
+	function peek(offset = 0) {
+		const idx = pos + offset;
+		return idx < text.length ? text[idx] ?? "" : "";
+	}
+	function charAt(i) {
+		return i < text.length ? text[i] ?? "" : "";
+	}
+	function advance(count = 1) {
+		for (let i = 0; i < count; i++) if (pos < text.length) {
+			if (text[pos] === "\n") {
+				line++;
+				col = 0;
+				lineIndentLocked = false;
+			} else col++;
+			pos++;
+		}
+	}
+	/** Lock the line indent to the current column (called on first non-ws char). */
+	function lockLineIndent() {
+		if (!lineIndentLocked) {
+			lineIndent = col;
+			lineIndentLocked = true;
+			for (const key of blockStarted.keys()) if (key > lineIndent) blockStarted.delete(key);
+		}
+	}
+	function makeToken(kind, value, offset, tokenLine, tokenCol, rawLength) {
+		return {
+			kind,
+			value,
+			offset,
+			length: rawLength ?? value.length,
+			line: tokenLine,
+			column: tokenCol
+		};
+	}
+	function isWhitespace(ch) {
+		return ch === " " || ch === "	";
+	}
+	function isNewline(ch) {
+		return ch === "\n" || ch === "\r";
+	}
+	function isFlowIndicator(ch) {
+		return ch === "{" || ch === "}" || ch === "[" || ch === "]" || ch === ",";
+	}
+	/**
+	* Check if the current position starts a document marker (--- or ...)
+	* followed by whitespace, newline, or EOF. Used inside quoted scalars
+	* to detect unterminated strings broken by document boundaries.
+	*/
+	function isDocumentMarkerAhead() {
+		const c0 = peek();
+		const c1 = peek(1);
+		const c2 = peek(2);
+		const c3 = peek(3);
+		if (c0 === "-" && c1 === "-" && c2 === "-" || c0 === "." && c1 === "." && c2 === ".") return c3 === "" || c3 === " " || c3 === "	" || c3 === "\n" || c3 === "\r";
+		return false;
+	}
+	/**
+	* Returns true if `ch` cannot appear in a plain scalar at the current position.
+	*/
+	function isPlainScalarBreak(ch) {
+		if (ch === "" || isNewline(ch)) return true;
+		if (flowDepth > 0 && isFlowIndicator(ch)) return true;
+		return false;
+	}
+	function ensureBlockMap(indent, offset, tokLine, _tokCol) {
+		if (flowDepth > 0) return;
+		const started = blockStarted.get(indent);
+		if (started === "map") return;
+		if (started === "seq") return;
+		blockStarted.set(indent, "map");
+		pending.push(makeToken("block-map-start", "", offset, tokLine, indent));
+	}
+	function ensureBlockSeq(indent, offset, tokLine, _tokCol) {
+		if (flowDepth > 0) return;
+		if (blockStarted.get(indent) === "seq") return;
+		blockStarted.set(indent, "seq");
+		pending.push(makeToken("block-seq-start", "", offset, tokLine, indent));
+	}
+	function scanNewline() {
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		if (peek() === "\r" && peek(1) === "\n") {
+			advance(2);
+			return makeToken("newline", "\r\n", start, sLine, sCol);
+		}
+		const ch = peek();
+		advance();
+		return makeToken("newline", ch, start, sLine, sCol);
+	}
+	function scanWhitespace() {
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		if (!lineIndentLocked && peek() === "	") {
+			if (flowDepth === 0) {
+				let lookPos = pos;
+				while (lookPos < text.length && (text[lookPos] === " " || text[lookPos] === "	")) lookPos++;
+				const nextCh = lookPos < text.length ? text[lookPos] : void 0;
+				if (!afterEmptyBlockScalar && (nextCh === void 0 || nextCh === "\n" || nextCh === "\r")) {
+					while (pos < text.length && isWhitespace(peek())) advance();
+					return makeToken("whitespace", text.slice(start, pos), start, sLine, sCol);
+				}
+				if (nextCh === "{" || nextCh === "[") {
+					while (pos < text.length && isWhitespace(peek())) advance();
+					return makeToken("whitespace", text.slice(start, pos), start, sLine, sCol);
+				}
+				if (blockStarted.size === 0) {
+					while (pos < text.length && isWhitespace(peek())) advance();
+					return makeToken("whitespace", text.slice(start, pos), start, sLine, sCol);
+				}
+				while (pos < text.length && isWhitespace(peek())) advance();
+				return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+			}
+			{
+				let lookPos = pos;
+				while (lookPos < text.length && (text[lookPos] === " " || text[lookPos] === "	")) lookPos++;
+				const nextCh = lookPos < text.length ? text[lookPos] : "";
+				if (nextCh === "]" || nextCh === "}" || nextCh === "\n" || nextCh === "\r" || nextCh === "") {
+					while (pos < text.length && isWhitespace(peek())) advance();
+					return makeToken("whitespace", text.slice(start, pos), start, sLine, sCol);
+				}
+			}
+			while (pos < text.length && isWhitespace(peek())) advance();
+			return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+		}
+		while (pos < text.length && isWhitespace(peek())) advance();
+		if (!lineIndentLocked && flowDepth === 0 && sCol === 0) {
+			const ws = text.slice(start, pos);
+			const tabIdx = ws.indexOf("	");
+			if (tabIdx > 0) {
+				const nextCh = pos < text.length ? text[pos] : void 0;
+				if (nextCh !== void 0 && nextCh !== "\n" && nextCh !== "\r") {
+					if (blockStarted.has(tabIdx)) return makeToken("error", ws, start, sLine, sCol);
+				}
+			}
+		}
+		return makeToken("whitespace", text.slice(start, pos), start, sLine, sCol);
+	}
+	function scanComment() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		while (pos < text.length && !isNewline(peek())) advance();
+		return makeToken("comment", text.slice(start, pos), start, sLine, sCol);
+	}
+	function scanDocumentStartOrEnd() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		const marker = text.slice(pos, pos + 3);
+		const after = charAt(pos + 3);
+		if (after !== "" && !isNewline(after) && !isWhitespace(after)) return null;
+		advance(3);
+		const kind = marker === "---" ? "document-start" : "document-end";
+		blockStarted.clear();
+		return makeToken(kind, marker, start, sLine, sCol);
+	}
+	function scanPlainScalar() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		while (pos < text.length) {
+			const ch = peek();
+			if (isPlainScalarBreak(ch)) break;
+			if (ch === ":" && (isWhitespace(peek(1)) || isNewline(peek(1)) || peek(1) === "" || flowDepth > 0 && isFlowIndicator(peek(1)))) break;
+			if (ch === " " && peek(1) === "#") break;
+			if (ch === "	" && peek(1) === "#") break;
+			advance();
+		}
+		let end = pos;
+		while (end > start && isWhitespace(text[end - 1] ?? "")) end--;
+		return makeToken("scalar", text.slice(start, end), start, sLine, sCol);
+	}
+	function scanSingleQuotedScalar() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		advance();
+		let value = "";
+		while (pos < text.length) {
+			const ch = peek();
+			if (ch === "'") if (peek(1) === "'") {
+				value += "'";
+				advance(2);
+			} else {
+				advance();
+				return makeToken("scalar", value, start, sLine, sCol, pos - start);
+			}
+			else if (ch === "\n" || ch === "\r" && peek(1) === "\n") {
+				value += " ";
+				if (ch === "\r") advance(2);
+				else advance();
+				if (col === 0 && isDocumentMarkerAhead()) return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+			} else {
+				value += ch;
+				advance();
+			}
+		}
+		return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+	}
+	function scanDoubleQuotedScalar() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		advance();
+		let value = "";
+		while (pos < text.length) {
+			const ch = peek();
+			if (ch === "\"") {
+				advance();
+				return makeToken("scalar", value, start, sLine, sCol, pos - start);
+			}
+			if (ch === "\\") {
+				advance();
+				switch (peek()) {
+					case "\\":
+						value += "\\";
+						advance();
+						break;
+					case "\"":
+						value += "\"";
+						advance();
+						break;
+					case "/":
+						value += "/";
+						advance();
+						break;
+					case "b":
+						value += "\b";
+						advance();
+						break;
+					case "f":
+						value += "\f";
+						advance();
+						break;
+					case "n":
+						value += "\n";
+						advance();
+						break;
+					case "r":
+						value += "\r";
+						advance();
+						break;
+					case "	":
+					case "t":
+						value += "	";
+						advance();
+						break;
+					case "0":
+						value += "\0";
+						advance();
+						break;
+					case "a":
+						value += "\x07";
+						advance();
+						break;
+					case "e":
+						value += "\x1B";
+						advance();
+						break;
+					case "v":
+						value += "\v";
+						advance();
+						break;
+					case " ":
+						value += " ";
+						advance();
+						break;
+					case "N":
+						value += "…";
+						advance();
+						break;
+					case "_":
+						value += "\xA0";
+						advance();
+						break;
+					case "L":
+						value += "\u2028";
+						advance();
+						break;
+					case "P":
+						value += "\u2029";
+						advance();
+						break;
+					case "x": {
+						advance();
+						const hex = text.slice(pos, pos + 2);
+						if (hex.length === 2 && /^[\da-fA-F]{2}$/.test(hex)) {
+							value += String.fromCharCode(Number.parseInt(hex, 16));
+							advance(2);
+						} else return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+						break;
+					}
+					case "u": {
+						advance();
+						const hex = text.slice(pos, pos + 4);
+						if (hex.length === 4 && /^[\da-fA-F]{4}$/.test(hex)) {
+							value += String.fromCodePoint(Number.parseInt(hex, 16));
+							advance(4);
+						} else return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+						break;
+					}
+					case "U": {
+						advance();
+						const hex = text.slice(pos, pos + 8);
+						const cp = hex.length === 8 && /^[\da-fA-F]{8}$/.test(hex) ? Number.parseInt(hex, 16) : NaN;
+						if (cp <= 1114111) {
+							value += String.fromCodePoint(cp);
+							advance(8);
+						} else return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+						break;
+					}
+					case "\n":
+						advance();
+						while (pos < text.length && isWhitespace(peek())) advance();
+						break;
+					case "\r":
+						advance();
+						if (peek() === "\n") advance();
+						while (pos < text.length && isWhitespace(peek())) advance();
+						break;
+					default: return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+				}
+			} else if (ch === "\n" || ch === "\r" && peek(1) === "\n") {
+				value += " ";
+				if (ch === "\r") advance(2);
+				else advance();
+				if (col === 0 && isDocumentMarkerAhead()) return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+				if (sCol > 0 && peek() === "	") {
+					advance();
+					return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+				}
+			} else {
+				value += ch;
+				advance();
+			}
+		}
+		return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+	}
+	function scanBlockScalar() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		const isFolded = peek() === ">";
+		advance();
+		let chomp = "clip";
+		let explicitIndent = 0;
+		for (let hc = 0; hc < 2 && pos < text.length && !isNewline(peek()); hc++) {
+			const ch = peek();
+			if (ch === "-") {
+				chomp = "strip";
+				advance();
+			} else if (ch === "+") {
+				chomp = "keep";
+				advance();
+			} else if (ch >= "1" && ch <= "9") {
+				explicitIndent = Number.parseInt(ch, 10);
+				advance();
+			} else break;
+		}
+		let hadHeaderWhitespace = false;
+		while (pos < text.length && isWhitespace(peek())) {
+			hadHeaderWhitespace = true;
+			advance();
+		}
+		if (pos < text.length && !isNewline(peek()) && peek() !== "") if (peek() === "#" && hadHeaderWhitespace) while (pos < text.length && !isNewline(peek())) advance();
+		else {
+			while (pos < text.length && !isNewline(peek())) advance();
+			return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+		}
+		if (pos < text.length && isNewline(peek())) if (peek() === "\r" && peek(1) === "\n") advance(2);
+		else advance();
+		let contentIndent = explicitIndent;
+		let foundContent = explicitIndent > 0;
+		if (explicitIndent > 0) {
+			let parentIndent = lineIndent;
+			let scanBack = start - 1;
+			while (scanBack >= 0 && (text[scanBack] === " " || text[scanBack] === "	")) scanBack--;
+			if (scanBack >= 0 && text[scanBack] === ":") {
+				let lineStart = scanBack;
+				while (lineStart > 0 && text[lineStart - 1] !== "\n" && text[lineStart - 1] !== "\r") lineStart--;
+				let spaces = 0;
+				while (lineStart + spaces < text.length && text[lineStart + spaces] === " ") spaces++;
+				if (lineStart + spaces < text.length && text[lineStart + spaces] === "-") {
+					const afterDash = lineStart + spaces + 1;
+					if (afterDash < text.length && (text[afterDash] === " " || text[afterDash] === "	")) parentIndent = spaces + 2;
+					else parentIndent = spaces;
+				} else parentIndent = spaces;
+			} else if (scanBack >= 0 && text[scanBack] === "-") {
+				let lineStart = scanBack;
+				while (lineStart > 0 && text[lineStart - 1] !== "\n" && text[lineStart - 1] !== "\r") lineStart--;
+				parentIndent = scanBack - lineStart;
+			} else {
+				let maxBlockIndent = 0;
+				for (const key of blockStarted.keys()) if (key > maxBlockIndent) maxBlockIndent = key;
+				parentIndent = maxBlockIndent;
+			}
+			contentIndent = parentIndent + explicitIndent;
+			foundContent = true;
+		}
+		if (contentIndent === 0) {
+			let scanAhead = pos;
+			while (scanAhead < text.length) {
+				let spaces = 0;
+				while (scanAhead < text.length && text[scanAhead] === " ") {
+					spaces++;
+					scanAhead++;
+				}
+				if (spaces === 0 && scanAhead + 2 < text.length) {
+					const sc0 = text[scanAhead];
+					const sc1 = text[scanAhead + 1];
+					const sc2 = text[scanAhead + 2];
+					const sc3 = scanAhead + 3 < text.length ? text[scanAhead + 3] : "";
+					if ((sc0 === "-" && sc1 === "-" && sc2 === "-" || sc0 === "." && sc1 === "." && sc2 === ".") && (sc3 === "" || sc3 === " " || sc3 === "	" || sc3 === "\n" || sc3 === "\r")) break;
+				}
+				if (scanAhead >= text.length || text[scanAhead] === "\n" || text[scanAhead] === "\r") {
+					if (scanAhead < text.length) {
+						scanAhead++;
+						if (text[scanAhead - 1] === "\r" && scanAhead < text.length && text[scanAhead] === "\n") scanAhead++;
+					}
+					continue;
+				}
+				contentIndent = spaces;
+				foundContent = true;
+				break;
+			}
+		}
+		if (!foundContent || contentIndent === 0 && blockStarted.size > 0) {
+			if (chomp === "keep") {
+				let count = 0;
+				while (pos < text.length) {
+					while (pos < text.length && (text[pos] === " " || text[pos] === "	")) {
+						pos++;
+						col++;
+					}
+					if (pos >= text.length) {
+						if (count === 0) count = 1;
+						break;
+					}
+					if (text[pos] === "\n") {
+						count++;
+						pos++;
+						line++;
+						col = 0;
+					} else if (text[pos] === "\r") {
+						count++;
+						pos++;
+						if (pos < text.length && text[pos] === "\n") pos++;
+						line++;
+						col = 0;
+					} else break;
+				}
+				return makeToken("scalar", "\n".repeat(count), start, sLine, sCol, pos - start);
+			}
+			afterEmptyBlockScalar = true;
+			return makeToken("scalar", "", start, sLine, sCol, pos - start);
+		}
+		const lines = [];
+		const trailingNewlines = [];
+		while (pos < text.length) {
+			let spaces = 0;
+			const lineStart = pos;
+			while (pos < text.length && text[pos] === " ") {
+				spaces++;
+				pos++;
+				col++;
+			}
+			if (spaces === 0 && pos + 2 < text.length) {
+				const c0 = text[pos];
+				const c1 = text[pos + 1];
+				const c2 = text[pos + 2];
+				const c3 = pos + 3 < text.length ? text[pos + 3] : "";
+				if ((c0 === "-" && c1 === "-" && c2 === "-" || c0 === "." && c1 === "." && c2 === ".") && (c3 === "" || c3 === " " || c3 === "	" || c3 === "\n" || c3 === "\r")) {
+					pos = lineStart;
+					col = 0;
+					break;
+				}
+			}
+			if (pos >= text.length || text[pos] === "\n" || text[pos] === "\r") {
+				if (spaces > contentIndent) {
+					for (const nl of trailingNewlines) lines.push(nl);
+					trailingNewlines.length = 0;
+					lines.push(" ".repeat(spaces - contentIndent));
+				} else trailingNewlines.push("");
+				if (pos < text.length) {
+					if (text[pos] === "\r" && pos + 1 < text.length && text[pos + 1] === "\n") pos += 2;
+					else pos++;
+					line++;
+					col = 0;
+				}
+				continue;
+			}
+			if (spaces < contentIndent) {
+				pos = lineStart;
+				col = 0;
+				break;
+			}
+			for (const nl of trailingNewlines) lines.push(nl);
+			trailingNewlines.length = 0;
+			const extra = " ".repeat(spaces - contentIndent);
+			const contentStart = pos;
+			while (pos < text.length && text[pos] !== "\n" && text[pos] !== "\r") {
+				pos++;
+				col++;
+			}
+			lines.push(extra + text.slice(contentStart, pos));
+			if (pos < text.length) {
+				if (text[pos] === "\r" && pos + 1 < text.length && text[pos + 1] === "\n") pos += 2;
+				else pos++;
+				line++;
+				col = 0;
+			}
+		}
+		let value;
+		if (isFolded) {
+			let result = "";
+			let prevMoreIndented = false;
+			let hadContent = false;
+			for (let i = 0; i < lines.length; i++) {
+				const ln = lines[i] ?? "";
+				const isMoreIndented = ln.length > 0 && (ln[0] === " " || ln[0] === "	");
+				if (ln === "") result += "\n";
+				else if (!hadContent) {
+					result += ln;
+					prevMoreIndented = isMoreIndented;
+					hadContent = true;
+				} else {
+					if (result[result.length - 1] === "\n") if (isMoreIndented || prevMoreIndented) result += `\n${ln}`;
+					else result += ln;
+					else if (isMoreIndented || prevMoreIndented) result += `\n${ln}`;
+					else result += ` ${ln}`;
+					prevMoreIndented = isMoreIndented;
+				}
+			}
+			if (chomp === "strip") {} else if (chomp === "keep") {
+				result += "\n";
+				for (const _nl of trailingNewlines) result += "\n";
+			} else result += "\n";
+			value = result;
+		} else {
+			value = lines.join("\n");
+			if (chomp === "strip") {} else if (chomp === "keep") {
+				value += "\n";
+				for (const _nl of trailingNewlines) value += "\n";
+			} else value += "\n";
+		}
+		return makeToken("scalar", value, start, sLine, sCol, pos - start);
+	}
+	function scanAnchor() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		advance();
+		const nameStart = pos;
+		while (pos < text.length && !isWhitespace(peek()) && !isNewline(peek()) && !isFlowIndicator(peek())) advance();
+		const name = text.slice(nameStart, pos);
+		if (name.length === 0) return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+		return makeToken("anchor", name, start, sLine, sCol, pos - start);
+	}
+	function scanAlias() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		advance();
+		const nameStart = pos;
+		while (pos < text.length && !isWhitespace(peek()) && !isNewline(peek()) && !isFlowIndicator(peek())) advance();
+		const name = text.slice(nameStart, pos);
+		if (name.length === 0) return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+		return makeToken("alias", name, start, sLine, sCol, pos - start);
+	}
+	function scanTag() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		advance();
+		if (peek() === "<") {
+			advance();
+			while (pos < text.length && peek() !== ">") {
+				if (isNewline(peek())) return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+				advance();
+			}
+			if (peek() === ">") advance();
+			else return makeToken("error", text.slice(start, pos), start, sLine, sCol);
+			return makeToken("tag", text.slice(start, pos), start, sLine, sCol);
+		}
+		if (peek() === "!") advance();
+		while (pos < text.length && !isWhitespace(peek()) && !isNewline(peek()) && !isFlowIndicator(peek())) advance();
+		return makeToken("tag", text.slice(start, pos), start, sLine, sCol);
+	}
+	function scanDirective() {
+		lockLineIndent();
+		const start = pos;
+		const sLine = line;
+		const sCol = col;
+		while (pos < text.length && !isNewline(peek())) advance();
+		return makeToken("directive", text.slice(start, pos), start, sLine, sCol);
+	}
+	function scanNext() {
+		if (pending.length > 0) {
+			const next = pending.shift();
+			if (next !== void 0) return next;
+		}
+		if (pos >= text.length) return null;
+		const prevWasQuoted = afterQuotedScalar;
+		const prevWasFlowEnd = afterFlowCollectionEnd;
+		if (flowDepth > 0) {
+			const ch0 = peek();
+			const isComment = ch0 === "#" && (pos === 0 || col === 0 || text[pos - 1] === " " || text[pos - 1] === "	" || text[pos - 1] === "\n" || text[pos - 1] === "\r");
+			if (!isWhitespace(ch0) && !isNewline(ch0) && !isComment) {
+				afterQuotedScalar = false;
+				afterFlowCollectionEnd = false;
+			}
+		} else {
+			afterQuotedScalar = false;
+			afterFlowCollectionEnd = false;
+		}
+		const ch = peek();
+		if (ch === "") {
+			const start = pos;
+			const sLine = line;
+			const sCol = col;
+			advance();
+			return makeToken("byte-order-mark", "", start, sLine, sCol);
+		}
+		if (isNewline(ch)) return scanNewline();
+		if (isWhitespace(ch)) return scanWhitespace();
+		afterEmptyBlockScalar = false;
+		if (ch === "#") {
+			if (pos === 0 || col === 0) return scanComment();
+			const prev = text[pos - 1];
+			if (prev === " " || prev === "	" || prev === "\n" || prev === "\r") return scanComment();
+			if (prevWasQuoted) {
+				const start = pos;
+				const sLine = line;
+				const sCol = col;
+				advance();
+				return makeToken("error", "#", start, sLine, sCol);
+			}
+		}
+		if (col === 0 && ch === "-" && peek(1) === "-" && peek(2) === "-") {
+			const marker = scanDocumentStartOrEnd();
+			if (marker) return marker;
+		}
+		if (col === 0 && ch === "." && peek(1) === "." && peek(2) === ".") {
+			const marker = scanDocumentStartOrEnd();
+			if (marker) return marker;
+		}
+		if (ch === "%" && col === 0 && flowDepth === 0) return scanDirective();
+		if ((ch === "|" || ch === ">") && flowDepth === 0) {
+			const next = peek(1);
+			if (next === "" || isNewline(next) || isWhitespace(next) || next === "-" || next === "+" || next >= "0" && next <= "9" || next === "#") return scanBlockScalar();
+		}
+		if (ch === "{") {
+			lockLineIndent();
+			const start = pos;
+			const sLine = line;
+			const sCol = col;
+			advance();
+			flowDepth++;
+			return makeToken("flow-map-start", "{", start, sLine, sCol);
+		}
+		if (ch === "}") {
+			lockLineIndent();
+			const start = pos;
+			const sLine = line;
+			const sCol = col;
+			advance();
+			if (flowDepth > 0) flowDepth--;
+			afterFlowCollectionEnd = true;
+			return makeToken("flow-map-end", "}", start, sLine, sCol);
+		}
+		if (ch === "[") {
+			lockLineIndent();
+			const start = pos;
+			const sLine = line;
+			const sCol = col;
+			advance();
+			flowDepth++;
+			return makeToken("flow-seq-start", "[", start, sLine, sCol);
+		}
+		if (ch === "]") {
+			lockLineIndent();
+			const start = pos;
+			const sLine = line;
+			const sCol = col;
+			advance();
+			if (flowDepth > 0) flowDepth--;
+			afterFlowCollectionEnd = true;
+			return makeToken("flow-seq-end", "]", start, sLine, sCol);
+		}
+		if (ch === ",") {
+			lockLineIndent();
+			const start = pos;
+			const sLine = line;
+			const sCol = col;
+			advance();
+			if (flowDepth === 0) return makeToken("error", ",", start, sLine, sCol);
+			return makeToken("flow-separator", ",", start, sLine, sCol);
+		}
+		if (ch === "&") return scanAnchor();
+		if (ch === "*") return scanAlias();
+		if (ch === "!") return scanTag();
+		if (ch === "?" && (isWhitespace(peek(1)) || isNewline(peek(1)) || peek(1) === "")) {
+			lockLineIndent();
+			const indent = lineIndent;
+			const start = pos;
+			const sLine = line;
+			const sCol = col;
+			advance();
+			if (flowDepth === 0) {
+				let lk = pos;
+				let hasTab = false;
+				let firstTabPos = -1;
+				while (lk < text.length && (text[lk] === " " || text[lk] === "	")) {
+					if (text[lk] === "	" && !hasTab) {
+						hasTab = true;
+						firstTabPos = lk;
+					}
+					lk++;
+				}
+				if (hasTab && lk < text.length && text[lk] !== "\n" && text[lk] !== "\r") {
+					while (pos < firstTabPos) advance();
+					const tabStart = pos;
+					const tabLine = line;
+					const tabCol = col;
+					advance();
+					ensureBlockMap(indent, start, sLine, sCol);
+					pending.push(makeToken("block-map-key", "?", start, sLine, sCol));
+					pending.push(makeToken("error", "	", tabStart, tabLine, tabCol));
+					const first = pending.shift();
+					if (first !== void 0) return first;
+					return makeToken("block-map-key", "?", start, sLine, sCol);
+				}
+			}
+			ensureBlockMap(indent, start, sLine, sCol);
+			if (pending.length > 0) {
+				pending.push(makeToken("block-map-key", "?", start, sLine, sCol));
+				const first = pending.shift();
+				if (first !== void 0) return first;
+			}
+			return makeToken("block-map-key", "?", start, sLine, sCol);
+		}
+		if (ch === "-" && (isWhitespace(peek(1)) || isNewline(peek(1)) || peek(1) === "") && flowDepth === 0) {
+			lockLineIndent();
+			const indent = lineIndent;
+			const start = pos;
+			const sLine = line;
+			const sCol = col;
+			advance();
+			{
+				let lk = pos;
+				let hasTab = false;
+				let firstTabPos = -1;
+				while (lk < text.length && (text[lk] === " " || text[lk] === "	")) {
+					if (text[lk] === "	" && !hasTab) {
+						hasTab = true;
+						firstTabPos = lk;
+					}
+					lk++;
+				}
+				if (hasTab && lk < text.length) {
+					const nc = text[lk];
+					const afterNc = lk + 1 < text.length ? text[lk + 1] : "";
+					if (nc === "-" && (afterNc === " " || afterNc === "	" || afterNc === "\n" || afterNc === "\r" || afterNc === "") || nc === "?" && (afterNc === " " || afterNc === "	" || afterNc === "\n" || afterNc === "\r" || afterNc === "")) {
+						while (pos < firstTabPos) advance();
+						const tabStart = pos;
+						const tabLine = line;
+						const tabCol = col;
+						advance();
+						ensureBlockSeq(indent, start, sLine, sCol);
+						pending.push(makeToken("block-seq-entry", "-", start, sLine, sCol));
+						pending.push(makeToken("error", "	", tabStart, tabLine, tabCol));
+						const first = pending.shift();
+						if (first !== void 0) return first;
+						return makeToken("block-seq-entry", "-", start, sLine, sCol);
+					}
+				}
+			}
+			ensureBlockSeq(indent, start, sLine, sCol);
+			if (pending.length > 0) {
+				pending.push(makeToken("block-seq-entry", "-", start, sLine, sCol));
+				const first = pending.shift();
+				if (first !== void 0) return first;
+			}
+			return makeToken("block-seq-entry", "-", start, sLine, sCol);
+		}
+		if (ch === ":" && (isWhitespace(peek(1)) || isNewline(peek(1)) || peek(1) === "" || flowDepth > 0 && isFlowIndicator(peek(1)) || flowDepth > 0 && prevWasQuoted || flowDepth > 0 && prevWasFlowEnd)) {
+			lockLineIndent();
+			const indent = lineIndent;
+			const start = pos;
+			const sLine = line;
+			const sCol = col;
+			advance();
+			if (flowDepth === 0) {
+				let lk = pos;
+				let hasTab = false;
+				let firstTabPos = -1;
+				while (lk < text.length && (text[lk] === " " || text[lk] === "	")) {
+					if (text[lk] === "	" && !hasTab) {
+						hasTab = true;
+						firstTabPos = lk;
+					}
+					lk++;
+				}
+				if (hasTab && lk < text.length) {
+					const nc = text[lk];
+					const afterNc = lk + 1 < text.length ? text[lk + 1] : "";
+					if (nc === "-" && (afterNc === " " || afterNc === "	" || afterNc === "\n" || afterNc === "\r" || afterNc === "") || nc === "?" && (afterNc === " " || afterNc === "	" || afterNc === "\n" || afterNc === "\r" || afterNc === "")) {
+						while (pos < firstTabPos) advance();
+						const tabStart = pos;
+						const tabLine = line;
+						const tabCol = col;
+						advance();
+						ensureBlockMap(indent, start, sLine, sCol);
+						pending.push(makeToken("block-map-value", ":", start, sLine, sCol));
+						pending.push(makeToken("error", "	", tabStart, tabLine, tabCol));
+						const first = pending.shift();
+						if (first !== void 0) return first;
+						return makeToken("block-map-value", ":", start, sLine, sCol);
+					}
+				}
+			}
+			ensureBlockMap(indent, start, sLine, sCol);
+			if (pending.length > 0) {
+				pending.push(makeToken("block-map-value", ":", start, sLine, sCol));
+				const first = pending.shift();
+				if (first !== void 0) return first;
+			}
+			return makeToken("block-map-value", ":", start, sLine, sCol);
+		}
+		if (ch === "'") {
+			afterQuotedScalar = true;
+			return scanSingleQuotedScalar();
+		}
+		if (ch === "\"") {
+			afterQuotedScalar = true;
+			return scanDoubleQuotedScalar();
+		}
+		afterQuotedScalar = false;
+		return scanPlainScalar();
+	}
+	return {
+		scan() {
+			const token = scanNext();
+			state.currentToken = token;
+			return token === null ? null : token.kind;
+		},
+		getToken() {
+			return state.currentToken === null ? null : state.currentToken.kind;
+		},
+		getTokenValue() {
+			return state.currentToken === null ? "" : state.currentToken.value;
+		},
+		getTokenOffset() {
+			return state.currentToken === null ? 0 : state.currentToken.offset;
+		},
+		getTokenLength() {
+			return state.currentToken === null ? 0 : state.currentToken.length;
+		},
+		getTokenLine() {
+			return state.currentToken === null ? 0 : state.currentToken.line;
+		},
+		getTokenColumn() {
+			return state.currentToken === null ? 0 : state.currentToken.column;
+		},
+		getPosition() {
+			return pos;
+		},
+		setPosition(newPos) {
+			pos = 0;
+			line = 0;
+			col = 0;
+			while (pos < newPos && pos < text.length) {
+				if (text[pos] === "\n") {
+					line++;
+					col = 0;
+				} else col++;
+				pos++;
+			}
+			lineIndent = 0;
+			lineIndentLocked = false;
+			flowDepth = 0;
+			blockStarted.clear();
+			pending.length = 0;
+			afterEmptyBlockScalar = false;
+			afterQuotedScalar = false;
+			afterFlowCollectionEnd = false;
+			state.currentToken = null;
+		}
+	};
+}
+/**
+* Tokenize a YAML source string into an array of {@link YamlToken} records.
+*
+* Lexer errors are embedded as `"error"` tokens in the result; the downstream
+* parser/composer collects them and records diagnostics as needed.
+*/
+function lexAll(text) {
+	const scanner = createScanner(text);
+	const tokens = [];
+	let kind = scanner.scan();
+	while (kind !== null) {
+		tokens.push({
+			kind,
+			value: scanner.getTokenValue(),
+			offset: scanner.getTokenOffset(),
+			length: scanner.getTokenLength(),
+			line: scanner.getTokenLine(),
+			column: scanner.getTokenColumn()
+		});
+		kind = scanner.scan();
+	}
+	return tokens;
+}
+
+//#endregion
+export { createScanner, lexAll };