@mrhenry/twig-tokenizer 0.1.0

package/src/lexer.js

@@ -0,0 +1,1766 @@
+// @ts-check
+/**
+ * The {@link Lexer} and {@link Source} classes.
+ *
+ * Mirrors `src/Lexer.php` and `src/Source.php` of the reference
+ * implementation. See `spec/02-lexical-structure.md`.
+ *
+ * @module twig-tokenizer
+ */
+import { Token, TokenType } from './token.js';
+import { TokenStream } from './token-stream.js';
+import { SyntaxError } from './errors.js';
+import { OPERATORS } from './operators.js';
+import {
+	APOSTROPHE,
+	CARRIAGE_RETURN,
+	FULL_STOP,
+	HYPHEN_MINUS,
+	LATIN_CAPITAL_LETTER_E,
+	LATIN_SMALL_LETTER_E,
+	LEFT_CURLY_BRACKET,
+	LINE_FEED,
+	LOW_LINE,
+	NUMBER_SIGN,
+	PLUS_SIGN,
+	QUOTATION_MARK,
+	REVERSE_SOLIDUS,
+	RIGHT_CURLY_BRACKET,
+	VERTICAL_LINE,
+} from './code-points/code-points.js';
+import {
+	isAsciiLetterCodePoint,
+	isDigitCodePoint,
+	isHexDigitCodePoint,
+	isNameCodePoint,
+	isNameStartCodePoint,
+	isOctalDigitCodePoint,
+	isOperatorDelimiterCodePoint,
+	isPhpTrimCodePoint,
+	isWhitespaceCodePoint,
+} from './code-points/ranges.js';
+
+/**
+ * Holds information about a non-compiled Twig template.
+ */
+export class Source {
+	/**
+	 * @param {string} code The template source code.
+	 * @param {string} name The template logical name.
+	 * @param {string} [path] The filesystem path of the template, if any.
+	 */
+	constructor(code, name, path = '') {
+		/** @type {string} */
+		this.code = code;
+		/** @type {string} */
+		this.name = name;
+		/** @type {string} */
+		this.path = path;
+	}
+
+	/**
+	 * @returns {string} The source code.
+	 */
+	getCode() {
+		return this.code;
+	}
+
+	/**
+	 * @returns {string} The template name.
+	 */
+	getName() {
+		return this.name;
+	}
+
+	/**
+	 * @returns {string} The template path.
+	 */
+	getPath() {
+		return this.path;
+	}
+
+	/**
+	 * Returns the 1-based column for a 0-based offset in the source code.
+	 *
+	 * Mirrors the byte-oriented `Source::getColumn()` of the reference
+	 * implementation: the column counts UTF-8 bytes since the last newline.
+	 *
+	 * @param {number} offset A negative offset means the position is unknown.
+	 * @returns {number|null} The 1-based column, or null.
+	 */
+	getColumn(offset) {
+		if (offset < 0) {
+			return null;
+		}
+		const before = this.code.slice(0, offset).replace(/\r\n|\r/g, '\n');
+		const lineStart = before.lastIndexOf('\n');
+		const prefix = lineStart === -1 ? before : before.slice(lineStart + 1);
+		return new TextEncoder().encode(prefix).length + 1;
+	}
+}
+
+/**
+ * Lexer options.
+ *
+ * @typedef {object} LexerOptions
+ * @property {string[]} [tag_comment]
+ * @property {string[]} [tag_block]
+ * @property {string[]} [tag_variable]
+ * @property {string} [whitespace_trim]
+ * @property {string} [whitespace_line_trim]
+ * @property {string} [whitespace_line_chars]
+ * @property {string[]} [interpolation]
+ * @property {string} [punctuation]
+ */
+
+/**
+ * The default lexer configuration, mirroring `Twig\Lexer`'s constructor
+ * defaults (see `spec/01-introduction.md` §1.2 and `spec/02-lexical-structure.md`).
+ *
+ * Exposed so tooling (grammar/config conformance tests) can read the canonical
+ * delimiters, whitespace-control markers and interpolation markers.
+ *
+ */
+export const DEFAULT_LEXER_OPTIONS = Object.freeze({
+	tag_comment: ['{#', '#}'],
+	tag_block: ['{%', '%}'],
+	tag_variable: ['{{', '}}'],
+	whitespace_trim: '-',
+	whitespace_line_trim: '~',
+	whitespace_line_chars: ' \t\0\x0B',
+	interpolation: ['#{', '}'],
+	punctuation: '',
+});
+
+/**
+ * Lexer state constants.
+ *
+ * @readonly
+ * @enum {number}
+ */
+export const LexerState = {
+	DATA: 0,
+	BLOCK: 1,
+	VARIABLE: 2,
+	STRING: 3,
+	INTERPOLATION: 4,
+};
+
+/** The word `verbatim`, as a literal string. */
+const VERBATIM = 'verbatim';
+
+/** The word `endverbatim`, as a literal string. */
+const ENDVERBATIM = 'endverbatim';
+
+/** The word `line`, as a literal string. */
+const LINE = 'line';
+
+/**
+ * Operators, matched longest-first like the reference implementation's
+ * operator regular expression (which sorts its alternatives by length).
+ * These tables are built once at module load, never per lexer instance.
+ */
+const OPERATORS_SORTED = [...OPERATORS].sort((a, b) => b.length - a.length);
+
+/** @type {Map<number, string[]>} Operators grouped by their first code unit. */
+const OPERATORS_BY_FIRST_CHAR = new Map();
+for (const operator of OPERATORS_SORTED) {
+	const first = operator.charCodeAt(0);
+	let list = OPERATORS_BY_FIRST_CHAR.get(first);
+	if (list === undefined) {
+		list = [];
+		OPERATORS_BY_FIRST_CHAR.set(first, list);
+	}
+	list.push(operator);
+}
+
+/** The few operators containing whitespace must use the whitespace-aware matcher. */
+const OPERATORS_WITH_WHITESPACE = new Set(
+	OPERATORS_SORTED.filter((operator) => operator.includes(' ')),
+);
+
+/** @type {string[]} */
+const OPENING_BRACKETS = ['{', '(', '['];
+/** @type {string[]} */
+const CLOSING_BRACKETS = ['}', ')', ']'];
+
+/** Default punctuation characters. */
+const PUNCTUATION = '()[]{}?:.,|';
+
+/**
+ * Whether a code unit is one of the default punctuation characters.
+ *
+ * @param {number} c The code unit to test.
+ * @returns {boolean}
+ */
+function isDefaultPunctuation(c) {
+	return PUNCTUATION.includes(String.fromCharCode(c));
+}
+
+/**
+ * Whether a code unit is one of the line-trim characters.
+ *
+ * Mirrors `whitespace_line_chars` (`" \t\0\x0B"` by default), which is used
+ * after a `~` trim marker and by `rtrim($text, " \t\0\x0B")`.
+ *
+ * @param {number} search The code unit to test.
+ * @param {string} chars The configured line-trim characters.
+ * @returns {boolean}
+ */
+function isLineTrimCodeUnit(search, chars) {
+	return chars.includes(String.fromCharCode(search));
+}
+
+/**
+ * Matches an operator literal at `cursor`, allowing whitespace inside the
+ * operator to match one or more whitespace code points.
+ *
+ * @param {string} code The template source.
+ * @param {number} cursor The offset to match at.
+ * @param {string} operator The operator literal.
+ * @returns {number} The offset just past the match, or -1.
+ */
+function matchOperatorText(code, cursor, operator) {
+	let i = cursor;
+	let j = 0;
+	const len = operator.length;
+
+	while (j < len) {
+		const c = operator.charCodeAt(j);
+		if (isWhitespaceCodePoint(c)) {
+			if (i >= code.length || !isWhitespaceCodePoint(code.charCodeAt(i))) {
+				return -1;
+			}
+			while (i < code.length && isWhitespaceCodePoint(code.charCodeAt(i))) {
+				i += 1;
+			}
+			j += 1;
+		} else {
+			if (code.charCodeAt(i) !== c) {
+				return -1;
+			}
+			i += 1;
+			j += 1;
+		}
+	}
+
+	return i;
+}
+
+/**
+ * Whether the two code points immediately before `cursor` look like a dot or
+ * pipe preceding the current position.
+ *
+ * Mirrors the `(?<![.|][\s]|.[.|])` lookbehind of the reference
+ * implementation.
+ *
+ * @param {string} code The template source.
+ * @param {number} cursor The offset of the operator start.
+ * @returns {boolean}
+ */
+function hasDotOrPipeBefore(code, cursor) {
+	if (cursor < 2) {
+		return false;
+	}
+	const a = code.charCodeAt(cursor - 2);
+	const b = code.charCodeAt(cursor - 1);
+	const aIsDotOrPipe = a === FULL_STOP || a === VERTICAL_LINE;
+	const bIsDotOrPipe = b === FULL_STOP || b === VERTICAL_LINE;
+	if (aIsDotOrPipe && isWhitespaceCodePoint(b)) {
+		return true;
+	}
+	return bIsDotOrPipe;
+}
+
+/**
+ * Trims trailing code units from `value` that satisfy `predicate`.
+ *
+ * @param {string} value
+ * @param {(search: number) => boolean} predicate
+ * @returns {string}
+ */
+function trimEnd(value, predicate) {
+	let end = value.length;
+	while (end > 0 && predicate(value.charCodeAt(end - 1))) {
+		end -= 1;
+	}
+	return value.slice(0, end);
+}
+
+/**
+ * PHP `trim()` with the default character list (`" \t\n\r\v\x00"`).
+ *
+ * PHP's default trim does *not* remove the form feed (`\f`); JS `String#trim()`
+ * does. Keeping the byte-identical behavior matters for documentation-comment
+ * attachment and trimming.
+ *
+ * @param {string} value
+ * @returns {string}
+ */
+function trimLikePhp(value) {
+	let start = 0;
+	let end = value.length;
+	while (start < end && isPhpTrimCodePoint(value.charCodeAt(start))) {
+		start += 1;
+	}
+	while (end > start && isPhpTrimCodePoint(value.charCodeAt(end - 1))) {
+		end -= 1;
+	}
+	return value.slice(start, end);
+}
+
+/**
+ * Normalizes `\r\n` and `\r` to `\n`.
+ *
+ * @param {string} text
+ * @returns {string}
+ */
+function normalizeNewlines(text) {
+	if (!text.includes('\r')) {
+		return text;
+	}
+	let result = '';
+	let i = 0;
+	const len = text.length;
+	while (i < len) {
+		const c = text.charCodeAt(i);
+		if (c === CARRIAGE_RETURN) {
+			if (text.charCodeAt(i + 1) === LINE_FEED) {
+				i += 1;
+			}
+			result += '\n';
+		} else {
+			result += text[i];
+		}
+		i += 1;
+	}
+	return result;
+}
+
+/**
+ * Processes escape sequences in a string fragment (PHP `stripcslashes`).
+ *
+ * @param {string} string The raw string content.
+ * @param {string} _quoteType The string delimiter (`'` or `"`).
+ * @returns {string} The unescaped string.
+ */
+function stripcslashes(string, _quoteType) {
+	let result = '';
+	let i = 0;
+	const length = string.length;
+
+	while (i < length) {
+		const position = string.indexOf('\\', i);
+		if (position === -1) {
+			result += string.slice(i);
+			break;
+		}
+		result += string.slice(i, position);
+		i = position + 1;
+
+		if (i >= length) {
+			result += '\\';
+			break;
+		}
+
+		const nextChar = string[i];
+		if (nextChar === 'f') {
+			result += '\f';
+		} else if (nextChar === 'n') {
+			result += '\n';
+		} else if (nextChar === 'r') {
+			result += '\r';
+		} else if (nextChar === 't') {
+			result += '\t';
+		} else if (nextChar === 'v') {
+			result += '\v';
+		} else if (nextChar === '\\') {
+			result += nextChar;
+		} else if (nextChar === "'" || nextChar === '"') {
+			result += nextChar;
+		} else if (nextChar === '#' && i + 1 < length && string[i + 1] === '{') {
+			result += '#{';
+			i += 1;
+		} else if (nextChar === 'x' && i + 1 < length && isHexDigitCodePoint(string.charCodeAt(i + 1))) {
+			let hexadecimal = string[i + 1];
+			i += 1;
+			if (i + 1 < length && isHexDigitCodePoint(string.charCodeAt(i + 1))) {
+				hexadecimal += string[i + 1];
+				i += 1;
+			}
+			result += String.fromCharCode(parseInt(hexadecimal, 16));
+		} else if (isOctalDigitCodePoint(nextChar.charCodeAt(0))) {
+			let octal = nextChar;
+			while (i + 1 < length && isOctalDigitCodePoint(string.charCodeAt(i + 1)) && octal.length < 3) {
+				octal += string[i + 1];
+				i += 1;
+			}
+			result += String.fromCharCode(parseInt(octal, 8) % 256);
+		} else {
+			result += nextChar;
+		}
+
+		i += 1;
+	}
+
+	return result;
+}
+
+/**
+ * Reads a numeric literal from `code[start, end)`, skipping underscores, and
+ * converts it to a number.
+ *
+ * Mirrors `0 + str_replace('_', '', $match[0])` of the reference
+ * implementation.
+ *
+ * @param {string} code The template source.
+ * @param {number} start The literal start.
+ * @param {number} end The literal end.
+ * @returns {number}
+ */
+function numberValue(code, start, end) {
+	let result = '';
+	for (let i = start; i < end; i += 1) {
+		const c = code.charCodeAt(i);
+		if (c !== LOW_LINE) {
+			result += code[i];
+		}
+	}
+	return Number(result);
+}
+
+/**
+ * Consumes zero or more `_` + digit groups starting at `i`.
+ *
+ * @param {string} code The template source.
+ * @param {number} i The offset to start from.
+ * @param {number} len The code length.
+ * @returns {number} The offset after the consumed groups.
+ */
+function consumeUnderscoreGroups(code, i, len) {
+	while (
+		i + 1 < len &&
+		code.charCodeAt(i) === LOW_LINE &&
+		isDigitCodePoint(code.charCodeAt(i + 1))
+	) {
+		i += 2;
+		while (i < len && isDigitCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+	}
+	return i;
+}
+
+/**
+ * The Twig lexer: tokenizes template source into a {@link TokenStream}.
+ *
+ * Operates in states: data (static text), block (`{%`), variable (`{{`),
+ * string, and interpolation. Lexing is a single forward pass over the code
+ * points of the source; no intermediate token-start table is built.
+ */
+export class Lexer {
+	/**
+	 * @param {LexerOptions} [options] Lexer options.
+	 */
+	constructor(options = {}) {
+		/** @type {LexerOptions & {tag_comment: string[], tag_block: string[], tag_variable: string[], whitespace_trim: string, whitespace_line_trim: string, whitespace_line_chars: string, interpolation: string[]}} */
+		this.options = {
+			...DEFAULT_LEXER_OPTIONS,
+			...options,
+		};
+
+		const o = this.options;
+
+		// delimiters: distinct first characters of every tag/comment start; when
+		// they all share a single character, the data scan can jump straight to
+		// its next occurrence instead of peeking at every code unit
+		const candidates = [
+			o.tag_variable[0],
+			o.tag_block[0],
+			o.tag_comment[0] + '#',
+			o.tag_comment[0],
+		];
+		const firstChars = [...new Set(candidates.map((c) => c[0]))];
+		/** @type {string} */
+		this.delimiterFirstChars = firstChars.join('');
+		/** @type {string|null} */
+		this.delimiterFirstChar = firstChars.length === 1 ? firstChars[0] : null;
+
+		/** The documentation-comment start marker (`{#` + `#`). */
+		this.documentationStart = o.tag_comment[0] + '#';
+
+		/** Keeps the empty comment (`{##}`) lexing as a regular comment. */
+		this.emptyCommentLookahead = o.tag_comment[1].startsWith('#')
+			? o.tag_comment[1].slice(1)
+			: null;
+
+		// comment closing markers, precomputed once: `[marker, kind]` where kind
+		// is 0 (followed by whitespace), 1 (followed by line-trim chars) or
+		// 2 (plain, optionally followed by a newline)
+		/** @type {Array<[string, number]>} */
+		this.commentMarkers = [
+			[o.whitespace_trim + o.tag_comment[1], 0],
+			[o.whitespace_line_trim + o.tag_comment[1], 1],
+			[o.tag_comment[1], 2],
+		];
+		/** @type {Array<[string, number]>} */
+		this.docCommentMarkers = [
+			[o.whitespace_trim + '#' + o.tag_comment[1], 0],
+			[o.whitespace_line_trim + '#' + o.tag_comment[1], 1],
+			[o.whitespace_trim + o.tag_comment[1], 0],
+			[o.whitespace_line_trim + o.tag_comment[1], 1],
+			[o.tag_comment[1], 2],
+		];
+		/** @type {number[]} First code units of every comment closing marker. */
+		this.commentFirstChars = [
+			o.whitespace_trim.charCodeAt(0),
+			o.whitespace_line_trim.charCodeAt(0),
+			o.tag_comment[1].charCodeAt(0),
+		];
+
+		/** @type {Source|null} */
+		this.source = null;
+		/** @type {string} */
+		this.code = '';
+		/** @type {number} */
+		this.cursor = 0;
+		/** @type {number} */
+		this.lineNumber = 1;
+		/** @type {number} */
+		this.end = 0;
+		/** @type {number} */
+		this.state = LexerState.DATA;
+		/** @type {number} */
+		this.currentVariableBlockLine = 1;
+		/** @type {Token[]} */
+		this.tokens = [];
+		/** @type {number[]} */
+		this.states = [];
+		/** @type {Array<[string, number]>} */
+		this.brackets = [];
+		/** @type {string[]} */
+		this.documentation = [];
+		/** @type {number} End offset of the previously pushed token's raw text. */
+		this.lastRawEnd = 0;
+	}
+
+	/**
+	 * Tokenizes a template source.
+	 *
+	 * @param {Source} source The template source.
+	 * @returns {TokenStream} The resulting token stream.
+	 */
+	tokenize(source) {
+		this.source = source;
+		this.code = source.getCode();
+		this.cursor = 0;
+		this.lineNumber = 1;
+		this.end = this.code.length;
+		this.tokens.length = 0;
+		this.states.length = 0;
+		this.brackets.length = 0;
+		this.documentation.length = 0;
+		this.state = LexerState.DATA;
+		this.currentVariableBlockLine = 1;
+		this.lastRawEnd = 0;
+
+		while (this.cursor < this.end) {
+			switch (this.state) {
+				case LexerState.DATA:
+					this.lexData();
+					break;
+				case LexerState.BLOCK:
+					this.lexBlock();
+					break;
+				case LexerState.VARIABLE:
+					this.lexVariable();
+					break;
+				case LexerState.STRING:
+					this.lexString();
+					break;
+				case LexerState.INTERPOLATION:
+					this.lexInterpolation();
+					break;
+			}
+		}
+
+		this.pushToken(TokenType.EOF);
+
+		if (this.brackets.length) {
+			const [expect, lineNumber] = this.brackets[this.brackets.length - 1];
+			throw new SyntaxError(`Unclosed "${expect}".`, lineNumber, this.source);
+		}
+
+		return new TokenStream(this.tokens, this.source, this.options);
+	}
+
+	/**
+	 * Moves the cursor to `to`, counting the newlines crossed in the source.
+	 *
+	 * `\n`, `\r` and `\r\n` each count as a single newline, exactly like
+	 * `normalizeNewlines($text)` in the reference implementation. The scan is
+	 * bounded to `[cursor, to)` so each code unit is examined at most once over
+	 * the whole tokenize call.
+	 *
+	 * @param {number} to The new cursor position (absolute offset).
+	 */
+	advance(to) {
+		const code = this.code;
+		let count = 0;
+		for (let i = this.cursor; i < to; i += 1) {
+			const c = code.charCodeAt(i);
+			if (c === LINE_FEED) {
+				count += 1;
+			} else if (c === CARRIAGE_RETURN) {
+				count += 1;
+				if (code.charCodeAt(i + 1) === LINE_FEED) {
+					i += 1;
+				}
+			}
+		}
+		this.lineNumber += count;
+		this.cursor = to;
+	}
+
+	/**
+	 * Lexes data (static text) up to the next tag/comment start.
+	 *
+	 * The next delimiter is found by scanning forward on demand, so no
+	 * precomputed table of token starts is ever built.
+	 */
+	lexData() {
+		const code = this.code;
+		const o = this.options;
+		const first = this.delimiterFirstChar;
+
+		// find the next delimiter start
+		/** @type {number} */
+		let start;
+		if (first !== null) {
+			start = code.indexOf(first, this.cursor);
+		} else {
+			const firstChars = this.delimiterFirstChars;
+			let s = this.cursor;
+			while (s < this.end && firstChars.indexOf(code[s]) === -1) {
+				s += 1;
+			}
+			start = s < this.end ? s : -1;
+		}
+
+		let kind = 0;
+		let end = 0;
+		while (start !== -1) {
+			if (code.startsWith(o.tag_variable[0], start)) {
+				kind = 1;
+				end = start + o.tag_variable[0].length;
+			} else if (code.startsWith(o.tag_block[0], start)) {
+				kind = 2;
+				end = start + o.tag_block[0].length;
+			} else if (
+				code.startsWith(this.documentationStart, start) &&
+				(this.emptyCommentLookahead === null ||
+					!code.startsWith(this.emptyCommentLookahead, start + this.documentationStart.length))
+			) {
+				kind = 4;
+				end = start + this.documentationStart.length;
+			} else if (code.startsWith(o.tag_comment[0], start)) {
+				kind = 3;
+				end = start + o.tag_comment[0].length;
+			}
+			if (kind !== 0) {
+				break;
+			}
+			// not a delimiter; look for the next candidate
+			if (first !== null) {
+				start = code.indexOf(first, start + 1);
+			} else {
+				let s = start + 1;
+				while (s < this.end && this.delimiterFirstChars.indexOf(code[s]) === -1) {
+					s += 1;
+				}
+				start = s < this.end ? s : -1;
+			}
+		}
+
+		if (start === -1) {
+			// rest of the template is static text
+			const text = code.slice(this.cursor);
+			this.pushToken(TokenType.TEXT, normalizeNewlines(text), this.cursor, null, text);
+			this.advance(this.end);
+			return;
+		}
+
+		// whitespace-trim marker on the delimiter trims the preceding text
+		let trim = '';
+		if (code.startsWith(o.whitespace_trim, end)) {
+			trim = o.whitespace_trim;
+			end += o.whitespace_trim.length;
+		} else if (code.startsWith(o.whitespace_line_trim, end)) {
+			trim = o.whitespace_line_trim;
+			end += o.whitespace_line_trim.length;
+		}
+
+		// text before the delimiter
+		const rawText = code.slice(this.cursor, start);
+		let text = rawText;
+		if (trim) {
+			text = o.whitespace_trim === trim
+				? trimEnd(text, isPhpTrimCodePoint)
+				: trimEnd(text, (search) => isLineTrimCodeUnit(search, o.whitespace_line_chars));
+		}
+		this.pushToken(TokenType.TEXT, normalizeNewlines(text), this.cursor, null, rawText);
+		this.advance(start);
+		this.advance(end);
+
+		switch (kind) {
+			case 4:
+				this.lexComment(true);
+				break;
+			case 3:
+				this.lexComment();
+				break;
+			case 2:
+				this.lexBlockStart(start);
+				break;
+			default:
+				this.lexVariableStart(start);
+				break;
+		}
+	}
+
+	/** Handles a `{%` opening: verbatim, line directive, or block start. */
+	/**
+	 * @param {number} rawStart The offset where the opening delimiter starts.
+	 */
+	lexBlockStart(rawStart) {
+		const lineNumber = this.lineNumber;
+		const cursor = this.cursor;
+
+		// raw data (verbatim)?
+		if (this.matchBlockRaw()) {
+			this.documentation.length = 0;
+			this.lexRawData(rawStart);
+			return;
+		}
+
+		// {% line \d+ %}
+		const line = this.matchBlockLine();
+		if (line !== -1) {
+			this.documentation.length = 0;
+			this.lineNumber = line;
+			return;
+		}
+
+		this.pushToken(TokenType.BLOCK_START, '', cursor, lineNumber, this.code.slice(rawStart, cursor), rawStart);
+		this.pushState(LexerState.BLOCK);
+		this.currentVariableBlockLine = this.lineNumber;
+	}
+
+	/** Handles a `{{` opening. */
+	/**
+	 * @param {number} rawStart The offset where the opening delimiter starts.
+	 */
+	lexVariableStart(rawStart) {
+		const lineNumber = this.lineNumber;
+		const cursor = this.cursor;
+		this.pushToken(TokenType.VAR_START, '', cursor, lineNumber, this.code.slice(rawStart, cursor), rawStart);
+		this.pushState(LexerState.VARIABLE);
+		this.currentVariableBlockLine = this.lineNumber;
+	}
+
+	/** Lexes inside a block tag. */
+	lexBlock() {
+		if (!this.brackets.length) {
+			const end = this.matchBlockEnd();
+			if (end !== -1) {
+				this.pushClosingToken(TokenType.BLOCK_END, end);
+				this.popState();
+				return;
+			}
+		}
+		this.lexExpression();
+	}
+
+	/** Lexes inside a print statement. */
+	lexVariable() {
+		if (!this.brackets.length) {
+			const end = this.matchVariableEnd();
+			if (end !== -1) {
+				this.pushClosingToken(TokenType.VAR_END, end);
+				this.popState();
+				return;
+			}
+		}
+		this.lexExpression();
+	}
+
+	/** Lexes a single expression token. */
+	lexExpression() {
+		// whitespace
+		if (this.lexWhitespace() && this.cursor >= this.end) {
+			throw new SyntaxError(
+				`Unclosed "${LexerState.BLOCK === this.state ? 'block' : 'variable'}.`,
+				this.currentVariableBlockLine,
+				this.source,
+			);
+		}
+
+		// operators
+		if (this.lexOperator()) {
+			return;
+		}
+		// names
+		if (this.lexName()) {
+			return;
+		}
+		// numbers
+		if (this.lexNumber()) {
+			return;
+		}
+		// punctuation
+		if (this.lexPunctuation()) {
+			return;
+		}
+		// strings
+		if (this.lexQuotedString()) {
+			return;
+		}
+		// opening double-quoted string
+		if (this.lexStringStateOpen()) {
+			return;
+		}
+		// inline comment
+		if (this.lexInlineComment()) {
+			return;
+		}
+		// unlexable
+		throw new SyntaxError(
+			`Unexpected character "${this.code[this.cursor]}".`,
+			this.lineNumber,
+			this.source,
+			this.source ? this.source.getColumn(this.cursor) : null,
+		);
+	}
+
+	/**
+	 * Consumes one or more whitespace code points at the cursor.
+	 *
+	 * Mirrors the `\s+` (PCRE byte mode) regular expression.
+	 *
+	 * @returns {boolean} Whether whitespace was consumed.
+	 */
+	lexWhitespace() {
+		const code = this.code;
+		let i = this.cursor;
+		while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+		if (i === this.cursor) {
+			return false;
+		}
+		this.advance(i);
+		return true;
+	}
+
+	/**
+	 * Matches an operator at the cursor, pushing an OPERATOR token.
+	 *
+	 * Mirrors the `operator` regular expression of the reference
+	 * implementation: operators are tried longest-first; whitespace inside an
+	 * operator matches one or more whitespace code points; an operator ending
+	 * with a letter must be followed by a delimiter character; an operator
+	 * starting with a letter must not have a dot or pipe immediately before it.
+	 *
+	 * @returns {boolean} Whether an operator was matched.
+	 */
+	lexOperator() {
+		const code = this.code;
+		const cursor = this.cursor;
+
+		// peek: only try operators whose first code unit matches the cursor
+		const operators = OPERATORS_BY_FIRST_CHAR.get(code.charCodeAt(cursor));
+		if (operators === undefined) {
+			return false;
+		}
+
+		// fast path: a lone single-character operator always matches here (its
+		// first code unit equals the cursor's, and no letter rules apply)
+		if (operators.length === 1 && operators[0].length === 1) {
+			const operator = operators[0];
+			if (OPENING_BRACKETS.includes(operator)) {
+				this.checkBrackets(operator);
+			}
+			this.pushToken(TokenType.OPERATOR, operator, cursor, null, operator, cursor);
+			this.advance(cursor + 1);
+			return true;
+		}
+
+		for (const operator of operators) {
+			let end;
+			if (OPERATORS_WITH_WHITESPACE.has(operator)) {
+				end = matchOperatorText(code, cursor, operator);
+				if (end === -1) {
+					continue;
+				}
+			} else {
+				if (!code.startsWith(operator, cursor)) {
+					continue;
+				}
+				end = cursor + operator.length;
+			}
+
+			// an operator ending with a letter must be followed by a delimiter
+			const last = operator.charCodeAt(operator.length - 1);
+			if (
+				isAsciiLetterCodePoint(last) &&
+				!isOperatorDelimiterCodePoint(code.codePointAt(end) ?? -1)
+			) {
+				continue;
+			}
+
+			// an operator starting with a letter must not have a dot or pipe before
+			const first = operator.charCodeAt(0);
+			if (isAsciiLetterCodePoint(first) && hasDotOrPipeBefore(code, cursor)) {
+				continue;
+			}
+
+			if (OPENING_BRACKETS.includes(operator)) {
+				this.checkBrackets(operator);
+			}
+			this.pushToken(TokenType.OPERATOR, operator, null, null, code.slice(cursor, end), cursor);
+			this.advance(end);
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * Matches a name at the cursor, pushing a NAME token.
+	 *
+	 * Mirrors the byte-oriented `REGEX_NAME` of the reference implementation:
+	 * a letter, underscore or any code point from U+007F onward, followed by
+	 * the same set plus digits.
+	 *
+	 * @returns {boolean} Whether a name was matched.
+	 */
+	lexName() {
+		const code = this.code;
+		const len = this.end;
+		const start = this.cursor;
+		let i = start;
+
+		// ASCII fast path: letters and underscore are always name-start code points
+		const firstUnit = code.charCodeAt(i);
+		if (
+			(firstUnit >= 0x0041 && firstUnit <= 0x005a) ||
+			(firstUnit >= 0x0061 && firstUnit <= 0x007a) ||
+			firstUnit === LOW_LINE
+		) {
+			i += 1;
+		} else {
+			const first = code.codePointAt(i) ?? -1;
+			if (!isNameStartCodePoint(first)) {
+				return false;
+			}
+			i += 1 + +(first > 0xffff);
+		}
+
+		while (i < len) {
+			const unit = code.charCodeAt(i);
+			if (
+				(unit >= 0x0041 && unit <= 0x005a) ||
+				(unit >= 0x0061 && unit <= 0x007a) ||
+				(unit >= 0x0030 && unit <= 0x0039) ||
+				unit === LOW_LINE
+			) {
+				i += 1;
+				continue;
+			}
+			// a non-ASCII code point (or a surrogate half): decode and test it
+			const cp = code.codePointAt(i) ?? -1;
+			if (!isNameCodePoint(cp)) {
+				break;
+			}
+			i += 1 + +(cp > 0xffff);
+		}
+
+		this.pushToken(TokenType.NAME, code.slice(start, i), null, null, code.slice(start, i), start);
+		this.advance(i);
+		return true;
+	}
+
+	/**
+	 * Matches a number literal at the cursor, pushing a NUMBER token.
+	 *
+	 * Mirrors the `REGEX_NUMBER` of the reference implementation: an integer
+	 * part (with optional underscore-separated digit groups), an optional
+	 * fractional part and an optional exponent part. Each optional part is
+	 * only consumed when it is fully valid.
+	 *
+	 * @returns {boolean} Whether a number was matched.
+	 */
+	lexNumber() {
+		const code = this.code;
+		const len = this.end;
+		const start = this.cursor;
+		let i = start;
+
+		if (i >= len || !isDigitCodePoint(code.charCodeAt(i))) {
+			return false;
+		}
+		while (i < len && isDigitCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+		i = consumeUnderscoreGroups(code, i, len);
+
+		// fractional part: only when a digit follows the dot
+		if (
+			i + 1 < len &&
+			code.charCodeAt(i) === FULL_STOP &&
+			isDigitCodePoint(code.charCodeAt(i + 1))
+		) {
+			i += 1;
+			while (i < len && isDigitCodePoint(code.charCodeAt(i))) {
+				i += 1;
+			}
+			i = consumeUnderscoreGroups(code, i, len);
+		}
+
+		// exponent part: only when a (possibly signed) digit follows
+		const exponent = code.charCodeAt(i);
+		if (exponent === LATIN_SMALL_LETTER_E || exponent === LATIN_CAPITAL_LETTER_E) {
+			let j = i + 1;
+			const sign = code.charCodeAt(j);
+			if (sign === PLUS_SIGN || sign === HYPHEN_MINUS) {
+				j += 1;
+			}
+			if (j < len && isDigitCodePoint(code.charCodeAt(j))) {
+				i = j;
+				while (i < len && isDigitCodePoint(code.charCodeAt(i))) {
+					i += 1;
+				}
+				i = consumeUnderscoreGroups(code, i, len);
+			}
+		}
+
+		this.pushToken(TokenType.NUMBER, numberValue(code, start, i), null, null, code.slice(start, i), start);
+		this.advance(i);
+		return true;
+	}
+
+	/**
+	 * Matches a punctuation character at the cursor, pushing a PUNCTUATION
+	 * token.
+	 *
+	 * @returns {boolean} Whether a punctuation character was matched.
+	 */
+	lexPunctuation() {
+		const code = this.code;
+		const c = code.charCodeAt(this.cursor);
+		const p = this.options.punctuation;
+		if (p ? !p.includes(code[this.cursor]) : !isDefaultPunctuation(c)) {
+			return false;
+		}
+		const character = code[this.cursor];
+		this.checkBrackets(character);
+		this.pushToken(TokenType.PUNCTUATION, character, this.cursor, null, character, this.cursor);
+		this.advance(this.cursor + 1);
+		return true;
+	}
+
+	/**
+	 * Matches a complete single- or double-quoted string at the cursor,
+	 * pushing a STRING token.
+	 *
+	 * Mirrors `REGEX_STRING`: within a double-quoted string an unescaped `#`
+	 * is not part of the content, so such a string is not matched here and is
+	 * instead handled via the string state. Within a single-quoted string `#`
+	 * is an ordinary content character.
+	 *
+	 * @returns {boolean} Whether a complete string was matched.
+	 */
+	lexQuotedString() {
+		const code = this.code;
+		const len = this.end;
+		const start = this.cursor;
+		const quote = code.charCodeAt(start);
+		if (quote !== QUOTATION_MARK && quote !== APOSTROPHE) {
+			return false;
+		}
+
+		let i = start + 1;
+		while (i < len) {
+			const c = code.charCodeAt(i);
+			if (c === quote) {
+				const content = code.slice(start + 1, i);
+				this.pushToken(TokenType.STRING, stripcslashes(normalizeNewlines(content), code[start]), start, null, code.slice(start, i + 1), start);
+				this.advance(i + 1);
+				return true;
+			}
+			if (c === REVERSE_SOLIDUS) {
+				if (i + 1 >= len) {
+					return false;
+				}
+				i += 2;
+				continue;
+			}
+			if (quote === QUOTATION_MARK && c === NUMBER_SIGN) {
+				return false;
+			}
+			i += 1;
+		}
+
+		return false;
+	}
+
+	/**
+	 * Matches an opening double-quote at the cursor, entering the string state.
+	 *
+	 * @returns {boolean} Whether a double quote was matched.
+	 */
+	lexStringStateOpen() {
+		if (this.code.charCodeAt(this.cursor) !== QUOTATION_MARK) {
+			return false;
+		}
+		this.brackets.push(['"', this.lineNumber]);
+		this.pushState(LexerState.STRING);
+		this.advance(this.cursor + 1);
+		return true;
+	}
+
+	/**
+	 * Matches an inline comment (running to the end of the line) at the cursor.
+	 *
+	 * Mirrors `REGEX_RAW_INLINE_COMMENT` (`#[^\r\n]*`).
+	 *
+	 * @returns {boolean} Whether an inline comment was matched.
+	 */
+	lexInlineComment() {
+		const code = this.code;
+		const len = this.end;
+		if (code.charCodeAt(this.cursor) !== NUMBER_SIGN) {
+			return false;
+		}
+		let i = this.cursor;
+		while (i < len) {
+			const c = code.charCodeAt(i);
+			if (c === LINE_FEED || c === CARRIAGE_RETURN) {
+				break;
+			}
+			i += 1;
+		}
+		const comment = code.slice(this.cursor, i);
+		if (comment.startsWith('##')) {
+			this.addDocumentation(comment.slice(2));
+		}
+		this.advance(i);
+		return true;
+	}
+
+	/**
+	 * Matches a `{% verbatim %}` block opening at the cursor, consuming it.
+	 *
+	 * Mirrors the sticky `lex_block_raw` regular expression.
+	 *
+	 * @returns {boolean} Whether a verbatim block opening was matched.
+	 */
+	matchBlockRaw() {
+		const code = this.code;
+		const o = this.options;
+		let i = this.cursor;
+
+		while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+		if (!code.startsWith(VERBATIM, i)) {
+			return false;
+		}
+		i += VERBATIM.length;
+		while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+
+		// closing tag with optional trim
+		const closingTrim = o.whitespace_trim + o.tag_block[1];
+		if (code.startsWith(closingTrim, i)) {
+			i += closingTrim.length;
+			while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+				i += 1;
+			}
+			this.advance(i);
+			return true;
+		}
+		const lineTrimEnd = o.whitespace_line_trim + o.tag_block[1];
+		if (code.startsWith(lineTrimEnd, i)) {
+			i += lineTrimEnd.length;
+			while (i < this.end && isLineTrimCodeUnit(code.charCodeAt(i), o.whitespace_line_chars)) {
+				i += 1;
+			}
+			this.advance(i);
+			return true;
+		}
+		if (code.startsWith(o.tag_block[1], i)) {
+			i += o.tag_block[1].length;
+			this.advance(i);
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * Matches a `{% line \d+ %}` directive at the cursor, consuming it.
+	 *
+	 * Mirrors the sticky `lex_block_line` regular expression.
+	 *
+	 * @returns {number} The line number, or -1 when no directive matches.
+	 */
+	matchBlockLine() {
+		const code = this.code;
+		const o = this.options;
+		let i = this.cursor;
+
+		while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+		if (!code.startsWith(LINE, i)) {
+			return -1;
+		}
+		i += LINE.length;
+
+		// `\s+` between the word and the number is required
+		if (i >= this.end || !isWhitespaceCodePoint(code.charCodeAt(i))) {
+			return -1;
+		}
+		while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+
+		if (i >= this.end || !isDigitCodePoint(code.charCodeAt(i))) {
+			return -1;
+		}
+		const digitsStart = i;
+		while (i < this.end && isDigitCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+		const lineNumber = parseInt(code.slice(digitsStart, i), 10);
+
+		while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+		if (!code.startsWith(o.tag_block[1], i)) {
+			return -1;
+		}
+		i += o.tag_block[1].length;
+
+		this.advance(i);
+		return lineNumber;
+	}
+
+	/**
+	 * Matches the closing `}}` of a print statement at the cursor.
+	 *
+	 * Mirrors the sticky `lex_var` regular expression.
+	 *
+	 * @returns {number} The end offset, or -1.
+	 */
+	matchVariableEnd() {
+		return this.matchClosingDelimiter(this.options.tag_variable[1], false);
+	}
+
+	/**
+	 * Matches the closing `%}` of a block tag at the cursor.
+	 *
+	 * Mirrors the sticky `lex_block` regular expression.
+	 *
+	 * @returns {number} The end offset, or -1.
+	 */
+	matchBlockEnd() {
+		return this.matchClosingDelimiter(this.options.tag_block[1], true);
+	}
+
+	/**
+	 * Matches a closing delimiter (`}}` or `%}`) at the cursor.
+	 *
+	 * Mirrors the `lex_var` / `lex_block` regular expressions: the `-` marker
+	 * is followed by optional whitespace (plus a single newline for block
+	 * tags), the `~` marker by the line-trim characters, and a plain closing
+	 * tag may be followed by a single newline for block tags.
+	 *
+	 * @param {string} tagEnd The closing delimiter literal.
+	 * @param {boolean} consumesTrailingNewline Whether a plain closing tag may
+	 *   be followed by a newline.
+	 * @returns {number} The end offset, or -1.
+	 */
+	matchClosingDelimiter(tagEnd, consumesTrailingNewline) {
+		const code = this.code;
+		const o = this.options;
+
+		// peek: the closing tag must start with whitespace, a trim marker or the
+		// delimiter itself; anything else cannot possibly be a closing tag
+		const first = code.charCodeAt(this.cursor);
+		if (
+			!isWhitespaceCodePoint(first) &&
+			first !== o.whitespace_trim.charCodeAt(0) &&
+			first !== o.whitespace_line_trim.charCodeAt(0) &&
+			first !== tagEnd.charCodeAt(0)
+		) {
+			return -1;
+		}
+
+		let i = this.cursor;
+		while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+
+		const closingTrim = o.whitespace_trim + tagEnd;
+		if (code.startsWith(closingTrim, i)) {
+			i += closingTrim.length;
+			while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+				i += 1;
+			}
+			return i;
+		}
+
+		const lineTrimEnd = o.whitespace_line_trim + tagEnd;
+		if (code.startsWith(lineTrimEnd, i)) {
+			i += lineTrimEnd.length;
+			while (i < this.end && isLineTrimCodeUnit(code.charCodeAt(i), o.whitespace_line_chars)) {
+				i += 1;
+			}
+			return i;
+		}
+
+		if (code.startsWith(tagEnd, i)) {
+			i += tagEnd.length;
+			if (consumesTrailingNewline) {
+				const c = code.charCodeAt(i);
+				if (c === CARRIAGE_RETURN) {
+					i += 1;
+					if (code.charCodeAt(i) === LINE_FEED) {
+						i += 1;
+					}
+				} else if (c === LINE_FEED) {
+					i += 1;
+				}
+			}
+			return i;
+		}
+
+		return -1;
+	}
+
+	/**
+	 * Matches the start of a string interpolation (`#{`) at the cursor.
+	 *
+	 * Mirrors the `interpolation_start` regular expression: `#{`, optional
+	 * whitespace and an optional closing brace.
+	 *
+	 * @returns {number} The end offset, or -1.
+	 */
+	matchInterpolationStart() {
+		const code = this.code;
+		if (
+			code.charCodeAt(this.cursor) !== NUMBER_SIGN ||
+			code.charCodeAt(this.cursor + 1) !== LEFT_CURLY_BRACKET
+		) {
+			return -1;
+		}
+		let i = this.cursor + 2;
+		while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+		if (code.charCodeAt(i) === RIGHT_CURLY_BRACKET) {
+			i += 1;
+		}
+		return i;
+	}
+
+	/**
+	 * Matches the end of a string interpolation (`}`) at the cursor.
+	 *
+	 * Mirrors the `interpolation_end` regular expression: optional whitespace
+	 * then `}`.
+	 *
+	 * @returns {number} The end offset, or -1.
+	 */
+	matchInterpolationEnd() {
+		const code = this.code;
+		let i = this.cursor;
+		while (i < this.end && isWhitespaceCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+		if (code.charCodeAt(i) !== RIGHT_CURLY_BRACKET) {
+			return -1;
+		}
+		return i + 1;
+	}
+
+	/**
+	 * Matches a part of a double-quoted string with no unescaped `#{`.
+	 *
+	 * Mirrors `REGEX_DQ_STRING_PART`: a `#` is consumed unless it is the start
+	 * of an interpolation (`#{`); a backslash escapes the following code
+	 * point. The match may be empty.
+	 *
+	 * @returns {number} The end offset (may equal the cursor).
+	 */
+	matchDoubleQuoteStringPart() {
+		const code = this.code;
+		const len = this.end;
+		let i = this.cursor;
+
+		while (i < len) {
+			const c = code.charCodeAt(i);
+			if (c === NUMBER_SIGN) {
+				if (code.charCodeAt(i + 1) === LEFT_CURLY_BRACKET) {
+					break;
+				}
+				i += 1;
+				continue;
+			}
+			if (c === REVERSE_SOLIDUS) {
+				if (i + 1 >= len) {
+					break;
+				}
+				i += 2;
+				continue;
+			}
+			if (c === QUOTATION_MARK) {
+				break;
+			}
+			i += 1;
+		}
+
+		return i;
+	}
+
+	/** Lexes the body of a `verbatim` block as literal text. */
+	/**
+	 * @param {number} rawStart The offset where the opening delimiter starts.
+	 */
+	lexRawData(rawStart) {
+		const code = this.code;
+		const o = this.options;
+		const blockStartFirst = o.tag_block[0][0];
+
+		// search for the `{% endverbatim %}` closing marker
+		let markerStart = -1;
+		let markerEnd = -1;
+		let trim = '';
+		let i = this.cursor;
+		while (i < code.length) {
+			const start = code.indexOf(blockStartFirst, i);
+			if (start === -1) {
+				break;
+			}
+			i = start + 1;
+			if (!code.startsWith(o.tag_block[0], start)) {
+				continue;
+			}
+			let j = start + o.tag_block[0].length;
+			if (code.startsWith(o.whitespace_trim, j)) {
+				trim = o.whitespace_trim;
+				j += o.whitespace_trim.length;
+			} else if (code.startsWith(o.whitespace_line_trim, j)) {
+				trim = o.whitespace_line_trim;
+				j += o.whitespace_line_trim.length;
+			}
+			while (j < code.length && isWhitespaceCodePoint(code.charCodeAt(j))) {
+				j += 1;
+			}
+			if (!code.startsWith(ENDVERBATIM, j)) {
+				continue;
+			}
+			j += ENDVERBATIM.length;
+			while (j < code.length && isWhitespaceCodePoint(code.charCodeAt(j))) {
+				j += 1;
+			}
+
+			// closing tag with optional trim
+			const closingTrim = o.whitespace_trim + o.tag_block[1];
+			if (code.startsWith(closingTrim, j)) {
+				j += closingTrim.length;
+				while (j < code.length && isWhitespaceCodePoint(code.charCodeAt(j))) {
+					j += 1;
+				}
+				markerStart = start;
+				markerEnd = j;
+				break;
+			}
+			const lineTrimEnd = o.whitespace_line_trim + o.tag_block[1];
+			if (code.startsWith(lineTrimEnd, j)) {
+				j += lineTrimEnd.length;
+				while (j < code.length && isLineTrimCodeUnit(code.charCodeAt(j), o.whitespace_line_chars)) {
+					j += 1;
+				}
+				markerStart = start;
+				markerEnd = j;
+				break;
+			}
+			if (code.startsWith(o.tag_block[1], j)) {
+				j += o.tag_block[1].length;
+				markerStart = start;
+				markerEnd = j;
+				break;
+			}
+		}
+
+		if (markerStart === -1) {
+			throw new SyntaxError(
+				'Unexpected end of file: Unclosed "verbatim" block.',
+				this.lineNumber,
+				this.source,
+			);
+		}
+
+		const offset = this.cursor;
+		let text = code.slice(this.cursor, markerStart);
+		this.advance(markerEnd);
+
+		// trim?
+		if (trim !== '') {
+			text = o.whitespace_trim === trim
+				? trimEnd(text, isPhpTrimCodePoint)
+				: trimEnd(text, (search) => isLineTrimCodeUnit(search, o.whitespace_line_chars));
+		}
+
+		this.pushToken(TokenType.TEXT, normalizeNewlines(text), offset, null, code.slice(rawStart, markerEnd), rawStart);
+	}
+
+	/**
+	 * Lexes a comment body.
+	 *
+	 * @param {boolean} [isDocumentation] Whether it is a documentation comment.
+	 */
+	lexComment(isDocumentation = false) {
+		const code = this.code;
+		const markers = isDocumentation ? this.docCommentMarkers : this.commentMarkers;
+		const firstChars = this.commentFirstChars;
+
+		let index = -1;
+		let end = -1;
+		for (let p = this.cursor; p < code.length; p += 1) {
+			const pc = code.charCodeAt(p);
+			if (pc !== firstChars[0] && pc !== firstChars[1] && pc !== firstChars[2]) {
+				continue;
+			}
+			for (let k = 0; k < markers.length; k += 1) {
+				const marker = markers[k][0];
+				if (!code.startsWith(marker, p)) {
+					continue;
+				}
+				let j = p + marker.length;
+				const kind = markers[k][1];
+				if (kind === 0) {
+					while (j < code.length && isWhitespaceCodePoint(code.charCodeAt(j))) {
+						j += 1;
+					}
+				} else if (kind === 1) {
+					while (j < code.length && isLineTrimCodeUnit(code.charCodeAt(j), this.options.whitespace_line_chars)) {
+						j += 1;
+					}
+				} else {
+					const c = code.charCodeAt(j);
+					if (c === CARRIAGE_RETURN) {
+						j += 1;
+						if (code.charCodeAt(j) === LINE_FEED) {
+							j += 1;
+						}
+					} else if (c === LINE_FEED) {
+						j += 1;
+					}
+				}
+				index = p;
+				end = j;
+				break;
+			}
+			if (index !== -1) {
+				break;
+			}
+		}
+
+		if (index === -1) {
+			throw new SyntaxError('Unclosed comment.', this.lineNumber, this.source);
+		}
+
+		const comment = code.slice(this.cursor, index);
+		if (isDocumentation) {
+			let documentation = comment;
+			// support a symmetric "##}" closing marker
+			if (documentation.endsWith('#')) {
+				documentation = documentation.slice(0, -1);
+			}
+			this.addDocumentation(documentation);
+		}
+
+		this.advance(end);
+	}
+
+	/** Lexes inside a double-quoted string. */
+	lexString() {
+		const code = this.code;
+
+		// interpolation start
+		const interpEnd = this.matchInterpolationStart();
+		if (interpEnd !== -1) {
+			this.brackets.push([this.options.interpolation[0], this.lineNumber]);
+			this.pushToken(TokenType.INTERPOLATION_START, '', this.cursor, null, this.code.slice(this.cursor, interpEnd), this.cursor);
+			this.advance(interpEnd);
+			this.pushState(LexerState.INTERPOLATION);
+			return;
+		}
+
+		// string part
+		const partEnd = this.matchDoubleQuoteStringPart();
+		if (partEnd > this.cursor) {
+			const content = code.slice(this.cursor, partEnd);
+			this.pushToken(TokenType.STRING, stripcslashes(normalizeNewlines(content), '"'), this.cursor, null, content, this.cursor);
+			this.advance(partEnd);
+			return;
+		}
+
+		// closing quote
+		if (code.charCodeAt(this.cursor) === QUOTATION_MARK) {
+			const top = this.brackets[this.brackets.length - 1];
+			const [expect, lineNumber] = top;
+			if ('"' !== code[this.cursor]) {
+				throw new SyntaxError(`Unclosed "${expect}".`, lineNumber, this.source);
+			}
+			this.brackets.pop();
+			this.popState();
+			this.advance(this.cursor + 1);
+			return;
+		}
+
+		// unlexable
+		throw new SyntaxError(
+			`Unexpected character "${code[this.cursor]}".`,
+			this.lineNumber,
+			this.source,
+			this.source ? this.source.getColumn(this.cursor) : null,
+		);
+	}
+
+	/** Lexes inside a string interpolation. */
+	lexInterpolation() {
+		const bracket = this.brackets[this.brackets.length - 1];
+		if (bracket && bracket[0] === this.options.interpolation[0]) {
+			const end = this.matchInterpolationEnd();
+			if (end !== -1) {
+				this.brackets.pop();
+				this.pushClosingToken(TokenType.INTERPOLATION_END, end);
+				this.popState();
+				return;
+			}
+		}
+		this.lexExpression();
+	}
+
+	/**
+	 * Pushes a token, skipping empty text tokens and attaching documentation.
+	 *
+	 * @param {number} type
+	 * @param {unknown} [value]
+	 * @param {number|null} [offset]
+	 * @param {number|null} [lineNumber]
+	 * @param {string|null} [raw] The exact source text of the token.
+	 * @param {number|null} [rawStart] The offset where the token's raw text begins.
+	 */
+	pushToken(type, value = '', offset = null, lineNumber = null, raw = null, rawStart = null) {
+		// do not push empty text tokens
+		if (TokenType.TEXT === type && value === '') {
+			return;
+		}
+
+		let documentation = null;
+		if (this.documentation.length && (TokenType.TEXT !== type || trimLikePhp(String(value)) !== '')) {
+			documentation = this.documentation.join('\n');
+			this.documentation.length = 0;
+		}
+
+		const start = rawStart ?? offset ?? this.cursor;
+		const rawText = raw ?? '';
+		const leading = this.code.slice(this.lastRawEnd, start);
+		this.lastRawEnd = start + rawText.length;
+
+		const token = new Token(type, value, lineNumber ?? this.lineNumber, offset ?? this.cursor, documentation);
+		token.withSource(rawText, leading, start - leading.length);
+		this.tokens.push(token);
+	}
+
+	/**
+	 * Records a documentation fragment.
+	 *
+	 * @param {string} documentation
+	 */
+	addDocumentation(documentation) {
+		documentation = trimLikePhp(normalizeNewlines(documentation));
+		if (documentation !== '') {
+			this.documentation.push(documentation);
+		}
+	}
+
+	/**
+	 * Pushes a closing delimiter token, consuming leading whitespace.
+	 *
+	 * @param {number} type
+	 * @param {number} end The end offset of the closing delimiter match.
+	 */
+	pushClosingToken(type, end) {
+		const code = this.code;
+		let i = this.cursor;
+		while (i < end && isPhpTrimCodePoint(code.charCodeAt(i))) {
+			i += 1;
+		}
+		if (i > this.cursor) {
+			this.advance(i);
+		}
+		this.pushToken(type, '', i, null, code.slice(i, end), i);
+		this.advance(end);
+	}
+
+	/**
+	 * Pushes a lexer state.
+	 *
+	 * @param {number} state
+	 */
+	pushState(state) {
+		this.states.push(this.state);
+		this.state = state;
+	}
+
+	/** Pops a lexer state. */
+	popState() {
+		if (this.states.length === 0) {
+			throw new Error('Cannot pop state without a previous state.');
+		}
+		this.state = this.states.pop() ?? 0;
+	}
+
+	/**
+	 * Tracks bracket matching.
+	 *
+	 * @param {string} code The bracket character.
+	 */
+	checkBrackets(code) {
+		if (OPENING_BRACKETS.includes(code)) {
+			this.brackets.push([code, this.lineNumber]);
+		} else if (CLOSING_BRACKETS.includes(code)) {
+			if (!this.brackets.length) {
+				throw new SyntaxError(
+					`Unexpected "${code}".`,
+					this.lineNumber,
+					this.source,
+					this.source ? this.source.getColumn(this.cursor) : null,
+				);
+			}
+			const [expect, lineNumber] = this.brackets.pop() ?? ['', 0];
+			const expectedClosing = CLOSING_BRACKETS[OPENING_BRACKETS.indexOf(expect)];
+			if (code !== expectedClosing) {
+				throw new SyntaxError(`Unclosed "${expect}".`, lineNumber, this.source);
+			}
+		}
+	}
+}