@mrhenry/twig-tokenizer 0.1.0

package/src/operators.js

@@ -0,0 +1,65 @@
+// @ts-check
+/**
+ * The set of operators recognized by the lexer.
+ *
+ * Matches the operators registered by `CoreExtension::getExpressionParsers()`
+ * in the reference implementation. Operators are matched longest-first; word
+ * operators are only matched when followed by a delimiter character and are
+ * not matched when immediately preceded by `.` or `|`.
+ *
+ * @module twig-tokenizer
+ */
+
+/**
+ * All operator tokens (names and aliases) known to the lexer.
+ *
+ * @type {string[]}
+ */
+export const OPERATORS = [
+	'not',
+	'...',
+	'-',
+	'+',
+	'?:',
+	'? :',
+	'??',
+	'or',
+	'xor',
+	'and',
+	'b-or',
+	'b-xor',
+	'b-and',
+	'==',
+	'!=',
+	'<=>',
+	'<',
+	'>',
+	'>=',
+	'<=',
+	'not in',
+	'in',
+	'matches',
+	'starts with',
+	'ends with',
+	'has some',
+	'has every',
+	'===',
+	'!==',
+	'..',
+	'~',
+	'*',
+	'/',
+	'//',
+	'%',
+	'**',
+	'?',
+	'=',
+	'is',
+	'is not',
+	'|',
+	'(',
+	'.',
+	'?.',
+	'[',
+	'=>',
+];

package/src/token-stream.js

@@ -0,0 +1,188 @@
+// @ts-check
+/**
+ * The {@link TokenStream} class.
+ *
+ * Mirrors `src/TokenStream.php` of the reference implementation.
+ *
+ * @module twig-tokenizer
+ */
+// eslint-disable-next-line no-unused-vars -- used in JSDoc type annotations
+import { Token, TokenType, typeToEnglish } from './token.js';
+import { SyntaxError } from './errors.js';
+
+/**
+ * A sequence of {@link Token} objects with a moving pointer.
+ */
+export class TokenStream {
+	/**
+	 * @param {Token[]} tokens The full token list (must end with an EOF token).
+	 * @param {import('./lexer.js').Source|null} [source] The template source.
+	 * @param {import('./lexer.js').LexerOptions|null} [options] The lexer options.
+	 */
+	constructor(tokens, source = null, options = null) {
+		/** @type {Token[]} */
+		this.tokens = tokens;
+		/** @type {import('./lexer.js').Source|null} */
+		this.source = source;
+		/** @type {import('./lexer.js').LexerOptions|null} */
+		this.options = options;
+		/** @type {number} The current pointer. */
+		this.current = 0;
+	}
+
+	/**
+	 * @returns {string} The tokens joined with newlines.
+	 */
+	toString() {
+		return this.tokens.join('\n');
+	}
+
+	/**
+	 * Serializes the stream back to template source.
+	 *
+	 * Each token emits its retained trivia followed by its source text. Token
+	 * values that were mutated are rendered from their current value, so
+	 * mutated, inserted and removed tokens are all reflected. When nothing was
+	 * mutated the result is byte-for-byte the source the stream was tokenized
+	 * from. Parser-synthesised tokens are omitted.
+	 *
+	 * @returns {string} The serialized template source.
+	 */
+	serialize() {
+		let out = '';
+		for (const token of this.tokens) {
+			if (token.synthetic) {
+				continue;
+			}
+			out += token.leading + token.serialize(this.options);
+		}
+		return out;
+	}
+
+	/**
+	 * Inserts tokens at the current position.
+	 *
+	 * @param {Token[]} tokens Tokens to inject.
+	 */
+	injectTokens(tokens) {
+		this.tokens = [
+			...this.tokens.slice(0, this.current),
+			...tokens,
+			...this.tokens.slice(this.current),
+		];
+	}
+
+	/**
+	 * Sets the pointer to the next token and returns the old one.
+	 *
+	 * @returns {Token} The previous current token.
+	 */
+	next() {
+		if (!this.tokens[++this.current]) {
+			throw new SyntaxError(
+				'Unexpected end of template.',
+				this.tokens[this.current - 1].getLine(),
+				this.source,
+			);
+		}
+		return this.tokens[this.current - 1];
+	}
+
+	/**
+	 * Tests a token, sets the pointer to the next one and returns it or null.
+	 *
+	 * @param {number|Array<string|number>|string} primary Type (or value) to test.
+	 * @param {Array<string|number>|string|number|null} [secondary] Value(s) to test.
+	 * @returns {Token|null} The next token if it matches, else null.
+	 */
+	nextIf(primary, secondary = null) {
+		return this.tokens[this.current].test(primary, secondary)
+			? this.next()
+			: null;
+	}
+
+	/**
+	 * Tests a token and returns it or throws a {@link SyntaxError}.
+	 *
+	 * @param {number|Array<string|number>|string} type Type to expect.
+	 * @param {Array<string|number>|string|number|null} [value] Expected value(s).
+	 * @param {string|null} [message] Optional message prefix.
+	 * @returns {Token} The expected token.
+	 */
+	expect(type, value = null, message = null) {
+		const token = this.tokens[this.current];
+		if (!token.test(type, value)) {
+			const line = token.getLine();
+			throw new SyntaxError(
+				`${message ? `${message}. ` : ''}Unexpected token "${token.toEnglish()}"${
+					token.getValue() ? ` of value "${token.getValue()}"` : ''
+				} ("${typeToEnglish(/** @type {number} */ (type))}" expected${
+					value ? ` with value "${value}"` : ''
+				}).`,
+				line,
+				this.source,
+				this.source ? this.source.getColumn(token.getOffset() ?? -1) : null,
+			);
+		}
+		this.next();
+		return token;
+	}
+
+	/**
+	 * Looks at the token `number` steps ahead without moving the pointer.
+	 *
+	 * @param {number} [number] How many tokens ahead (>= 1).
+	 * @returns {Token} The looked-ahead token.
+	 */
+	look(number = 1) {
+		if (!this.tokens[this.current + number]) {
+			throw new SyntaxError(
+				'Unexpected end of template.',
+				this.tokens[this.current + number - 1].getLine(),
+				this.source,
+			);
+		}
+		return this.tokens[this.current + number];
+	}
+
+	/**
+	 * Tests the current token.
+	 *
+	 * @param {number|Array<string|number>|string} primary
+	 * @param {Array<string|number>|string|number|null} [secondary]
+	 * @returns {boolean} Whether the current token matches.
+	 */
+	test(primary, secondary = null) {
+		return this.tokens[this.current].test(primary, secondary);
+	}
+
+	/**
+	 * @returns {boolean} Whether the end of the stream was reached.
+	 */
+	isEOF() {
+		return this.tokens[this.current].test(TokenType.EOF);
+	}
+
+	/**
+	 * @returns {Token} The current token.
+	 */
+	getCurrent() {
+		return this.tokens[this.current];
+	}
+
+	/**
+	 * @returns {import('./lexer.js').Source|null} The source context.
+	 */
+	getSourceContext() {
+		return this.source;
+	}
+
+	/**
+	 * @returns {Token[]} The underlying token array (used by the parser).
+	 */
+	getTokens() {
+		return this.tokens;
+	}
+}
+
+export { TokenType };

package/src/token.js

@@ -0,0 +1,356 @@
+// @ts-check
+/**
+ * Token type constants and the {@link Token} class.
+ *
+ * Mirrors `src/Token.php` of the reference implementation.
+ *
+ * @module twig-tokenizer
+ */
+
+/**
+ * Enumeration of token types. Values match the PHP reference implementation
+ * (`Twig\Token::*_TYPE`).
+ *
+ * @readonly
+ * @enum {number}
+ */
+export const TokenType = {
+	/** End of the template stream. */
+	EOF: -1,
+	/** Static template text. */
+	TEXT: 0,
+	/** `{%` opening delimiter of a block tag. */
+	BLOCK_START: 1,
+	/** `{{` opening delimiter of a print statement. */
+	VAR_START: 2,
+	/** `%}` closing delimiter of a block tag. */
+	BLOCK_END: 3,
+	/** `}}` closing delimiter of a print statement. */
+	VAR_END: 4,
+	/** An identifier or keyword. */
+	NAME: 5,
+	/** An integer or floating-point literal. */
+	NUMBER: 6,
+	/** A quoted string fragment. */
+	STRING: 7,
+	/** An operator symbol or word. */
+	OPERATOR: 8,
+	/** One of `) ] { } : ,`. */
+	PUNCTUATION: 9,
+	/** `#{` inside a double-quoted string. */
+	INTERPOLATION_START: 10,
+	/** `}` closing an interpolation. */
+	INTERPOLATION_END: 11,
+};
+
+/** @typedef {number} TokenTypeValue */
+
+/**
+ * A lexed token: a type, a value and source position information.
+ *
+ * @template {unknown} [T=unknown]
+ */
+export class Token {
+	/**
+	 * @param {TokenTypeValue} type The token type (one of {@link TokenType}).
+	 * @param {T} value The token value.
+	 * @param {number} lineNumber The 1-based line number.
+	 * @param {number|null} [offset] The 0-based offset in the source (byte/char).
+	 * @param {string|null} [documentation] Documentation comment attached to this token.
+	 */
+	constructor(type, value, lineNumber, offset = null, documentation = null) {
+		/** @type {TokenTypeValue} */
+		this.type = type;
+		/** @type {T} */
+		this.value = value;
+		/** @type {number} */
+		this.lineNumber = lineNumber;
+		/** @type {number|null} */
+		this.offset = offset;
+		/** @type {string|null} */
+		this.documentation = documentation;
+
+		/** The exact source text of this token (null when synthetic). */
+		this.raw = null;
+		/** Trivia (whitespace/comments/delimiters) preceding this token. */
+		this.leading = '';
+		/** Absolute source offset where {@link Token#leading} starts. */
+		this.sourceStart = offset ?? 0;
+		/** Absolute source offset where {@link Token#raw} starts. */
+		this.rawStart = offset ?? 0;
+		/** Absolute source offset just past {@link Token#raw}. */
+		this.rawEnd = offset ?? 0;
+		/** Whether this token was synthesised by the parser and has no source. */
+		this.synthetic = false;
+		/** The value at construction time, used to detect mutations. */
+		this.originalValue = value;
+	}
+
+	/**
+	 * Records the source text and trivia of this token.
+	 *
+	 * @param {string} raw The exact source text of the token (may be empty).
+	 * @param {string} leading Trivia preceding the token.
+	 * @param {number} sourceStart Absolute offset where the trivia starts.
+	 * @returns {this} This token (for chaining).
+	 */
+	withSource(raw, leading, sourceStart) {
+		this.raw = raw;
+		this.leading = leading;
+		this.sourceStart = sourceStart;
+		this.rawStart = sourceStart + leading.length;
+		this.rawEnd = this.rawStart + raw.length;
+		this.originalValue = this.value;
+		return this;
+	}
+
+	/**
+	 * Whether the token's value differs from the value it was created with.
+	 *
+	 * @returns {boolean} True when the value was mutated.
+	 */
+	isMutated() {
+		return this.value !== this.originalValue;
+	}
+
+	/**
+	 * Tests the token for a type and/or value.
+	 *
+	 * Parameters may be:
+	 *  * just type
+	 *  * type and value (or array of possible values)
+	 *  * just value (or array of possible values); NAME type is used then.
+	 *
+	 * For backwards compatibility, an OPERATOR expectation also matches a
+	 * PUNCTUATION token whose value is one of `(`, `[`, `|`, `.`, `?`, `?:`.
+	 *
+	 * @param {TokenTypeValue|Array<string|number>|string} type The type to test.
+	 * @param {Array<string|number>|string|number|null} [values] The token value(s).
+	 * @returns {boolean} Whether the token matches.
+	 */
+	test(type, values = null) {
+		let t = type;
+		let v = values;
+		if (v === null && typeof t !== 'number') {
+			v = t;
+			t = TokenType.NAME;
+		}
+
+		let typeMatches = this.type === t;
+		if (typeMatches && t === TokenType.PUNCTUATION && Array.isArray(v)) {
+			const legacy = ['(', '[', '|', '.', '?', '?:'];
+			if (v.some((x) => legacy.includes(String(x)))) {
+				// punctuation that is now an operator token; keep matching
+			}
+		}
+		if (!typeMatches && t === TokenType.OPERATOR && this.type === TokenType.PUNCTUATION) {
+			if (v === null) {
+				typeMatches = true;
+			} else if (Array.isArray(v)) {
+				typeMatches = v.some((x) => ['(', '[', '|', '.', '?', '?:'].includes(String(x)));
+			} else {
+				typeMatches = ['(', '[', '|', '.', '?', '?:'].includes(String(v));
+			}
+		}
+
+		return (
+			typeMatches &&
+			(v === null ||
+				(Array.isArray(v) && v.includes(/** @type {string|number} */ (this.value))) ||
+				// eslint-disable-next-line eqeqeq -- intentionally compares string and number token values
+				this.value == v)
+		);
+	}
+
+	/**
+	 * @returns {number} The 1-based line number of the token.
+	 */
+	getLine() {
+		return this.lineNumber;
+	}
+
+	/**
+	 * @returns {T} The token value.
+	 */
+	getValue() {
+		return this.value;
+	}
+
+	/**
+	 * @returns {TokenTypeValue} The token type.
+	 */
+	getType() {
+		return this.type;
+	}
+
+	/**
+	 * @returns {number|null} The 0-based offset, or null when not tied to a position.
+	 */
+	getOffset() {
+		return this.offset;
+	}
+
+	/**
+	 * @returns {string|null} Attached documentation, if any.
+	 */
+	getDocumentation() {
+		return this.documentation;
+	}
+
+	/**
+	 * @returns {string} An English description of the token type.
+	 */
+	toEnglish() {
+		return typeToEnglish(this.type);
+	}
+
+	/**
+	 * @returns {string} `TYPE(value)` representation, e.g. `NAME_TYPE(foo)`.
+	 */
+	toString() {
+		return `${typeToString(this.type, true)}(${this.value})`;
+	}
+
+	/**
+	 * Serializes this token back to template source.
+	 *
+	 * When the token still holds the value it was lexed with, its exact original
+	 * source text is returned (lossless). When the value was mutated, the token
+	 * is rendered from its current value instead, so mutations are reflected.
+	 * Parser-synthesised tokens serialize to the empty string.
+	 *
+	 * @param {import('./lexer.js').LexerOptions|null} [options] Lexer options
+	 *   (used to render mutated delimiters).
+	 * @returns {string} The source text of the token.
+	 */
+	serialize(options = null) {
+		if (this.synthetic) {
+			return '';
+		}
+		if (this.raw !== null && !this.isMutated()) {
+			return this.raw;
+		}
+		return this.render(options);
+	}
+
+	/**
+	 * Renders a canonical source representation from the current type/value.
+	 *
+	 * @param {import('./lexer.js').LexerOptions|null} [options] Lexer options.
+	 * @returns {string} The rendered token text.
+	 */
+	render(options = null) {
+		const tagBlock = options?.tag_block ?? ['{%', '%}'];
+		const tagVariable = options?.tag_variable ?? ['{{', '}}'];
+		const interpolation = options?.interpolation ?? ['#{', '}'];
+		switch (this.type) {
+			case TokenType.TEXT:
+			case TokenType.NAME:
+			case TokenType.OPERATOR:
+			case TokenType.PUNCTUATION:
+				return String(this.value);
+			case TokenType.NUMBER:
+				return formatNumber(this.value);
+			case TokenType.STRING:
+				return quoteString(this.value);
+			case TokenType.BLOCK_START:
+				return tagBlock[0];
+			case TokenType.BLOCK_END:
+				return tagBlock[1];
+			case TokenType.VAR_START:
+				return tagVariable[0];
+			case TokenType.VAR_END:
+				return tagVariable[1];
+			case TokenType.INTERPOLATION_START:
+				return interpolation[0];
+			case TokenType.INTERPOLATION_END:
+				return interpolation[1];
+			case TokenType.EOF:
+				return '';
+			default:
+				return String(this.value);
+		}
+	}
+}
+
+/**
+ * Renders a number as it would appear in Twig source.
+ *
+ * @param {unknown} value The number value.
+ * @returns {string} The rendered number.
+ */
+function formatNumber(value) {
+	const numeric = Number(value);
+	if (Number.isNaN(numeric)) {
+		return '0';
+	}
+	return String(numeric);
+}
+
+/**
+ * Renders a string as a single-quoted Twig literal.
+ *
+ * @param {unknown} value The string value.
+ * @returns {string} The quoted literal.
+ */
+function quoteString(value) {
+	const text = String(value);
+	return `'${text.replace(/\\/g, '\\\\').replace(/'/g, "\\'")}'`;
+}
+
+/**
+ * Converts a token type value to its constant name.
+ *
+ * @param {TokenTypeValue} type
+ * @param {boolean} [short] When true, omit the `Twig\Token::` prefix.
+ * @returns {string} The type name.
+ */
+export function typeToString(type, short = false) {
+	/** @type {Record<number, string>} */
+	const names = {
+		[TokenType.EOF]: 'EOF_TYPE',
+		[TokenType.TEXT]: 'TEXT_TYPE',
+		[TokenType.BLOCK_START]: 'BLOCK_START_TYPE',
+		[TokenType.VAR_START]: 'VAR_START_TYPE',
+		[TokenType.BLOCK_END]: 'BLOCK_END_TYPE',
+		[TokenType.VAR_END]: 'VAR_END_TYPE',
+		[TokenType.NAME]: 'NAME_TYPE',
+		[TokenType.NUMBER]: 'NUMBER_TYPE',
+		[TokenType.STRING]: 'STRING_TYPE',
+		[TokenType.OPERATOR]: 'OPERATOR_TYPE',
+		[TokenType.PUNCTUATION]: 'PUNCTUATION_TYPE',
+		[TokenType.INTERPOLATION_START]: 'INTERPOLATION_START_TYPE',
+		[TokenType.INTERPOLATION_END]: 'INTERPOLATION_END_TYPE',
+	};
+	const name = names[type];
+	if (name === undefined) {
+		throw new Error(`Token of type "${type}" does not exist.`);
+	}
+	return short ? name : `Twig\\Token::${name}`;
+}
+
+/**
+ * Converts a token type value to an English description.
+ *
+ * @param {TokenTypeValue} type
+ * @returns {string} English description.
+ */
+export function typeToEnglish(type) {
+	/** @type {Record<number, string>} */
+	const names = {
+		[TokenType.EOF]: 'end of template',
+		[TokenType.TEXT]: 'text',
+		[TokenType.BLOCK_START]: 'begin of statement block',
+		[TokenType.VAR_START]: 'begin of print statement',
+		[TokenType.BLOCK_END]: 'end of statement block',
+		[TokenType.VAR_END]: 'end of print statement',
+		[TokenType.NAME]: 'name',
+		[TokenType.NUMBER]: 'number',
+		[TokenType.STRING]: 'string',
+		[TokenType.OPERATOR]: 'operator',
+		[TokenType.PUNCTUATION]: 'punctuation',
+		[TokenType.INTERPOLATION_START]: 'begin of string interpolation',
+		[TokenType.INTERPOLATION_END]: 'end of string interpolation',
+	};
+	return names[type] ?? `token of type ${type}`;
+}