@mrhenry/twig-tokenizer 0.1.0

package/LICENSE

@@ -0,0 +1,24 @@
+MIT License
+
+Copyright (c) 2026 Mr. Henry
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+This license applies only to the code in this repository.
+Images are explicitly excluded.

package/package.json

@@ -0,0 +1,10 @@
+{
+	"name": "@mrhenry/twig-tokenizer",
+	"version": "0.1.0",
+	"license": "MIT",
+	"type": "module",
+	"main": "src/index.js",
+	"exports": {
+		".": "./src/index.js"
+	}
+}

package/src/code-points/code-points.js

@@ -0,0 +1,54 @@
+// @ts-check
+/**
+ * Named constants for the code points used by the Twig lexer.
+ *
+ * @module twig-tokenizer
+ */
+
+/** \x00 */
+export const NULL = 0x0000;
+/** \t */
+export const CHARACTER_TABULATION = 0x0009;
+/** \n */
+export const LINE_FEED = 0x000a;
+/** \v */
+export const LINE_TABULATION = 0x000b;
+/** \f */
+export const FORM_FEED = 0x000c;
+/** \r */
+export const CARRIAGE_RETURN = 0x000d;
+/** \u20 */
+export const SPACE = 0x0020;
+
+/** " */
+export const QUOTATION_MARK = 0x0022;
+/** # */
+export const NUMBER_SIGN = 0x0023;
+/** ' */
+export const APOSTROPHE = 0x0027;
+/** ( */
+export const LEFT_PARENTHESIS = 0x0028;
+/** ) */
+export const RIGHT_PARENTHESIS = 0x0029;
+/** + */
+export const PLUS_SIGN = 0x002b;
+/** - */
+export const HYPHEN_MINUS = 0x002d;
+/** . */
+export const FULL_STOP = 0x002e;
+/** E */
+export const LATIN_CAPITAL_LETTER_E = 0x0045;
+/** [ */
+export const LEFT_SQUARE_BRACKET = 0x005b;
+/** \ */
+export const REVERSE_SOLIDUS = 0x005c;
+/** _ */
+export const LOW_LINE = 0x005f;
+/** e */
+export const LATIN_SMALL_LETTER_E = 0x0065;
+/** { */
+export const LEFT_CURLY_BRACKET = 0x007b;
+/** | */
+export const VERTICAL_LINE = 0x007c;
+/** } */
+export const RIGHT_CURLY_BRACKET = 0x007d;

package/src/code-points/ranges.js

@@ -0,0 +1,161 @@
+// @ts-check
+/**
+ * Predicates for classifying code points during lexing.
+ *
+ * @module twig-tokenizer
+ */
+import {
+	CARRIAGE_RETURN,
+	CHARACTER_TABULATION,
+	FORM_FEED,
+	LEFT_CURLY_BRACKET,
+	LEFT_PARENTHESIS,
+	LEFT_SQUARE_BRACKET,
+	LINE_FEED,
+	LINE_TABULATION,
+	NULL,
+	RIGHT_PARENTHESIS,
+	SPACE,
+} from './code-points.js';
+
+/**
+ * Whether the code point is an ASCII decimal digit (`0` .. `9`).
+ *
+ * @param {number} search The code point to test.
+ * @returns {boolean}
+ */
+export function isDigitCodePoint(search) {
+	return search >= 0x0030 && search <= 0x0039;
+}
+
+/**
+ * Whether the code point is a hexadecimal digit (`0` .. `9`, `a` .. `f`, `A` .. `F`).
+ *
+ * @param {number} search The code point to test.
+ * @returns {boolean}
+ */
+export function isHexDigitCodePoint(search) {
+	return (
+		(search >= 0x0030 && search <= 0x0039) ||
+		(search >= 0x0061 && search <= 0x0066) ||
+		(search >= 0x0041 && search <= 0x0046)
+	);
+}
+
+/**
+ * Whether the code point is an octal digit (`0` .. `7`).
+ *
+ * @param {number} search The code point to test.
+ * @returns {boolean}
+ */
+export function isOctalDigitCodePoint(search) {
+	return search >= 0x0030 && search <= 0x0037;
+}
+
+/**
+ * Whether the code point is an ASCII letter (`a` .. `z`, `A` .. `Z`).
+ *
+ * @param {number} search The code point to test.
+ * @returns {boolean}
+ */
+export function isAsciiLetterCodePoint(search) {
+	return (
+		(search >= 0x0061 && search <= 0x007a) ||
+		(search >= 0x0041 && search <= 0x005a)
+	);
+}
+
+/**
+ * Whether the code point starts a Twig name.
+ *
+ * Mirrors the reference implementation's byte-oriented
+ * `[a-zA-Z_\x7f-\xff]`: in byte mode any UTF-8 byte in `\x7f-\xff` is an
+ * identifier start character, which corresponds to every code point from
+ * U+007F onward here (astral code points are matched via their surrogate
+ * halves in byte terms, and via the code point itself here).
+ *
+ * @param {number} search The code point to test.
+ * @returns {boolean}
+ */
+export function isNameStartCodePoint(search) {
+	return (
+		(search >= 0x0041 && search <= 0x005a) ||
+		(search >= 0x0061 && search <= 0x007a) ||
+		search === 0x005f ||
+		search >= 0x007f
+	);
+}
+
+/**
+ * Whether the code point continues a Twig name.
+ *
+ * Mirrors the byte-oriented `[a-zA-Z0-9_\x7f-\xff]`.
+ *
+ * @param {number} search The code point to test.
+ * @returns {boolean}
+ */
+export function isNameCodePoint(search) {
+	return isNameStartCodePoint(search) || isDigitCodePoint(search);
+}
+
+/**
+ * Whether the code point is whitespace as matched by PCRE `\s` in byte mode.
+ *
+ * Space, tab, line feed, vertical tab, form feed and carriage return. JS
+ * `\s` also matches Unicode whitespace (NBSP, en/em spaces, U+2028/U+2029, …)
+ * that the reference implementation treats as identifier characters in byte
+ * mode; using this explicit set keeps the lexer byte-for-byte compatible
+ * with `src/Lexer.php`.
+ *
+ * @param {number} search The code point to test.
+ * @returns {boolean}
+ */
+export function isWhitespaceCodePoint(search) {
+	return (
+		search === SPACE ||
+		search === CHARACTER_TABULATION ||
+		search === LINE_FEED ||
+		search === LINE_TABULATION ||
+		search === FORM_FEED ||
+		search === CARRIAGE_RETURN
+	);
+}
+
+/**
+ * Whether the code point is stripped by PHP's default `ltrim()`/`rtrim()`.
+ *
+ * The default character list is `" \t\n\r\v\x00"`, which notably does *not*
+ * include the form feed `\f`.
+ *
+ * @param {number} search The code point to test.
+ * @returns {boolean}
+ */
+export function isPhpTrimCodePoint(search) {
+	return (
+		search === SPACE ||
+		search === CHARACTER_TABULATION ||
+		search === LINE_FEED ||
+		search === CARRIAGE_RETURN ||
+		search === LINE_TABULATION ||
+		search === NULL
+	);
+}
+
+/**
+ * Whether the code point may follow an operator that ends with a letter.
+ *
+ * Mirrors the `(?=[\s()\[{])` lookahead of the reference implementation.
+ *
+ * @param {number} search The code point to test.
+ * @returns {boolean}
+ */
+export function isOperatorDelimiterCodePoint(search) {
+	return (
+		isWhitespaceCodePoint(search) ||
+		search === LEFT_PARENTHESIS ||
+		search === RIGHT_PARENTHESIS ||
+		search === LEFT_SQUARE_BRACKET ||
+		search === LEFT_CURLY_BRACKET
+	);
+}
+

package/src/errors.js

@@ -0,0 +1,283 @@
+// @ts-check
+/**
+ * Error classes mirroring `Twig\Error\*`.
+ *
+ * @module twig-tokenizer
+ */
+
+/**
+ * A source context (template name + code).
+ *
+ * @typedef {object} Source
+ * @property {string} name The template name.
+ * @property {string} code The template source code.
+ */
+
+/**
+ * Base class for all Twig errors.
+ *
+ * Mirrors `Twig\Error\Error`: the raw message is stored separately and the
+ * exposed message appends the template location (`in "name" at line N column M.`).
+ */
+export class TwigError extends Error {
+	/**
+	 * @param {string} message The raw error message.
+	 * @param {number} [lineNumber] The 1-based template line, or -1.
+	 * @param {Source|null} [source] The template source context.
+	 * @param {Error|null} [previous] The previous (wrapped) exception.
+	 * @param {number|null} [columnNumber] The 1-based column, if known.
+	 */
+	constructor(message, lineNumber = -1, source = null, previous = null, columnNumber = null) {
+		super(message);
+		this.name = this.constructor.name;
+		/** @type {number} */
+		this.lineNumber = lineNumber;
+		/** @type {Source|null} */
+		this.source = source;
+		/** @type {Error|null} */
+		this.previous = previous;
+		/** @type {number|null} */
+		this.columnNumber = columnNumber;
+		/** @type {string} */
+		this.rawMessage = message;
+		this.updateRepresentation();
+	}
+
+	/** Rebuilds the formatted message with template location operatorInfo. */
+	updateRepresentation() {
+		let message = this.rawMessage;
+		const last = message.slice(-1);
+		const punctuation = last === '.' || last === '?' ? last : '';
+		if (punctuation) {
+			message = message.slice(0, -1);
+		}
+		if (this.source && this.source.name) {
+			message += ` in "${this.source.name}"`;
+		}
+		if (this.lineNumber > 0) {
+			message += ` at line ${this.lineNumber}`;
+			if (this.columnNumber !== null) {
+				message += ` column ${this.columnNumber}`;
+			}
+		}
+		if (punctuation) {
+			message += punctuation;
+		}
+		this.message = message;
+	}
+
+	/**
+	 * @returns {string} The raw message (without location).
+	 */
+	getRawMessage() {
+		return this.rawMessage;
+	}
+
+	/**
+	 * The fully-qualified PHP class name this error maps to.
+	 *
+	 * @returns {string} e.g. `Twig\Error\SyntaxError`.
+	 */
+	getPhpClass() {
+		return `Twig\\Error\\${this.name.replace(/Error$/, 'Error')}`;
+	}
+
+	/**
+	 * Formats the error like PHP's `sprintf('%s: %s', $e::class, $e->getMessage())`.
+	 *
+	 * @returns {string} `Class: message`.
+	 */
+	toPhpFormat() {
+		return `${this.getPhpClass()}: ${this.message}`;
+	}
+
+	/**
+	 * @returns {number} The template line, or -1.
+	 */
+	getTemplateLine() {
+		return this.lineNumber;
+	}
+
+	/**
+	 * Appends text to the raw message and rebuilds the formatted message.
+	 *
+	 * Mirrors `Twig\Error\Error::appendMessage()`.
+	 *
+	 * @param {string} rawMessage
+	 */
+	appendMessage(rawMessage) {
+		this.rawMessage += rawMessage;
+		this.updateRepresentation();
+	}
+
+	/**
+	 * @param {number} line Sets the template line.
+	 */
+	setTemplateLine(line) {
+		this.lineNumber = line;
+		this.updateRepresentation();
+	}
+
+	/**
+	 * @returns {Source|null} The source context.
+	 */
+	getSourceContext() {
+		return this.source;
+	}
+
+	/**
+	 * @param {Source|null} source Sets the source context.
+	 */
+	setSourceContext(source) {
+		this.source = source;
+		this.updateRepresentation();
+	}
+
+	/**
+	 * @param {number|null} columnNumber Sets the column.
+	 */
+	setTemplateColumn(columnNumber) {
+		this.columnNumber = columnNumber;
+		this.updateRepresentation();
+	}
+}
+
+/**
+ * Raised for lexer/parser errors, unknown tags or callables.
+ */
+export class SyntaxError extends TwigError {
+	/**
+	 * @param {string} message
+	 * @param {number} [lineNumber]
+	 * @param {Source|null} [source]
+	 * @param {number|null} [columnNumber]
+	 */
+	constructor(message, lineNumber = -1, source = null, columnNumber = null) {
+		super(message, lineNumber, source, null, columnNumber);
+	}
+
+	/**
+	 * @override
+	 * @returns {string}
+	 */
+	getPhpClass() {
+		return 'Twig\\Error\\SyntaxError';
+	}
+
+	/**
+	 * Appends "Did you mean …?" suggestions to the message.
+	 *
+	 * Mirrors `Twig\Error\SyntaxError::addSuggestions()`, including its
+	 * byte-oriented semantics: the Levenshtein distance and the length threshold
+	 * are computed over the UTF-8 byte sequences (PHP `levenshtein()` /
+	 * `strlen()`).
+	 *
+	 * @param {string} name The name that does not exist.
+	 * @param {string[]} items The possible names.
+	 */
+	addSuggestions(name, items) {
+		const nameBytes = toBytes(name);
+		/** @type {Record<string, number>} */
+		const alternatives = {};
+		for (const item of items) {
+			const lev = byteLevenshtein(nameBytes, toBytes(item));
+			if (lev <= nameBytes.length / 3 || item.includes(name)) {
+				alternatives[item] = lev;
+			}
+		}
+		const keys = Object.keys(alternatives);
+		if (!keys.length) {
+			return;
+		}
+		keys.sort((a, b) => alternatives[a] - alternatives[b]);
+		this.appendMessage(` Did you mean "${keys.join('", "')}"?`);
+	}
+}
+
+/**
+ * @param {string} value
+ * @returns {Uint8Array} The UTF-8 bytes of the string.
+ */
+function toBytes(value) {
+	return new TextEncoder().encode(value);
+}
+
+/**
+ * The Levenshtein distance between two byte sequences (mirrors PHP
+ * `levenshtein()`, which operates on bytes).
+ *
+ * @param {Uint8Array} a
+ * @param {Uint8Array} b
+ * @returns {number}
+ */
+function byteLevenshtein(a, b) {
+	if (a.length === 0) {
+		return b.length;
+	}
+	if (b.length === 0) {
+		return a.length;
+	}
+	const matrix = new Uint16Array((a.length + 1) * (b.length + 1));
+	for (let i = 0; i <= a.length; i++) {
+		matrix[i * (b.length + 1)] = i;
+	}
+	for (let j = 0; j <= b.length; j++) {
+		matrix[j] = j;
+	}
+	for (let i = 1; i <= a.length; i++) {
+		for (let j = 1; j <= b.length; j++) {
+			const cost = a[i - 1] === b[j - 1] ? 0 : 1;
+			matrix[i * (b.length + 1) + j] = Math.min(
+				matrix[(i - 1) * (b.length + 1) + j] + 1,
+				matrix[i * (b.length + 1) + j - 1] + 1,
+				matrix[(i - 1) * (b.length + 1) + j - 1] + cost,
+			);
+		}
+	}
+	return matrix[a.length * (b.length + 1) + b.length];
+}
+
+/**
+ * Raised for errors during rendering.
+ */
+export class RuntimeError extends TwigError {
+	/**
+	 * @param {string} message
+	 * @param {number} [lineNumber]
+	 * @param {Source|null} [source]
+	 * @param {Error|null} [previous]
+	 */
+	constructor(message, lineNumber = -1, source = null, previous = null) {
+		super(message, lineNumber, source, previous);
+	}
+
+	/**
+	 * @override
+	 * @returns {string}
+	 */
+	getPhpClass() {
+		return 'Twig\\Error\\RuntimeError';
+	}
+}
+
+/**
+ * Raised when a template cannot be found or read.
+ */
+export class LoaderError extends TwigError {
+	/**
+	 * @param {string} message
+	 * @param {number} [lineNumber]
+	 * @param {Source|null} [source]
+	 */
+	constructor(message, lineNumber = -1, source = null) {
+		super(message, lineNumber, source);
+	}
+
+	/**
+	 * @override
+	 * @returns {string}
+	 */
+	getPhpClass() {
+		return 'Twig\\Error\\LoaderError';
+	}
+}

package/src/index.js

@@ -0,0 +1,11 @@
+// @ts-check
+/**
+ * Public API of the `@mrhenry/twig-tokenizer` package.
+ *
+ * @module twig-tokenizer
+ */
+
+export { Token, TokenType, typeToString, typeToEnglish } from './token.js';
+export { TokenStream } from './token-stream.js';
+export { Lexer, LexerState, Source, DEFAULT_LEXER_OPTIONS } from './lexer.js';
+export { TwigError, SyntaxError, RuntimeError, LoaderError } from './errors.js';