@mrhenry/twig-html-parser 0.1.0

package/LICENSE

@@ -0,0 +1,24 @@
+MIT License
+
+Copyright (c) 2026 Mr. Henry
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+This license applies only to the code in this repository.
+Images are explicitly excluded.

package/package.json

@@ -0,0 +1,16 @@
+{
+	"name": "@mrhenry/twig-html-parser",
+	"version": "0.1.0",
+	"license": "MIT",
+	"type": "module",
+	"main": "src/index.js",
+	"exports": {
+		".": "./src/index.js"
+	},
+	"scripts": {
+		"test": "node --test test/*.test.js"
+	},
+	"dependencies": {
+		"@mrhenry/twig-tokenizer": "0.1.0"
+	}
+}

package/src/atoms.js

@@ -0,0 +1,335 @@
+// @ts-check
+/**
+ * Atom extraction for the unified HTML+Twig parser.
+ *
+ * The existing `@mrhenry/twig-tokenizer` lexer walks the raw template source
+ * and isolates every Twig construct (prints, block tags, comments, verbatim
+ * blocks) from the surrounding static text. That token stream is the single
+ * source of truth for source positions, so this module turns it into a flat,
+ * source-ordered list of *atoms*:
+ *
+ *  * `text` atoms — a `TEXT` token (a static HTML region), except verbatim
+ *    blocks which stay opaque so their content is never parsed as HTML;
+ *  * `twig` atoms — a maximal `{{ … }}` or `{% … %}` construct, with its
+ *    opening tag name extracted from the first `NAME` token;
+ *  * `comment` atoms — `{# … #}` / `{## … ##}` comments, which the lexer
+ *    *skips* (they surface as the `leading` trivia of the following token) and
+ *    which are therefore carved back out of that trivia here.
+ *
+ * Every atom keeps the exact source text (`raw`), the trivia preceding it
+ * (`leading`) and absolute offsets, so concatenating `leading + raw` across
+ * all atoms reproduces the template source byte-for-byte. The HTML tokenizer
+ * consumes these atoms directly — Twig constructs are first-class units and
+ * the HTML parser never sees a stray `{%` or `{{`.
+ *
+ * @module twig-html-parser
+ */
+import { TokenType } from '@mrhenry/twig-tokenizer';
+
+// Delimiter configuration matching the default lexer options. The lexer is
+// always constructed with these defaults in practice.
+const COMMENT_START = '{#';
+const COMMENT_END = '#}';
+const WHITESPACE_TRIM = '-';
+const WHITESPACE_LINE_TRIM = '~';
+const DOC_START = '{##';
+
+/**
+ * An atom of the unified template stream.
+ *
+ * @typedef {object} Atom
+ * @property {'text'|'twig'|'comment'} kind
+ * @property {string} leading Trivia (whitespace) preceding the atom.
+ * @property {string} raw The atom's own exact source text.
+ * @property {number} sourceStart Absolute offset where `leading` starts.
+ * @property {number} rawStart Absolute offset where `raw` starts.
+ * @property {number} rawEnd Absolute offset just past `raw`.
+ * @property {boolean} [verbatim] Whether a text atom is a verbatim block.
+ * @property {boolean} [isPrint] For twig atoms: `{{ }}` print vs `{% %}` block.
+ * @property {string|null} [tag] For twig atoms: the tag name (or null for prints).
+ */
+
+// comment closing markers: `[marker, kind]` where kind is 0 (skip following
+// whitespace), 1 (skip following line-trim chars) or 2 (skip one newline).
+// Mirrors `Lexer#commentMarkers` / `Lexer#docCommentMarkers`.
+/** @type {Array<[string, 0|1|2]>} */
+const COMMENT_MARKERS = [
+	[WHITESPACE_TRIM + COMMENT_END, 0],
+	[WHITESPACE_LINE_TRIM + COMMENT_END, 1],
+	[COMMENT_END, 2],
+];
+/** @type {Array<[string, 0|1|2]>} */
+const DOC_COMMENT_MARKERS = [
+	[WHITESPACE_TRIM + '#' + COMMENT_END, 0],
+	[WHITESPACE_LINE_TRIM + '#' + COMMENT_END, 1],
+	[WHITESPACE_TRIM + COMMENT_END, 0],
+	[WHITESPACE_LINE_TRIM + COMMENT_END, 1],
+	[COMMENT_END, 2],
+];
+
+/**
+ * @param {Atom['kind']} kind
+ * @param {string} raw
+ * @param {number} sourceStart
+ * @param {number} rawStart
+ * @param {number} rawEnd
+ * @param {string} leading
+ * @param {object} [extra]
+ * @returns {Atom}
+ */
+function atom(kind, raw, sourceStart, rawStart, rawEnd, leading, extra = {}) {
+	return { kind, raw, sourceStart, rawStart, rawEnd, leading, ...extra };
+}
+
+/**
+ * Scans the source at a `{#` position and returns the offset just past the
+ * comment's closing marker. The comment span is the comment text only; any
+ * whitespace the lexer happens to consume after the marker stays in the trivia
+ * so the split always reproduces the source exactly.
+ *
+ * @param {string} code The template source.
+ * @param {number} start Offset of the `{#` comment start.
+ * @returns {number} Offset just past the comment, or `-1` when unclosed.
+ */
+function scanCommentEnd(code, start) {
+	const isDoc = code.startsWith(DOC_START, start) && code.charCodeAt(start + 3) !== /* } */ 0x7d;
+	const markers = isDoc ? DOC_COMMENT_MARKERS : COMMENT_MARKERS;
+	const firstChars = [WHITESPACE_TRIM.charCodeAt(0), WHITESPACE_LINE_TRIM.charCodeAt(0), COMMENT_END.charCodeAt(0)];
+
+	for (let p = start + 2; p < code.length; p += 1) {
+		const pc = code.charCodeAt(p);
+		if (pc !== firstChars[0] && pc !== firstChars[1] && pc !== firstChars[2]) {
+			continue;
+		}
+		for (let k = 0; k < markers.length; k += 1) {
+			const marker = markers[k][0];
+			if (code.startsWith(marker, p)) {
+				return p + marker.length;
+			}
+		}
+	}
+	return -1;
+}
+
+/**
+ * Splits a trivia span `[from, to)` of the source into whitespace and comment
+ * pieces. Comments are the only non-whitespace tokens the lexer can leave in
+ * the gap between two tokens.
+ *
+ * @param {string} code The template source.
+ * @param {number} from Absolute start of the trivia.
+ * @param {number} to Absolute end of the trivia.
+ * @returns {Array<{kind: 'ws'|'comment', start: number, end: number}>} Pieces.
+ */
+function splitTrivia(code, from, to) {
+	/** @type {Array<{kind: 'ws'|'comment', start: number, end: number}>} */
+	const pieces = [];
+	let p = from;
+	while (p < to) {
+		const commentStart = code.indexOf(COMMENT_START, p);
+		if (commentStart === -1 || commentStart >= to) {
+			break;
+		}
+		if (commentStart > p) {
+			pieces.push({ kind: 'ws', start: p, end: commentStart });
+		}
+		let commentEnd = scanCommentEnd(code, commentStart);
+		if (commentEnd === -1 || commentEnd > to) {
+			commentEnd = to;
+		}
+		pieces.push({ kind: 'comment', start: commentStart, end: commentEnd });
+		p = commentEnd;
+	}
+	if (p < to) {
+		pieces.push({ kind: 'ws', start: p, end: to });
+	}
+	return pieces;
+}
+
+/**
+ * Emits comment atoms for the comments in a trivia span, accumulating the
+ * whitespace that precedes each comment into its `leading`. Returns the
+ * whitespace remaining after the last comment, to be used as the `leading` of
+ * the following atom, together with the absolute offset where it starts.
+ *
+ * @param {Atom[]} out The atom list to append to.
+ * @param {string} code The template source.
+ * @param {number} from Absolute start of the trivia.
+ * @param {number} to Absolute end of the trivia.
+ * @returns {{text: string, start: number}} The trailing whitespace trivia.
+ */
+function emitComments(out, code, from, to) {
+	let leading = '';
+	let leadingStart = from;
+	for (const piece of splitTrivia(code, from, to)) {
+		if (piece.kind === 'comment') {
+			out.push(
+				atom(
+					'comment',
+					code.slice(piece.start, piece.end),
+					piece.start - leading.length,
+					piece.start,
+					piece.end,
+					leading,
+				),
+			);
+			leading = '';
+			leadingStart = piece.end;
+		} else {
+			leading = code.slice(piece.start, piece.end);
+			leadingStart = piece.start;
+		}
+	}
+	return { text: leading, start: leadingStart };
+}
+
+/**
+ * Extracts the source-ordered atom list from a token stream.
+ *
+ * The stream must be produced from `source` (the same source string is passed
+ * in so exact raw text can be sliced for twig groups).
+ *
+ * @param {string} source The template source code.
+ * @param {Array<import('@mrhenry/twig-tokenizer').Token>} tokens The lexer's token list.
+ * @returns {Atom[]} The atoms, in source order.
+ */
+export function extractAtoms(source, tokens) {
+	/** @type {Atom[]} */
+	const atoms = [];
+
+	// Whitespace that a `-%}`/`~%}`/`-%}`-style closing marker caused the lexer
+	// to consume into the closing token's raw. It is carried over and becomes
+	// trivia of the following atom so twig atoms keep a clean raw.
+	let carryStart = -1;
+
+	/**
+	 * The trivia span preceding the current token, including any carried
+	 * trim-consumed whitespace.
+	 *
+	 * @param {import('@mrhenry/twig-tokenizer').Token} token
+	 * @returns {number} Absolute start of the trivia.
+	 */
+	function triviaFrom(token) {
+		return carryStart !== -1 ? carryStart : token.sourceStart;
+	}
+
+	let i = 0;
+	while (i < tokens.length) {
+		const token = tokens[i];
+		if (token.type === TokenType.EOF) {
+			const from = triviaFrom(token);
+			const trailing = emitComments(atoms, source, from, source.length);
+			if (trailing.text !== '') {
+				atoms.push(
+					atom(
+						'text',
+						trailing.text,
+						trailing.start,
+						trailing.start,
+						source.length,
+						'',
+					),
+				);
+			}
+			carryStart = -1;
+			break;
+		}
+
+		if (token.type === TokenType.TEXT) {
+			const rawText = token.raw ?? '';
+			const from = triviaFrom(token);
+			const leading = emitComments(atoms, source, from, token.rawStart);
+			// fold the trailing whitespace into the raw so the atom's text content
+			// is contiguous (the cursor reads `raw` only)
+			const fullRaw = leading.text + rawText;
+			const rawStart = leading.start;
+			const isVerbatim = /^\s*{%[~-]?\s*verbatim\b/.test(fullRaw);
+			atoms.push(atom('text', fullRaw, rawStart, rawStart, token.rawEnd, '', isVerbatim ? { verbatim: true } : {}));
+			carryStart = -1;
+			i += 1;
+			continue;
+		}
+
+		if (token.type === TokenType.BLOCK_START || token.type === TokenType.VAR_START) {
+			const startToken = token;
+			const isPrint = token.type === TokenType.VAR_START;
+			const endType = isPrint ? TokenType.VAR_END : TokenType.BLOCK_END;
+			i += 1;
+			let tag = null;
+			let endToken = null;
+			while (i < tokens.length) {
+				const t = tokens[i];
+				// the first NAME token names a block tag (`if`, `endif`, ...); print
+				// statements have no tag name
+				if (t.type === TokenType.NAME && tag === null && !isPrint) {
+					tag = /** @type {string} */ (t.value);
+				}
+				if (t.type === endType || t.type === TokenType.EOF) {
+					endToken = t;
+					break;
+				}
+				i += 1;
+			}
+			if (endToken === null) {
+				endToken = tokens[tokens.length - 1];
+			}
+			const from = triviaFrom(startToken);
+			const leading = emitComments(atoms, source, from, startToken.rawStart);
+			// the closing token's raw may include whitespace that a `-%}`/`~%}`
+			// (or plain `%}` + newline) marker trimmed; the twig tag itself ends at
+			// the closing delimiter. An unclosed construct runs to the end of the
+			// source (the EOF token has no closing delimiter).
+			let trueEnd;
+			if (endToken.type === TokenType.EOF) {
+				trueEnd = endToken.rawStart;
+			} else {
+				const endRaw = endToken.raw ?? '';
+				const markerLength = endRaw.startsWith(WHITESPACE_TRIM) || endRaw.startsWith(WHITESPACE_LINE_TRIM) ? 3 : 2;
+				trueEnd = endToken.rawStart + markerLength;
+			}
+			atoms.push(
+				atom(
+					'twig',
+					source.slice(startToken.rawStart, trueEnd),
+					leading.start,
+					startToken.rawStart,
+					trueEnd,
+					leading.text,
+					{ isPrint, tag },
+				),
+			);
+			const carried = source.slice(trueEnd, endToken.rawEnd);
+			carryStart = carried === '' ? -1 : trueEnd;
+			i += 1;
+			continue;
+		}
+
+		// Parser-synthesised or unexpected tokens are ignored; their source is
+		// covered by the surrounding tokens' trivia.
+		i += 1;
+	}
+
+	return atoms;
+}
+
+/**
+ * Serializes an atom list back to template source.
+ *
+ * @param {Atom[]} atoms
+ * @returns {string} The reconstructed source.
+ */
+export function serializeAtoms(atoms) {
+	let out = '';
+	for (const a of atoms) {
+		out += a.leading + a.raw;
+	}
+	return out;
+}
+
+/**
+ * @param {Atom} a
+ * @returns {string} A one-line description (for debugging).
+ */
+export function atomToDebugString(a) {
+	return `[${a.kind}] ${a.rawStart}-${a.rawEnd} ${JSON.stringify(a.raw.slice(0, 40))}`;
+}