@mrhenry/twig-html-parser 0.1.0

package/src/index.js

@@ -0,0 +1,11 @@
+// @ts-check
+/**
+ * Public API of the `@mrhenry/twig-html-parser` package.
+ *
+ * @module twig-html-parser
+ */
+
+export { extractAtoms, serializeAtoms, atomToDebugString } from './atoms.js';
+export { tokenizeHtml } from './html-tokenizer.js';
+export { buildTree, pairTwigBlocks, isBlockOpen } from './parser.js';
+export { parse } from './parse.js';

package/src/parse.js

@@ -0,0 +1,36 @@
+// @ts-check
+/**
+ * End-to-end parsing of a Twig template into the hybrid HTML+Twig AST.
+ *
+ * This is the single entry point used by the Prettier plugin:
+ *
+ *   1. the `@mrhenry/twig-tokenizer` lexer produces the source-ordered token
+ *      stream with exact offsets;
+ *   2. `extractAtoms` flattens it into text/twig/comment atoms;
+ *   3. `tokenizeHtml` runs the HTML5-style tokenizer over the atoms;
+ *   4. `buildTree` assembles the hybrid AST and pairs twig blocks.
+ *
+ * Because the source is never transformed, every node keeps exact offsets in
+ * the original text (needed for report-only mode, range formatting and cursor
+ * mapping).
+ *
+ * @module twig-html-parser
+ */
+import { Lexer, Source } from '@mrhenry/twig-tokenizer';
+import { extractAtoms } from './atoms.js';
+import { tokenizeHtml } from './html-tokenizer.js';
+import { buildTree } from './parser.js';
+
+/**
+ * Parses a template source string into the hybrid AST.
+ *
+ * @param {string} source The template source code.
+ * @param {string} [name] The template name (for diagnostics).
+ * @returns {ReturnType<typeof buildTree>} The root AST node.
+ */
+export function parse(source, name = 'index.twig') {
+	const stream = new Lexer().tokenize(new Source(source, name));
+	const atoms = extractAtoms(source, stream.getTokens());
+	const tokens = tokenizeHtml(atoms, source);
+	return buildTree(tokens);
+}

package/src/parser.js

@@ -0,0 +1,330 @@
+// @ts-check
+/**
+ * Builds the hybrid HTML+Twig AST from the tokenizer's token list.
+ *
+ * The tree mirrors the source structure exactly: elements nest according to
+ * the tags the author wrote (no implied end tags, no foster parenting), and
+ * twig atoms appear in data position, inside attribute values and between
+ * attributes. Twig block tags (`if`/`for`/`block`/…) are then paired into
+ * `twigBlock` nodes so the printer can indent their bodies.
+ *
+ * @module twig-html-parser
+ */
+
+/**
+ * @typedef {import('./atoms.js').Atom} Atom
+ * @typedef {import('./html-tokenizer.js').HtmlToken} HtmlToken
+ * @typedef {import('./html-tokenizer.js').TagItem} TagItem
+ * @typedef {import('./html-tokenizer.js').HtmlAttribute} HtmlAttribute
+ * @typedef {import('./html-tokenizer.js').ValueChunk} ValueChunk
+ */
+
+/**
+ * Block tags that introduce a body terminated by a matching `end*` tag.
+ *
+ * Mirrors `@mrhenry/twig-parser`'s `BLOCK_TAGS` / `TAG_END_NAMES` (and
+ * `spec/04-tags.md` §4.21). `set` is handled separately by {@link isBlockOpen}
+ * because only its capture form (`{% set x %}...{% endset %}`) has a body.
+ */
+const BLOCK_OPEN_TAGS = new Set([
+	'if', 'for', 'block', 'macro', 'embed', 'apply', 'autoescape', 'with',
+	'sandbox', 'guard',
+]);
+/** Closing tag name → opening tag name. */
+/** @type {Record<string, string>} */
+const BLOCK_CLOSE_TAGS = {
+	endif: 'if',
+	endfor: 'for',
+	endblock: 'block',
+	endset: 'set',
+	endmacro: 'macro',
+	endembed: 'embed',
+	endapply: 'apply',
+	endautoescape: 'autoescape',
+	endwith: 'with',
+	endsandbox: 'sandbox',
+	endguard: 'guard',
+};
+/** Tags that separate sections inside a block body. */
+const BLOCK_MID_TAGS = new Set(['else', 'elseif']);
+
+/**
+ * A hybrid AST node.
+ *
+ * @typedef {object} AstNode
+ * @property {string} type
+ * @property {number} rawStart Absolute offset where the node starts.
+ * @property {number} rawEnd Absolute offset just past the node.
+ * @property {string} [raw] The exact source text (text/twig/comment/doctype).
+ * @property {string} [name] The element name.
+ * @property {string} [nameRaw] The exact source text of the element name.
+ * @property {boolean} [selfClosing] Whether the start tag is self-closing.
+ * @property {string} [startTagRaw] The start tag source.
+ * @property {number} [startTagStart] The start tag offset.
+ * @property {number} [startTagEnd] Offset just past the start tag.
+ * @property {string} [endTagRaw] The end tag source.
+ * @property {number} [endTagStart] The end tag offset.
+ * @property {number} [endTagEnd] Offset just past the end tag.
+ * @property {Array<AstNode>} [children] Child nodes of an element or block.
+ * @property {Array<AstNode | HtmlAttribute>} [attrs] Items inside a start tag.
+ * @property {Array<{head: AstNode|null, body: Array<AstNode | HtmlAttribute>}>} [sections] Block sections.
+ * @property {Atom} [atom] The source atom of a twig leaf.
+ * @property {string} [tag] The twig block tag name.
+ * @property {AstNode} [open] The opening twig leaf of a block.
+ * @property {AstNode} [close] The closing twig leaf of a block.
+ */
+
+/**
+ * @param {string} type
+ * @param {number} rawStart
+ * @param {number} rawEnd
+ * @param {object} [extra]
+ * @returns {AstNode}
+ */
+function node(type, rawStart, rawEnd, extra = {}) {
+	return { type, rawStart, rawEnd, ...extra };
+}
+
+/**
+ * Whether a twig leaf atom opens a block (has a body).
+ *
+ * @param {Atom} atom
+ * @returns {boolean}
+ */
+export function isBlockOpen(atom) {
+	if (atom.isPrint || atom.kind !== 'twig' || !atom.tag) {
+		return false;
+	}
+	if (BLOCK_OPEN_TAGS.has(atom.tag)) {
+		return true;
+	}
+	// `{% set x %}...{% endset %}` is a capture block; `{% set x = … %}` is not
+	if (atom.tag === 'set') {
+		return !/=/.test(atom.raw);
+	}
+	return false;
+}
+
+/**
+ * Whether a twig leaf atom closes a block opened by `openTag`.
+ *
+ * @param {Atom} atom
+ * @param {string} openTag
+ * @returns {boolean}
+ */
+function isBlockClose(atom, openTag) {
+	const tag = atom.tag;
+	if (tag === null || tag === undefined) {
+		return false;
+	}
+	return BLOCK_CLOSE_TAGS[tag] === openTag;
+}
+
+/**
+ * Builds the hybrid AST from the HTML token list.
+ *
+ * @param {HtmlToken[]} tokens
+ * @returns {AstNode} The root node.
+ */
+export function buildTree(tokens) {
+	/** @type {AstNode} */
+	const root = node('root', tokens.length ? tokens[0].rawStart : 0, tokens.length ? tokens[tokens.length - 1].rawEnd : 0, { children: [] });
+	/** @type {Array<AstNode>} Stack of open elements. */
+	const stack = [root];
+
+	/**
+	 * @returns {AstNode} The current container (root or element).
+	 */
+	function current() {
+		return stack[stack.length - 1];
+	}
+
+	/**
+	 * @param {AstNode} child
+	 */
+	function append(child) {
+		/** @type {AstNode} */
+		const container = current();
+		if (!Array.isArray(container.children)) {
+			container.children = [];
+		}
+		if (child.type === 'text') {
+			const siblings = /** @type {Array<AstNode>} */ (container.children);
+			const last = siblings[siblings.length - 1];
+			if (last && last.type === 'text') {
+				last.raw = /** @type {string} */ (last.raw) + /** @type {string} */ (child.raw);
+				last.rawEnd = child.rawEnd;
+				return;
+			}
+		}
+		container.children.push(child);
+	}
+
+	for (const token of tokens) {
+		switch (token.type) {
+			case 'startTag': {
+				const element = node('element', token.rawStart, token.rawEnd, {
+					name: token.name,
+					nameRaw: token.nameRaw,
+					selfClosing: token.selfClosing,
+					startTagRaw: token.raw,
+					startTagStart: token.rawStart,
+					startTagEnd: token.rawEnd,
+					children: [],
+					attrs: /** @type {Array<AstNode | HtmlAttribute>} */ (
+						(token.attrs ?? []).map((item) =>
+							item.type === 'twig'
+								? node('twig', item.atom.sourceStart, item.atom.rawEnd, {
+									atom: item.atom,
+									raw: item.atom.leading + item.atom.raw,
+								})
+								: item,
+						)
+					),
+				});
+				append(element);
+				if (!token.selfClosing && !VOID_ELEMENTS.has(token.name ?? '')) {
+					stack.push(element);
+				}
+				break;
+			}
+			case 'endTag': {
+				// pop the matching open element (lenient: closes implicit ancestors)
+				let closed = false;
+				for (let i = stack.length - 1; i > 0; i -= 1) {
+					if (stack[i].name === token.name) {
+						const element = stack[i];
+						element.endTagRaw = token.raw;
+						element.endTagStart = token.rawStart;
+						element.endTagEnd = token.rawEnd;
+						element.rawEnd = token.rawEnd;
+						stack.length = i;
+						closed = true;
+						break;
+					}
+				}
+				if (!closed) {
+					// dangling end tag: keep it as text so nothing is lost
+					append(node('text', token.rawStart, token.rawEnd, { raw: token.raw }));
+				}
+				break;
+			}
+			case 'text':
+				append(node('text', token.rawStart, token.rawEnd, { raw: token.raw }));
+				break;
+			case 'comment': {
+				if (token.atom && token.atom.kind === 'comment') {
+					// twig comment
+					append(node('twigComment', token.rawStart, token.rawEnd, { atom: token.atom, raw: token.raw }));
+				} else {
+					append(node('comment', token.rawStart, token.rawEnd, { raw: token.raw }));
+				}
+				break;
+			}
+			case 'doctype':
+				append(node('doctype', token.rawStart, token.rawEnd, { raw: token.raw }));
+				break;
+			case 'twig': {
+				const atom = /** @type {Atom} */ (token.atom);
+				append(node('twig', token.rawStart, token.rawEnd, { atom, raw: token.raw }));
+				break;
+			}
+			default:
+				break;
+		}
+	}
+
+	pairTree(root);
+	return root;
+}
+
+/** Void elements that never have an end tag. */
+const VOID_ELEMENTS = new Set([
+	'area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link',
+	'meta', 'param', 'source', 'track', 'wbr',
+]);
+
+/**
+ * Pairs twig block tags into `twigBlock` nodes at every container level and
+ * inside attribute lists.
+ *
+ * @param {AstNode} root
+ */
+function pairTree(root) {
+	/** @param {AstNode} n */
+	const walk = (n) => {
+		if (Array.isArray(n.children)) {
+			n.children = /** @type {AstNode[]} */ (pairTwigBlocks(n.children));
+			for (const child of n.children) {
+				walk(child);
+			}
+		}
+		if (n.type === 'element' && Array.isArray(n.attrs)) {
+			n.attrs = pairTwigBlocks(n.attrs);
+		}
+	};
+	const children = root.children;
+	if (Array.isArray(children)) {
+		root.children = /** @type {AstNode[]} */ (pairTwigBlocks(children));
+		for (const child of root.children) {
+			walk(child);
+		}
+	}
+}
+
+/**
+ * Replaces runs of block-open/close twig leaves in a flat item list with
+ * `twigBlock` nodes.
+ *
+ * @param {Array<AstNode | HtmlAttribute>} items
+ * @returns {Array<AstNode | HtmlAttribute>}
+ */
+export function pairTwigBlocks(items) {
+	/** @type {Array<{open: AstNode, sections: Array<{head: AstNode|null, body: Array<AstNode | HtmlAttribute>}>}>} */
+	const stack = [];
+	/** @type {Array<AstNode | HtmlAttribute>} */
+	const out = [];
+
+	for (const item of items) {
+		if (/** @type {any} */ (item).type === 'twig') {
+			const twig = /** @type {AstNode} */ (item);
+			const atom = /** @type {Atom} */ (twig.atom);
+			if (isBlockOpen(atom)) {
+				stack.push({ open: twig, sections: [{ head: null, body: [] }] });
+				continue;
+			}
+			if (stack.length > 0 && atom.tag && BLOCK_MID_TAGS.has(atom.tag)) {
+				stack[stack.length - 1].sections.push({ head: twig, body: [] });
+				continue;
+			}
+			if (stack.length > 0 && isBlockClose(atom, /** @type {string} */ ((/** @type {any} */ (stack[stack.length - 1]).open).atom.tag ?? ''))) {
+				const frame = /** @type {any} */ (stack.pop());
+				const block = node('twigBlock', frame.open.rawStart, twig.rawEnd, {
+					tag: frame.open.atom.tag,
+					open: frame.open,
+					sections: frame.sections,
+					close: twig,
+					children: frame.sections.flatMap((/** @type {any} */ s) => s.body),
+				});
+				if (stack.length > 0) {
+					stack[stack.length - 1].sections[stack[stack.length - 1].sections.length - 1].body.push(block);
+				} else {
+					out.push(block);
+				}
+				continue;
+			}
+		}
+		if (stack.length > 0) {
+			stack[stack.length - 1].sections[stack[stack.length - 1].sections.length - 1].body.push(item);
+		} else {
+			out.push(item);
+		}
+	}
+
+	// flush any unclosed blocks (malformed source) back as leaves in order
+	for (const frame of stack) {
+		out.push(frame.open, ...frame.sections.flatMap((/** @type {any} */ s) => s.body));
+	}
+
+	return out;
+}

package/test/atoms.test.js

@@ -0,0 +1,161 @@
+// @ts-check
+/**
+ * Round-trip and structure tests for `@mrhenry/twig-html-parser` atom
+ * extraction.
+ *
+ * Every tokenizable source in the corpus (fixture templates, fuzzer cases and
+ * hand-written edge cases) must produce an atom list that serializes back to
+ * the exact source it was extracted from. Atoms must also carry exact offsets
+ * and correctly classify twig constructs, comments and verbatim blocks.
+ *
+ * @module test
+ */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { Lexer, Source } from '@mrhenry/twig-tokenizer';
+import { extractAtoms, serializeAtoms } from '@mrhenry/twig-html-parser';
+import { collectSources, EDGE_CASES } from '../../test/serialize-sources.js';
+
+/**
+ * @param {string} source
+ * @returns {ReturnType<typeof extractAtoms>}
+ */
+function atoms(source) {
+	const tokens = new Lexer().tokenize(new Source(source, 'index.twig')).getTokens();
+	return extractAtoms(source, tokens);
+}
+
+/**
+ * @param {ReturnType<typeof extractAtoms>} list
+ * @returns {string[]}
+ */
+function kinds(list) {
+	return list.map((a) => a.kind);
+}
+
+let ran = 0;
+let skipped = 0;
+
+for (const { source, name } of collectSources()) {
+	test(`atoms round trip: ${name}`, () => {
+		let stream;
+		try {
+			stream = new Lexer().tokenize(new Source(source, name));
+		} catch (e) {
+			skipped += 1;
+			return; // not a tokenizable source
+		}
+		ran += 1;
+		const list = extractAtoms(source, stream.getTokens());
+		assert.equal(serializeAtoms(list), source, `lossy serialization for ${name}`);
+	});
+}
+
+for (const [label, source] of Object.entries(EDGE_CASES)) {
+	test(`atoms edge case: ${label}`, () => {
+		const list = atoms(source);
+		ran += 1;
+		assert.equal(serializeAtoms(list), source, `lossy serialization for ${label}`);
+	});
+}
+
+test('atoms classify a mixed template', () => {
+	const source = 'a {# c #} <div {% if x %} id="{{ i }}" {% endif %}>b</div> {{ y }}';
+	const list = atoms(source);
+	assert.deepEqual(kinds(list), ['text', 'comment', 'text', 'twig', 'text', 'twig', 'text', 'twig', 'text', 'twig']);
+});
+
+test('atoms extract tag names and print flags', () => {
+	const source = '{% if x %}{{ y }}{% endif %}';
+	const list = atoms(source);
+	const twigs = list.filter((a) => a.kind === 'twig');
+	assert.deepEqual(
+		twigs.map((a) => [a.tag, a.isPrint]),
+		[['if', false], [null, true], ['endif', false]],
+	);
+});
+
+test('atoms carve comments out of leading trivia', () => {
+	const source = 'foo\n{# c #}\nbar';
+	const list = atoms(source);
+	const comment = list.find((a) => a.kind === 'comment');
+	assert.ok(comment);
+	assert.equal(comment.raw, '{# c #}');
+	assert.equal(comment.leading, '');
+	// the comment precedes `bar`; the whitespace it consumed folds into the
+	// following text atom's raw
+	assert.equal(comment.rawStart, 4);
+	assert.equal(list[list.length - 1].raw, '\nbar');
+	assert.equal(list[list.length - 1].leading, '');
+});
+
+test('atoms carve multiple comments with interleaved whitespace', () => {
+	const source = 'a {# one #} b {# two #} c';
+	const list = atoms(source);
+	const comments = list.filter((a) => a.kind === 'comment');
+	assert.equal(comments.length, 2);
+	// the spaces around the comments live inside the adjacent text atom raws
+	assert.equal(comments[0].raw, '{# one #}');
+	assert.equal(comments[0].leading, '');
+	assert.equal(comments[1].raw, '{# two #}');
+	assert.equal(comments[1].leading, '');
+	assert.equal(list[list.length - 1].raw, ' c');
+	assert.equal(list[list.length - 1].leading, '');
+});
+
+test('atoms keep a whitespace text atom between comments', () => {
+	const source = '{# a #} {# b #}x';
+	const list = atoms(source);
+	assert.deepEqual(kinds(list), ['comment', 'text', 'comment', 'text']);
+	assert.equal(list[1].raw, ' ');
+	assert.equal(list[2].raw, '{# b #}');
+	assert.equal(serializeAtoms(list), source);
+});
+
+test('atoms keep a trailing comment and whitespace', () => {
+	const source = 'foo {# c #}';
+	const list = atoms(source);
+	assert.deepEqual(kinds(list), ['text', 'comment']);
+	assert.equal(serializeAtoms(list), source);
+});
+
+test('atoms keep a leading comment', () => {
+	const source = '{# c #}\n<div>hi</div>';
+	const list = atoms(source);
+	assert.deepEqual(kinds(list), ['comment', 'text']);
+	assert.equal(list[0].rawStart, 0);
+	assert.equal(list[1].raw, '\n<div>hi</div>');
+	assert.equal(serializeAtoms(list), source);
+});
+
+test('atoms flag verbatim blocks as opaque text', () => {
+	const source = '{% verbatim %}<div>{{ not parsed }}</div>{% endverbatim %}';
+	const list = atoms(source);
+	const verbatim = list.find((a) => a.verbatim);
+	assert.ok(verbatim);
+	assert.equal(verbatim.kind, 'text');
+	assert.equal(verbatim.raw, source);
+});
+
+test('atoms preserve offsets across a tag group', () => {
+	const source = '<a href="{{ url }}">x</a>';
+	const list = atoms(source);
+	const print = list.find((a) => a.kind === 'twig' && a.isPrint);
+	assert.ok(print);
+	assert.equal(print.raw, '{{ url }}');
+	assert.equal(source.slice(print.rawStart, print.rawEnd), '{{ url }}');
+	assert.equal(print.rawStart, 9);
+});
+
+test('atoms round trip trim markers and whitespace control', () => {
+	const source = '{%- if x -%}\n\t{{- y -}}\n{%- endif -%}';
+	assert.equal(serializeAtoms(atoms(source)), source);
+});
+
+test('atoms summary', () => {
+	assert.ok(ran > 0, 'no sources were extracted');
+});
+
+process.on('exit', () => {
+	console.log(`\nAtom extraction: ran=${ran} skipped=${skipped}`);
+});

package/test/html-tokenizer.test.js

@@ -0,0 +1,161 @@
+// @ts-check
+/**
+ * Structure and coverage tests for the `@mrhenry/twig-html-parser` HTML
+ * tokenizer.
+ *
+ * Every source in the corpus must tokenize to a contiguous, source-ordered
+ * token list (no gaps, no out-of-order spans) with exact offsets — the
+ * guarantee that report-only / range / cursor modes rely on. Focused tests
+ * cover the interesting tokenizer behaviour: twig between attributes, twig in
+ * attribute values, raw text elements, comments, doctype and void elements.
+ *
+ * @module test
+ */
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { Lexer, Source } from '@mrhenry/twig-tokenizer';
+import { extractAtoms, tokenizeHtml } from '@mrhenry/twig-html-parser';
+import { collectSources, EDGE_CASES } from '../../test/serialize-sources.js';
+
+/**
+ * Tokenizes a source into HTML tokens.
+ *
+ * @param {string} source
+ * @returns {ReturnType<typeof tokenizeHtml>}
+ */
+function html(source) {
+	const tokens = new Lexer().tokenize(new Source(source, 'index.twig')).getTokens();
+	return tokenizeHtml(extractAtoms(source, tokens), source);
+}
+
+/**
+ * Asserts the token list covers the source contiguously and in order.
+ *
+ * @param {string} source
+ * @param {ReturnType<typeof tokenizeHtml>} tokens
+ */
+function assertContiguous(source, tokens) {
+	let prev = 0;
+	for (const t of tokens) {
+		assert.ok(t.rawStart >= prev, `token out of order at ${t.rawStart} (prev ${prev})`);
+		assert.ok(t.rawStart === prev, `gap at ${t.rawStart} (prev ${prev})`);
+		assert.ok(t.rawEnd >= t.rawStart, `negative span at ${t.rawStart}`);
+		prev = t.rawEnd;
+	}
+	assert.equal(prev, source.length, `trailing gap (covered ${prev}/${source.length})`);
+}
+
+/**
+ * @param {ReturnType<typeof tokenizeHtml>} tokens
+ * @returns {string[]}
+ */
+function types(tokens) {
+	return tokens.map((t) => t.type);
+}
+
+let ran = 0;
+let skipped = 0;
+
+for (const { source, name } of collectSources()) {
+	test(`html tokenizer contiguous: ${name}`, () => {
+		try {
+			new Lexer().tokenize(new Source(source, name));
+		} catch (e) {
+			skipped += 1;
+			return; // not tokenizable
+		}
+		ran += 1;
+		assertContiguous(source, html(source));
+	});
+}
+
+for (const [label, source] of Object.entries(EDGE_CASES)) {
+	test(`html tokenizer edge case: ${label}`, () => {
+		assertContiguous(source, html(source));
+		ran += 1;
+	});
+}
+
+test('tokenizes elements and text', () => {
+	const tokens = html('<div class="a">hello</div>');
+	assert.deepEqual(types(tokens), ['startTag', 'text', 'endTag']);
+	assert.equal(tokens[0].name, 'div');
+	assert.equal(tokens[0].raw, '<div class="a">');
+	assert.equal(tokens[1].raw, 'hello');
+	assert.equal(tokens[2].raw, '</div>');
+});
+
+test('tokenizes a twig print in an attribute value', () => {
+	const tokens = html('<a href="{{ url }}">x</a>');
+	const tag = /** @type {any} */ (tokens[0]);
+	const attr = /** @type {any} */ (tag.attrs[0]);
+	assert.equal(attr.nameRaw, 'href');
+	assert.equal(attr.valueChunks.length, 1);
+	assert.equal(attr.valueChunks[0].type, 'twig');
+	assert.equal(/** @type {any} */ (attr.valueChunks[0]).atom.raw, '{{ url }}');
+});
+
+test('tokenizes conditional attributes via twig between attributes', () => {
+	const source = '<div class="row" {% if x %} id="y" {% endif %} data-x>';
+	const tokens = html(source);
+	assert.deepEqual(types(tokens), ['startTag']);
+	const tag = /** @type {any} */ (tokens[0]);
+	const kinds = tag.attrs.map((/** @type {any} */ a) => a.type ?? 'attr');
+	assert.deepEqual(kinds, ['attr', 'twig', 'attr', 'twig', 'attr']);
+	assert.equal(tag.attrs[1].atom.tag, 'if');
+	assert.equal(tag.attrs[3].atom.tag, 'endif');
+	assert.equal(tag.raw, source);
+});
+
+test('tokenizes doctype and comments', () => {
+	const tokens = html('<!doctype html>\n<!-- note --><p>x</p>');
+	assert.deepEqual(types(tokens), ['doctype', 'text', 'comment', 'startTag', 'text', 'endTag']);
+	assert.equal(tokens[0].raw, '<!doctype html>');
+	assert.equal(tokens[2].raw, '<!-- note -->');
+});
+
+test('tokenizes raw text elements (script) with twig inside', () => {
+	const source = '<script>var x = "{{ v }}";</script>';
+	const tokens = html(source);
+	assert.deepEqual(types(tokens), ['startTag', 'text', 'twig', 'text', 'endTag']);
+	assert.equal(tokens[1].raw, 'var x = "');
+	assert.equal(tokens[2].raw, '{{ v }}');
+	assert.equal(tokens[3].raw, '";');
+	assert.equal(tokens[4].raw, '</script>');
+});
+
+test('does not parse markup inside style content', () => {
+	const source = '<style>.a { color: red; }</style>';
+	const tokens = html(source);
+	assert.deepEqual(types(tokens), ['startTag', 'text', 'endTag']);
+	assert.equal(tokens[1].raw, '.a { color: red; }');
+});
+
+test('handles void elements without an end tag', () => {
+	const tokens = html('<img src="a" alt="b">tail');
+	assert.deepEqual(types(tokens), ['startTag', 'text']);
+	assert.equal(tokens[0].name, 'img');
+	assert.equal(tokens[1].raw, 'tail');
+});
+
+test('preserves self-closing tags', () => {
+	const tokens = html('<link rel="x" />');
+	assert.equal(tokens[0].type, 'startTag');
+	assert.equal(/** @type {any} */ (tokens[0]).selfClosing, true);
+	assert.equal(tokens[0].raw, '<link rel="x" />');
+});
+
+test('treats a stray < followed by twig as text', () => {
+	const source = '< {{ x }}';
+	const tokens = html(source);
+	assert.deepEqual(types(tokens), ['text', 'text', 'twig']);
+	assert.equal(tokens[0].raw, '<');
+});
+
+test('tokenizer summary', () => {
+	assert.ok(ran > 0, 'no sources were tokenized');
+});
+
+process.on('exit', () => {
+	console.log(`\nHTML tokenizer: ran=${ran} skipped=${skipped}`);
+});