@mrhenry/twig-html-parser 0.1.0

package/src/html-tokenizer.js

@@ -0,0 +1,1032 @@
+// @ts-check
+/**
+ * HTML tokenizer over the unified atom stream.
+ *
+ * The tokenizer implements the HTML5 tokenizer states (data, tag open/name,
+ * attribute states, raw text, comments, doctype, …) but consumes *atoms*
+ * instead of a raw character buffer. Text atoms supply the actual HTML text;
+ * twig and comment atoms are first-class units that are never mistaken for
+ * HTML — they are emitted as twig/comment tokens in data position and as
+ * inline items inside start tags (conditional attributes) or attribute values.
+ *
+ * Because the input is the original source (never a substituted copy), every
+ * emitted token carries exact absolute offsets that can be fed straight to
+ * Prettier's `locStart`/`locEnd` and to range/cursor mapping.
+ *
+ * @module twig-html-parser
+ */
+
+/**
+ * @typedef {import('./atoms.js').Atom} Atom
+ */
+
+/** Whitespace characters inside HTML tags. */
+const WS = ' \t\n\f\r';
+/** Elements whose content is raw text (not parsed as markup). */
+const RAWTEXT_ELEMENTS = new Set(['script', 'style']);
+/** Elements whose content is RCDATA (treated as raw text here). */
+const RCDATA_ELEMENTS = new Set(['textarea', 'title']);
+
+/**
+ * @param {number} c
+ * @returns {boolean} Whether the code unit is HTML whitespace inside a tag.
+ */
+function isWhitespace(c) {
+	return WS.includes(String.fromCharCode(c));
+}
+
+/**
+ * A cursor over the atom list. Text atoms are consumed char by char; twig and
+ * comment atoms are taken whole.
+ */
+class Cursor {
+	/**
+	 * @param {Atom[]} atoms
+	 */
+	constructor(atoms) {
+		/** @type {Atom[]} */
+		this.atoms = atoms;
+		/** @type {number} Index of the current atom. */
+		this.index = 0;
+		/** @type {number} Offset within the current text atom's raw. */
+		this.pos = 0;
+		/** @type {number} Last absolute offset returned (used at end of input). */
+		this.lastOffset = atoms.length ? atoms[0].rawStart : 0;
+	}
+
+	/**
+	 * Advances past exhausted text atoms.
+	 */
+	ensureText() {
+		let a = this.atoms[this.index];
+		while (a && a.kind === 'text' && this.pos >= a.raw.length) {
+			this.index += 1;
+			this.pos = 0;
+			a = this.atoms[this.index];
+		}
+	}
+
+	/**
+	 * @returns {boolean} Whether the cursor reached the end of the atoms.
+	 */
+	atEnd() {
+		this.ensureText();
+		return this.index >= this.atoms.length;
+	}
+
+	/**
+	 * @returns {boolean} Whether the current atom is a twig/comment atom.
+	 */
+	atTwig() {
+		this.ensureText();
+		const a = this.atoms[this.index];
+		return !!a && a.kind !== 'text';
+	}
+
+	/**
+	 * @returns {Atom|null} The current text atom, or null at a twig atom / end.
+	 */
+	textAtom() {
+		this.ensureText();
+		const a = this.atoms[this.index];
+		return a && a.kind === 'text' ? a : null;
+	}
+
+	/**
+	 * @returns {string|null} The current character, or null at twig/end.
+	 */
+	peek() {
+		const a = this.textAtom();
+		if (!a) {
+			return null;
+		}
+		return a.raw[this.pos];
+	}
+
+	/**
+	 * @returns {number} Absolute source offset of the current character.
+	 */
+	offset() {
+		const a = this.atoms[this.index];
+		if (!a) {
+			return this.lastOffset;
+		}
+		const off = a.rawStart + this.pos;
+		this.lastOffset = off;
+		return off;
+	}
+
+	/**
+	 * Advances one character.
+	 */
+	next() {
+		this.pos += 1;
+	}
+
+	/**
+	 * Consumes the current twig/comment atom and returns it.
+	 *
+	 * @returns {Atom} The consumed atom.
+	 */
+	takeTwig() {
+		const a = this.atoms[this.index];
+		this.index += 1;
+		this.pos = 0;
+		return a;
+	}
+}
+
+/**
+ * An attribute value chunk: either literal text or a twig/comment atom.
+ *
+ * @typedef {{type: 'text', text: string, start: number, end: number} | {type: 'twig', atom: Atom}} ValueChunk
+ */
+
+/**
+ * A parsed attribute.
+ *
+ * @typedef {object} HtmlAttribute
+ * @property {'attr'} type
+ * @property {string} nameRaw The exact source text of the attribute name.
+ * @property {number} nameStart Absolute offset of the attribute name.
+ * @property {number} nameEnd Absolute offset just past the attribute name.
+ * @property {string|null} quote The quote character, or null when valueless/unquoted.
+ * @property {ValueChunk[]} valueChunks The attribute value (empty when valueless).
+ * @property {number} valueStart Absolute offset of the value content.
+ * @property {number} valueEnd Absolute offset just past the value content.
+ * @property {number} rawStart Absolute offset of the whole attribute source.
+ * @property {number} rawEnd Absolute offset just past the whole attribute.
+ */
+
+/**
+ * The start tag being assembled by the tokenizer.
+ *
+ * @typedef {object} TagState
+ * @property {string} name
+ * @property {string} nameRaw
+ * @property {number} nameStart
+ * @property {number} nameEnd
+ * @property {TagItem[]} attrs
+ * @property {boolean} selfClosing
+ * @property {number} rawStart
+ */
+
+/**
+ * The attribute being assembled by the tokenizer.
+ *
+ * @typedef {object} AttrState
+ * @property {string} nameRaw
+ * @property {number} nameStart
+ * @property {number} nameEnd
+ * @property {string|null} quote
+ * @property {ValueChunk[]} valueChunks
+ * @property {number} valueStart
+ * @property {number} valueEnd
+ * @property {number} rawStart
+ * @property {number} rawEnd
+ * @property {number} chunkStart Offset where the current literal value run started.
+ */
+
+/**
+ * An item inside a start tag: either an attribute or a twig atom.
+ *
+ * @typedef {HtmlAttribute | {type: 'twig', atom: Atom}} TagItem
+ */
+
+/**
+ * A single HTML token.
+ *
+ * @typedef {object} HtmlToken
+ * @property {'text'|'twig'|'comment'|'doctype'|'startTag'|'endTag'} type
+ * @property {string} raw The exact source text of the token.
+ * @property {number} rawStart Absolute offset of the token's raw text.
+ * @property {number} rawEnd Absolute offset just past the token's raw text.
+ * @property {Atom} [atom] For twig/comment tokens, the source atom.
+ * @property {string} [name] For start/end tags, the tag name (lowercased).
+ * @property {string} [nameRaw] For start/end tags, the exact tag name source.
+ * @property {boolean} [selfClosing] For start tags.
+ * @property {TagItem[]} [attrs] For start tags.
+ */
+
+/**
+ * @param {string} type
+ * @param {string} raw
+ * @param {number} rawStart
+ * @param {number} rawEnd
+ * @param {object} [extra]
+ * @returns {HtmlToken}
+ */
+function htmlToken(type, raw, rawStart, rawEnd, extra = {}) {
+	return /** @type {HtmlToken} */ ({ type, raw, rawStart, rawEnd, ...extra });
+}
+
+/**
+ * Tokenizes the atom stream into HTML tokens.
+ *
+ * @param {Atom[]} atoms
+ * @param {string} source The original template source.
+ * @returns {HtmlToken[]} The HTML tokens.
+ */
+export function tokenizeHtml(atoms, source) {
+	const cursor = new Cursor(atoms);
+	/** @type {HtmlToken[]} */
+	const tokens = [];
+
+	/** The start/end tag being assembled (null outside a tag). */
+	/** @type {TagState|null} */
+	let tag = null;
+	/** The attribute being assembled (null outside an attribute). */
+	/** @type {AttrState|null} */
+	let attr = null;
+	/** The raw-text element name, when scanning raw content. */
+	let rawtextTag = null;
+
+	let state = 'data';
+
+	// ---- scanning helpers ----
+
+	/**
+	 * Reads characters while the predicate holds, across text atoms, stopping at
+	 * twig atoms (which are left for the state machine).
+	 *
+	 * @param {(c: string) => boolean} pred
+	 * @returns {{text: string, start: number, end: number}}
+	 */
+	function scanWhile(pred) {
+		const start = cursor.offset();
+		let out = '';
+		let end = start;
+		for (;;) {
+			const a = cursor.textAtom();
+			if (!a) {
+				break; // twig or end of input
+			}
+			const raw = a.raw;
+			let i = cursor.pos;
+			while (i < raw.length && pred(raw[i])) {
+				i += 1;
+			}
+			out += raw.slice(cursor.pos, i);
+			cursor.pos = i;
+			end = a.rawStart + i;
+			if (i < raw.length) {
+				break; // stopped by the predicate inside this atom
+			}
+		}
+		return { text: out, start, end };
+	}
+
+	/**
+	 * Scans a run of data text, stopping just before `<` or a twig atom.
+	 *
+	 * @returns {{text: string, start: number, end: number}}
+	 */
+	function scanDataText() {
+		const start = cursor.offset();
+		let out = '';
+		let end = start;
+		for (;;) {
+			const a = cursor.textAtom();
+			if (!a) {
+				break;
+			}
+			const raw = a.raw;
+			const idx = raw.indexOf('<', cursor.pos);
+			const limit = idx === -1 ? raw.length : idx;
+			out += raw.slice(cursor.pos, limit);
+			cursor.pos = limit;
+			end = a.rawStart + limit;
+			if (idx !== -1) {
+				break;
+			}
+		}
+		return { text: out, start, end };
+	}
+
+	/**
+	 * Scans a tag or attribute name (any char except whitespace, `/`, `>`, `=`).
+	 *
+	 * @returns {{text: string, end: number}}
+	 */
+	function scanName() {
+		const r = scanWhile((c) => !isWhitespace(c.charCodeAt(0)) && c !== '/' && c !== '>' && c !== '=');
+		return { text: r.text, end: r.end };
+	}
+
+	/**
+	 * Consumes up to and including the next `>` (or the end of the current run),
+	 * including any twig atoms in between.
+	 *
+	 * @returns {{text: string, start: number, end: number}}
+	 */
+	function readTagRest() {
+		const start = cursor.offset();
+		let out = '';
+		let end = start;
+		for (;;) {
+			const a = cursor.textAtom();
+			if (!a) {
+				if (cursor.atTwig()) {
+					const twig = cursor.takeTwig();
+					out += twig.raw;
+					end = twig.rawEnd;
+					continue;
+				}
+				break;
+			}
+			const raw = a.raw;
+			const idx = raw.indexOf('>', cursor.pos);
+			const limit = idx === -1 ? raw.length : idx + 1;
+			out += raw.slice(cursor.pos, limit);
+			cursor.pos = limit;
+			end = a.rawStart + limit;
+			if (idx !== -1) {
+				break;
+			}
+		}
+		return { text: out, start, end };
+	}
+
+	/**
+	 * Consumes a `<!-- ... -->` comment across text and twig atoms. Returns the
+	 * raw text consumed; when the comment is unterminated the rest of the input
+	 * is consumed.
+	 *
+	 * @returns {{text: string, end: number}}
+	 */
+	function scanHtmlComment() {
+		let out = '';
+		let end = cursor.offset();
+		for (;;) {
+			const a = cursor.textAtom();
+			if (!a) {
+				if (cursor.atTwig()) {
+					const twig = cursor.takeTwig();
+					out += twig.raw;
+					end = twig.rawEnd;
+					continue;
+				}
+				break;
+			}
+			const raw = a.raw;
+			const idx = raw.indexOf('-->', cursor.pos);
+			const limit = idx === -1 ? raw.length : idx + 3;
+			out += raw.slice(cursor.pos, limit);
+			cursor.pos = limit;
+			end = a.rawStart + limit;
+			if (idx !== -1) {
+				break;
+			}
+		}
+		return { text: out, end };
+	}
+
+	/**
+	 * Scans raw text up to `</tag` (case-insensitive) or a twig atom.
+	 *
+	 * @param {string} tagName
+	 * @returns {{text: string, textStart: number, textEnd: number, found: boolean, endTag: string, endTagStart: number, endTagEnd: number}}
+	 */
+	function scanRawText(tagName) {
+		const lowerTag = tagName.toLowerCase();
+		const start = cursor.offset();
+		let out = '';
+		let textEnd = start;
+		for (;;) {
+			const a = cursor.textAtom();
+			if (!a) {
+				break; // twig or end of input
+			}
+			const raw = a.raw;
+			const idx = raw.toLowerCase().indexOf(`</${lowerTag}`, cursor.pos);
+			const limit = idx === -1 ? raw.length : idx;
+			out += raw.slice(cursor.pos, limit);
+			cursor.pos = limit;
+			textEnd = a.rawStart + limit;
+			if (idx !== -1) {
+				// consume `</tag` plus whitespace up to `>`
+				let i = idx;
+				while (i < raw.length && raw[i] !== '>') {
+					i += 1;
+				}
+				const endTagEnd = i < raw.length ? a.rawStart + i + 1 : a.rawStart + raw.length;
+				const endTag = raw.slice(idx, endTagEnd - a.rawStart);
+				cursor.pos = endTagEnd - a.rawStart;
+				return { text: out, textStart: start, textEnd, found: true, endTag, endTagStart: a.rawStart + idx, endTagEnd };
+			}
+		}
+		return { text: out, textStart: start, textEnd, found: false, endTag: '', endTagStart: 0, endTagEnd: 0 };
+	}
+
+	/**
+	 * Commits the current attribute (if any) to the tag's attribute list.
+	 */
+	function commitAttr() {
+		if (attr) {
+			/** @type {HtmlAttribute} */
+			const a = {
+				type: 'attr',
+				nameRaw: /** @type {AttrState} */ (attr).nameRaw,
+				nameStart: /** @type {AttrState} */ (attr).nameStart,
+				nameEnd: /** @type {AttrState} */ (attr).nameEnd,
+				quote: /** @type {AttrState} */ (attr).quote,
+				valueChunks: /** @type {AttrState} */ (attr).valueChunks,
+				valueStart: /** @type {AttrState} */ (attr).valueStart,
+				valueEnd: /** @type {AttrState} */ (attr).valueEnd,
+				rawStart: /** @type {AttrState} */ (attr).rawStart,
+				rawEnd: /** @type {AttrState} */ (attr).rawEnd,
+			};
+			/** @type {TagState} */ (tag).attrs.push(a);
+			attr = null;
+		}
+	}
+
+	/**
+	 * Emits the current start tag once its closing `>` has been consumed.
+	 */
+	function emitStartTag() {
+		commitAttr();
+		tokens.push(
+			htmlToken('startTag', source.slice(/** @type {TagState} */ (tag).rawStart, tagEnd), /** @type {TagState} */ (tag).rawStart, tagEnd, {
+				name: /** @type {TagState} */ (tag).name,
+				nameRaw: /** @type {TagState} */ (tag).nameRaw,
+				selfClosing: /** @type {TagState} */ (tag).selfClosing,
+				attrs: /** @type {TagState} */ (tag).attrs,
+			}),
+		);
+		const name = /** @type {TagState} */ (tag).name;
+		tag = null;
+		if (RAWTEXT_ELEMENTS.has(name) || RCDATA_ELEMENTS.has(name)) {
+			rawtextTag = name;
+		}
+	}
+
+	/**
+	 * Closes the current literal attribute value run as a text chunk.
+	 */
+	function addTextChunk() {
+		const a = /** @type {AttrState} */ (attr);
+		if (a && a.chunkStart < cursor.offset()) {
+			a.valueChunks.push({
+				type: 'text',
+				text: source.slice(a.chunkStart, cursor.offset()),
+				start: a.chunkStart,
+				end: cursor.offset(),
+			});
+		}
+		if (a) {
+			a.chunkStart = cursor.offset();
+		}
+	}
+
+	/**
+	 * Emits a twig or comment token covering the atom's leading trivia and raw
+	 * so the token stream stays contiguous with the source.
+	 *
+	 * @param {Atom} atom
+	 */
+	function emitAtomToken(atom) {
+		tokens.push(
+			htmlToken(
+				atom.kind === 'comment' ? 'comment' : 'twig',
+				atom.leading + atom.raw,
+				atom.sourceStart,
+				atom.rawEnd,
+				{ atom },
+			),
+		);
+	}
+
+	// The tag end offset is tracked separately so `emitStartTag` can slice the
+	// exact `<...>` source.
+	let tagEnd = -1;
+	/** Absolute offset of the `<` of the current markup declaration. */
+	let declStart = -1;
+
+	while (!cursor.atEnd()) {
+		switch (state) {
+			case 'data': {
+				if (rawtextTag) {
+					if (cursor.atTwig()) {
+						emitAtomToken(cursor.takeTwig());
+						break;
+					}
+					const r = scanRawText(rawtextTag);
+					if (r.text !== '') {
+						tokens.push(htmlToken('text', r.text, r.textStart, r.textEnd));
+					}
+					if (r.found) {
+						tokens.push(
+							htmlToken('endTag', r.endTag, r.endTagStart, r.endTagEnd, {
+								name: rawtextTag,
+								nameRaw: rawtextTag,
+							}),
+						);
+						rawtextTag = null;
+						state = 'data';
+					}
+					break;
+				}
+				if (cursor.atTwig()) {
+					emitAtomToken(cursor.takeTwig());
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				if (c === '<') {
+					const start = cursor.offset();
+					cursor.next();
+					tag = {
+						name: '',
+						nameRaw: '',
+						attrs: [],
+						selfClosing: false,
+						rawStart: start,
+						nameStart: -1,
+						nameEnd: -1,
+					};
+					state = 'tagOpen';
+					break;
+				}
+				const t = scanDataText();
+				tokens.push(htmlToken('text', t.text, t.start, t.end));
+				break;
+			}
+
+			case 'tagOpen': {
+				if (cursor.atTwig()) {
+					// `<` directly followed by twig: not a tag, treat as text
+					tokens.push(htmlToken('text', '<', /** @type {TagState} */ (tag).rawStart, /** @type {TagState} */ (tag).rawStart + 1));
+					tag = null;
+					state = 'data';
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				if (c === '/') {
+					cursor.next();
+					state = 'endTagOpen';
+					break;
+				}
+				if (c === '!') {
+					cursor.next();
+					declStart = /** @type {TagState} */ (tag).rawStart;
+					state = 'markupDeclarationOpen';
+					break;
+				}
+				if (c === '?') {
+					cursor.next();
+					declStart = /** @type {TagState} */ (tag).rawStart;
+					tag = null;
+					state = 'bogusComment';
+					break;
+				}
+				if (isWhitespace(c.charCodeAt(0)) || c === '>') {
+					tokens.push(htmlToken('text', '<', /** @type {TagState} */ (tag).rawStart, /** @type {TagState} */ (tag).rawStart + 1));
+					tag = null;
+					state = 'data';
+					break;
+				}
+				/** @type {TagState} */ (tag).nameStart = cursor.offset();
+				state = 'tagName';
+				break;
+			}
+
+			case 'tagName': {
+				const nameEnd = scanName();
+				/** @type {TagState} */ (tag).nameRaw = nameEnd.text;
+				/** @type {TagState} */ (tag).name = nameEnd.text.toLowerCase();
+				/** @type {TagState} */ (tag).nameEnd = nameEnd.end;
+				state = 'beforeAttributeName';
+				break;
+			}
+
+			case 'endTagOpen': {
+				if (cursor.atTwig()) {
+					tokens.push(htmlToken('text', '</', /** @type {TagState} */ (tag).rawStart, /** @type {TagState} */ (tag).rawStart + 2));
+					tag = null;
+					state = 'data';
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				if (c === '>' || isWhitespace(c.charCodeAt(0))) {
+					tokens.push(htmlToken('text', '</', /** @type {TagState} */ (tag).rawStart, /** @type {TagState} */ (tag).rawStart + 2));
+					tag = null;
+					state = 'data';
+					break;
+				}
+				/** @type {TagState} */ (tag).nameStart = cursor.offset();
+				const nameEnd = scanName();
+				/** @type {TagState} */ (tag).nameRaw = nameEnd.text;
+				/** @type {TagState} */ (tag).name = nameEnd.text.toLowerCase();
+				/** @type {TagState} */ (tag).nameEnd = nameEnd.end;
+				state = 'beforeEndTagName';
+				break;
+			}
+
+			case 'beforeEndTagName': {
+				if (cursor.atTwig()) {
+					emitAtomToken(cursor.takeTwig());
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				if (isWhitespace(c.charCodeAt(0))) {
+					cursor.next();
+					break;
+				}
+				if (c === '>') {
+					const end = cursor.offset() + 1;
+					cursor.next();
+					tokens.push(
+						htmlToken('endTag', source.slice(/** @type {TagState} */ (tag).rawStart, end), /** @type {TagState} */ (tag).rawStart, end, {
+							name: /** @type {TagState} */ (tag).name,
+							nameRaw: /** @type {TagState} */ (tag).nameRaw,
+						}),
+					);
+					tag = null;
+					state = 'data';
+					break;
+				}
+				// unexpected character: keep scanning the end tag name
+				const end = scanName();
+				/** @type {TagState} */ (tag).nameRaw = end.text;
+				/** @type {TagState} */ (tag).name = end.text.toLowerCase();
+				/** @type {TagState} */ (tag).nameEnd = end.end;
+				break;
+			}
+
+			case 'beforeAttributeName': {
+				if (cursor.atTwig()) {
+					const atom = cursor.takeTwig();
+					/** @type {TagState} */ (tag).attrs.push({ type: 'twig', atom });
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				const cc = c.charCodeAt(0);
+				if (isWhitespace(cc)) {
+					cursor.next();
+					break;
+				}
+				if (c === '>') {
+					tagEnd = cursor.offset() + 1;
+					cursor.next();
+					emitStartTag();
+					state = 'data';
+					break;
+				}
+				if (c === '/') {
+					cursor.next();
+					state = 'selfClosingStartTag';
+					break;
+				}
+				attr = {
+					nameRaw: '',
+					nameStart: cursor.offset(),
+					nameEnd: -1,
+					quote: null,
+					valueChunks: [],
+					valueStart: -1,
+					valueEnd: -1,
+					rawStart: cursor.offset(),
+					rawEnd: -1,
+					chunkStart: -1,
+				};
+				state = 'attributeName';
+				break;
+			}
+
+			case 'attributeName': {
+				if (cursor.atTwig()) {
+					// a twig atom ends the attribute name
+					/** @type {AttrState} */ (attr).nameRaw = source.slice(/** @type {AttrState} */ (attr).nameStart, cursor.offset());
+					/** @type {AttrState} */ (attr).nameEnd = cursor.offset();
+					commitAttr();
+					const atom = cursor.takeTwig();
+					/** @type {TagState} */ (tag).attrs.push({ type: 'twig', atom });
+					state = 'beforeAttributeName';
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				const cc = c.charCodeAt(0);
+				if (isWhitespace(cc)) {
+					/** @type {AttrState} */ (attr).nameRaw = source.slice(/** @type {AttrState} */ (attr).nameStart, cursor.offset());
+					/** @type {AttrState} */ (attr).nameEnd = cursor.offset();
+					state = 'afterAttributeName';
+					break;
+				}
+				if (c === '=') {
+					/** @type {AttrState} */ (attr).nameRaw = source.slice(/** @type {AttrState} */ (attr).nameStart, cursor.offset());
+					/** @type {AttrState} */ (attr).nameEnd = cursor.offset();
+					cursor.next();
+					state = 'beforeAttributeValue';
+					break;
+				}
+				if (c === '>') {
+					/** @type {AttrState} */ (attr).nameRaw = source.slice(/** @type {AttrState} */ (attr).nameStart, cursor.offset());
+					/** @type {AttrState} */ (attr).nameEnd = cursor.offset();
+					/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).nameEnd;
+					commitAttr();
+					tagEnd = cursor.offset() + 1;
+					cursor.next();
+					emitStartTag();
+					state = 'data';
+					break;
+				}
+				if (c === '/') {
+					/** @type {AttrState} */ (attr).nameRaw = source.slice(/** @type {AttrState} */ (attr).nameStart, cursor.offset());
+					/** @type {AttrState} */ (attr).nameEnd = cursor.offset();
+					/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).nameEnd;
+					commitAttr();
+					cursor.next();
+					state = 'selfClosingStartTag';
+					break;
+				}
+				cursor.next();
+				break;
+			}
+
+			case 'afterAttributeName': {
+				if (cursor.atTwig()) {
+					/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).nameEnd;
+					commitAttr();
+					const atom = cursor.takeTwig();
+					/** @type {TagState} */ (tag).attrs.push({ type: 'twig', atom });
+					state = 'beforeAttributeName';
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				const cc = c.charCodeAt(0);
+				if (isWhitespace(cc)) {
+					cursor.next();
+					break;
+				}
+				if (c === '=') {
+					cursor.next();
+					state = 'beforeAttributeValue';
+					break;
+				}
+				if (c === '>') {
+					/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).nameEnd;
+					commitAttr();
+					tagEnd = cursor.offset() + 1;
+					cursor.next();
+					emitStartTag();
+					state = 'data';
+					break;
+				}
+				if (c === '/') {
+					/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).nameEnd;
+					commitAttr();
+					cursor.next();
+					state = 'selfClosingStartTag';
+					break;
+				}
+				/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).nameEnd;
+				commitAttr();
+				attr = {
+					nameRaw: '',
+					nameStart: cursor.offset(),
+					nameEnd: -1,
+					quote: null,
+					valueChunks: [],
+					valueStart: -1,
+					valueEnd: -1,
+					rawStart: cursor.offset(),
+					rawEnd: -1,
+					chunkStart: -1,
+				};
+				state = 'attributeName';
+				break;
+			}
+
+			case 'beforeAttributeValue': {
+				if (cursor.atTwig()) {
+					/** @type {AttrState} */ (attr).valueStart = cursor.offset();
+					/** @type {AttrState} */ (attr).valueChunks.push({ type: 'twig', atom: cursor.takeTwig() });
+					/** @type {AttrState} */ (attr).valueEnd = cursor.offset();
+					/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).valueEnd;
+					commitAttr();
+					state = 'beforeAttributeName';
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				const cc = c.charCodeAt(0);
+				if (isWhitespace(cc)) {
+					cursor.next();
+					break;
+				}
+				if (c === '"' || c === "'") {
+					/** @type {AttrState} */ (attr).quote = c;
+					cursor.next();
+					/** @type {AttrState} */ (attr).valueStart = cursor.offset();
+					/** @type {AttrState} */ (attr).chunkStart = cursor.offset();
+					state = 'attributeValueQuoted';
+					break;
+				}
+				if (c === '>') {
+					/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).nameEnd;
+					commitAttr();
+					tagEnd = cursor.offset() + 1;
+					cursor.next();
+					emitStartTag();
+					state = 'data';
+					break;
+				}
+				/** @type {AttrState} */ (attr).valueStart = cursor.offset();
+				/** @type {AttrState} */ (attr).chunkStart = cursor.offset();
+				state = 'attributeValueUnquoted';
+				break;
+			}
+
+			case 'attributeValueQuoted': {
+				if (cursor.atTwig()) {
+					addTextChunk();
+					/** @type {AttrState} */ (attr).valueChunks.push({ type: 'twig', atom: cursor.takeTwig() });
+					/** @type {AttrState} */ (attr).chunkStart = cursor.offset();
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				if (c === /** @type {AttrState} */ (attr).quote) {
+					const end = cursor.offset();
+					addTextChunk();
+					cursor.next();
+					/** @type {AttrState} */ (attr).valueEnd = end;
+					/** @type {AttrState} */ (attr).rawEnd = end + 1;
+					commitAttr();
+					state = 'afterAttributeValueQuoted';
+					break;
+				}
+				cursor.next();
+				break;
+			}
+
+			case 'attributeValueUnquoted': {
+				if (cursor.atTwig()) {
+					addTextChunk();
+					/** @type {AttrState} */ (attr).valueChunks.push({ type: 'twig', atom: cursor.takeTwig() });
+					/** @type {AttrState} */ (attr).chunkStart = cursor.offset();
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				const cc = c.charCodeAt(0);
+				if (isWhitespace(cc)) {
+					addTextChunk();
+					/** @type {AttrState} */ (attr).valueEnd = cursor.offset();
+					/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).valueEnd;
+					commitAttr();
+					state = 'beforeAttributeName';
+					break;
+				}
+				if (c === '>') {
+					addTextChunk();
+					/** @type {AttrState} */ (attr).valueEnd = cursor.offset();
+					/** @type {AttrState} */ (attr).rawEnd = /** @type {AttrState} */ (attr).valueEnd;
+					commitAttr();
+					tagEnd = cursor.offset() + 1;
+					cursor.next();
+					emitStartTag();
+					state = 'data';
+					break;
+				}
+				cursor.next();
+				break;
+			}
+
+			case 'afterAttributeValueQuoted': {
+				if (cursor.atTwig()) {
+					state = 'beforeAttributeName';
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					state = 'beforeAttributeName';
+					break;
+				}
+				if (isWhitespace(c.charCodeAt(0))) {
+					cursor.next();
+					state = 'beforeAttributeName';
+					break;
+				}
+				if (c === '>') {
+					tagEnd = cursor.offset() + 1;
+					cursor.next();
+					emitStartTag();
+					state = 'data';
+					break;
+				}
+				if (c === '/') {
+					cursor.next();
+					state = 'selfClosingStartTag';
+					break;
+				}
+				state = 'beforeAttributeName';
+				break;
+			}
+
+			case 'selfClosingStartTag': {
+				if (!cursor.atTwig() && cursor.peek() === '>') {
+					tagEnd = cursor.offset() + 1;
+					cursor.next();
+					/** @type {TagState} */ (tag).selfClosing = true;
+					emitStartTag();
+					state = 'data';
+					break;
+				}
+				state = 'beforeAttributeName';
+				break;
+			}
+
+			case 'markupDeclarationOpen': {
+				if (cursor.atTwig()) {
+					cursor.takeTwig();
+					break;
+				}
+				const c = cursor.peek();
+				if (c === null) {
+					cursor.next();
+					break;
+				}
+				if (c === '-') {
+					cursor.next();
+					if (cursor.peek() === '-') {
+						cursor.next();
+						const r = scanHtmlComment();
+						tokens.push(htmlToken('comment', source.slice(declStart, r.end), declStart, r.end));
+						tag = null;
+						state = 'data';
+					} else {
+						state = 'bogusComment';
+					}
+					break;
+				}
+				// doctype or bogus comment, read until `>`
+				const rest = readTagRest();
+				const trimmed = rest.text.trim().toLowerCase();
+				const isDoctype = /^doctype(\s|$)/.test(trimmed);
+				tokens.push(
+					htmlToken(isDoctype ? 'doctype' : 'comment', source.slice(declStart, rest.end), declStart, rest.end),
+				);
+				tag = null;
+				state = 'data';
+				break;
+			}
+
+			case 'bogusComment': {
+				const rest = readTagRest();
+				tokens.push(htmlToken('comment', source.slice(declStart, rest.end), declStart, rest.end));
+				state = 'data';
+				break;
+			}
+
+			default:
+				state = 'data';
+				break;
+		}
+	}
+
+	// flush an unterminated tag as raw text
+	if (tag) {
+		tokens.push(htmlToken('text', source.slice(/** @type {TagState} */ (tag).rawStart), /** @type {TagState} */ (tag).rawStart, source.length));
+		tag = null;
+	}
+
+	return tokens;
+}