@graphty/graph-io 0.3.25 → 0.3.27

package/src/common/xml.ts

@@ -8,11 +8,13 @@
  * instead of re-reading the token from its `<`, and line numbers are counted from a cached next
  * line-break position, so a document without line breaks costs the same as one with them.
  *
- * What it handles: the XML declaration and processing instructions (skipped), comments (skipped),
+ * What it handles: the XML declaration and processing instructions (skipped), comments (skipped;
+ * a `--` inside one, which XML 1.0 forbids, is accepted: a comment carries no data and banner
+ * comments such as `<!-- ----- -->` are common in hand-written files),
  * CDATA sections (text), a DOCTYPE with an internal subset (skipped; entity declarations are not
  * expanded, so an unknown entity reference is a syntax error), the five predefined entities and
  * numeric character references (decimal and hexadecimal) in text and attribute values, attribute
- * value whitespace normalisation, end-of-line normalisation (CR LF and lone CR become LF), and
+ * value whitespace normalization, end-of-line normalization (CR LF and lone CR become LF), and
  * well-formedness: matching end tags, one root element, no text outside it, no duplicate
  * attributes, no unterminated markup at the end of the input. Namespaces are not resolved; element
  * and attribute names are reported as written (with their prefix).
@@ -30,16 +32,23 @@ import { type Column, type GraphSnapshot } from "@graphty/graph-format";
 import { type LossNote } from "../types.js";
 import { XML_ILLEGAL_CHAR_CODE } from "./codes.js";
 import { isNameChar } from "./export.js";
+import { agree, plural } from "./plural.js";
 
-/** The events of the tokenizer; every callback is synchronous. */
+/**
+ * The events of the tokenizer; every callback is synchronous.
+ * @category Plugin helpers
+ */
 export interface XmlHandler {
     /**
      * An element starts (a self-closing element produces start then end).
      * @param name - the element name as written, prefix included
      * @param attrs - the attributes, entities decoded, in document order
      * @param line - the 1-based line of the `<`
+     * @param ns - the namespace URI the element's prefix (or the default namespace) is bound to:
+     * "" when the element is in no namespace, null when its prefix is undeclared or the name is not
+     * namespace-well-formed (a leading or trailing colon, two colons)
      */
-    start(name: string, attrs: ReadonlyMap<string, string>, line: number): void;
+    start(name: string, attrs: ReadonlyMap<string, string>, line: number, ns: string | null): void;
     /**
      * An element ends.
      * @param name - the element name
@@ -53,9 +62,18 @@ export interface XmlHandler {
      * @param line - the 1-based line where the run starts
      */
     text(text: string, line: number): void;
+    /**
+     * A DOCTYPE declaration (optional; its internal subset is never expanded).
+     * @param text - the declaration as written, from `<!DOCTYPE` to its `>`
+     * @param line - the 1-based line of the `<`
+     */
+    doctype?(text: string, line: number): void;
 }
 
-/** A well-formedness or syntax error, with the line it was found on. */
+/**
+ * A well-formedness or syntax error, with the line it was found on.
+ * @category Plugin helpers
+ */
 export class XmlSyntaxError extends Error {
     /** The 1-based line. */
     readonly line: number;
@@ -73,10 +91,26 @@ export class XmlSyntaxError extends Error {
 }
 
 /**
- * Opt-in repairs of two defects the Cytoscape XGMML writer is known to produce (research note
- * `research-xgmml.md` 3.8 and 5). Each is off unless its callback is given; GEXF and GraphML never
+ * The input holds no markup at all (empty, whitespace only, a byte order mark only): an importer
+ * reports it as E_EMPTY_INPUT rather than as a syntax error.
+ * @category Plugin helpers
+ */
+export class XmlEmptyInputError extends XmlSyntaxError {
+    /**
+     * Create the error.
+     * @param line - the 1-based line where the input ended
+     */
+    constructor(line: number) {
+        super("the input is empty: no root element", line);
+        this.name = "XmlEmptyInputError";
+    }
+}
+
+/**
+ * Opt-in repairs of two defects the Cytoscape XGMML writer is known to produce. Each is off unless its callback is given; GEXF and GraphML never
  * pass them, so their documents stay strictly well-formed. The callback is told the line of each
  * repair, so the importer can warn per occurrence.
+ * @category Plugin helpers
  */
 export interface XmlRepairs {
     /**
@@ -122,6 +156,7 @@ const OPEN_BRACKET = 91;
 const CLOSE_BRACKET = 93;
 const BANG = 33;
 const DASH = 45;
+const QUESTION = 63;
 
 /** The longest entity reference held back at a chunk boundary (`&#x10FFFF;` is 10 characters). */
 const MAX_ENTITY_LENGTH = 16;
@@ -178,11 +213,22 @@ const ILLEGAL_CHAR = new RegExp(
  * written even as a character reference, so a conforming parser rejects the whole document.
  * @param text - the text
  * @returns true when the text cannot appear in an XML 1.0 document
+ * @category Plugin helpers
  */
 export function hasIllegalXmlChar(text: string): boolean {
     return ILLEGAL_CHAR.test(text);
 }
 
+/**
+ * Where the first character XML 1.0 forbids is in a text.
+ * @param text - the text
+ * @returns its index, or -1
+ */
+function illegalCharIndex(text: string): number {
+    const match = ILLEGAL_CHAR.exec(text);
+    return match === null ? -1 : match.index;
+}
+
 /**
  * The loss notes of the texts a snapshot holds that no XML 1.0 document can carry (see
  * hasIllegalXmlChar): one E_XML_ILLEGAL_CHAR note per column (node, edge and graph string / dict /
@@ -190,6 +236,7 @@ export function hasIllegalXmlChar(text: string): boolean {
  * the first such text. Shared by the GEXF and GraphML exporters' check().
  * @param snapshot - the snapshot
  * @returns the notes, empty when every text is writable
+ * @category Plugin helpers
  */
 export function xmlIllegalTextNotes(snapshot: GraphSnapshot): LossNote[] {
     const notes: LossNote[] = [];
@@ -204,7 +251,7 @@ export function xmlIllegalTextNotes(snapshot: GraphSnapshot): LossNote[] {
         notes.push(
             Object.freeze({
                 code: XML_ILLEGAL_CHAR_CODE,
-                message: `${ids} node id(s) hold a character XML 1.0 cannot carry; export() will throw`,
+                message: `${ids} node id${plural(ids)} ${agree(ids, "holds", "hold")} a character XML 1.0 cannot carry; the save fails`,
                 column: null,
                 count: ids,
             }),
@@ -221,7 +268,7 @@ export function xmlIllegalTextNotes(snapshot: GraphSnapshot): LossNote[] {
                 notes.push(
                     Object.freeze({
                         code: XML_ILLEGAL_CHAR_CODE,
-                        message: `${domain} column "${column.meta.name}": ${bad} value(s) hold a character XML 1.0 cannot carry; export() will throw`,
+                        message: `${domain} column "${column.meta.name}": ${bad} value${plural(bad)} ${agree(bad, "holds", "hold")} a character XML 1.0 cannot carry; the save fails`,
                         column: column.meta.name,
                         count: bad,
                     }),
@@ -258,7 +305,10 @@ function countIllegalRows(column: Column): number {
     return bad;
 }
 
-/** The encoding pseudo-attribute of an XML declaration at the very start of a document. */
+/**
+ * The encoding pseudo-attribute of an XML declaration at the very start of a document (one
+ * anywhere else is a syntax error of the tokenizer).
+ */
 const XML_DECLARED_ENCODING = /^<\?xml\s[^>]*?\bencoding\s*=\s*["']([A-Za-z][A-Za-z0-9._-]*)["']/;
 
 /**
@@ -266,16 +316,35 @@ const XML_DECLARED_ENCODING = /^<\?xml\s[^>]*?\bencoding\s*=\s*["']([A-Za-z][A-Z
  * for the shared byte decoder (common/input.ts).
  * @param head - the start of the document, decoded as windows-1252
  * @returns the declared label, or null when there is no declaration or it names no encoding
+ * @category Plugin helpers
  */
 export function xmlDeclaredEncoding(head: string): string | null {
     const match = XML_DECLARED_ENCODING.exec(head);
     return match === null ? null : match[1];
 }
 
+/**
+ * The head of a document as text for an XML format's sniff(): UTF-16 when a byte order mark (or
+ * BOM-less UTF-16 `<?`, XML 1.0 Appendix F) says so, else UTF-8, never failing.
+ * @param head - the first bytes
+ * @returns the text, a leading byte order mark removed
+ * @category Plugin helpers
+ */
+export function sniffXmlText(head: Uint8Array): string {
+    let label = "utf-8";
+    if ((head[0] === 0xff && head[1] === 0xfe) || (head[0] === 0x3c && head[1] === 0x00 && head[2] === 0x3f)) {
+        label = "utf-16le";
+    } else if ((head[0] === 0xfe && head[1] === 0xff) || (head[0] === 0x00 && head[1] === 0x3c && head[3] === 0x3f)) {
+        label = "utf-16be";
+    }
+    return new TextDecoder(label, { fatal: false }).decode(head);
+}
+
 /**
  * Whether a text is an XML Name.
  * @param text - the text
  * @returns true for a well-formed name
+ * @category Plugin helpers
  */
 export function isXmlName(text: string): boolean {
     if (text.length === 0) {
@@ -314,6 +383,7 @@ function isXmlChar(cp: number): boolean {
  * @param line - the line, for errors
  * @param repairs - the opt-in repairs (XGMML only); none by default
  * @returns the decoded text
+ * @category Plugin helpers
  */
 export function decodeEntities(raw: string, line: number, repairs?: XmlRepairs): string {
     let amp = raw.indexOf("&");
@@ -338,13 +408,14 @@ export function decodeEntities(raw: string, line: number, repairs?: XmlRepairs):
             continue;
         }
         if (semi < 0) {
-            throw new XmlSyntaxError("unterminated entity reference", line);
+            throw new XmlSyntaxError("unterminated entity reference", lineAt(raw, amp, line));
         }
         const name = raw.slice(amp + 1, semi);
         if (name.startsWith("#")) {
             const hex = name.startsWith("#x") || name.startsWith("#X");
             const digits = name.slice(hex ? 2 : 1);
-            const ok = hex ? /^[0-9a-fA-F]{1,6}$/.test(digits) : /^[0-9]{1,7}$/.test(digits);
+            // XML 1.0 puts no limit on leading zeros (&#x0000000041; is "A")
+            const ok = hex ? /^0*[\da-fA-F]{1,6}$/.test(digits) : /^0*\d{1,7}$/.test(digits);
             const cp = ok ? Number.parseInt(digits, hex ? 16 : 10) : -1;
             const low = cp >= 0xd800 && cp <= 0xdbff ? lowSurrogateAfter(raw, semi + 1, repairs) : null;
             if (low !== null) {
@@ -355,13 +426,13 @@ export function decodeEntities(raw: string, line: number, repairs?: XmlRepairs):
                 continue;
             }
             if (cp < 0 || !isXmlChar(cp)) {
-                throw new XmlSyntaxError(`invalid character reference &${name};`, line);
+                throw new XmlSyntaxError(`invalid character reference &${name};`, lineAt(raw, amp, line));
             }
             out += String.fromCodePoint(cp);
         } else {
             const value = NAMED_ENTITIES[name];
             if (value === undefined) {
-                throw new XmlSyntaxError(`unknown entity &${name};`, line);
+                throw new XmlSyntaxError(`unknown entity &${name};`, lineAt(raw, amp, line));
             }
             out += value;
         }
@@ -414,6 +485,7 @@ function lowSurrogateAfter(
  * Whether a text is whitespace only.
  * @param text - the text
  * @returns true when every character is XML whitespace (also for an empty text)
+ * @category Plugin helpers
  */
 export function isWhitespace(text: string): boolean {
     for (let i = 0; i < text.length; i++) {
@@ -428,6 +500,7 @@ export function isWhitespace(text: string): boolean {
  * The local part of a possibly prefixed XML name.
  * @param name - the name as written
  * @returns the text after the last colon, or the name itself
+ * @category Plugin helpers
  */
 export function localName(name: string): string {
     const colon = name.lastIndexOf(":");
@@ -440,6 +513,7 @@ export function localName(name: string): string {
  * @param chunks - the text (already UTF-8 decoded, BOM removed)
  * @param handler - the event sink
  * @param repairs - the opt-in repairs (XGMML only); none by default
+ * @category Plugin helpers
  */
 export async function tokenizeXml(
     chunks: AsyncIterable<string>,
@@ -447,10 +521,40 @@ export async function tokenizeXml(
     repairs?: XmlRepairs,
 ): Promise<void> {
     const tokenizer = new XmlTokenizer(handler, repairs);
-    for await (const chunk of chunks) {
-        tokenizer.push(chunk);
+    // XML 1.0 section 2.8: nothing may precede the XML declaration, not even whitespace (template
+    // output often adds a line break, and the declared encoding would then go unread)
+    let skipped = 0;
+    let lines = 1;
+    let leading = true;
+    try {
+        for await (const chunk of chunks) {
+            if (leading) {
+                const start = chunk.search(/\S/);
+                const blank = start < 0 ? chunk : chunk.slice(0, start);
+                skipped += blank.length;
+                lines += blank.split("\n").length - 1;
+                if (start >= 0) {
+                    leading = false;
+                    if (skipped > 0 && /^<\?xml\s/.test(chunk.slice(start, start + 6))) {
+                        throw new XmlSyntaxError(
+                            "the XML declaration must be at the very start of the document; whitespace precedes it",
+                            lines,
+                        );
+                    }
+                }
+            }
+            tokenizer.push(chunk);
+        }
+        tokenizer.finish();
+    } catch (err) {
+        if (err instanceof XmlSyntaxError && tokenizer.declaresXml11 && !err.message.includes("XML 1.1")) {
+            throw new XmlSyntaxError(
+                `${err.message}; the document declares XML 1.1, which graph-io does not read (it reads XML 1.0)`,
+                err.line,
+            );
+        }
+        throw err;
     }
-    tokenizer.finish();
 }
 
 /** The result of parsing one start tag out of the buffer. */
@@ -461,6 +565,8 @@ interface StartTag {
     readonly attrs: Map<string, string>;
     /** Whether the tag ends with `/>`. */
     readonly selfClosing: boolean;
+    /** Whether an attribute name starts with `xmlns` (a namespace declaration may be among them). */
+    readonly declaresNamespace: boolean;
     /** The buffer index one past the `>`. */
     readonly end: number;
 }
@@ -481,7 +587,7 @@ const TERMINATORS: Readonly<Partial<Record<PendingKind, string>>> = {
  * until the end is found, so a token spanning many chunks costs its length once) and the state
  * the terminator scan is in. `tail` holds the last characters of the pieces a fixed terminator
  * could straddle; `quote` is the open quote character (0 outside a value) of a start tag or a
- * DOCTYPE; `depth`, `comment` and `run` track a DOCTYPE's internal subset.
+ * DOCTYPE; `depth`, `comment`, `pi` and `run` track a DOCTYPE's internal subset.
  */
 interface Pending {
     kind: PendingKind;
@@ -490,12 +596,30 @@ interface Pending {
     quote: number;
     depth: number;
     comment: boolean;
+    pi: boolean;
     run: number;
 }
 
+/** One `xmlns` / `xmlns:prefix` declaration in scope. */
+interface NamespaceBinding {
+    /** The prefix; "" for the default namespace. */
+    readonly prefix: string;
+    /** The namespace URI; "" undeclares the default namespace. */
+    readonly uri: string;
+    /** The element depth that declared it (the stack length before the element was pushed). */
+    readonly depth: number;
+}
+
+/** The namespace the `xml` prefix is bound to by definition. */
+const XML_NAMESPACE = "http://www.w3.org/XML/1998/namespace";
+
+/** The markup declarations a truncated `<!...` at the end of the input may have been cut from. */
+const DECLARATION_OPENERS: readonly string[] = ["<!--", "<![CDATA[", "<!DOCTYPE"];
+
 /**
  * The tokenizer state: the unconsumed tail of the input, the current line, the open element
  * stack and the text run being accumulated. `push()` chunks, then `finish()`.
+ * @category Plugin helpers
  */
 export class XmlTokenizer {
     private readonly handler: XmlHandler;
@@ -532,6 +656,27 @@ export class XmlTokenizer {
 
     private rootClosed = false;
 
+    /** Whether any markup was consumed (an input with none is empty, not malformed). */
+    private markupSeen = false;
+
+    /** Whether nothing has been consumed yet: the XML declaration may only appear here. */
+    private atStart = true;
+
+    /** Whether a DOCTYPE declaration was read (one is allowed, in the prolog). */
+    private doctypeSeen = false;
+
+    /** The version of the XML declaration, or null. */
+    private version: string | null = null;
+
+    /** The last two characters of the character data since the last markup, for the `]]>` check. */
+    private textTail = "";
+
+    /** A high surrogate held back from the end of the previous chunk (its low half may start the next). */
+    private pendingHigh = "";
+
+    /** The namespace declarations in scope, innermost last. */
+    private readonly bindings: NamespaceBinding[] = [];
+
     private readonly repairs: XmlRepairs | undefined;
 
     /**
@@ -544,12 +689,31 @@ export class XmlTokenizer {
         this.repairs = repairs;
     }
 
+    /**
+     * Whether the document's XML declaration says version 1.1 (read by the XML 1.0 rules all the
+     * same; the syntax error of a 1.1-only construct then says why).
+     * @returns true after a `<?xml version="1.1"?>` declaration
+     */
+    get declaresXml11(): boolean {
+        return this.version === "1.1";
+    }
+
     /**
      * Feed one chunk and emit every complete token in it.
      * @param chunk - the text
      */
     push(chunk: string): void {
         let text = chunk;
+        if (this.pendingHigh.length > 0) {
+            text = this.pendingHigh + text;
+            this.pendingHigh = "";
+        }
+        const last = text.charCodeAt(text.length - 1); // NOSONAR(S7758): reads UTF-16 code units on purpose
+        if (last >= 0xd800 && last <= 0xdbff) {
+            // a text chunk may end between the two halves of an astral character
+            this.pendingHigh = text.slice(-1);
+            text = text.slice(0, -1);
+        }
         if (this.pendingCr) {
             text = `\r${text}`;
             this.pendingCr = false;
@@ -559,7 +723,7 @@ export class XmlTokenizer {
             text = text.slice(0, -1);
         }
         if (text.includes("\r")) {
-            text = text.replace(/\r\n?/g, "\n");
+            text = text.replaceAll(/\r\n?/g, "\n");
         }
         this.feed(text);
     }
@@ -567,11 +731,13 @@ export class XmlTokenizer {
     /** Signal the end of the input: flush the last text run and check well-formedness. */
     finish(): void {
         this.final = true;
+        const held = this.pendingHigh;
+        this.pendingHigh = "";
         if (this.pendingCr) {
             this.pendingCr = false;
-            this.feed("\n");
+            this.feed(`\n${held}`);
         } else {
-            this.feed("");
+            this.feed(held);
         }
         if (this.pending !== null || this.buffer.length > 0) {
             throw new XmlSyntaxError("unexpected end of input inside markup", this.line);
@@ -581,14 +747,17 @@ export class XmlTokenizer {
             throw new XmlSyntaxError(`unclosed element <${this.stack[this.stack.length - 1]}>`, this.line);
         }
         if (!this.rootSeen) {
+            if (!this.markupSeen) {
+                throw new XmlEmptyInputError(this.line);
+            }
             throw new XmlSyntaxError("no root element", this.line);
         }
     }
 
     /**
-     * Append normalised text: continue a pending token's terminator scan over the new text alone,
+     * Append normalized text: continue a pending token's terminator scan over the new text alone,
      * and once the token is complete (or when none is pending) scan the buffer for tokens.
-     * @param text - the text, line breaks normalised
+     * @param text - the text, line breaks normalized
      */
     private feed(text: string): void {
         const { pending } = this;
@@ -697,9 +866,19 @@ export class XmlTokenizer {
      * @returns the index one past the `>`, or -1
      */
     private scanDoctype(pending: Pending, text: string, from: number): number {
-        let { quote, depth, comment, run } = pending;
+        let { quote, depth, comment, pi, run } = pending;
         for (let i = from; i < text.length; i++) {
             const c = text.charCodeAt(i);
+            if (pi) {
+                // a processing instruction in the subset ends at `?>`; quotes inside it are text
+                if (c === GT && run === 1) {
+                    pi = false;
+                    run = 0;
+                } else {
+                    run = c === QUESTION ? 1 : 0;
+                }
+                continue;
+            }
             if (comment) {
                 if (c === DASH) {
                     run++;
@@ -731,6 +910,11 @@ export class XmlTokenizer {
                     run = 0;
                     continue;
                 }
+                if (run === 1 && c === QUESTION) {
+                    pi = true;
+                    run = 0;
+                    continue;
+                }
                 run = 0;
                 if (c === LT) {
                     run = 1;
@@ -747,6 +931,7 @@ export class XmlTokenizer {
                 pending.quote = 0;
                 pending.depth = 0;
                 pending.comment = false;
+                pending.pi = false;
                 pending.run = 0;
                 return i + 1;
             }
@@ -754,6 +939,7 @@ export class XmlTokenizer {
         pending.quote = quote;
         pending.depth = depth;
         pending.comment = comment;
+        pending.pi = pi;
         pending.run = run;
         return -1;
     }
@@ -775,7 +961,13 @@ export class XmlTokenizer {
                 if (!this.final) {
                     // an entity reference may be split across chunks: hold back from its "&"
                     const amp = buffer.lastIndexOf("&");
-                    if (amp >= pos && amp >= length - MAX_ENTITY_LENGTH && buffer.indexOf(";", amp) < 0) {
+                    if (
+                        amp >= pos &&
+                        !buffer.includes(";", amp) &&
+                        // ponytail: a zero-padded reference longer than MAX_ENTITY_LENGTH is held back
+                        // whole, so an absurdly long run of zeros is re-scanned once per chunk
+                        (amp >= length - MAX_ENTITY_LENGTH || /^&#[xX]?0[\da-fA-F]*$/.test(buffer.slice(amp)))
+                    ) {
                         end = amp;
                     }
                     if (this.repairs?.surrogatePair !== undefined) {
@@ -822,6 +1014,7 @@ export class XmlTokenizer {
             quote: 0,
             depth: 0,
             comment: false,
+            pi: false,
             run: 0,
         };
         if (piece.length < MIN_DECIDABLE_MARKUP) {
@@ -866,60 +1059,32 @@ export class XmlTokenizer {
      * @returns the index after the markup, or -1 when the buffer ends before the markup does
      */
     private consumeMarkup(pos: number): number {
+        const end = this.markupEnd(pos);
+        if (end >= 0) {
+            this.markupSeen = true;
+            this.atStart = false;
+            // markup ends a run of character data: a `]]>` cannot straddle it
+            this.textTail = "";
+        }
+        return end;
+    }
+
+    /**
+     * Read the markup starting at `pos` (a `<`) and deliver what it means.
+     * @param pos - the index of the `<`
+     * @returns the index after the markup, or -1 when the buffer ends before the markup does
+     */
+    private markupEnd(pos: number): number {
         const { buffer } = this;
         if (buffer.startsWith("<!", pos)) {
-            if (buffer.length - pos < MIN_DECIDABLE_MARKUP && !this.final) {
-                // "<![CDATA[" is 9 characters; wait until the kind of declaration is decidable
-                return -1;
-            }
-            if (buffer.startsWith("<!--", pos)) {
-                const end = buffer.indexOf("-->", pos + 4);
-                if (end < 0) {
-                    return -1;
-                }
-                this.advanceLine(pos, end + 3);
-                return end + 3;
-            }
-            if (buffer.startsWith("<![CDATA[", pos)) {
-                const end = buffer.indexOf("]]>", pos + 9);
-                if (end < 0) {
-                    return -1;
-                }
-                if (this.text.length === 0) {
-                    this.textLine = this.line;
-                }
-                const cdata = buffer.slice(pos + 9, end);
-                if (hasIllegalXmlChar(cdata)) {
-                    throw new XmlSyntaxError("a character XML 1.0 forbids appears in a CDATA section", this.line);
-                }
-                this.text += cdata;
-                this.advanceLine(pos, end + 3);
-                return end + 3;
-            }
-            if (!buffer.startsWith("<!DOCTYPE", pos)) {
-                throw new XmlSyntaxError("unexpected markup declaration", this.line);
-            }
-            const state: Pending = {
-                kind: "doctype",
-                pieces: [],
-                tail: "",
-                quote: 0,
-                depth: 0,
-                comment: false,
-                run: 0,
-            };
-            const end = this.scanDoctype(state, buffer, pos + 9);
-            if (end < 0) {
-                return -1;
-            }
-            this.advanceLine(pos, end);
-            return end;
+            return this.declarationEnd(pos);
         }
         if (buffer.startsWith("<?", pos)) {
             const end = buffer.indexOf("?>", pos + 2);
             if (end < 0) {
                 return -1;
             }
+            this.processingInstruction(pos, end);
             this.advanceLine(pos, end + 2);
             return end + 2;
         }
@@ -928,7 +1093,8 @@ export class XmlTokenizer {
             if (end < 0) {
                 return -1;
             }
-            const name = buffer.slice(pos + 2, end).trim();
+            // whitespace may follow the name, never precede it
+            const name = buffer.slice(pos + 2, end).trimEnd();
             if (!isXmlName(name)) {
                 throw new XmlSyntaxError(`malformed end tag </${name}>`, this.line);
             }
@@ -942,7 +1108,7 @@ export class XmlTokenizer {
             return -1;
         }
         this.flushText();
-        this.startElement(tag.name, tag.attrs);
+        this.startElement(tag.name, tag.attrs, tag.declaresNamespace);
         if (tag.selfClosing) {
             this.endElement(tag.name);
         }
@@ -950,6 +1116,161 @@ export class XmlTokenizer {
         return tag.end;
     }
 
+    /**
+     * Read the CDATA section at `pos` into the text.
+     * @param pos - the index of the `<`
+     * @returns the index after the section, or -1 when the buffer ends before the section does
+     */
+    private cdataEnd(pos: number): number {
+        const end = this.buffer.indexOf("]]>", pos + 9);
+        if (end < 0) {
+            return -1;
+        }
+        if (this.text.length === 0) {
+            this.textLine = this.line;
+        }
+        const cdata = this.buffer.slice(pos + 9, end);
+        const bad = illegalCharIndex(cdata);
+        if (bad >= 0) {
+            throw new XmlSyntaxError(
+                "a character XML 1.0 forbids appears in a CDATA section",
+                this.lineIn(pos, pos + 9 + bad),
+            );
+        }
+        this.text += cdata;
+        this.advanceLine(pos, end + 3);
+        return end + 3;
+    }
+
+    /**
+     * Read the `<!...>` markup at `pos`: a comment, a CDATA section or the DOCTYPE.
+     * @param pos - the index of the `<`
+     * @returns the index after the markup, or -1 when the buffer ends before the markup does
+     */
+    private declarationEnd(pos: number): number {
+        const { buffer } = this;
+        if (buffer.length - pos < MIN_DECIDABLE_MARKUP) {
+            // "<![CDATA[" is 9 characters; wait until the kind of declaration is decidable, and at
+            // the end of the input read the start of one cut short as a truncation
+            const head = buffer.slice(pos);
+            if (!this.final || DECLARATION_OPENERS.some((opener) => opener.startsWith(head))) {
+                return -1;
+            }
+        }
+        if (buffer.startsWith("<!--", pos)) {
+            const end = buffer.indexOf("-->", pos + 4);
+            if (end < 0) {
+                return -1;
+            }
+            // a "--" inside, which XML 1.0 forbids, is accepted: the comment carries no data
+            this.advanceLine(pos, end + 3);
+            return end + 3;
+        }
+        if (buffer.startsWith("<![CDATA[", pos)) {
+            return this.cdataEnd(pos);
+        }
+        if (!buffer.startsWith("<!DOCTYPE", pos)) {
+            if (buffer.slice(pos + 2, pos + 9).toUpperCase() === "DOCTYPE") {
+                throw new XmlSyntaxError(
+                    `<!${buffer.slice(pos + 2, pos + 9)}> is not XML (it must be written <!DOCTYPE>); the input looks like an HTML document`,
+                    this.line,
+                );
+            }
+            throw new XmlSyntaxError("unexpected markup declaration", this.line);
+        }
+        if (this.rootSeen || this.doctypeSeen) {
+            throw new XmlSyntaxError("a DOCTYPE declaration is allowed once, before the root element", this.line);
+        }
+        const state: Pending = {
+            kind: "doctype",
+            pieces: [],
+            tail: "",
+            quote: 0,
+            depth: 0,
+            comment: false,
+            pi: false,
+            run: 0,
+        };
+        const end = this.scanDoctype(state, buffer, pos + 9);
+        if (end < 0) {
+            return -1;
+        }
+        this.doctypeSeen = true;
+        this.handler.doctype?.(buffer.slice(pos, end), this.line);
+        this.advanceLine(pos, end);
+        return end;
+    }
+
+    /**
+     * Check a processing instruction `<?target ...?>` (skipped otherwise): its target must be a
+     * name, and the reserved target `xml` is the XML declaration, allowed only at the very start
+     * of the document with a 1.x version.
+     * @param pos - the index of the `<`
+     * @param end - the index of the `?>`
+     */
+    private processingInstruction(pos: number, end: number): void {
+        const body = this.buffer.slice(pos + 2, end);
+        const target = /^[^\s]*/.exec(body)?.[0] ?? "";
+        if (!isXmlName(target)) {
+            throw new XmlSyntaxError(`a processing instruction needs a target name, not "${target}"`, this.line);
+        }
+        if (target.toLowerCase() !== "xml") {
+            return;
+        }
+        if (target !== "xml") {
+            throw new XmlSyntaxError(
+                `<?${target} is not an XML declaration (which is written <?xml in lower case); the target is reserved`,
+                this.line,
+            );
+        }
+        if (!this.atStart || pos !== 0) {
+            throw new XmlSyntaxError(
+                "an XML declaration (<?xml ...?>) may appear only at the very start of the document; the target xml is reserved",
+                this.line,
+            );
+        }
+        const version = /\bversion\s*=\s*(["'])([^"']*)\1/.exec(body)?.[2];
+        if (version === undefined) {
+            throw new XmlSyntaxError("the XML declaration has no version", this.line);
+        }
+        if (!/^1\.\d+$/.test(version)) {
+            throw new XmlSyntaxError(`XML version ${version} is not supported (only 1.x)`, this.line);
+        }
+        this.version = version;
+    }
+
+    /**
+     * The line of a buffer position after `start`, the position the current line count refers to.
+     * Used on the error path only.
+     * @param start - a position on line `this.line`
+     * @param at - the position
+     * @returns its line
+     */
+    private lineIn(start: number, at: number): number {
+        return lineAt(this.buffer.slice(start, at), at - start, this.line);
+    }
+
+    /**
+     * Decode the entities of a text, naming XML 1.1 when a character reference XML 1.0 forbids
+     * appears in a document that declares version 1.1.
+     * @param raw - the text as written
+     * @param line - the line the text starts on
+     * @returns the decoded text
+     */
+    private decode(raw: string, line: number): string {
+        try {
+            return decodeEntities(raw, line, this.repairs);
+        } catch (err) {
+            if (err instanceof XmlSyntaxError && this.version !== null && this.version !== "1.0") {
+                throw new XmlSyntaxError(
+                    `${err.message} (the document declares XML ${this.version}, whose character rules are not supported; it is read as XML 1.0)`,
+                    err.line,
+                );
+            }
+            throw err;
+        }
+    }
+
     /**
      * Parse a start tag at `pos`.
      * @param pos - the index of the `<`
@@ -969,7 +1290,9 @@ export class XmlTokenizer {
         const name = buffer.slice(i, nameEnd);
         i = nameEnd;
         const attrs = new Map<string, string>();
+        let declaresNamespace = false;
         for (;;) {
+            const before = i;
             while (i < length && isSpace(buffer.charCodeAt(i))) {
                 i++;
             }
@@ -978,23 +1301,30 @@ export class XmlTokenizer {
             }
             const c = buffer.charCodeAt(i);
             if (c === GT) {
-                return { name, attrs, selfClosing: false, end: i + 1 };
+                return { name, attrs, selfClosing: false, declaresNamespace, end: i + 1 };
             }
             if (c === SLASH) {
                 if (i + 1 >= length) {
                     return null;
                 }
                 if (buffer.charCodeAt(i + 1) !== GT) {
-                    throw new XmlSyntaxError(`unexpected "/" in <${name}>`, this.line);
+                    throw new XmlSyntaxError(`unexpected "/" in <${name}>`, this.lineIn(pos, i));
                 }
-                return { name, attrs, selfClosing: true, end: i + 2 };
+                return { name, attrs, selfClosing: true, declaresNamespace, end: i + 2 };
+            }
+            if (i === before) {
+                // the name, or the previous value's closing quote, runs into the next attribute
+                throw new XmlSyntaxError(
+                    `the attributes of <${name}> must be separated by whitespace`,
+                    this.lineIn(pos, i),
+                );
             }
             const attrEnd = this.readName(i);
             if (attrEnd < 0) {
                 return null;
             }
             if (attrEnd === i) {
-                throw new XmlSyntaxError(`malformed attribute in <${name}>`, this.line);
+                throw new XmlSyntaxError(`malformed attribute in <${name}>`, this.lineIn(pos, i));
             }
             const attrName = buffer.slice(i, attrEnd);
             i = attrEnd;
@@ -1005,7 +1335,7 @@ export class XmlTokenizer {
                 return null;
             }
             if (buffer.charCodeAt(i) !== EQUALS) {
-                throw new XmlSyntaxError(`attribute ${attrName} of <${name}> has no value`, this.line);
+                throw new XmlSyntaxError(`attribute ${attrName} of <${name}> has no value`, this.lineIn(pos, i));
             }
             i++;
             while (i < length && isSpace(buffer.charCodeAt(i))) {
@@ -1016,23 +1346,30 @@ export class XmlTokenizer {
             }
             const quote = buffer.charCodeAt(i);
             if (quote !== QUOTE && quote !== APOS) {
-                throw new XmlSyntaxError(`attribute ${attrName} of <${name}> is not quoted`, this.line);
+                throw new XmlSyntaxError(`attribute ${attrName} of <${name}> is not quoted`, this.lineIn(pos, i));
             }
             const close = buffer.indexOf(quote === QUOTE ? '"' : "'", i + 1);
             if (close < 0) {
                 return null;
             }
+            const line = this.lineIn(pos, i);
             if (attrs.has(attrName)) {
-                throw new XmlSyntaxError(`duplicate attribute ${attrName} in <${name}>`, this.line);
+                throw new XmlSyntaxError(`duplicate attribute ${attrName} in <${name}>`, line);
             }
             const raw = buffer.slice(i + 1, close);
             if (hasIllegalXmlChar(raw)) {
                 throw new XmlSyntaxError(
                     `a character XML 1.0 forbids appears in attribute ${attrName} of <${name}>`,
-                    this.line,
+                    line,
                 );
             }
-            attrs.set(attrName, decodeEntities(normalizeAttributeValue(raw), this.line, this.repairs));
+            if (raw.includes("<")) {
+                throw new XmlSyntaxError(`a "<" in the value of attribute ${attrName} of <${name}> (write &lt;)`, line);
+            }
+            if (attrName.startsWith("xmlns")) {
+                declaresNamespace = true;
+            }
+            attrs.set(attrName, this.decode(normalizeAttributeValue(raw), line));
             i = close + 1;
         }
     }
@@ -1096,10 +1433,26 @@ export class XmlTokenizer {
             this.textLine = this.line;
         }
         const raw = this.buffer.slice(start, end);
-        if (hasIllegalXmlChar(raw)) {
-            throw new XmlSyntaxError("a character XML 1.0 forbids appears in character data", this.line);
+        const bad = illegalCharIndex(raw);
+        if (bad >= 0) {
+            throw new XmlSyntaxError(
+                "a character XML 1.0 forbids appears in character data",
+                lineAt(raw, bad, this.line),
+            );
         }
-        this.text += decodeEntities(raw, this.line, this.repairs);
+        if (raw.includes(">")) {
+            const probe = this.textTail + raw;
+            const close = probe.indexOf("]]>");
+            if (close >= 0) {
+                throw new XmlSyntaxError(
+                    '"]]>" is not allowed in character data (write ]]&gt;)',
+                    lineAt(raw, Math.max(0, close - this.textTail.length), this.line),
+                );
+            }
+        }
+        this.textTail = raw.length >= 2 ? raw.slice(-2) : (this.textTail + raw).slice(-2);
+        this.atStart = false;
+        this.text += this.decode(raw, this.line);
         this.advanceLine(start, end);
     }
 
@@ -1145,16 +1498,49 @@ export class XmlTokenizer {
      * Open an element.
      * @param name - the element name
      * @param attrs - its attributes
+     * @param declaresNamespace - whether an attribute name starts with `xmlns`
      */
-    private startElement(name: string, attrs: Map<string, string>): void {
+    private startElement(name: string, attrs: Map<string, string>, declaresNamespace: boolean): void {
         if (this.stack.length === 0) {
             if (this.rootClosed) {
                 throw new XmlSyntaxError(`a second root element <${name}> follows the document element`, this.line);
             }
             this.rootSeen = true;
         }
+        if (declaresNamespace) {
+            for (const [key, uri] of attrs) {
+                if (key === "xmlns") {
+                    this.bindings.push({ prefix: "", uri, depth: this.stack.length });
+                } else if (key.startsWith("xmlns:")) {
+                    this.bindings.push({ prefix: key.slice(6), uri, depth: this.stack.length });
+                }
+            }
+        }
         this.stack.push(name);
-        this.handler.start(name, attrs, this.line);
+        this.handler.start(name, attrs, this.line, this.namespaceOf(name));
+    }
+
+    /**
+     * The namespace an element name is in (Namespaces in XML 1.0 section 6).
+     * @param name - the element name as written
+     * @returns the URI; "" for no namespace; null for an undeclared prefix or a name that is not
+     * namespace-well-formed
+     */
+    private namespaceOf(name: string): string | null {
+        const colon = name.indexOf(":");
+        if (colon >= 0 && (colon === 0 || colon === name.length - 1 || name.includes(":", colon + 1))) {
+            return null;
+        }
+        const prefix = colon < 0 ? "" : name.slice(0, colon);
+        for (let i = this.bindings.length - 1; i >= 0; i--) {
+            if (this.bindings[i].prefix === prefix) {
+                return this.bindings[i].uri;
+            }
+        }
+        if (prefix === "xml") {
+            return XML_NAMESPACE;
+        }
+        return prefix === "" ? "" : null;
     }
 
     /**
@@ -1170,6 +1556,9 @@ export class XmlTokenizer {
             throw new XmlSyntaxError(`end tag </${name}> does not match <${open}>`, this.line);
         }
         this.stack.pop();
+        while ((this.bindings.at(-1)?.depth ?? -1) >= this.stack.length) {
+            this.bindings.pop();
+        }
         if (this.stack.length === 0) {
             this.rootClosed = true;
         }
@@ -1178,11 +1567,11 @@ export class XmlTokenizer {
 }
 
 /**
- * Attribute value normalisation (XML 1.0 section 3.3.3): a literal tab or line break becomes a
+ * Attribute value normalization (XML 1.0 section 3.3.3): a literal tab or line break becomes a
  * space; a character reference to one is kept, which is why this runs before entity decoding.
  * @param raw - the value between the quotes
- * @returns the normalised value
+ * @returns the normalized value
  */
 function normalizeAttributeValue(raw: string): string {
-    return raw.includes("\n") || raw.includes("\t") ? raw.replace(/[\n\t]/g, " ") : raw;
+    return raw.includes("\n") || raw.includes("\t") ? raw.replaceAll(/[\n\t]/g, " ") : raw;
 }