@willbooster/tree-sitter-xml 0.0.0-stage → 1.0.0

package/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2023 ObserverOfTime
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -1,3 +1,150 @@
-# Temporary Holding Version
+# @willbooster/tree-sitter-xml
 
-This version is a temporary placeholder for this package. An operational version to replace this has been submitted for review and is awaiting a staged release.
+[![npm version](https://img.shields.io/npm/v/@willbooster/tree-sitter-xml.svg)](https://www.npmjs.com/package/@willbooster/tree-sitter-xml)
+[![license](https://img.shields.io/npm/l/@willbooster/tree-sitter-xml.svg)](https://www.npmjs.com/package/@willbooster/tree-sitter-xml)
+[![Test](https://github.com/WillBooster/tree-sitter-xml/actions/workflows/test.yml/badge.svg)](https://github.com/WillBooster/tree-sitter-xml/actions/workflows/test.yml)
+[![Test rust](https://github.com/WillBooster/tree-sitter-xml/actions/workflows/test-rust.yml/badge.svg)](https://github.com/WillBooster/tree-sitter-xml/actions/workflows/test-rust.yml)
+[![semantic-release](https://img.shields.io/badge/%20%20%F0%9F%93%A6%F0%9F%9A%80-semantic--release-e10079.svg)](https://github.com/semantic-release/semantic-release)
+[![wbfy](https://img.shields.io/badge/wbfy-20.41.8-1e90ff.svg)](https://github.com/WillBooster/shared/tree/main/packages/wbfy)
+[![crates.io](https://img.shields.io/crates/v/willbooster-tree-sitter-xml.svg)](https://crates.io/crates/willbooster-tree-sitter-xml)
+
+XML and DTD grammars for [tree-sitter](https://github.com/tree-sitter/tree-sitter), forked from
+[tree-sitter-grammars/tree-sitter-xml](https://github.com/tree-sitter-grammars/tree-sitter-xml). We are grateful
+to its authors and contributors. This is not an official release of that project.
+
+This fork builds on the grammars and queries of upstream's version 0.7.0 and generates the parsers with the
+[WillBooster/tree-sitter](https://github.com/WillBooster/tree-sitter) generator. The parsers carry that fork's own ABI
+number (`0x57420001`) and load only with that fork's runtime, version 2.0.0 or later: upstream's tree-sitter runtime
+rejects them. The XML scanner adapts to the generated array API with the change upstream itself made after 0.7.0. The
+parsers parse as the parsers in upstream's 0.7.0 packages do, except for NUL characters, which are ordinary characters:
+
+- A NUL character inside a quoted literal, such as an attribute value (`AttValue`), an entity value (`EntityValue`),
+  or a system literal (`SystemLiteral`), is part of that token, where upstream's lexer reports an error.
+- Where a NUL character cannot be part of any token, the `ERROR` node covers it as it covers any other unexpected
+  character, without the unexpected-character node that upstream's parsers put inside it.
+
+It holds two grammars, as upstream does: `xml` parses XML documents, including an internal DTD subset, and `dtd`
+parses external DTD files.
+
+## Usage
+
+The npm package ships `tree-sitter-xml.wasm` and `tree-sitter-dtd.wasm` for
+[@willbooster/web-tree-sitter](https://www.npmjs.com/package/@willbooster/web-tree-sitter) version 2.0.0 or later, which runs in Node.js, Bun, browsers, and Cloudflare Workers. In Node.js and Bun, load the grammar from its path:
+
+```js
+import { fileURLToPath } from 'node:url';
+import { Language, Parser } from '@willbooster/web-tree-sitter';
+
+await Parser.init();
+const parser = new Parser();
+const wasmPath = fileURLToPath(import.meta.resolve('@willbooster/tree-sitter-xml/tree-sitter-xml.wasm'));
+parser.setLanguage(await Language.load(wasmPath));
+const tree = parser.parse('<?xml version="1.0"?>\n<note><to>Tove</to></note>\n');
+```
+
+In browsers, serve both `.wasm` files and load them by URL (the example uses Vite's `?url` imports):
+
+```js
+import { Language, Parser } from '@willbooster/web-tree-sitter';
+import runtimeUrl from '@willbooster/web-tree-sitter/web-tree-sitter.wasm?url';
+import xmlUrl from '@willbooster/tree-sitter-xml/tree-sitter-xml.wasm?url';
+
+await Parser.init({ locateFile: () => runtimeUrl });
+const parser = new Parser();
+parser.setLanguage(await Language.load(xmlUrl));
+```
+
+Cloudflare Workers do not allow compiling Wasm at run time, so import both `.wasm` files as modules, with or without
+Node.js compatibility:
+
+```js
+import { Language, Parser } from '@willbooster/web-tree-sitter';
+import runtime from '@willbooster/web-tree-sitter/web-tree-sitter.wasm';
+import xml from '@willbooster/tree-sitter-xml/tree-sitter-xml.wasm';
+
+await Parser.init({ wasmModule: runtime });
+const parser = new Parser();
+parser.setLanguage(await Language.load(xml));
+```
+
+The package also ships the grammar definitions (`common/common.mjs`, `xml/grammar.js`, and `dtd/grammar.js`, with the
+`package.json` files that mark the latter two as ES modules), the queries in `queries/`, and the node types in
+`xml/src/node-types.json` and `dtd/src/node-types.json`.
+
+In Rust, depend on the [crate](https://crates.io/crates/willbooster-tree-sitter-xml) and on
+[willbooster-tree-sitter](https://crates.io/crates/willbooster-tree-sitter), the runtime this package is tested and
+fuzzed with:
+
+```toml
+[dependencies]
+tree-sitter = { package = "willbooster-tree-sitter", version = "2" }
+tree-sitter-xml = { package = "willbooster-tree-sitter-xml", version = "1" }
+```
+
+```rust
+let mut parser = tree_sitter::Parser::new();
+parser.set_language(&tree_sitter_xml::LANGUAGE_XML.into())?;
+```
+
+`tree_sitter_xml::LANGUAGE_DTD` is the `dtd` grammar. The crate also exports upstream's `XML_HIGHLIGHT_QUERY`,
+`DTD_HIGHLIGHT_QUERY`, `XML_NODE_TYPES`, and `DTD_NODE_TYPES`.
+
+## Development
+
+```sh
+mise install
+bun install --frozen-lockfile
+bun run test/ci-setup # installs Chromium for the browser test
+bun run build/ci
+bun run test
+script/parse-examples
+cargo test
+```
+
+The scripts and tests generate, build, test, and parse with `script/tree-sitter`, the tree-sitter CLI of the
+WillBooster/tree-sitter runtime version locked in `Cargo.lock`, since the generator and the runtime of upstream's CLI are
+not the ones this package ships with. Its first run downloads that CLI from the runtime's GitHub Release, or builds it
+with `cargo` (whose build runs the CMake that `mise.toml` pins) when the download fails or the release has no binary that runs here. Run other CLI commands through it as
+well (e.g. `script/tree-sitter parse --grammar-path xml file.xml`).
+
+`bun run generate` records a fresh generation profile for each grammar from the `test/corpus` cases of that grammar,
+then generates compact parser tables. It would also profile files tracked in `examples/`, but this repository tracks
+none. The corpus is upstream's, with the blank line between each case's name and its attributes removed, since the
+WillBooster/tree-sitter CLI, like upstream's current CLI, reads no case whose header has one, plus
+`null-characters.txt`, which pins this parser's reading of NUL characters. The parsers also embed metadata from
+`tree-sitter.json`. After changing a grammar, `tree-sitter.json`, or a corpus case, regenerate and commit `xml/src/`
+and `dtd/src/`. Profiles in `.tmp/generation-profiles/` are temporary and must not be committed. `bun run build-wasm`,
+`bun run build/ci`, and the release build regenerate the parsers before compiling them.
+
+`bun run test` runs:
+
+- the corpus in `test/corpus`, with the native build and with the Wasm build (the first run downloads the WASI SDK).
+  Each case names its grammar with `:language(xml)` or `:language(dtd)`;
+- an incremental-parsing check (`test/unit/incremental.test.ts`): `script/fuzz-corpus` runs `tree-sitter fuzz` for a
+  grammar, which edits each of its corpus cases at random, reparses it, undoes the edits, and reparses again.
+  `TREE_SITTER_SEED`, `TREE_SITTER_ITERATIONS`, and `TREE_SITTER_EDITS` run other or more edits;
+- a check that the real-world files in `examples/` fail to parse exactly as listed in `script/known-failures.txt`, each
+  parsed with the grammar its extension belongs to in `tree-sitter.json`. The first run clones the repositories. The
+  example repositories are pinned to commits in `script/parse-examples`. After a grammar change or a moved pin alters
+  that list, `script/parse-examples` rewrites it; review its diff before committing;
+- a performance check (`test/unit/performance.test.ts`) that recovering from an error on each line takes linear
+  time with both grammars, since consumers parse files while they are being edited. It loads the Wasm builds through
+  @willbooster/web-tree-sitter, which `bun run build/ci` rebuilds after regenerating the parsers;
+- a check (`test/unit/runtimeVersion.test.ts`) that `@willbooster/web-tree-sitter` in `package.json` and
+  `willbooster-tree-sitter` in `Cargo.lock` are the same version, since the Wasm tests run on the former and the Rust
+  tests, the fuzzer, and the CLI on the latter;
+- tests that load the Wasm builds with @willbooster/web-tree-sitter in Chromium (`test/e2e/browser.test.ts`) and in
+  Cloudflare Workers with and without Node.js compatibility (`test/e2e/workers.test.ts`).
+
+The tests compile the parsers into `.tmp/tree-sitter-lib` rather than the CLI's cache shared by every checkout;
+`script/fuzz-corpus` and `script/parse-examples` build parsers of their own in `.tmp/` for each run and delete them
+afterwards.
+
+CI also runs these tests on Linux arm64 and macOS, where the CLI compiles the parsers and the scanners natively, and
+fuzzes both parsers with libFuzzer and sanitizers (`.github/workflows/robustness.yml`).
+
+### References
+
+- [Extensible Markup Language (XML) 1.0](https://www.w3.org/TR/xml/)
+- [Associating Schemas with XML documents 1.0](https://www.w3.org/TR/xml-model/)
+- [Associating Style Sheets with XML documents 1.0](https://www.w3.org/TR/xml-stylesheet/)

package/common/common.mjs

@@ -0,0 +1,375 @@
+/**
+ * @file Reusable constructs
+ * @author ObserverOfTime
+ * @license MIT
+ */
+
+/// <reference types="tree-sitter-cli/dsl" />
+// @ts-check
+
+/**
+ * @param {"'" | ''} q
+ * @see {@link https://www.w3.org/TR/xml/#NT-PubidChar}
+ */
+export const pubid_char = (q) =>
+  new RegExp(`[ \\r\\na-zA-Z0-9\\-${q}()+,./:=?;!*#@$_%]*`);
+
+/**
+ * @param {GrammarSymbols<any>} $
+ * @param {'"' | "'"} q
+ */
+export const att_value = ($, q) =>
+  seq(
+    q,
+    field(
+      'content',
+      repeat(choice(
+        new RegExp(`[^<&${q}]`),
+        $._Reference
+      ))
+    ),
+    q
+  );
+
+/**
+ * @param {GrammarSymbols<any>} $
+ * @param {'"' | "'"} q
+ */
+export const entity_value = ($, q) =>
+  seq(
+    q,
+    field(
+      'content',
+      repeat(choice(
+        new RegExp(`[^<%&${q}]`),
+        $.PEReference,
+        $._Reference
+      ))
+    ),
+    q
+  );
+
+
+/** @param {RuleOrLiteral[]} choices */
+export const str = (...choices) =>
+  choice(
+    seq("'", ...choices, "'"),
+    seq('"', ...choices, '"')
+  );
+
+/**
+ * @param {GrammarSymbols<any>} $
+ * @param {RuleOrLiteral[]} choices
+ */
+export const ref = ($, ...choices) =>
+  choice(...choices, $.PEReference);
+
+/** @param {RuleOrLiteral[]} rules */
+export const rseq = (...rules) => repeat(seq(...rules));
+
+/** @param {RuleOrLiteral[]} rules */
+export const rseq1 = (...rules) => repeat1(seq(...rules));
+
+const O = optional;
+
+/** @type {Record<string, ($: GrammarSymbols<any>) => RuleOrLiteral>} */
+export const rules = {
+    _markupdecl: $ => choice(
+      $.elementdecl,
+      $.AttlistDecl,
+      $._EntityDecl,
+      $.NotationDecl,
+      $.PI,
+      $.Comment
+    ),
+
+    _DeclSep: $ => choice($.PEReference, $._S),
+
+    elementdecl: $ => seq(
+      '<!',
+      'ELEMENT',
+      $._S,
+      ref($, $.Name),
+      $._S,
+      $.contentspec,
+      O($._S),
+      '>'
+    ),
+
+    contentspec: $ => choice(
+      'EMPTY',
+      'ANY',
+      $.Mixed,
+      $.children,
+      $.PEReference
+    ),
+
+    Mixed: $ => choice(
+      seq(
+        '(',
+        O($._S),
+        ref($, '#PCDATA'),
+        rseq(
+          O($._S),
+          '|',
+          O($._S),
+          ref($, $.Name),
+        ),
+        O($._S),
+        rseq(
+          $.PEReference,
+          O($._S)
+        ),
+        ')',
+        '*'
+      ),
+      prec(-1, seq(
+        '(',
+        O($._S),
+        ref($, '#PCDATA'),
+        O($._S),
+        rseq(
+          $.PEReference,
+          O($._S)
+        ),
+        ')'
+      ))
+    ),
+
+    children: $ => prec(1, seq(
+      $._choice,
+      O(choice('?', '*', '+'))
+    )),
+
+    _cp: $ => prec.left(seq(
+      ref($, $.Name, $._choice),
+      O(choice('?', '*', '+'))
+    )),
+
+    _choice: $ => seq(
+      '(',
+      O($._S),
+      $._cp,
+      rseq(
+        O($._S),
+        choice('|', ','),
+        O($._S),
+        $._cp
+      ),
+      rseq(
+        O($._S),
+        $.PEReference,
+      ),
+      O($._S),
+      ')'
+    ),
+
+    AttlistDecl: $ => seq(
+      '<!',
+      'ATTLIST',
+      $._S,
+      ref($, $.Name),
+      repeat(choice(
+        $.AttDef,
+        seq($._S, $.PEReference)
+      )),
+      O($._S),
+      '>'
+    ),
+
+    AttDef: $ => prec.right(seq(
+      $._S,
+      ref($, $.Name),
+      $._S,
+      $._AttType,
+      O(seq($._S, $.DefaultDecl))
+    )),
+
+    _AttType: $ => choice(
+      $.StringType,
+      $.TokenizedType,
+      $._EnumeratedType,
+      $.PEReference
+    ),
+
+    StringType: _ => 'CDATA',
+
+    TokenizedType: _ => token(choice(
+      'ID',
+      'IDREF',
+      'IDREFS',
+      'ENTITY',
+      'ENTITIES',
+      'NMTOKEN',
+      'NMTOKENS',
+    )),
+
+    _EnumeratedType: $ => choice(
+      $.NotationType,
+      $.Enumeration
+    ),
+
+    NotationType: $ => seq(
+      'NOTATION',
+      $._S,
+      '(',
+      O($._S),
+      ref($, $.Name),
+      rseq(
+        O($._S),
+        '|',
+        O($._S)
+      ),
+      ref($, $.Name),
+      O($._S),
+      ')'
+    ),
+
+    Enumeration: $ => seq(
+      '(',
+      O($._S),
+      $.Nmtoken,
+      rseq(
+        O($._S),
+        '|',
+        O($._S),
+        $.Nmtoken
+      ),
+      O($._S),
+      ')'
+    ),
+
+    DefaultDecl: $ => choice(
+      '#REQUIRED',
+      '#IMPLIED',
+      seq(
+        O(seq('#FIXED', $._S)),
+        $.AttValue
+      ),
+      $.PEReference
+    ),
+
+    _EntityDecl: $ => choice(
+      $.GEDecl,
+      $.PEDecl
+    ),
+
+    GEDecl: $ => seq(
+      '<!',
+      'ENTITY',
+      $._S,
+      ref($, $.Name),
+      $._S,
+      choice(
+        $.EntityValue,
+        seq(
+          $.ExternalID,
+          O($.NDataDecl)
+        )
+      ),
+      O($._S),
+      '>'
+    ),
+
+    PEDecl: $ => seq(
+      '<!',
+      'ENTITY',
+      $._S,
+      '%',
+      $._S,
+      $.Name,
+      $._S,
+      choice(
+        $.EntityValue,
+        $.ExternalID
+      ),
+      O($._S),
+      '>'
+    ),
+
+    EntityValue: $ => choice(
+      entity_value($, '"'),
+      entity_value($, "'")
+    ),
+
+    NDataDecl: $ => seq($._S, 'NDATA', $._S, ref($, $.Name)),
+
+    NotationDecl: $ => seq(
+      '<!',
+      'NOTATION',
+      $._S,
+      ref($, $.Name),
+      $._S,
+      choice($.ExternalID, $.PublicID),
+      O($._S),
+      '>'
+    ),
+
+    PEReference: $ => seq('%', $.Name, ';'),
+
+    _S: _ => /[ \t\r\n]+/,
+
+    Name: _ => /[a-zA-Z_][a-zA-Z0-9_:.·-]*/,
+
+    Nmtoken: _ => /[a-zA-Z0-9_:.·-]+/,
+
+    _Reference: $ => choice($.EntityRef, $.CharRef),
+
+    EntityRef: $ => seq('&', $.Name, ';'),
+
+    CharRef: _ => choice(
+      seq('&#', /[0-9]+/, ';'),
+      seq('&#x', /[0-9a-fA-F]+/, ';')
+    ),
+
+    AttValue: $ => choice(
+      att_value($, '"'),
+      att_value($, "'")
+    ),
+
+    ExternalID: $ => choice(
+      seq('SYSTEM', $._S, $.SystemLiteral),
+      seq('PUBLIC', $._S, $.PubidLiteral, $._S, $.SystemLiteral)
+    ),
+
+    PublicID: $ => prec.right(
+      seq(ref($, 'PUBLIC'), $._S, $.PubidLiteral)
+    ),
+
+    SystemLiteral: $ => choice(
+      seq('"', alias(/[^"]*/, $.URI), '"'),
+      seq("'", alias(/[^']*/, $.URI), "'")
+    ),
+
+    PubidLiteral: _ => choice(
+      seq('"', pubid_char("'"), '"'),
+      seq("'", pubid_char(''), "'")
+    ),
+
+    _VersionInfo: $ => seq(
+      $._S,
+      'version',
+      $._Eq,
+      str($.VersionNum)
+    ),
+
+    VersionNum: _ => /1\.[0-9]+/,
+
+    _EncodingDecl: $ => seq(
+      $._S,
+      'encoding',
+      $._Eq,
+      str($.EncName)
+    ),
+
+    EncName: _ => /[A-Za-z][A-Za-z0-9._\-]*/,
+
+    PI: $ => seq(
+      '<?',
+      $.PITarget,
+      O(seq($._S, $._pi_content)),
+      '?>'
+    ),
+
+    _Eq: $ => seq(O($._S), '=', O($._S))
+}

package/dtd/grammar.js

@@ -0,0 +1,72 @@
+/**
+ * @file Tree-sitter grammar definition for DTD
+ * @author ObserverOfTime
+ * @license MIT
+ */
+
+/// <reference types="tree-sitter-cli/dsl" />
+// @ts-check
+
+import * as c from '../common/common.mjs';
+
+const O = optional;
+
+export default grammar({
+  name: 'dtd',
+
+  externals: $ => [
+    $.PITarget,
+    $._pi_content,
+    $.Comment,
+  ],
+
+  extras: _ => [],
+
+  supertypes: $ => [
+    $._markupdecl,
+    $._AttType,
+    $._EnumeratedType,
+    $._EntityDecl,
+    $._Reference,
+  ],
+
+  conflicts: $ => [
+    [$.AttlistDecl, $.AttDef]
+  ],
+
+  word: $ => $.Name,
+
+  rules: {
+    extSubset: $ => seq(
+      O($.TextDecl),
+      repeat1($._extSubsetDecl)
+    ),
+
+    TextDecl: $ => seq(
+      '<?',
+      'xml',
+      $._VersionInfo,
+      O($._EncodingDecl),
+      O($._S),
+      '?>'
+    ),
+
+    _extSubsetDecl: $ => choice(
+      $._markupdecl,
+      $.conditionalSect,
+      $._DeclSep
+    ),
+
+    conditionalSect: $ => seq(
+      '<![',
+      O($._S),
+      c.ref($, 'IGNORE', 'INCLUDE'),
+      O($._S),
+      '[',
+      repeat($._extSubsetDecl),
+      ']]>'
+    ),
+
+    ...c.rules
+  }
+});

package/dtd/package.json

@@ -0,0 +1,4 @@
+{
+  "private": true,
+  "type": "module"
+}