@willbooster/tree-sitter-html 2.0.1 → 3.1.0

package/README.md

@@ -14,13 +14,13 @@ to its authors and contributors. This is not an official release of that project
 
 This fork is based on upstream's version 0.23.2 grammar and external scanner and generates the parser with the
 [WillBooster/tree-sitter](https://github.com/WillBooster/tree-sitter) generator. The parser carries that fork's own ABI
-number, so it loads only with the WillBooster runtime, version 2.0.0 or later in the 2.x series
+number, so it loads only with the WillBooster runtime, version 2.x
 ([willbooster-tree-sitter](https://crates.io/crates/willbooster-tree-sitter),
 [@willbooster/web-tree-sitter](https://www.npmjs.com/package/@willbooster/web-tree-sitter)); upstream's runtime rejects
 it. Input that contains NUL characters can parse to a different tree than with upstream's 0.23.2 parser, because the
 lexers that generator emits treat a NUL character as an ordinary character. This grammar also accepts literal `>`
 characters and bare `&` in text nodes, which upstream 0.23.2 rejects. A bare `&` becomes its own `text` node beside
-surrounding text, while character references remain `entity` nodes. Text nodes include leading, trailing, and whitespace-only content, including whitespace between markup and character references. Its scanner requires an HTML tag-name boundary after
+surrounding text, while character references remain `entity` nodes. Text nodes include leading, trailing, and whitespace-only content, including whitespace between markup and character references. A DOCTYPE declaration exposes its uninterpreted body as a `doctype_value` child, including legacy PUBLIC or SYSTEM identifiers. The grammar's scanner requires an HTML tag-name boundary after
 `</script` and `</style`, so longer names and overlapping prefixes remain raw text. Closing-name comparisons
 fold only ASCII letters, and embedded NUL characters remain part of raw-text content. Script content follows
 HTML escaped and double-escaped states, so closing-tag text inside a double escape remains raw text. A script that reaches
@@ -34,10 +34,17 @@ exceeds its fixed-size buffer, the outermost tag identities are lost, which can
 tags close. Custom tag names longer than 255 bytes are truncated in serialized state.
 Malformed input after a void element can replace its enclosing element with an `ERROR` node during recovery.
 
+Attribute values retain their complete `attribute_value` range and expose recognized named and numeric character
+references as `entity` children. Quoted values retain their `quoted_attribute_value` wrapper. Other value content,
+including literal ampersands, has no child nodes, so injections targeting `attribute_value` retain it by default. Set `injection.include-children` when the injected language needs the original reference
+spellings. Legacy names without a semicolon remain literal before an ASCII letter, digit, or `=`; unknown
+names and incomplete numeric prefixes remain literal. Comments are document or element content, and are not accepted
+between attributes. Comment-shaped text inside a quoted value remains part of that value.
+
 ## Usage
 
 The npm package ships `tree-sitter-html.wasm` for
-[@willbooster/web-tree-sitter](https://www.npmjs.com/package/@willbooster/web-tree-sitter) version 2.0.0 or later in the 2.x series, which runs in Node.js, Bun, browsers, and Cloudflare Workers. In Node.js and Bun, load the grammar from its path:
+[@willbooster/web-tree-sitter](https://www.npmjs.com/package/@willbooster/web-tree-sitter) version 2.x, which runs in Node.js, Bun, browsers, and Cloudflare Workers. In Node.js and Bun, load the grammar from its path:
 
 ```js
 import { fileURLToPath } from 'node:url';
@@ -84,7 +91,7 @@ fuzzed with:
 ```toml
 [dependencies]
 tree-sitter = { package = "willbooster-tree-sitter", version = "2.0.0" }
-tree-sitter-html = { package = "willbooster-tree-sitter-html", version = "2" }
+tree-sitter-html = { package = "willbooster-tree-sitter-html", version = "3" }
 ```
 
 ```rust
@@ -143,6 +150,9 @@ afterwards.
 CI also runs these tests on Linux arm64 and macOS, where the CLI compiles the parser and the scanner natively, and fuzzes
 both with libFuzzer and sanitizers (`.github/workflows/robustness.yml`).
 
+Run `script/generate-entities` with Python 3.8 or later when refreshing the HTML Standard's named-reference data, then regenerate
+and commit `src/`. The command reads the official `entities.json` over HTTPS.
+
 ### References
 
 - [The HTML Standard](https://html.spec.whatwg.org/multipage/)

package/grammar.js

@@ -12,7 +12,6 @@ module.exports = grammar({
   name: 'html',
 
   extras: $ => [
-    $.comment,
     /\s+/,
   ],
 
@@ -26,6 +25,10 @@ module.exports = grammar({
     $._implicit_end_tag,
     $.raw_text,
     $.comment,
+    $._attribute_entity,
+    $._unquoted_attribute_entity,
+    $._attribute_ampersand,
+    $._unquoted_attribute_fragment,
   ],
 
   rules: {
@@ -34,14 +37,17 @@ module.exports = grammar({
     doctype: $ => seq(
       '<!',
       alias($._doctype, 'doctype'),
-      /[^>]+/,
+      $.doctype_value,
       '>',
     ),
 
     _doctype: _ => /[Dd][Oo][Cc][Tt][Yy][Pp][Ee]/,
 
+    doctype_value: _ => repeat1(/[^>]+/),
+
     _node: $ => choice(
       $.doctype,
+      $.comment,
       $.entity,
       $.text,
       $.element,
@@ -124,13 +130,32 @@ module.exports = grammar({
 
     attribute_name: _ => /[^<>"'/=\s]+/,
 
-    attribute_value: _ => /[^<>"'=\s]+/,
+    attribute_value: $ => seq(
+      choice(
+        alias($._unquoted_attribute_entity, $.entity),
+        /[^<>"'=\s&]+/,
+        $._attribute_ampersand,
+      ),
+      repeat(choice(
+        alias($._attribute_entity, $.entity),
+        $._unquoted_attribute_fragment,
+        $._attribute_ampersand,
+      )),
+    ),
 
     entity: _ => token(/&(#([xX][0-9a-fA-F]+|[0-9]+)|[A-Za-z][A-Za-z0-9]*);?/),
 
     quoted_attribute_value: $ => choice(
-      seq('\'', optional(alias(/[^']+/, $.attribute_value)), '\''),
-      seq('"', optional(alias(/[^"]+/, $.attribute_value)), '"'),
+      seq('\'', optional(alias(repeat1(choice(
+        alias($._attribute_entity, $.entity),
+        token.immediate(/[^'&]+/),
+        $._attribute_ampersand,
+      )), $.attribute_value)), '\''),
+      seq('"', optional(alias(repeat1(choice(
+        alias($._attribute_entity, $.entity),
+        token.immediate(/[^"&]+/),
+        $._attribute_ampersand,
+      )), $.attribute_value)), '"'),
     ),
 
     text: _ => token.immediate(/[^<&]+|&/),

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@willbooster/tree-sitter-html",
-  "version": "2.0.1",
+  "version": "3.1.0",
   "description": "HTML grammar for tree-sitter",
   "keywords": [
     "incremental",

package/src/node-types.json

@@ -22,9 +22,39 @@
       ]
     }
   },
+  {
+    "type": "attribute_value",
+    "named": true,
+    "fields": {},
+    "children": {
+      "multiple": true,
+      "required": false,
+      "types": [
+        {
+          "type": "entity",
+          "named": true
+        }
+      ]
+    }
+  },
   {
     "type": "doctype",
     "named": true,
+    "fields": {},
+    "children": {
+      "multiple": false,
+      "required": true,
+      "types": [
+        {
+          "type": "doctype_value",
+          "named": true
+        }
+      ]
+    }
+  },
+  {
+    "type": "doctype_value",
+    "named": true,
     "fields": {}
   },
   {
@@ -36,6 +66,10 @@
       "multiple": true,
       "required": false,
       "types": [
+        {
+          "type": "comment",
+          "named": true
+        },
         {
           "type": "doctype",
           "named": true
@@ -75,6 +109,10 @@
       "multiple": true,
       "required": true,
       "types": [
+        {
+          "type": "comment",
+          "named": true
+        },
         {
           "type": "doctype",
           "named": true
@@ -283,14 +321,9 @@
     "type": "attribute_name",
     "named": true
   },
-  {
-    "type": "attribute_value",
-    "named": true
-  },
   {
     "type": "comment",
-    "named": true,
-    "extra": true
+    "named": true
   },
   {
     "type": "doctype",

package/tree-sitter.json

@@ -7,7 +7,9 @@
       "scope": "source.html",
       "path": ".",
       "external-files": [
-        "src/tag.h"
+        "src/tag.h",
+        "src/attributeEntity.h",
+        "src/entities.h"
       ],
       "file-types": [
         "html"
@@ -18,7 +20,7 @@
     }
   ],
   "metadata": {
-    "version": "2.0.1",
+    "version": "3.1.0",
     "license": "MIT",
     "description": "HTML grammar for tree-sitter",
     "authors": [