@codemirror/language 6.9.1 → 6.9.3

package/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 6.9.3 (2023-11-27)
+
+### Bug fixes
+
+Fix an issue in `StreamLanguage` where it ran out of node type ids if you repeatedly redefined a language with the same token table.
+
+## 6.9.2 (2023-10-24)
+
+### Bug fixes
+
+Allow `StreamParser` tokens get multiple highlighting tags.
+
 ## 6.9.1 (2023-09-20)
 
 ### Bug fixes

package/dist/index.cjs

@@ -2452,6 +2452,8 @@ const noTokens = Object.create(null);
 const typeArray = [common.NodeType.none];
 const nodeSet = new common.NodeSet(typeArray);
 const warned = [];
+// Cache of node types by name and tags
+const byTag = Object.create(null);
 const defaultTable = Object.create(null);
 for (let [legacyName, name] of [
     ["variable", "variableName"],
@@ -2485,31 +2487,40 @@ function warnForPart(part, msg) {
     console.warn(msg);
 }
 function createTokenType(extra, tagStr) {
-    let tag = null;
-    for (let part of tagStr.split(".")) {
-        let value = (extra[part] || highlight.tags[part]);
-        if (!value) {
-            warnForPart(part, `Unknown highlighting tag ${part}`);
-        }
-        else if (typeof value == "function") {
-            if (!tag)
-                warnForPart(part, `Modifier ${part} used at start of tag`);
-            else
-                tag = value(tag);
-        }
-        else {
-            if (tag)
-                warnForPart(part, `Tag ${part} used as modifier`);
-            else
-                tag = value;
+    let tags = [];
+    for (let name of tagStr.split(" ")) {
+        let found = [];
+        for (let part of name.split(".")) {
+            let value = (extra[part] || highlight.tags[part]);
+            if (!value) {
+                warnForPart(part, `Unknown highlighting tag ${part}`);
+            }
+            else if (typeof value == "function") {
+                if (!found.length)
+                    warnForPart(part, `Modifier ${part} used at start of tag`);
+                else
+                    found = found.map(value);
+            }
+            else {
+                if (found.length)
+                    warnForPart(part, `Tag ${part} used as modifier`);
+                else
+                    found = Array.isArray(value) ? value : [value];
+            }
         }
+        for (let tag of found)
+            tags.push(tag);
     }
-    if (!tag)
+    if (!tags.length)
         return 0;
-    let name = tagStr.replace(/ /g, "_"), type = common.NodeType.define({
+    let name = tagStr.replace(/ /g, "_"), key = name + " " + tags.map(t => t.id);
+    let known = byTag[key];
+    if (known)
+        return known.id;
+    let type = byTag[key] = common.NodeType.define({
         id: typeArray.length,
         name,
-        props: [highlight.styleTags({ [name]: tag })]
+        props: [highlight.styleTags({ [name]: tags })]
     });
     typeArray.push(type);
     return type.id;

package/dist/index.d.cts

@@ -1139,11 +1139,13 @@ interface StreamParser<State> {
     Read one token, advancing the stream past it, and returning a
     string indicating the token's style tag—either the name of one
     of the tags in
-    [`tags`](https://lezer.codemirror.net/docs/ref#highlight.tags),
-    or such a name suffixed by one or more tag
+    [`tags`](https://lezer.codemirror.net/docs/ref#highlight.tags)
+    or [`tokenTable`](https://codemirror.net/6/docs/ref/#language.StreamParser.tokenTable), or such a
+    name suffixed by one or more tag
     [modifier](https://lezer.codemirror.net/docs/ref#highlight.Tag^defineModifier)
     names, separated by periods. For example `"keyword"` or
-    "`variableName.constant"`.
+    "`variableName.constant"`, or a space-separated set of such
+    token types.
     
     It is okay to return a zero-length token, but only if that
     updates the state so that the next call will return a non-empty
@@ -1175,10 +1177,10 @@ interface StreamParser<State> {
     /**
     Extra tokens to use in this parser. When the tokenizer returns a
     token name that exists as a property in this object, the
-    corresponding tag will be assigned to the token.
+    corresponding tags will be assigned to the token.
     */
     tokenTable?: {
-        [name: string]: Tag;
+        [name: string]: Tag | readonly Tag[];
     };
 }
 /**

package/dist/index.d.ts

@@ -1139,11 +1139,13 @@ interface StreamParser<State> {
     Read one token, advancing the stream past it, and returning a
     string indicating the token's style tag—either the name of one
     of the tags in
-    [`tags`](https://lezer.codemirror.net/docs/ref#highlight.tags),
-    or such a name suffixed by one or more tag
+    [`tags`](https://lezer.codemirror.net/docs/ref#highlight.tags)
+    or [`tokenTable`](https://codemirror.net/6/docs/ref/#language.StreamParser.tokenTable), or such a
+    name suffixed by one or more tag
     [modifier](https://lezer.codemirror.net/docs/ref#highlight.Tag^defineModifier)
     names, separated by periods. For example `"keyword"` or
-    "`variableName.constant"`.
+    "`variableName.constant"`, or a space-separated set of such
+    token types.
     
     It is okay to return a zero-length token, but only if that
     updates the state so that the next call will return a non-empty
@@ -1175,10 +1177,10 @@ interface StreamParser<State> {
     /**
     Extra tokens to use in this parser. When the tokenizer returns a
     token name that exists as a property in this object, the
-    corresponding tag will be assigned to the token.
+    corresponding tags will be assigned to the token.
     */
     tokenTable?: {
-        [name: string]: Tag;
+        [name: string]: Tag | readonly Tag[];
     };
 }
 /**

package/dist/index.js

@@ -2450,6 +2450,8 @@ const noTokens = /*@__PURE__*/Object.create(null);
 const typeArray = [NodeType.none];
 const nodeSet = /*@__PURE__*/new NodeSet(typeArray);
 const warned = [];
+// Cache of node types by name and tags
+const byTag = /*@__PURE__*/Object.create(null);
 const defaultTable = /*@__PURE__*/Object.create(null);
 for (let [legacyName, name] of [
     ["variable", "variableName"],
@@ -2483,31 +2485,40 @@ function warnForPart(part, msg) {
     console.warn(msg);
 }
 function createTokenType(extra, tagStr) {
-    let tag = null;
-    for (let part of tagStr.split(".")) {
-        let value = (extra[part] || tags[part]);
-        if (!value) {
-            warnForPart(part, `Unknown highlighting tag ${part}`);
-        }
-        else if (typeof value == "function") {
-            if (!tag)
-                warnForPart(part, `Modifier ${part} used at start of tag`);
-            else
-                tag = value(tag);
-        }
-        else {
-            if (tag)
-                warnForPart(part, `Tag ${part} used as modifier`);
-            else
-                tag = value;
+    let tags$1 = [];
+    for (let name of tagStr.split(" ")) {
+        let found = [];
+        for (let part of name.split(".")) {
+            let value = (extra[part] || tags[part]);
+            if (!value) {
+                warnForPart(part, `Unknown highlighting tag ${part}`);
+            }
+            else if (typeof value == "function") {
+                if (!found.length)
+                    warnForPart(part, `Modifier ${part} used at start of tag`);
+                else
+                    found = found.map(value);
+            }
+            else {
+                if (found.length)
+                    warnForPart(part, `Tag ${part} used as modifier`);
+                else
+                    found = Array.isArray(value) ? value : [value];
+            }
         }
+        for (let tag of found)
+            tags$1.push(tag);
     }
-    if (!tag)
+    if (!tags$1.length)
         return 0;
-    let name = tagStr.replace(/ /g, "_"), type = NodeType.define({
+    let name = tagStr.replace(/ /g, "_"), key = name + " " + tags$1.map(t => t.id);
+    let known = byTag[key];
+    if (known)
+        return known.id;
+    let type = byTag[key] = NodeType.define({
         id: typeArray.length,
         name,
-        props: [styleTags({ [name]: tag })]
+        props: [styleTags({ [name]: tags$1 })]
     });
     typeArray.push(type);
     return type.id;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codemirror/language",
-  "version": "6.9.1",
+  "version": "6.9.3",
   "description": "Language support infrastructure for the CodeMirror code editor",
   "scripts": {
     "test": "cm-runtests",