@codemirror/language 6.10.8 → 6.11.0

package/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 6.11.0 (2025-03-13)
+
+### New features
+
+Stream parsers now support a `mergeTokens` option that can be used to turn off automatic merging of adjacent tokens.
+
 ## 6.10.8 (2024-12-23)
 
 ### Bug fixes

package/README.md

@@ -16,3 +16,51 @@ We aim to be an inclusive, welcoming community. To make that explicit,
 we have a [code of
 conduct](http://contributor-covenant.org/version/1/1/0/) that applies
 to communication around the project.
+
+## Usage
+
+Setting up a language from a [Lezer](https://lezer.codemirror.net)
+parser looks like this:
+
+```javascript
+import {parser} from "@lezer/json"
+import {LRLanguage, continuedIndent, indentNodeProp,
+        foldNodeProp, foldInside} from "@codemirror/language"
+
+export const jsonLanguage = LRLanguage.define({
+  name: "json",
+  parser: parser.configure({
+    props: [
+      indentNodeProp.add({
+        Object: continuedIndent({except: /^\s*\}/}),
+        Array: continuedIndent({except: /^\s*\]/})
+      }),
+      foldNodeProp.add({
+        "Object Array": foldInside
+      })
+    ]
+  }),
+  languageData: {
+    closeBrackets: {brackets: ["[", "{", '"']},
+    indentOnInput: /^\s*[\}\]]$/
+  }
+})
+```
+
+Often, you'll also use this package just to access some specific
+language-related features, such as accessing the editor's syntax
+tree...
+
+```javascript
+import {syntaxTree} from "@codemirror/language"
+
+const tree = syntaxTree(view)
+```
+
+... or computing the appriate indentation at a given point.
+
+```javascript
+import {getIndentation} from "@codemirror/language"
+
+console.log(getIndentation(view.state, view.state.selection.main.head))
+```

package/dist/index.cjs

@@ -2181,7 +2181,8 @@ function fullParser(spec) {
         copyState: spec.copyState || defaultCopyState,
         indent: spec.indent || (() => null),
         languageData: spec.languageData || {},
-        tokenTable: spec.tokenTable || noTokens
+        tokenTable: spec.tokenTable || noTokens,
+        mergeTokens: spec.mergeTokens !== false
     };
 }
 function defaultCopyState(state) {
@@ -2404,7 +2405,8 @@ class Parse {
             size += this.chunk.length - len0;
         }
         let last = this.chunk.length - 4;
-        if (size == 4 && last >= 0 && this.chunk[last] == id && this.chunk[last + 2] == from)
+        if (this.lang.streamParser.mergeTokens && size == 4 && last >= 0 &&
+            this.chunk[last] == id && this.chunk[last + 2] == from)
             this.chunk[last + 2] = to;
         else
             this.chunk.push(id, from, to, size);

package/dist/index.d.cts

@@ -1182,6 +1182,11 @@ interface StreamParser<State> {
     tokenTable?: {
         [name: string]: Tag | readonly Tag[];
     };
+    /**
+    By default, adjacent tokens of the same type are merged in the
+    output tree. Set this to false to disable that.
+    */
+    mergeTokens?: boolean;
 }
 /**
 A [language](https://codemirror.net/6/docs/ref/#language.Language) class based on a CodeMirror

package/dist/index.d.ts

@@ -1182,6 +1182,11 @@ interface StreamParser<State> {
     tokenTable?: {
         [name: string]: Tag | readonly Tag[];
     };
+    /**
+    By default, adjacent tokens of the same type are merged in the
+    output tree. Set this to false to disable that.
+    */
+    mergeTokens?: boolean;
 }
 /**
 A [language](https://codemirror.net/6/docs/ref/#language.Language) class based on a CodeMirror

package/dist/index.js

@@ -2179,7 +2179,8 @@ function fullParser(spec) {
         copyState: spec.copyState || defaultCopyState,
         indent: spec.indent || (() => null),
         languageData: spec.languageData || {},
-        tokenTable: spec.tokenTable || noTokens
+        tokenTable: spec.tokenTable || noTokens,
+        mergeTokens: spec.mergeTokens !== false
     };
 }
 function defaultCopyState(state) {
@@ -2402,7 +2403,8 @@ class Parse {
             size += this.chunk.length - len0;
         }
         let last = this.chunk.length - 4;
-        if (size == 4 && last >= 0 && this.chunk[last] == id && this.chunk[last + 2] == from)
+        if (this.lang.streamParser.mergeTokens && size == 4 && last >= 0 &&
+            this.chunk[last] == id && this.chunk[last + 2] == from)
             this.chunk[last + 2] = to;
         else
             this.chunk.push(id, from, to, size);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codemirror/language",
-  "version": "6.10.8",
+  "version": "6.11.0",
   "description": "Language support infrastructure for the CodeMirror code editor",
   "scripts": {
     "test": "cm-runtests",