@codemirror/language 6.10.3 → 6.10.5

package/CHANGELOG.md

@@ -1,3 +1,19 @@
+## 6.10.5 (2024-11-27)
+
+### Bug fixes
+
+Fix an issue where a `StreamLanguage` could get confused when trying to reuse existing parse data when the parsed range changed.
+
+## 6.10.4 (2024-11-24)
+
+### Bug fixes
+
+Join adjacent tokens of the same type into a single token in .
+
+Call stream language indent functions even when the language is used as a nested parser.
+
+Fix a crash in `StreamParser` when a parse was resumed with different input ranges.
+
 ## 6.10.3 (2024-09-19)
 
 ### Bug fixes

package/dist/index.cjs

@@ -2208,8 +2208,8 @@ class StreamLanguage extends Language {
                 return new Parse(self, input, fragments, ranges);
             }
         };
-        super(data, impl, [indentService.of((cx, pos) => this.getIndent(cx, pos))], parser.name);
-        this.topNode = docID(data);
+        super(data, impl, [], parser.name);
+        this.topNode = docID(data, this);
         self = this;
         this.streamParser = p;
         this.stateAfter = new common.NodeProp({ perNode: true });
@@ -2219,20 +2219,18 @@ class StreamLanguage extends Language {
     Define a stream language.
     */
     static define(spec) { return new StreamLanguage(spec); }
-    getIndent(cx, pos) {
-        let tree = syntaxTree(cx.state), at = tree.resolve(pos);
-        while (at && at.type != this.topNode)
-            at = at.parent;
-        if (!at)
-            return null;
+    /**
+    @internal
+    */
+    getIndent(cx) {
         let from = undefined;
         let { overrideIndentation } = cx.options;
         if (overrideIndentation) {
             from = IndentedFrom.get(cx.state);
-            if (from != null && from < pos - 1e4)
+            if (from != null && from < cx.pos - 1e4)
                 from = undefined;
         }
-        let start = findState(this, tree, 0, at.from, from !== null && from !== void 0 ? from : pos), statePos, state;
+        let start = findState(this, cx.node.tree, 0, cx.node.from, from !== null && from !== void 0 ? from : cx.pos), statePos, state;
         if (start) {
             state = start.state;
             statePos = start.pos + 1;
@@ -2241,10 +2239,10 @@ class StreamLanguage extends Language {
             state = this.streamParser.startState(cx.unit);
             statePos = 0;
         }
-        if (pos - statePos > 10000 /* C.MaxIndentScanDist */)
+        if (cx.pos - statePos > 10000 /* C.MaxIndentScanDist */)
             return null;
-        while (statePos < pos) {
-            let line = cx.state.doc.lineAt(statePos), end = Math.min(pos, line.to);
+        while (statePos < cx.pos) {
+            let line = cx.state.doc.lineAt(statePos), end = Math.min(cx.pos, line.to);
             if (line.length) {
                 let indentation = overrideIndentation ? overrideIndentation(line.from) : -1;
                 let stream = new StringStream(line.text, cx.state.tabSize, cx.unit, indentation < 0 ? undefined : indentation);
@@ -2254,11 +2252,11 @@ class StreamLanguage extends Language {
             else {
                 this.streamParser.blankLine(state, cx.unit);
             }
-            if (end == pos)
+            if (end == cx.pos)
                 break;
             statePos = line.to + 1;
         }
-        let line = cx.lineAt(pos);
+        let line = cx.lineAt(cx.pos);
         if (overrideIndentation && from == null)
             IndentedFrom.set(cx.state, line.from);
         return this.streamParser.indent(state, /^\s*(.*)/.exec(line.text)[1], cx);
@@ -2280,7 +2278,7 @@ function findState(lang, tree, off, startPos, before) {
 function cutTree(lang, tree, from, to, inside) {
     if (inside && from <= 0 && to >= tree.length)
         return tree;
-    if (!inside && tree.type == lang.topNode)
+    if (!inside && from == 0 && tree.type == lang.topNode)
         inside = true;
     for (let i = tree.children.length - 1; i >= 0; i--) {
         let pos = tree.positions[i], child = tree.children[i], inner;
@@ -2293,11 +2291,11 @@ function cutTree(lang, tree, from, to, inside) {
     }
     return null;
 }
-function findStartInFragments(lang, fragments, startPos, editorState) {
+function findStartInFragments(lang, fragments, startPos, endPos, editorState) {
     for (let f of fragments) {
         let from = f.from + (f.openStart ? 25 : 0), to = f.to - (f.openEnd ? 25 : 0);
         let found = from <= startPos && to > startPos && findState(lang, f.tree, 0 - f.offset, startPos, to), tree;
-        if (found && (tree = cutTree(lang, f.tree, startPos + f.offset, found.pos + f.offset, false)))
+        if (found && found.pos <= endPos && (tree = cutTree(lang, f.tree, startPos + f.offset, found.pos + f.offset, false)))
             return { state: found.state, tree };
     }
     return { state: lang.streamParser.startState(editorState ? getIndentUnit(editorState) : 4), tree: common.Tree.empty };
@@ -2316,7 +2314,7 @@ class Parse {
         this.rangeIndex = 0;
         this.to = ranges[ranges.length - 1].to;
         let context = ParseContext.get(), from = ranges[0].from;
-        let { state, tree } = findStartInFragments(lang, fragments, from, context === null || context === void 0 ? void 0 : context.state);
+        let { state, tree } = findStartInFragments(lang, fragments, from, ranges[ranges.length - 1].to, context === null || context === void 0 ? void 0 : context.state);
         this.state = state;
         this.parsedPos = this.chunkStart = from + tree.length;
         for (let i = 0; i < tree.children.length; i++) {
@@ -2394,7 +2392,8 @@ class Parse {
         while (this.ranges[this.rangeIndex].to < this.parsedPos)
             this.rangeIndex++;
     }
-    emitToken(id, from, to, size, offset) {
+    emitToken(id, from, to, offset) {
+        let size = 4;
         if (this.ranges.length > 1) {
             offset = this.skipGapsTo(from, offset, 1);
             from += offset;
@@ -2403,7 +2402,11 @@ class Parse {
             to += offset;
             size += this.chunk.length - len0;
         }
-        this.chunk.push(id, from, to, size);
+        let last = this.chunk.length - 4;
+        if (size == 4 && last >= 0 && this.chunk[last] == id && this.chunk[last + 2] == from)
+            this.chunk[last + 2] = to;
+        else
+            this.chunk.push(id, from, to, size);
         return offset;
     }
     parseLine(context) {
@@ -2416,7 +2419,7 @@ class Parse {
             while (!stream.eol()) {
                 let token = readToken(streamParser.token, stream, this.state);
                 if (token)
-                    offset = this.emitToken(this.lang.tokenTable.resolve(token), this.parsedPos + stream.start, this.parsedPos + stream.pos, 4, offset);
+                    offset = this.emitToken(this.lang.tokenTable.resolve(token), this.parsedPos + stream.start, this.parsedPos + stream.pos, offset);
                 if (stream.start > 10000 /* C.MaxLineLength */)
                     break;
             }
@@ -2533,8 +2536,11 @@ function createTokenType(extra, tagStr) {
     typeArray.push(type);
     return type.id;
 }
-function docID(data) {
-    let type = common.NodeType.define({ id: typeArray.length, name: "Document", props: [languageDataProp.add(() => data)], top: true });
+function docID(data, lang) {
+    let type = common.NodeType.define({ id: typeArray.length, name: "Document", props: [
+            languageDataProp.add(() => data),
+            indentNodeProp.add(() => cx => lang.getIndent(cx))
+        ], top: true });
     typeArray.push(type);
     return type;
 }

package/dist/index.d.cts

@@ -1193,7 +1193,6 @@ declare class StreamLanguage<State> extends Language {
     Define a stream language.
     */
     static define<State>(spec: StreamParser<State>): StreamLanguage<State>;
-    private getIndent;
     get allowsNesting(): boolean;
 }
 

package/dist/index.d.ts

@@ -1193,7 +1193,6 @@ declare class StreamLanguage<State> extends Language {
     Define a stream language.
     */
     static define<State>(spec: StreamParser<State>): StreamLanguage<State>;
-    private getIndent;
     get allowsNesting(): boolean;
 }
 

package/dist/index.js

@@ -2206,8 +2206,8 @@ class StreamLanguage extends Language {
                 return new Parse(self, input, fragments, ranges);
             }
         };
-        super(data, impl, [indentService.of((cx, pos) => this.getIndent(cx, pos))], parser.name);
-        this.topNode = docID(data);
+        super(data, impl, [], parser.name);
+        this.topNode = docID(data, this);
         self = this;
         this.streamParser = p;
         this.stateAfter = new NodeProp({ perNode: true });
@@ -2217,20 +2217,18 @@ class StreamLanguage extends Language {
     Define a stream language.
     */
     static define(spec) { return new StreamLanguage(spec); }
-    getIndent(cx, pos) {
-        let tree = syntaxTree(cx.state), at = tree.resolve(pos);
-        while (at && at.type != this.topNode)
-            at = at.parent;
-        if (!at)
-            return null;
+    /**
+    @internal
+    */
+    getIndent(cx) {
         let from = undefined;
         let { overrideIndentation } = cx.options;
         if (overrideIndentation) {
             from = IndentedFrom.get(cx.state);
-            if (from != null && from < pos - 1e4)
+            if (from != null && from < cx.pos - 1e4)
                 from = undefined;
         }
-        let start = findState(this, tree, 0, at.from, from !== null && from !== void 0 ? from : pos), statePos, state;
+        let start = findState(this, cx.node.tree, 0, cx.node.from, from !== null && from !== void 0 ? from : cx.pos), statePos, state;
         if (start) {
             state = start.state;
             statePos = start.pos + 1;
@@ -2239,10 +2237,10 @@ class StreamLanguage extends Language {
             state = this.streamParser.startState(cx.unit);
             statePos = 0;
         }
-        if (pos - statePos > 10000 /* C.MaxIndentScanDist */)
+        if (cx.pos - statePos > 10000 /* C.MaxIndentScanDist */)
             return null;
-        while (statePos < pos) {
-            let line = cx.state.doc.lineAt(statePos), end = Math.min(pos, line.to);
+        while (statePos < cx.pos) {
+            let line = cx.state.doc.lineAt(statePos), end = Math.min(cx.pos, line.to);
             if (line.length) {
                 let indentation = overrideIndentation ? overrideIndentation(line.from) : -1;
                 let stream = new StringStream(line.text, cx.state.tabSize, cx.unit, indentation < 0 ? undefined : indentation);
@@ -2252,11 +2250,11 @@ class StreamLanguage extends Language {
             else {
                 this.streamParser.blankLine(state, cx.unit);
             }
-            if (end == pos)
+            if (end == cx.pos)
                 break;
             statePos = line.to + 1;
         }
-        let line = cx.lineAt(pos);
+        let line = cx.lineAt(cx.pos);
         if (overrideIndentation && from == null)
             IndentedFrom.set(cx.state, line.from);
         return this.streamParser.indent(state, /^\s*(.*)/.exec(line.text)[1], cx);
@@ -2278,7 +2276,7 @@ function findState(lang, tree, off, startPos, before) {
 function cutTree(lang, tree, from, to, inside) {
     if (inside && from <= 0 && to >= tree.length)
         return tree;
-    if (!inside && tree.type == lang.topNode)
+    if (!inside && from == 0 && tree.type == lang.topNode)
         inside = true;
     for (let i = tree.children.length - 1; i >= 0; i--) {
         let pos = tree.positions[i], child = tree.children[i], inner;
@@ -2291,11 +2289,11 @@ function cutTree(lang, tree, from, to, inside) {
     }
     return null;
 }
-function findStartInFragments(lang, fragments, startPos, editorState) {
+function findStartInFragments(lang, fragments, startPos, endPos, editorState) {
     for (let f of fragments) {
         let from = f.from + (f.openStart ? 25 : 0), to = f.to - (f.openEnd ? 25 : 0);
         let found = from <= startPos && to > startPos && findState(lang, f.tree, 0 - f.offset, startPos, to), tree;
-        if (found && (tree = cutTree(lang, f.tree, startPos + f.offset, found.pos + f.offset, false)))
+        if (found && found.pos <= endPos && (tree = cutTree(lang, f.tree, startPos + f.offset, found.pos + f.offset, false)))
             return { state: found.state, tree };
     }
     return { state: lang.streamParser.startState(editorState ? getIndentUnit(editorState) : 4), tree: Tree.empty };
@@ -2314,7 +2312,7 @@ class Parse {
         this.rangeIndex = 0;
         this.to = ranges[ranges.length - 1].to;
         let context = ParseContext.get(), from = ranges[0].from;
-        let { state, tree } = findStartInFragments(lang, fragments, from, context === null || context === void 0 ? void 0 : context.state);
+        let { state, tree } = findStartInFragments(lang, fragments, from, ranges[ranges.length - 1].to, context === null || context === void 0 ? void 0 : context.state);
         this.state = state;
         this.parsedPos = this.chunkStart = from + tree.length;
         for (let i = 0; i < tree.children.length; i++) {
@@ -2392,7 +2390,8 @@ class Parse {
         while (this.ranges[this.rangeIndex].to < this.parsedPos)
             this.rangeIndex++;
     }
-    emitToken(id, from, to, size, offset) {
+    emitToken(id, from, to, offset) {
+        let size = 4;
         if (this.ranges.length > 1) {
             offset = this.skipGapsTo(from, offset, 1);
             from += offset;
@@ -2401,7 +2400,11 @@ class Parse {
             to += offset;
             size += this.chunk.length - len0;
         }
-        this.chunk.push(id, from, to, size);
+        let last = this.chunk.length - 4;
+        if (size == 4 && last >= 0 && this.chunk[last] == id && this.chunk[last + 2] == from)
+            this.chunk[last + 2] = to;
+        else
+            this.chunk.push(id, from, to, size);
         return offset;
     }
     parseLine(context) {
@@ -2414,7 +2417,7 @@ class Parse {
             while (!stream.eol()) {
                 let token = readToken(streamParser.token, stream, this.state);
                 if (token)
-                    offset = this.emitToken(this.lang.tokenTable.resolve(token), this.parsedPos + stream.start, this.parsedPos + stream.pos, 4, offset);
+                    offset = this.emitToken(this.lang.tokenTable.resolve(token), this.parsedPos + stream.start, this.parsedPos + stream.pos, offset);
                 if (stream.start > 10000 /* C.MaxLineLength */)
                     break;
             }
@@ -2531,8 +2534,11 @@ function createTokenType(extra, tagStr) {
     typeArray.push(type);
     return type.id;
 }
-function docID(data) {
-    let type = NodeType.define({ id: typeArray.length, name: "Document", props: [languageDataProp.add(() => data)], top: true });
+function docID(data, lang) {
+    let type = NodeType.define({ id: typeArray.length, name: "Document", props: [
+            languageDataProp.add(() => data),
+            indentNodeProp.add(() => cx => lang.getIndent(cx))
+        ], top: true });
     typeArray.push(type);
     return type;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codemirror/language",
-  "version": "6.10.3",
+  "version": "6.10.5",
   "description": "Language support infrastructure for the CodeMirror code editor",
   "scripts": {
     "test": "cm-runtests",