@theia/monaco 1.76.0-next.4 → 1.76.0-next.42

package/lib/browser/textmate/textmate-tokenizer.d.ts

@@ -12,7 +12,8 @@ export declare class TokenizerState implements monaco.languages.IState {
 export interface TokenizerOption {
     /**
      * Maximum line length that will be handled by the TextMate tokenizer. If the length of the actual line exceeds this
-     * limit, the tokenizer terminates and the tokenization of any subsequent lines might be broken.
+     * limit, the line is skipped and tokenization continues from the preceding state. Constructs opened on a skipped
+     * line (e.g. a block comment) are therefore not accounted for.
      *
      * If the `lineLimit` is not defined, it means, there are no line length limits. Otherwise, it must be a positive
      * integer or an error will be thrown.

package/lib/browser/textmate/textmate-tokenizer.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"textmate-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAW,QAAQ,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,KAAK,MAAM,MAAM,2BAA2B,CAAC;AAEpD,qBAAa,cAAe,YAAW,MAAM,CAAC,SAAS,CAAC,MAAM;aAGtC,UAAU,EAAE,UAAU;gBAAtB,UAAU,EAAE,UAAU;IAG1C,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM;IAIhC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,OAAO;CAIlD;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAE5B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,SAAS,CAAC,qBAAqB,GAAG,MAAM,CAAC,SAAS,CAAC,cAAc,CAgC7J"}
+{"version":3,"file":"textmate-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAW,QAAQ,EAAE,UAAU,EAAE,MAAM,iBAAiB,CAAC;AAChE,OAAO,KAAK,MAAM,MAAM,2BAA2B,CAAC;AAEpD,qBAAa,cAAe,YAAW,MAAM,CAAC,SAAS,CAAC,MAAM;aAGtC,UAAU,EAAE,UAAU;gBAAtB,UAAU,EAAE,UAAU;IAG1C,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM;IAIhC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,OAAO;CAIlD;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAE5B;;;;;;;OAOG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,SAAS,CAAC,qBAAqB,GAAG,MAAM,CAAC,SAAS,CAAC,cAAc,CAkC7J"}

package/lib/browser/textmate/textmate-tokenizer.js

@@ -38,8 +38,10 @@ function createTextmateTokenizer(grammar, options) {
         getInitialState: () => new TokenizerState(vscode_textmate_1.INITIAL),
         tokenizeEncoded(line, state) {
             if (options.lineLimit !== undefined && line.length > options.lineLimit) {
-                // Skip tokenizing the line if it exceeds the line limit.
-                return { endState: state.stateStack, tokens: new Uint32Array() };
+                // Skip tokenizing the line if it exceeds the line limit. The state must be
+                // handed back as-is: `state.stateStack` is the raw `vscode-textmate` stack, and
+                // Monaco passes whatever is returned here straight into the next line.
+                return { endState: state, tokens: new Uint32Array() };
             }
             const result = grammar.tokenizeLine2(line, state.stateStack, 500);
             return {
@@ -49,8 +51,8 @@ function createTextmateTokenizer(grammar, options) {
         },
         tokenize(line, state) {
             if (options.lineLimit !== undefined && line.length > options.lineLimit) {
-                // Skip tokenizing the line if it exceeds the line limit.
-                return { endState: state.stateStack, tokens: [] };
+                // Skip tokenizing the line if it exceeds the line limit. See `tokenizeEncoded`.
+                return { endState: state, tokens: [] };
             }
             const result = grammar.tokenizeLine(line, state.stateStack, 500);
             return {

package/lib/browser/textmate/textmate-tokenizer.js.map

@@ -1 +1 @@
-{"version":3,"file":"textmate-tokenizer.js","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,0CAA0C;AAC1C,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;;;AAqChF,0DAgCC;AAnED,qDAAgE;AAGhE,MAAa,cAAc;IAEvB,YACoB,UAAsB;QAAtB,eAAU,GAAV,UAAU,CAAY;IACtC,CAAC;IAEL,KAAK;QACD,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,CAAC,KAA8B;QACjC,OAAO,KAAK,YAAY,cAAc,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,CAAC,CAAC;IACvG,CAAC;CAEJ;AAdD,wCAcC;AAkBD,SAAgB,uBAAuB,CAAC,OAAiB,EAAE,OAAwB;IAC/E,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QACtG,MAAM,IAAI,KAAK,CAAC,sDAAsD,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;IAChG,CAAC;IACD,OAAO;QACH,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,cAAc,CAAC,yBAAO,CAAC;QAClD,eAAe,CAAC,IAAY,EAAE,KAAqB;YAC/C,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;gBACrE,yDAAyD;gBACzD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,IAAI,WAAW,EAAE,EAAE,CAAC;YACrE,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YAClE,OAAO;gBACH,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM;aACxB,CAAC;QACN,CAAC;QACD,QAAQ,CAAC,IAAY,EAAE,KAAqB;YACxC,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;gBACrE,yDAAyD;gBACzD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,UAAU,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YACtD,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACjE,OAAO;gBACH,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;iBACvC,CAAC,CAAC;aACN,CAAC;QACN,CAAC;KACJ,CAAC;AACN,CAAC"}
+{"version":3,"file":"textmate-tokenizer.js","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,0CAA0C;AAC1C,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;;;AAsChF,0DAkCC;AAtED,qDAAgE;AAGhE,MAAa,cAAc;IAEvB,YACoB,UAAsB;QAAtB,eAAU,GAAV,UAAU,CAAY;IACtC,CAAC;IAEL,KAAK;QACD,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IAC/C,CAAC;IAED,MAAM,CAAC,KAA8B;QACjC,OAAO,KAAK,YAAY,cAAc,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,IAAI,CAAC,UAAU,CAAC,CAAC;IACvG,CAAC;CAEJ;AAdD,wCAcC;AAmBD,SAAgB,uBAAuB,CAAC,OAAiB,EAAE,OAAwB;IAC/E,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE,CAAC;QACtG,MAAM,IAAI,KAAK,CAAC,sDAAsD,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;IAChG,CAAC;IACD,OAAO;QACH,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,cAAc,CAAC,yBAAO,CAAC;QAClD,eAAe,CAAC,IAAY,EAAE,KAAqB;YAC/C,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;gBACrE,2EAA2E;gBAC3E,gFAAgF;gBAChF,uEAAuE;gBACvE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,WAAW,EAAE,EAAE,CAAC;YAC1D,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YAClE,OAAO;gBACH,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM;aACxB,CAAC;QACN,CAAC;QACD,QAAQ,CAAC,IAAY,EAAE,KAAqB;YACxC,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE,CAAC;gBACrE,gFAAgF;gBAChF,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YAC3C,CAAC;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC;YACjE,OAAO;gBACH,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;iBACvC,CAAC,CAAC;aACN,CAAC;QACN,CAAC;KACJ,CAAC;AACN,CAAC"}

package/lib/browser/textmate/textmate-tokenizer.spec.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=textmate-tokenizer.spec.d.ts.map

package/lib/browser/textmate/textmate-tokenizer.spec.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"textmate-tokenizer.spec.d.ts","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.spec.ts"],"names":[],"mappings":""}

package/lib/browser/textmate/textmate-tokenizer.spec.js

@@ -0,0 +1,57 @@
+"use strict";
+// *****************************************************************************
+// Copyright (C) 2026 Matthew Farrow and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+Object.defineProperty(exports, "__esModule", { value: true });
+const chai_1 = require("chai");
+const textmate_tokenizer_1 = require("./textmate-tokenizer");
+describe('createTextmateTokenizer', () => {
+    // A grammar that hands back the rule stack it was given, so the tests only observe
+    // what the tokenizer itself does with the state.
+    const grammar = {
+        tokenizeLine: (line, ruleStack) => ({ tokens: [], ruleStack }),
+        tokenizeLine2: (line, ruleStack) => ({ tokens: new Uint32Array(), ruleStack })
+    };
+    const tokenizer = (0, textmate_tokenizer_1.createTextmateTokenizer)(grammar, { lineLimit: 8 });
+    const longLine = 'x'.repeat(16);
+    // Monaco stores the end state and passes it into the next line. Returning anything
+    // other than a `TokenizerState` breaks the line after the skipped one: a short line
+    // silently restarts the grammar, and a second skipped line returns `undefined`,
+    // which Monaco rejects with 'Cannot set null/undefined state', stopping tokenization
+    // for the whole model.
+    it('should keep the state when tokenizeEncoded skips a line over the limit', () => {
+        const state = tokenizer.getInitialState();
+        const { endState } = tokenizer.tokenizeEncoded(longLine, state);
+        (0, chai_1.expect)(endState).to.be.an.instanceOf(textmate_tokenizer_1.TokenizerState);
+        (0, chai_1.expect)(endState).to.equal(state);
+    });
+    it('should keep the state when tokenize skips a line over the limit', () => {
+        const state = tokenizer.getInitialState();
+        const { endState } = tokenizer.tokenize(longLine, state);
+        (0, chai_1.expect)(endState).to.be.an.instanceOf(textmate_tokenizer_1.TokenizerState);
+        (0, chai_1.expect)(endState).to.equal(state);
+    });
+    it('should tokenize the line after a skipped one from the same state', () => {
+        const initial = tokenizer.getInitialState();
+        const afterSkipped = tokenizer.tokenizeEncoded(longLine, initial).endState;
+        // Two skipped lines in a row is where the broken state surfaced as an exception.
+        const afterSecondSkipped = tokenizer.tokenizeEncoded(longLine, afterSkipped).endState;
+        (0, chai_1.expect)(afterSecondSkipped).to.be.an.instanceOf(textmate_tokenizer_1.TokenizerState);
+        const { endState } = tokenizer.tokenizeEncoded('short', afterSecondSkipped);
+        (0, chai_1.expect)(endState).to.be.an.instanceOf(textmate_tokenizer_1.TokenizerState);
+        (0, chai_1.expect)(endState.stateStack).to.equal(initial.stateStack);
+    });
+});
+//# sourceMappingURL=textmate-tokenizer.spec.js.map

package/lib/browser/textmate/textmate-tokenizer.spec.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"textmate-tokenizer.spec.js","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.spec.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,gDAAgD;AAChD,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,gFAAgF;AAChF,gFAAgF;;AAEhF,+BAA8B;AAE9B,6DAA+E;AAE/E,QAAQ,CAAC,yBAAyB,EAAE,GAAG,EAAE;IAErC,mFAAmF;IACnF,iDAAiD;IACjD,MAAM,OAAO,GAAsB;QAC/B,YAAY,EAAE,CAAC,IAAY,EAAE,SAAqB,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,CAAC;QAClF,aAAa,EAAE,CAAC,IAAY,EAAE,SAAqB,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,WAAW,EAAE,EAAE,SAAS,EAAE,CAAC;KACrG,CAAC;IAEF,MAAM,SAAS,GAAG,IAAA,4CAAuB,EAAC,OAAO,EAAE,EAAE,SAAS,EAAE,CAAC,EAAE,CAAC,CAAC;IACrE,MAAM,QAAQ,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAEhC,mFAAmF;IACnF,oFAAoF;IACpF,gFAAgF;IAChF,qFAAqF;IACrF,uBAAuB;IACvB,EAAE,CAAC,wEAAwE,EAAE,GAAG,EAAE;QAC9E,MAAM,KAAK,GAAG,SAAS,CAAC,eAAe,EAAE,CAAC;QAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QAChE,IAAA,aAAM,EAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,mCAAc,CAAC,CAAC;QACrD,IAAA,aAAM,EAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,iEAAiE,EAAE,GAAG,EAAE;QACvE,MAAM,KAAK,GAAG,SAAS,CAAC,eAAe,EAAE,CAAC;QAC1C,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,QAAQ,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAC;QACzD,IAAA,aAAM,EAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,mCAAc,CAAC,CAAC;QACrD,IAAA,aAAM,EAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,kEAAkE,EAAE,GAAG,EAAE;QACxE,MAAM,OAAO,GAAG,SAAS,CAAC,eAAe,EAAE,CAAC;QAC5C,MAAM,YAAY,GAAG,SAAS,CAAC,eAAe,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,QAAQ,CAAC;QAC3E,iFAAiF;QACjF,MAAM,kBAAkB,GAAG,SAAS,CAAC,eAAe,CAAC,QAAQ,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC;QACtF,IAAA,aAAM,EAAC,kBAAkB,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,mCAAc,CAAC,CAAC;QAC/D,MAAM,EAAE,QAAQ,EAAE,GAAG,SAAS,CAAC,eAAe,CAAC,OAAO,EAAE,kBAAkB,CAAC,CAAC;QAC5E,IAAA,aAAM,EAAC,QAAQ,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,CAAC,mCAAc,CAAC,CAAC;QACrD,IAAA,aAAM,EAAkB,QAAS,CAAC,UAAU,CAAC,CAAC,EAAE,CAAC,KAAK,CAAkB,OAAQ,CAAC,UAAU,CAAC,CAAC;IACjG,CAAC,CAAC,CAAC;AAEP,CAAC,CAAC,CAAC"}

package/package.json

@@ -1,15 +1,15 @@
 {
   "name": "@theia/monaco",
-  "version": "1.76.0-next.4+21b9b5342",
+  "version": "1.76.0-next.42+b0f9e63a6",
   "description": "Theia - Monaco Extension",
   "dependencies": {
-    "@theia/core": "1.76.0-next.4+21b9b5342",
-    "@theia/editor": "1.76.0-next.4+21b9b5342",
-    "@theia/filesystem": "1.76.0-next.4+21b9b5342",
-    "@theia/markers": "1.76.0-next.4+21b9b5342",
+    "@theia/core": "1.76.0-next.42+b0f9e63a6",
+    "@theia/editor": "1.76.0-next.42+b0f9e63a6",
+    "@theia/filesystem": "1.76.0-next.42+b0f9e63a6",
+    "@theia/markers": "1.76.0-next.42+b0f9e63a6",
     "@theia/monaco-editor-core": "1.108.201",
-    "@theia/outline-view": "1.76.0-next.4+21b9b5342",
-    "@theia/workspace": "1.76.0-next.4+21b9b5342",
+    "@theia/outline-view": "1.76.0-next.42+b0f9e63a6",
+    "@theia/workspace": "1.76.0-next.42+b0f9e63a6",
     "fast-plist": "^0.1.3",
     "idb": "^4.0.5",
     "jsonc-parser": "^3.3.1",
@@ -57,5 +57,5 @@
   "nyc": {
     "extends": "../../configs/nyc.json"
   },
-  "gitHead": "21b9b534224c55152fca0b67af7f8a8283dad2bd"
+  "gitHead": "b0f9e63a6d331135265869a341dce7c7f1eef158"
 }

package/src/browser/textmate/textmate-tokenizer.spec.ts

@@ -0,0 +1,63 @@
+// *****************************************************************************
+// Copyright (C) 2026 Matthew Farrow and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { expect } from 'chai';
+import { IGrammar, StateStack } from 'vscode-textmate';
+import { createTextmateTokenizer, TokenizerState } from './textmate-tokenizer';
+
+describe('createTextmateTokenizer', () => {
+
+    // A grammar that hands back the rule stack it was given, so the tests only observe
+    // what the tokenizer itself does with the state.
+    const grammar = <IGrammar><unknown>{
+        tokenizeLine: (line: string, ruleStack: StateStack) => ({ tokens: [], ruleStack }),
+        tokenizeLine2: (line: string, ruleStack: StateStack) => ({ tokens: new Uint32Array(), ruleStack })
+    };
+
+    const tokenizer = createTextmateTokenizer(grammar, { lineLimit: 8 });
+    const longLine = 'x'.repeat(16);
+
+    // Monaco stores the end state and passes it into the next line. Returning anything
+    // other than a `TokenizerState` breaks the line after the skipped one: a short line
+    // silently restarts the grammar, and a second skipped line returns `undefined`,
+    // which Monaco rejects with 'Cannot set null/undefined state', stopping tokenization
+    // for the whole model.
+    it('should keep the state when tokenizeEncoded skips a line over the limit', () => {
+        const state = tokenizer.getInitialState();
+        const { endState } = tokenizer.tokenizeEncoded(longLine, state);
+        expect(endState).to.be.an.instanceOf(TokenizerState);
+        expect(endState).to.equal(state);
+    });
+
+    it('should keep the state when tokenize skips a line over the limit', () => {
+        const state = tokenizer.getInitialState();
+        const { endState } = tokenizer.tokenize(longLine, state);
+        expect(endState).to.be.an.instanceOf(TokenizerState);
+        expect(endState).to.equal(state);
+    });
+
+    it('should tokenize the line after a skipped one from the same state', () => {
+        const initial = tokenizer.getInitialState();
+        const afterSkipped = tokenizer.tokenizeEncoded(longLine, initial).endState;
+        // Two skipped lines in a row is where the broken state surfaced as an exception.
+        const afterSecondSkipped = tokenizer.tokenizeEncoded(longLine, afterSkipped).endState;
+        expect(afterSecondSkipped).to.be.an.instanceOf(TokenizerState);
+        const { endState } = tokenizer.tokenizeEncoded('short', afterSecondSkipped);
+        expect(endState).to.be.an.instanceOf(TokenizerState);
+        expect((<TokenizerState>endState).stateStack).to.equal((<TokenizerState>initial).stateStack);
+    });
+
+});

package/src/browser/textmate/textmate-tokenizer.ts

@@ -40,7 +40,8 @@ export interface TokenizerOption {
 
     /**
      * Maximum line length that will be handled by the TextMate tokenizer. If the length of the actual line exceeds this
-     * limit, the tokenizer terminates and the tokenization of any subsequent lines might be broken.
+     * limit, the line is skipped and tokenization continues from the preceding state. Constructs opened on a skipped
+     * line (e.g. a block comment) are therefore not accounted for.
      *
      * If the `lineLimit` is not defined, it means, there are no line length limits. Otherwise, it must be a positive
      * integer or an error will be thrown.
@@ -57,8 +58,10 @@ export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOpt
         getInitialState: () => new TokenizerState(INITIAL),
         tokenizeEncoded(line: string, state: TokenizerState): monaco.languages.IEncodedLineTokens {
             if (options.lineLimit !== undefined && line.length > options.lineLimit) {
-                // Skip tokenizing the line if it exceeds the line limit.
-                return { endState: state.stateStack, tokens: new Uint32Array() };
+                // Skip tokenizing the line if it exceeds the line limit. The state must be
+                // handed back as-is: `state.stateStack` is the raw `vscode-textmate` stack, and
+                // Monaco passes whatever is returned here straight into the next line.
+                return { endState: state, tokens: new Uint32Array() };
             }
             const result = grammar.tokenizeLine2(line, state.stateStack, 500);
             return {
@@ -68,8 +71,8 @@ export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOpt
         },
         tokenize(line: string, state: TokenizerState): monaco.languages.ILineTokens {
             if (options.lineLimit !== undefined && line.length > options.lineLimit) {
-                // Skip tokenizing the line if it exceeds the line limit.
-                return { endState: state.stateStack, tokens: [] };
+                // Skip tokenizing the line if it exceeds the line limit. See `tokenizeEncoded`.
+                return { endState: state, tokens: [] };
             }
             const result = grammar.tokenizeLine(line, state.stateStack, 500);
             return {