@theia/monaco 1.28.0-next.13 → 1.28.0-next.18

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (6) hide show
  1. package/lib/browser/textmate/textmate-tokenizer.d.ts +0 -8
  2. package/lib/browser/textmate/textmate-tokenizer.d.ts.map +1 -1
  3. package/lib/browser/textmate/textmate-tokenizer.js +7 -20
  4. package/lib/browser/textmate/textmate-tokenizer.js.map +1 -1
  5. package/package.json +7 -7
  6. package/src/browser/textmate/textmate-tokenizer.ts +6 -19
package/lib/browser/textmate/textmate-tokenizer.d.ts CHANGED
@@ -19,13 +19,5 @@ export interface TokenizerOption {
19
19
  */
20
20
  lineLimit?: number;
21
21
  }
22
- export declare namespace TokenizerOption {
23
- /**
24
- * The default TextMate tokenizer option.
25
- *
26
- * @deprecated Use the current value of `editor.maxTokenizationLineLength` preference instead.
27
- */
28
- const DEFAULT: TokenizerOption;
29
- }
30
22
  export declare function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOption): monaco.languages.EncodedTokensProvider & monaco.languages.TokensProvider;
31
23
  //# sourceMappingURL=textmate-tokenizer.d.ts.map
package/lib/browser/textmate/textmate-tokenizer.d.ts.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"textmate-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAW,YAAY,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAClE,OAAO,KAAK,MAAM,MAAM,2BAA2B,CAAC;AAEpD,qBAAa,cAAe,YAAW,MAAM,CAAC,SAAS,CAAC,MAAM;aAGtC,SAAS,EAAE,YAAY;gBAAvB,SAAS,EAAE,YAAY;IAG3C,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM;IAIhC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,OAAO;CAIlD;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAE5B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,yBAAiB,eAAe,CAAC;IAC7B;;;;OAIG;IACI,MAAM,OAAO,EAAE,eAErB,CAAC;CACL;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,SAAS,CAAC,qBAAqB,GAAG,MAAM,CAAC,SAAS,CAAC,cAAc,CAkC7J"}
1
+ {"version":3,"file":"textmate-tokenizer.d.ts","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.ts"],"names":[],"mappings":"AAgBA,OAAO,EAAW,YAAY,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAClE,OAAO,KAAK,MAAM,MAAM,2BAA2B,CAAC;AAEpD,qBAAa,cAAe,YAAW,MAAM,CAAC,SAAS,CAAC,MAAM;aAGtC,SAAS,EAAE,YAAY;gBAAvB,SAAS,EAAE,YAAY;IAG3C,KAAK,IAAI,MAAM,CAAC,SAAS,CAAC,MAAM;IAIhC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,SAAS,CAAC,MAAM,GAAG,OAAO;CAIlD;AAED;;GAEG;AACH,MAAM,WAAW,eAAe;IAE5B;;;;;;OAMG;IACH,SAAS,CAAC,EAAE,MAAM,CAAC;CAEtB;AAED,wBAAgB,uBAAuB,CAAC,OAAO,EAAE,QAAQ,EAAE,OAAO,EAAE,eAAe,GAAG,MAAM,CAAC,SAAS,CAAC,qBAAqB,GAAG,MAAM,CAAC,SAAS,CAAC,cAAc,CAgC7J"}
package/lib/browser/textmate/textmate-tokenizer.js CHANGED
@@ -15,7 +15,7 @@
15
15
  // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
16
16
  // *****************************************************************************
17
17
  Object.defineProperty(exports, "__esModule", { value: true });
18
- exports.createTextmateTokenizer = exports.TokenizerOption = exports.TokenizerState = void 0;
18
+ exports.createTextmateTokenizer = exports.TokenizerState = void 0;
19
19
  const vscode_textmate_1 = require("vscode-textmate");
20
20
  class TokenizerState {
21
21
  constructor(ruleStack) {
@@ -29,17 +29,6 @@ class TokenizerState {
29
29
  }
30
30
  }
31
31
  exports.TokenizerState = TokenizerState;
32
- var TokenizerOption;
33
- (function (TokenizerOption) {
34
- /**
35
- * The default TextMate tokenizer option.
36
- *
37
- * @deprecated Use the current value of `editor.maxTokenizationLineLength` preference instead.
38
- */
39
- TokenizerOption.DEFAULT = {
40
- lineLimit: 400
41
- };
42
- })(TokenizerOption = exports.TokenizerOption || (exports.TokenizerOption = {}));
43
32
  function createTextmateTokenizer(grammar, options) {
44
33
  if (options.lineLimit !== undefined && (options.lineLimit <= 0 || !Number.isInteger(options.lineLimit))) {
45
34
  throw new Error(`The 'lineLimit' must be a positive integer. It was ${options.lineLimit}.`);
@@ -47,24 +36,22 @@ function createTextmateTokenizer(grammar, options) {
47
36
  return {
48
37
  getInitialState: () => new TokenizerState(vscode_textmate_1.INITIAL),
49
38
  tokenizeEncoded(line, state) {
50
- let processedLine = line;
51
39
  if (options.lineLimit !== undefined && line.length > options.lineLimit) {
52
- // Line is too long to be tokenized
53
- processedLine = line.substring(0, options.lineLimit);
40
+ // Skip tokenizing the line if it exceeds the line limit.
41
+ return { endState: state.ruleStack, tokens: new Uint32Array() };
54
42
  }
55
- const result = grammar.tokenizeLine2(processedLine, state.ruleStack);
43
+ const result = grammar.tokenizeLine2(line, state.ruleStack, 500);
56
44
  return {
57
45
  endState: new TokenizerState(result.ruleStack),
58
46
  tokens: result.tokens
59
47
  };
60
48
  },
61
49
  tokenize(line, state) {
62
- let processedLine = line;
63
50
  if (options.lineLimit !== undefined && line.length > options.lineLimit) {
64
- // Line is too long to be tokenized
65
- processedLine = line.substring(0, options.lineLimit);
51
+ // Skip tokenizing the line if it exceeds the line limit.
52
+ return { endState: state.ruleStack, tokens: [] };
66
53
  }
67
- const result = grammar.tokenizeLine(processedLine, state.ruleStack);
54
+ const result = grammar.tokenizeLine(line, state.ruleStack, 500);
68
55
  return {
69
56
  endState: new TokenizerState(result.ruleStack),
70
57
  tokens: result.tokens.map(t => ({
package/lib/browser/textmate/textmate-tokenizer.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"textmate-tokenizer.js","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,0CAA0C;AAC1C,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,2EAA2E;AAC3E,gFAAgF;;;AAEhF,qDAAkE;AAGlE,MAAa,cAAc;IAEvB,YACoB,SAAuB;QAAvB,cAAS,GAAT,SAAS,CAAc;IACvC,CAAC;IAEL,KAAK;QACD,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,KAA8B;QACjC,OAAO,KAAK,YAAY,cAAc,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC;IACrG,CAAC;CAEJ;AAdD,wCAcC;AAkBD,IAAiB,eAAe,CAS/B;AATD,WAAiB,eAAe;IAC5B;;;;OAIG;IACU,uBAAO,GAAoB;QACpC,SAAS,EAAE,GAAG;KACjB,CAAC;AACN,CAAC,EATgB,eAAe,GAAf,uBAAe,KAAf,uBAAe,QAS/B;AAED,SAAgB,uBAAuB,CAAC,OAAiB,EAAE,OAAwB;IAC/E,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE;QACrG,MAAM,IAAI,KAAK,CAAC,sDAAsD,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;KAC/F;IACD,OAAO;QACH,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,cAAc,CAAC,yBAAO,CAAC;QAClD,eAAe,CAAC,IAAY,EAAE,KAAqB;YAC/C,IAAI,aAAa,GAAG,IAAI,CAAC;YACzB,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE;gBACpE,mCAAmC;gBACnC,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;aACxD;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,aAAa,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YACrE,OAAO;gBACH,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM;aACxB,CAAC;QACN,CAAC;QACD,QAAQ,CAAC,IAAY,EAAE,KAAqB;YACxC,IAAI,aAAa,GAAG,IAAI,CAAC;YACzB,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE;gBACpE,mCAAmC;gBACnC,aAAa,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,OAAO,CAAC,SAAS,CAAC,CAAC;aACxD;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,aAAa,EAAE,KAAK,CAAC,SAAS,CAAC,CAAC;YACpE,OAAO;gBACH,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;iBACvC,CAAC,CAAC;aACN,CAAC;QACN,CAAC;KACJ,CAAC;AACN,CAAC;AAlCD,0DAkCC"}
1
+ {"version":3,"file":"textmate-tokenizer.js","sourceRoot":"","sources":["../../../src/browser/textmate/textmate-tokenizer.ts"],"names":[],"mappings":";AAAA,gFAAgF;AAChF,0CAA0C;AAC1C,EAAE;AACF,2EAA2E;AAC3E,mEAAmE;AACnE,wCAAwC;AACxC,EAAE;AACF,4EAA4E;AAC5E,8EAA8E;AAC9E,6EAA6E;AAC7E,yDAAyD;AACzD,uDAAuD;AACvD,EAAE;AACF,2EAA2E;AAC3E,gFAAgF;;;AAEhF,qDAAkE;AAGlE,MAAa,cAAc;IAEvB,YACoB,SAAuB;QAAvB,cAAS,GAAT,SAAS,CAAc;IACvC,CAAC;IAEL,KAAK;QACD,OAAO,IAAI,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9C,CAAC;IAED,MAAM,CAAC,KAA8B;QACjC,OAAO,KAAK,YAAY,cAAc,IAAI,CAAC,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,SAAS,KAAK,IAAI,CAAC,SAAS,CAAC,CAAC;IACrG,CAAC;CAEJ;AAdD,wCAcC;AAkBD,SAAgB,uBAAuB,CAAC,OAAiB,EAAE,OAAwB;IAC/E,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,OAAO,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC,EAAE;QACrG,MAAM,IAAI,KAAK,CAAC,sDAAsD,OAAO,CAAC,SAAS,GAAG,CAAC,CAAC;KAC/F;IACD,OAAO;QACH,eAAe,EAAE,GAAG,EAAE,CAAC,IAAI,cAAc,CAAC,yBAAO,CAAC;QAClD,eAAe,CAAC,IAAY,EAAE,KAAqB;YAC/C,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE;gBACpE,yDAAyD;gBACzD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,WAAW,EAAE,EAAE,CAAC;aACnE;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,aAAa,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YACjE,OAAO;gBACH,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM;aACxB,CAAC;QACN,CAAC;QACD,QAAQ,CAAC,IAAY,EAAE,KAAqB;YACxC,IAAI,OAAO,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,OAAO,CAAC,SAAS,EAAE;gBACpE,yDAAyD;gBACzD,OAAO,EAAE,QAAQ,EAAE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;aACpD;YACD,MAAM,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,IAAI,EAAE,KAAK,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;YAChE,OAAO;gBACH,QAAQ,EAAE,IAAI,cAAc,CAAC,MAAM,CAAC,SAAS,CAAC;gBAC9C,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;oBAC5B,UAAU,EAAE,CAAC,CAAC,UAAU;oBACxB,MAAM,EAAE,CAAC,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC;iBACvC,CAAC,CAAC;aACN,CAAC;QACN,CAAC;KACJ,CAAC;AACN,CAAC;AAhCD,0DAgCC"}
package/package.json CHANGED
@@ -1,14 +1,14 @@
1
1
  {
2
2
  "name": "@theia/monaco",
3
- "version": "1.28.0-next.13+2784ecd9b11",
3
+ "version": "1.28.0-next.18+22f698bd56a",
4
4
  "description": "Theia - Monaco Extension",
5
5
  "dependencies": {
6
- "@theia/core": "1.28.0-next.13+2784ecd9b11",
7
- "@theia/editor": "1.28.0-next.13+2784ecd9b11",
8
- "@theia/filesystem": "1.28.0-next.13+2784ecd9b11",
9
- "@theia/markers": "1.28.0-next.13+2784ecd9b11",
6
+ "@theia/core": "1.28.0-next.18+22f698bd56a",
7
+ "@theia/editor": "1.28.0-next.18+22f698bd56a",
8
+ "@theia/filesystem": "1.28.0-next.18+22f698bd56a",
9
+ "@theia/markers": "1.28.0-next.18+22f698bd56a",
10
10
  "@theia/monaco-editor-core": "1.67.2",
11
- "@theia/outline-view": "1.28.0-next.13+2784ecd9b11",
11
+ "@theia/outline-view": "1.28.0-next.18+22f698bd56a",
12
12
  "fast-plist": "^0.1.2",
13
13
  "idb": "^4.0.5",
14
14
  "jsonc-parser": "^2.2.0",
@@ -54,5 +54,5 @@
54
54
  "nyc": {
55
55
  "extends": "../../configs/nyc.json"
56
56
  },
57
- "gitHead": "2784ecd9b114fd6839656b24eeee6111f64ac2e8"
57
+ "gitHead": "22f698bd56a43f72da15bff0841de1914557684f"
58
58
  }
package/src/browser/textmate/textmate-tokenizer.ts CHANGED
@@ -49,17 +49,6 @@ export interface TokenizerOption {
49
49
 
50
50
  }
51
51
 
52
- export namespace TokenizerOption {
53
- /**
54
- * The default TextMate tokenizer option.
55
- *
56
- * @deprecated Use the current value of `editor.maxTokenizationLineLength` preference instead.
57
- */
58
- export const DEFAULT: TokenizerOption = {
59
- lineLimit: 400
60
- };
61
- }
62
-
63
52
  export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOption): monaco.languages.EncodedTokensProvider & monaco.languages.TokensProvider {
64
53
  if (options.lineLimit !== undefined && (options.lineLimit <= 0 || !Number.isInteger(options.lineLimit))) {
65
54
  throw new Error(`The 'lineLimit' must be a positive integer. It was ${options.lineLimit}.`);
@@ -67,24 +56,22 @@ export function createTextmateTokenizer(grammar: IGrammar, options: TokenizerOpt
67
56
  return {
68
57
  getInitialState: () => new TokenizerState(INITIAL),
69
58
  tokenizeEncoded(line: string, state: TokenizerState): monaco.languages.IEncodedLineTokens {
70
- let processedLine = line;
71
59
  if (options.lineLimit !== undefined && line.length > options.lineLimit) {
72
- // Line is too long to be tokenized
73
- processedLine = line.substring(0, options.lineLimit);
60
+ // Skip tokenizing the line if it exceeds the line limit.
61
+ return { endState: state.ruleStack, tokens: new Uint32Array() };
74
62
  }
75
- const result = grammar.tokenizeLine2(processedLine, state.ruleStack);
63
+ const result = grammar.tokenizeLine2(line, state.ruleStack, 500);
76
64
  return {
77
65
  endState: new TokenizerState(result.ruleStack),
78
66
  tokens: result.tokens
79
67
  };
80
68
  },
81
69
  tokenize(line: string, state: TokenizerState): monaco.languages.ILineTokens {
82
- let processedLine = line;
83
70
  if (options.lineLimit !== undefined && line.length > options.lineLimit) {
84
- // Line is too long to be tokenized
85
- processedLine = line.substring(0, options.lineLimit);
71
+ // Skip tokenizing the line if it exceeds the line limit.
72
+ return { endState: state.ruleStack, tokens: [] };
86
73
  }
87
- const result = grammar.tokenizeLine(processedLine, state.ruleStack);
74
+ const result = grammar.tokenizeLine(line, state.ruleStack, 500);
88
75
  return {
89
76
  endState: new TokenizerState(result.ruleStack),
90
77
  tokens: result.tokens.map(t => ({