@codingame/monaco-vscode-textmate-service-override 25.1.1 → 26.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (22) hide show
  1. package/_virtual/main2.js +25 -0
  2. package/_virtual/main3.js +3 -0
  3. package/external/vscode-oniguruma/release/main.js +14 -0
  4. package/external/vscode-oniguruma/release/onig.wasm +0 -0
  5. package/index.js +1 -1
  6. package/package.json +3 -5
  7. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.js +42 -26
  8. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.d.ts +1 -1
  9. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.js +32 -24
  10. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.d.ts +1 -1
  11. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js +9 -9
  12. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.js +28 -12
  13. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.contribution.js +14 -8
  14. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.d.ts +1 -1
  15. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js +121 -86
  16. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.d.ts +1 -1
  17. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.js +14 -5
  18. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.js +2 -3
  19. package/vscode/src/vs/workbench/services/textMate/common/TMGrammarFactory.d.ts +2 -2
  20. package/vscode/src/vs/workbench/services/textMate/common/TMGrammarFactory.js +15 -14
  21. package/vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.d.ts +0 -3
  22. package/vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.js +0 -241
@@ -0,0 +1,25 @@
1
+ import { getDefaultExportFromCjs } from '@codingame/monaco-vscode-api/_virtual/_commonjsHelpers';
2
+ import { __require as requireMain } from '../external/vscode-oniguruma/release/main.js';
3
+
4
+ function _mergeNamespaces$1(n, m) {
5
+ m.forEach(function (e) {
6
+ e && typeof e !== 'string' && !Array.isArray(e) && Object.keys(e).forEach(function (k) {
7
+ if (k !== 'default' && !(k in n)) {
8
+ var d = Object.getOwnPropertyDescriptor(e, k);
9
+ Object.defineProperty(n, k, d.get ? d : {
10
+ enumerable: true,
11
+ get: function () { return e[k]; }
12
+ });
13
+ }
14
+ });
15
+ });
16
+ return Object.freeze(n);
17
+ }
18
+ var mainExports = requireMain();
19
+ var main = getDefaultExportFromCjs(mainExports);
20
+ var main$1 = _mergeNamespaces$1({
21
+ __proto__: null,
22
+ default: main
23
+ }, [mainExports]);
24
+
25
+ export { main as default, main$1 as main };
@@ -0,0 +1,3 @@
1
+ var main = {exports: {}};
2
+
3
+ export { main as __module };
@@ -0,0 +1,14 @@
1
+ import { __module as main$1 } from '../../../_virtual/main3.js';
2
+
3
+ var main = main$1.exports;
4
+ var hasRequiredMain;
5
+ function requireMain () {
6
+ if (hasRequiredMain) return main$1.exports;
7
+ hasRequiredMain = 1;
8
+ (function (module, exports$1) {
9
+ !function(t,n){module.exports=n();}(main,(()=>{return t={770:function(t,n,e){var r=this&&this.__importDefault||function(t){return t&&t.__esModule?t:{default:t}};Object.defineProperty(n,"__esModule",{value:true}),n.setDefaultDebugCall=n.createOnigScanner=n.createOnigString=n.loadWASM=n.OnigScanner=n.OnigString=void 0;const i=r(e(418));let o=null,a=false;class f{static _utf8ByteLength(t){let n=0;for(let e=0,r=t.length;e<r;e++){const i=t.charCodeAt(e);let o=i,a=false;if(i>=55296&&i<=56319&&e+1<r){const n=t.charCodeAt(e+1);n>=56320&&n<=57343&&(o=65536+(i-55296<<10)|n-56320,a=true);}n+=o<=127?1:o<=2047?2:o<=65535?3:4,a&&e++;}return n}constructor(t){const n=t.length,e=f._utf8ByteLength(t),r=e!==n,i=r?new Uint32Array(n+1):null;r&&(i[n]=e);const o=r?new Uint32Array(e+1):null;r&&(o[e]=n);const a=new Uint8Array(e);let s=0;for(let e=0;e<n;e++){const f=t.charCodeAt(e);let u=f,c=false;if(f>=55296&&f<=56319&&e+1<n){const n=t.charCodeAt(e+1);n>=56320&&n<=57343&&(u=65536+(f-55296<<10)|n-56320,c=true);}r&&(i[e]=s,c&&(i[e+1]=s),u<=127?o[s+0]=e:u<=2047?(o[s+0]=e,o[s+1]=e):u<=65535?(o[s+0]=e,o[s+1]=e,o[s+2]=e):(o[s+0]=e,o[s+1]=e,o[s+2]=e,o[s+3]=e)),u<=127?a[s++]=u:u<=2047?(a[s++]=192|(1984&u)>>>6,a[s++]=128|(63&u)>>>0):u<=65535?(a[s++]=224|(61440&u)>>>12,a[s++]=128|(4032&u)>>>6,a[s++]=128|(63&u)>>>0):(a[s++]=240|(1835008&u)>>>18,a[s++]=128|(258048&u)>>>12,a[s++]=128|(4032&u)>>>6,a[s++]=128|(63&u)>>>0),c&&e++;}this.utf16Length=n,this.utf8Length=e,this.utf16Value=t,this.utf8Value=a,this.utf16OffsetToUtf8=i,this.utf8OffsetToUtf16=o;}createString(t){const n=t._omalloc(this.utf8Length);return t.HEAPU8.set(this.utf8Value,n),n}}class s{constructor(t){if(this.id=++s.LAST_ID,!o)throw new Error("Must invoke loadWASM first.");this._onigBinding=o,this.content=t;const n=new f(t);this.utf16Length=n.utf16Length,this.utf8Length=n.utf8Length,this.utf16OffsetToUtf8=n.utf16OffsetToUtf8,this.utf8OffsetToUtf16=n.utf8OffsetToUtf16,this.utf8Length<1e4&&!s._sharedPtrInUse?(s._sharedPtr||(s._sharedPtr=o._omalloc(1e4)),s._sharedPtrInUse=true,o.HEAPU8.set(n.utf8Value,s._sharedPtr),this.ptr=s._sharedPtr):this.ptr=n.createString(o);}convertUtf8OffsetToUtf16(t){return this.utf8OffsetToUtf16?t<0?0:t>this.utf8Length?this.utf16Length:this.utf8OffsetToUtf16[t]:t}convertUtf16OffsetToUtf8(t){return this.utf16OffsetToUtf8?t<0?0:t>this.utf16Length?this.utf8Length:this.utf16OffsetToUtf8[t]:t}dispose(){this.ptr===s._sharedPtr?s._sharedPtrInUse=false:this._onigBinding._ofree(this.ptr);}}n.OnigString=s,s.LAST_ID=0,s._sharedPtr=0,s._sharedPtrInUse=false;class u{constructor(t){if(!o)throw new Error("Must invoke loadWASM first.");const n=[],e=[];for(let r=0,i=t.length;r<i;r++){const i=new f(t[r]);n[r]=i.createString(o),e[r]=i.utf8Length;}const r=o._omalloc(4*t.length);o.HEAPU32.set(n,r/4);const i=o._omalloc(4*t.length);o.HEAPU32.set(e,i/4);const a=o._createOnigScanner(r,i,t.length);for(let e=0,r=t.length;e<r;e++)o._ofree(n[e]);o._ofree(i),o._ofree(r),0===a&&function(t){throw new Error(t.UTF8ToString(t._getLastOnigError()))}(o),this._onigBinding=o,this._ptr=a;}dispose(){this._onigBinding._freeOnigScanner(this._ptr);}findNextMatchSync(t,n,e){let r=a,i=0;if("number"==typeof e?(8&e&&(r=true),i=e):"boolean"==typeof e&&(r=e),"string"==typeof t){t=new s(t);const e=this._findNextMatchSync(t,n,r,i);return t.dispose(),e}return this._findNextMatchSync(t,n,r,i)}_findNextMatchSync(t,n,e,r){const i=this._onigBinding;let o;if(o=e?i._findNextOnigScannerMatchDbg(this._ptr,t.id,t.ptr,t.utf8Length,t.convertUtf16OffsetToUtf8(n),r):i._findNextOnigScannerMatch(this._ptr,t.id,t.ptr,t.utf8Length,t.convertUtf16OffsetToUtf8(n),r),0===o)return null;const a=i.HEAPU32;let f=o/4;const s=a[f++],u=a[f++];let c=[];for(let n=0;n<u;n++){const e=t.convertUtf8OffsetToUtf16(a[f++]),r=t.convertUtf8OffsetToUtf16(a[f++]);c[n]={start:e,end:r,length:r-e};}return {index:s,captureIndices:c}}}n.OnigScanner=u;let c=false,l=null;n.loadWASM=function(t){if(c)return l;let n,e,r,a;if(c=true,function(t){return "function"==typeof t.instantiator}(t))n=t.instantiator,e=t.print;else {let r;!function(t){return void 0!==t.data}(t)?r=t:(r=t.data,e=t.print),n=function(t){return "undefined"!=typeof Response&&t instanceof Response}(r)?"function"==typeof WebAssembly.instantiateStreaming?function(t){return n=>WebAssembly.instantiateStreaming(t,n)}(r):function(t){return async n=>{const e=await t.arrayBuffer();return WebAssembly.instantiate(e,n)}}(r):function(t){return n=>WebAssembly.instantiate(t,n)}(r);}return l=new Promise(((t,n)=>{r=t,a=n;})),function(t,n,e,r){(0, i.default)({print:n,instantiateWasm:(n,e)=>{if("undefined"==typeof performance){const t=()=>Date.now();n.env.emscripten_get_now=t,n.wasi_snapshot_preview1.emscripten_get_now=t;}return t(n).then((t=>e(t.instance)),r),{}}}).then((t=>{o=t,e();}));}(n,e,r,a),l},n.createOnigString=function(t){return new s(t)},n.createOnigScanner=function(t){return new u(t)},n.setDefaultDebugCall=function(t){a=t;};},418:t=>{var n=(function(t){var n,e,r=void 0!==(t=t||{})?t:{};r.ready=new Promise((function(t,r){n=t,e=r;}));var i,o=Object.assign({},r),c="";function l(t){return r.locateFile?r.locateFile(t,c):c+t}(i=function(t){let n;return "function"==typeof readbuffer?new Uint8Array(readbuffer(t)):(n=read(t,"binary"),m("object"==typeof n),n)},"undefined"!=typeof onig_print&&("undefined"==typeof console&&(console={}),console.log=onig_print,console.warn=console.error="undefined"!=typeof printErr?printErr:onig_print));var h,p,d=r.print||console.log.bind(console),g=r.printErr||console.warn.bind(console);Object.assign(r,o),o=null,r.wasmBinary&&(h=r.wasmBinary),"object"!=typeof WebAssembly&&k("no native wasm support detected");var _=false;function m(t,n){t||k(n);}var y,w,S,v="undefined"!=typeof TextDecoder?new TextDecoder("utf8"):void 0;function A(t,n,e){for(var r=n+e,i=n;t[i]&&!(i>=r);)++i;if(i-n>16&&t.buffer&&v)return v.decode(t.subarray(n,i));for(var o="";n<i;){var a=t[n++];if(128&a){var f=63&t[n++];if(192!=(224&a)){var s=63&t[n++];if((a=224==(240&a)?(15&a)<<12|f<<6|s:(7&a)<<18|f<<12|s<<6|63&t[n++])<65536)o+=String.fromCharCode(a);else {var u=a-65536;o+=String.fromCharCode(55296|u>>10,56320|1023&u);}}else o+=String.fromCharCode((31&a)<<6|f);}else o+=String.fromCharCode(a);}return o}function b(t,n){return t?A(w,t,n):""}function O(t){y=t,r.HEAP8=new Int8Array(t),r.HEAP16=new Int16Array(t),r.HEAP32=new Int32Array(t),r.HEAPU8=w=new Uint8Array(t),r.HEAPU16=new Uint16Array(t),r.HEAPU32=S=new Uint32Array(t),r.HEAPF32=new Float32Array(t),r.HEAPF64=new Float64Array(t);}var U=[],P=[],R=[];function x(){if(r.preRun)for("function"==typeof r.preRun&&(r.preRun=[r.preRun]);r.preRun.length;)M(r.preRun.shift());G(U);}function T(){G(P);}function E(){if(r.postRun)for("function"==typeof r.postRun&&(r.postRun=[r.postRun]);r.postRun.length;)I(r.postRun.shift());G(R);}function M(t){U.unshift(t);}function L(t){P.unshift(t);}function I(t){R.unshift(t);}var W=0,C=null;function N(t){W++,r.monitorRunDependencies&&r.monitorRunDependencies(W);}function j(t){if(W--,r.monitorRunDependencies&&r.monitorRunDependencies(W),0==W&&(C)){var n=C;C=null,n();}}function k(t){r.onAbort&&r.onAbort(t),g(t="Aborted("+t+")"),_=true,t+=". Build with -sASSERTIONS for more info.";var n=new WebAssembly.RuntimeError(t);throw e(n),n}var B,H,F="data:application/octet-stream;base64,";function V(t){return t.startsWith(F)}function z(t){try{if(t==B&&h)return new Uint8Array(h);if(i)return i(t);throw "both async and sync fetching of the wasm failed"}catch(t){k(t);}}function q(){return Promise.resolve().then((function(){return z(B)}))}function Y(){var t={env:nt,wasi_snapshot_preview1:nt};function n(t,n){var e=t.exports;r.asm=e,O((p=r.asm.memory).buffer),L(r.asm.__wasm_call_ctors),j();}function i(t){n(t.instance);}function o(n){return q().then((function(n){return WebAssembly.instantiate(n,t)})).then((function(t){return t})).then(n,(function(t){g("failed to asynchronously prepare wasm: "+t),k(t);}))}if(N(),r.instantiateWasm)try{return r.instantiateWasm(t,n)}catch(t){g("Module.instantiateWasm callback failed with error: "+t),e(t);}return (h||"function"!=typeof WebAssembly.instantiateStreaming||V(B)||"function"!=typeof fetch?o(i):fetch(B,{credentials:"same-origin"}).then((function(n){return WebAssembly.instantiateStreaming(n,t).then(i,(function(t){return g("wasm streaming compile failed: "+t),g("falling back to ArrayBuffer instantiation"),o(i)}))}))).catch(e),{}}function G(t){for(;t.length>0;)t.shift()(r);}function J(t,n,e){w.copyWithin(t,n,n+e);}function K(t){try{return p.grow(t-y.byteLength+65535>>>16),O(p.buffer),1}catch(t){}}function Q(t){var n,e=w.length,r=2147483648;if((t>>>=0)>r)return false;for(var i=1;i<=4;i*=2){var o=e*(1+.2/i);if(o=Math.min(o,t+100663296),K(Math.min(r,(n=Math.max(t,o))+(65536-n%65536)%65536)))return true}return false}V(B="onig.wasm")||(B=l(B)),H="undefined"!=typeof dateNow?dateNow:()=>performance.now();var X=[null,[],[]];function Z(t,n){var e=X[t];0===n||10===n?((1===t?d:g)(A(e,0)),e.length=0):e.push(n);}function $(t,n,e,r){for(var i=0,o=0;o<e;o++){var a=S[n>>2],f=S[n+4>>2];n+=8;for(var s=0;s<f;s++)Z(t,w[a+s]);i+=f;}return S[r>>2]=i,0}var tt,nt={emscripten_get_now:H,emscripten_memcpy_big:J,emscripten_resize_heap:Q,fd_write:$};function et(t){function e(){tt||(tt=true,r.calledRun=true,_||(T(),n(r),r.onRuntimeInitialized&&r.onRuntimeInitialized(),E()));}W>0||(x(),W>0||(r.setStatus?(r.setStatus("Running..."),setTimeout((function(){setTimeout((function(){r.setStatus("");}),1),e();}),1)):e()));}if(Y(),r.___wasm_call_ctors=function(){return (r.___wasm_call_ctors=r.asm.__wasm_call_ctors).apply(null,arguments)},r.___errno_location=function(){return (r.___errno_location=r.asm.__errno_location).apply(null,arguments)},r._omalloc=function(){return (r._omalloc=r.asm.omalloc).apply(null,arguments)},r._ofree=function(){return (r._ofree=r.asm.ofree).apply(null,arguments)},r._getLastOnigError=function(){return (r._getLastOnigError=r.asm.getLastOnigError).apply(null,arguments)},r._createOnigScanner=function(){return (r._createOnigScanner=r.asm.createOnigScanner).apply(null,arguments)},r._freeOnigScanner=function(){return (r._freeOnigScanner=r.asm.freeOnigScanner).apply(null,arguments)},r._findNextOnigScannerMatch=function(){return (r._findNextOnigScannerMatch=r.asm.findNextOnigScannerMatch).apply(null,arguments)},r._findNextOnigScannerMatchDbg=function(){return (r._findNextOnigScannerMatchDbg=r.asm.findNextOnigScannerMatchDbg).apply(null,arguments)},r.stackSave=function(){return (r.stackSave=r.asm.stackSave).apply(null,arguments)},r.stackRestore=function(){return (r.stackRestore=r.asm.stackRestore).apply(null,arguments)},r.stackAlloc=function(){return (r.stackAlloc=r.asm.stackAlloc).apply(null,arguments)},r.dynCall_jiji=function(){return (r.dynCall_jiji=r.asm.dynCall_jiji).apply(null,arguments)},r.UTF8ToString=b,C=function t(){tt||et(),tt||(C=t);},r.preInit)for("function"==typeof r.preInit&&(r.preInit=[r.preInit]);r.preInit.length>0;)r.preInit.pop()();return et(),t.ready});t.exports=n;}},n={},function e(r){var i=n[r];if(void 0!==i)return i.exports;var o=n[r]={exports:{}};return t[r].call(o.exports,o,o.exports,e),o.exports}(770);var t,n;}));
10
+ } (main$1));
11
+ return main$1.exports;
12
+ }
13
+
14
+ export { requireMain as __require };
package/index.js CHANGED
@@ -5,7 +5,7 @@ export { ITextMateTokenizationService } from '@codingame/monaco-vscode-api/vscod
5
5
  import { SyncDescriptor } from '@codingame/monaco-vscode-api/vscode/vs/platform/instantiation/common/descriptors';
6
6
  import { TextMateTokenizationFeature } from './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js';
7
7
  import getServiceOverride$1 from '@codingame/monaco-vscode-files-service-override';
8
- import './vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.js';
8
+ import '@codingame/monaco-vscode-api/vscode/vs/workbench/services/themes/common/tokenClassificationExtensionPoint';
9
9
  import '@codingame/monaco-vscode-api/vscode/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens';
10
10
  import './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeature.contribution.js';
11
11
 
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@codingame/monaco-vscode-textmate-service-override",
3
- "version": "25.1.1",
3
+ "version": "26.0.0",
4
4
  "private": false,
5
5
  "description": "VSCode public API plugged on the monaco editor - textmate service-override",
6
6
  "keywords": [],
@@ -15,10 +15,8 @@
15
15
  },
16
16
  "type": "module",
17
17
  "dependencies": {
18
- "@codingame/monaco-vscode-api": "25.1.1",
19
- "@codingame/monaco-vscode-files-service-override": "25.1.1",
20
- "vscode-oniguruma": "1.7.0",
21
- "vscode-textmate": "9.3.1"
18
+ "@codingame/monaco-vscode-api": "26.0.0",
19
+ "@codingame/monaco-vscode-files-service-override": "26.0.0"
22
20
  },
23
21
  "main": "index.js",
24
22
  "module": "index.js",
@@ -16,8 +16,17 @@ import { keepObserved } from '@codingame/monaco-vscode-api/vscode/vs/base/common
16
16
  import { autorun } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/reactions/autorun';
17
17
 
18
18
  class TextMateWorkerTokenizerController extends Disposable {
19
- static { this._id = 0; }
20
- constructor(_model, _worker, _languageIdCodec, _backgroundTokenizationStore, _configurationService, _maxTokenizationLineLength) {
19
+ static {
20
+ this._id = 0;
21
+ }
22
+ constructor(
23
+ _model,
24
+ _worker,
25
+ _languageIdCodec,
26
+ _backgroundTokenizationStore,
27
+ _configurationService,
28
+ _maxTokenizationLineLength
29
+ ) {
21
30
  super();
22
31
  this._model = _model;
23
32
  this._worker = _worker;
@@ -28,19 +37,23 @@ class TextMateWorkerTokenizerController extends Disposable {
28
37
  this.controllerId = TextMateWorkerTokenizerController._id++;
29
38
  this._pendingChanges = [];
30
39
  this._states = ( new TokenizationStateStore());
31
- this._loggingEnabled = observableConfigValue('editor.experimental.asyncTokenizationLogging', false, this._configurationService);
40
+ this._loggingEnabled = observableConfigValue(
41
+ "editor.experimental.asyncTokenizationLogging",
42
+ false,
43
+ this._configurationService
44
+ );
32
45
  this._register(keepObserved(this._loggingEnabled));
33
- this._register(this._model.onDidChangeContent((e) => {
46
+ this._register(this._model.onDidChangeContent(e => {
34
47
  if (this._shouldLog) {
35
- console.log('model change', {
36
- fileName: this._model.uri.fsPath.split('\\').pop(),
37
- changes: changesToString(e.changes),
48
+ console.log("model change", {
49
+ fileName: this._model.uri.fsPath.split("\\").pop(),
50
+ changes: changesToString(e.changes)
38
51
  });
39
52
  }
40
53
  this._worker.$acceptModelChanged(this.controllerId, e);
41
54
  this._pendingChanges.push(e);
42
55
  }));
43
- this._register(this._model.onDidChangeLanguage((e) => {
56
+ this._register(this._model.onDidChangeLanguage(e => {
44
57
  const languageId = this._model.getLanguageId();
45
58
  const encodedLanguageId = this._languageIdCodec.encodeLanguageId(languageId);
46
59
  this._worker.$acceptModelLanguageChanged(this.controllerId, languageId, encodedLanguageId);
@@ -55,7 +68,7 @@ class TextMateWorkerTokenizerController extends Disposable {
55
68
  languageId,
56
69
  encodedLanguageId,
57
70
  maxTokenizationLineLength: this._maxTokenizationLineLength.get(),
58
- controllerId: this.controllerId,
71
+ controllerId: this.controllerId
59
72
  });
60
73
  this._register(autorun(reader => {
61
74
  const maxTokenizationLineLength = this._maxTokenizationLineLength.read(reader);
@@ -76,27 +89,26 @@ class TextMateWorkerTokenizerController extends Disposable {
76
89
  let tokens = ContiguousMultilineTokensBuilder.deserialize(( new Uint8Array(rawTokens)));
77
90
  const fontTokensUpdate = AnnotationsUpdate.deserialize(fontTokens, deserializeFontTokenOptions());
78
91
  if (this._shouldLog) {
79
- console.log('received background tokenization result', {
80
- fileName: this._model.uri.fsPath.split('\\').pop(),
81
- updatedTokenLines: ( tokens.map((t) => t.getLineRange())).join(' & '),
82
- updatedStateLines: ( stateDeltas.map((s) => ( ( new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length)).toString()))).join(' & '),
92
+ console.log("received background tokenization result", {
93
+ fileName: this._model.uri.fsPath.split("\\").pop(),
94
+ updatedTokenLines: ( tokens.map(t => t.getLineRange())).join(" & "),
95
+ updatedStateLines: ( stateDeltas.map(s => ( ( new LineRange(s.startLineNumber, s.startLineNumber + s.stateDeltas.length)).toString()))).join(" & ")
83
96
  });
84
97
  }
85
98
  if (this._shouldLog) {
86
- const changes = ( ( this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes)).map(c => changesToString(c))).join(' then ');
87
- console.log('Applying changes to local states', changes);
99
+ const changes = ( ( this._pendingChanges.filter(c => c.versionId <= versionId).map(c => c.changes)).map(c => changesToString(c))).join(" then ");
100
+ console.log("Applying changes to local states", changes);
88
101
  }
89
- while (this._pendingChanges.length > 0 &&
90
- this._pendingChanges[0].versionId <= versionId) {
102
+ while (this._pendingChanges.length > 0 && this._pendingChanges[0].versionId <= versionId) {
91
103
  const change = this._pendingChanges.shift();
92
104
  this._states.acceptChanges(change.changes);
93
105
  }
94
106
  if (this._pendingChanges.length > 0) {
95
107
  if (this._shouldLog) {
96
- const changes = ( ( this._pendingChanges.map(c => c.changes)).map(c => changesToString(c))).join(' then ');
97
- console.log('Considering non-processed changes', changes);
108
+ const changes = ( ( this._pendingChanges.map(c => c.changes)).map(c => changesToString(c))).join(" then ");
109
+ console.log("Considering non-processed changes", changes);
98
110
  }
99
- const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => linesLengthEditFromModelContentChange(c.changes))));
111
+ const curToFutureTransformerTokens = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map(c => linesLengthEditFromModelContentChange(c.changes))));
100
112
  const b = ( new ContiguousMultilineTokensBuilder());
101
113
  for (const t of tokens) {
102
114
  for (let i = t.startLineNumber; i <= t.endLineNumber; i++) {
@@ -116,9 +128,12 @@ class TextMateWorkerTokenizerController extends Disposable {
116
128
  }
117
129
  fontTokensUpdate.rebase(this._stringEditFromChanges(this._model, this._pendingChanges));
118
130
  }
119
- const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map((c) => linesLengthEditFromModelContentChange(c.changes))));
131
+ const curToFutureTransformerStates = MonotonousIndexTransformer.fromMany(( this._pendingChanges.map(c => linesLengthEditFromModelContentChange(c.changes))));
120
132
  if (!this._applyStateStackDiffFn || !this._initialState) {
121
- const { applyStateStackDiff, INITIAL } = await import('vscode-textmate').then(module => module.default ?? module);
133
+ const {
134
+ applyStateStackDiff,
135
+ INITIAL
136
+ } = await import('@codingame/monaco-vscode-api/_virtual/main').then(function (n) { return n.main; });
122
137
  this._applyStateStackDiffFn = applyStateStackDiff;
123
138
  this._initialState = INITIAL;
124
139
  }
@@ -130,8 +145,7 @@ class TextMateWorkerTokenizerController extends Disposable {
130
145
  if (delta) {
131
146
  state = this._applyStateStackDiffFn(prevState, delta);
132
147
  this._states.setEndState(d.startLineNumber + i, state);
133
- }
134
- else {
148
+ } else {
135
149
  state = this._states.getEndState(d.startLineNumber + i);
136
150
  }
137
151
  const offset = curToFutureTransformerStates.transform(d.startLineNumber + i - 1);
@@ -160,10 +174,12 @@ class TextMateWorkerTokenizerController extends Disposable {
160
174
  }
161
175
  return StringEdit.compose(edits);
162
176
  }
163
- get _shouldLog() { return this._loggingEnabled.get(); }
177
+ get _shouldLog() {
178
+ return this._loggingEnabled.get();
179
+ }
164
180
  }
165
181
  function changesToString(changes) {
166
- return ( changes.map(c => ( Range.lift(c.range).toString()) + ' => ' + c.text)).join(' & ');
182
+ return ( changes.map(c => ( Range.lift(c.range).toString()) + " => " + c.text)).join(" & ");
167
183
  }
168
184
 
169
185
  export { TextMateWorkerTokenizerController };
@@ -8,7 +8,7 @@ import { IExtensionResourceLoaderService } from "@codingame/monaco-vscode-api/vs
8
8
  import { INotificationService } from "@codingame/monaco-vscode-api/vscode/vs/platform/notification/common/notification.service";
9
9
  import { ITelemetryService } from "@codingame/monaco-vscode-api/vscode/vs/platform/telemetry/common/telemetry.service";
10
10
  import { IValidGrammarDefinition } from "../../common/TMScopeRegistry.js";
11
- import type { IRawTheme } from "vscode-textmate";
11
+ import type { IRawTheme } from "@codingame/monaco-vscode-api/_virtual/main";
12
12
  import { IWebWorkerService } from "@codingame/monaco-vscode-api/vscode/vs/platform/webWorker/browser/webWorkerService.service";
13
13
  export declare class ThreadedBackgroundTokenizerFactory implements IDisposable {
14
14
  private readonly _reportTokenizationTime;
@@ -15,9 +15,22 @@ import { FileAccess } from '@codingame/monaco-vscode-api/vscode/vs/base/common/n
15
15
 
16
16
  var ThreadedBackgroundTokenizerFactory_1;
17
17
  let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactory {
18
- static { ThreadedBackgroundTokenizerFactory_1 = this; }
19
- static { this._reportedMismatchingTokens = false; }
20
- constructor(_reportTokenizationTime, _shouldTokenizeAsync, _extensionResourceLoaderService, _configurationService, _languageService, _notificationService, _telemetryService, _webWorkerService) {
18
+ static {
19
+ ThreadedBackgroundTokenizerFactory_1 = this;
20
+ }
21
+ static {
22
+ this._reportedMismatchingTokens = false;
23
+ }
24
+ constructor(
25
+ _reportTokenizationTime,
26
+ _shouldTokenizeAsync,
27
+ _extensionResourceLoaderService,
28
+ _configurationService,
29
+ _languageService,
30
+ _notificationService,
31
+ _telemetryService,
32
+ _webWorkerService
33
+ ) {
21
34
  this._reportTokenizationTime = _reportTokenizationTime;
22
35
  this._shouldTokenizeAsync = _shouldTokenizeAsync;
23
36
  this._extensionResourceLoaderService = _extensionResourceLoaderService;
@@ -42,11 +55,14 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
42
55
  return undefined;
43
56
  }
44
57
  const store = ( new DisposableStore());
45
- const controllerContainer = this._getWorkerProxy().then((workerProxy) => {
58
+ const controllerContainer = this._getWorkerProxy().then(workerProxy => {
46
59
  if (store.isDisposed || !workerProxy) {
47
60
  return undefined;
48
61
  }
49
- const controllerContainer = { controller: undefined, worker: this._worker };
62
+ const controllerContainer = {
63
+ controller: undefined,
64
+ worker: this._worker
65
+ };
50
66
  store.add(keepAliveWhenAttached(textModel, () => {
51
67
  const controller = ( new TextMateWorkerTokenizerController(
52
68
  textModel,
@@ -76,17 +92,17 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
76
92
  container.controller.requestTokens(startLineNumber, endLineNumberExclusive);
77
93
  }
78
94
  },
79
- reportMismatchingTokens: (lineNumber) => {
95
+ reportMismatchingTokens: lineNumber => {
80
96
  if (ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens) {
81
97
  return;
82
98
  }
83
99
  ThreadedBackgroundTokenizerFactory_1._reportedMismatchingTokens = true;
84
100
  this._notificationService.error({
85
- message: 'Async Tokenization Token Mismatch in line ' + lineNumber,
86
- name: 'Async Tokenization Token Mismatch',
101
+ message: "Async Tokenization Token Mismatch in line " + lineNumber,
102
+ name: "Async Tokenization Token Mismatch"
87
103
  });
88
- this._telemetryService.publicLog2('asyncTokenizationMismatchingTokens', {});
89
- },
104
+ this._telemetryService.publicLog2("asyncTokenizationMismatchingTokens", {});
105
+ }
90
106
  };
91
107
  }
92
108
  setGrammarDefinitions(grammarDefinitions) {
@@ -109,16 +125,16 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
109
125
  async _createWorkerProxy() {
110
126
  const createData = {
111
127
  grammarDefinitions: this._grammarDefinitions,
112
- onigurumaWASMUri: ( new URL('vscode-oniguruma/release/onig.wasm', import.meta.url)).href,
128
+ onigurumaWASMUri: ( new URL('../../../../../../../../external/vscode-oniguruma/release/onig.wasm', import.meta.url)).href
113
129
  };
114
130
  const worker = this._worker = this._webWorkerService.createWorkerClient(( new WebWorkerDescriptor({
115
131
  esmModuleLocation: ( FileAccess.asBrowserUri(
116
- 'vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.workerMain.js'
132
+ "vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.workerMain.js"
117
133
  )),
118
- label: 'TextMateWorker'
134
+ label: "TextMateWorker"
119
135
  })));
120
136
  TextMateWorkerHost.setChannel(worker, {
121
- $readFile: async (_resource) => {
137
+ $readFile: async _resource => {
122
138
  const resource = URI.revive(_resource);
123
139
  return this._extensionResourceLoaderService.readExtensionResource(resource);
124
140
  },
@@ -155,22 +171,14 @@ let ThreadedBackgroundTokenizerFactory = class ThreadedBackgroundTokenizerFactor
155
171
  this._workerProxyPromise = null;
156
172
  }
157
173
  };
158
- ThreadedBackgroundTokenizerFactory = ThreadedBackgroundTokenizerFactory_1 = ( __decorate([
159
- ( __param(2, IExtensionResourceLoaderService)),
160
- ( __param(3, IConfigurationService)),
161
- ( __param(4, ILanguageService)),
162
- ( __param(5, INotificationService)),
163
- ( __param(6, ITelemetryService)),
164
- ( __param(7, IWebWorkerService))
165
- ], ThreadedBackgroundTokenizerFactory));
174
+ ThreadedBackgroundTokenizerFactory = ThreadedBackgroundTokenizerFactory_1 = ( __decorate([( __param(2, IExtensionResourceLoaderService)), ( __param(3, IConfigurationService)), ( __param(4, ILanguageService)), ( __param(5, INotificationService)), ( __param(6, ITelemetryService)), ( __param(7, IWebWorkerService))], ThreadedBackgroundTokenizerFactory));
166
175
  function keepAliveWhenAttached(textModel, factory) {
167
176
  const disposableStore = ( new DisposableStore());
168
177
  const subStore = disposableStore.add(( new DisposableStore()));
169
178
  function checkAttached() {
170
179
  if (textModel.isAttachedToEditor()) {
171
180
  subStore.add(factory());
172
- }
173
- else {
181
+ } else {
174
182
  subStore.clear();
175
183
  }
176
184
  }
@@ -2,7 +2,7 @@ import { UriComponents } from "@codingame/monaco-vscode-api/vscode/vs/base/commo
2
2
  import { LanguageId } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/encodedTokenAttributes";
3
3
  import { IModelChangedEvent } from "@codingame/monaco-vscode-api/vscode/vs/editor/common/model/mirrorTextModel";
4
4
  import { IValidEmbeddedLanguagesMap, IValidTokenTypeMap } from "../../../common/TMScopeRegistry.js";
5
- import type { IRawTheme, StackDiff } from "vscode-textmate";
5
+ import type { IRawTheme, StackDiff } from "@codingame/monaco-vscode-api/_virtual/main";
6
6
  import { IWebWorkerServerRequestHandler, IWebWorkerServer } from "@codingame/monaco-vscode-api/vscode/vs/base/common/worker/webWorker";
7
7
  export declare function create(workerServer: IWebWorkerServer): TextMateTokenizationWorker;
8
8
  export interface ICreateData {
@@ -16,7 +16,7 @@ class TextMateTokenizationWorker {
16
16
  this._host = TextMateWorkerHost.getChannel(workerServer);
17
17
  }
18
18
  async $init(_createData) {
19
- const grammarDefinitions = ( _createData.grammarDefinitions.map((def) => {
19
+ const grammarDefinitions = ( _createData.grammarDefinitions.map(def => {
20
20
  return {
21
21
  location: URI.revive(def.location),
22
22
  language: def.language,
@@ -26,25 +26,25 @@ class TextMateTokenizationWorker {
26
26
  injectTo: def.injectTo,
27
27
  balancedBracketSelectors: def.balancedBracketSelectors,
28
28
  unbalancedBracketSelectors: def.unbalancedBracketSelectors,
29
- sourceExtensionId: def.sourceExtensionId,
29
+ sourceExtensionId: def.sourceExtensionId
30
30
  };
31
31
  }));
32
32
  this._grammarFactory = this._loadTMGrammarFactory(grammarDefinitions, _createData.onigurumaWASMUri);
33
33
  }
34
34
  async _loadTMGrammarFactory(grammarDefinitions, onigurumaWASMUri) {
35
- const vscodeTextmate = await import('vscode-textmate').then(module => module.default ?? module);
36
- const vscodeOniguruma = await import('vscode-oniguruma').then(module => module.default ?? module);
35
+ const vscodeTextmate = await import('@codingame/monaco-vscode-api/_virtual/main').then(function (n) { return n.main; });
36
+ const vscodeOniguruma = await import('../../../../../../../../../_virtual/main2.js').then(function (n) { return n.main; });
37
37
  const response = await fetch(onigurumaWASMUri);
38
38
  const bytes = await response.arrayBuffer();
39
39
  await vscodeOniguruma.loadWASM(bytes);
40
40
  const onigLib = Promise.resolve({
41
- createOnigScanner: (sources) => vscodeOniguruma.createOnigScanner(sources),
42
- createOnigString: (str) => vscodeOniguruma.createOnigString(str)
41
+ createOnigScanner: sources => vscodeOniguruma.createOnigScanner(sources),
42
+ createOnigString: str => vscodeOniguruma.createOnigString(str)
43
43
  });
44
44
  return ( new TMGrammarFactory({
45
- logTrace: (msg) => { },
45
+ logTrace: msg => {},
46
46
  logError: (msg, err) => console.error(msg, err),
47
- readFile: (resource) => this._host.$readFile(resource)
47
+ readFile: resource => this._host.$readFile(resource)
48
48
  }, grammarDefinitions, vscodeTextmate, onigLib));
49
49
  }
50
50
  $acceptNewModel(data) {
@@ -66,7 +66,7 @@ class TextMateTokenizationWorker {
66
66
  },
67
67
  reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) {
68
68
  that._host.$reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, isRandomSample);
69
- },
69
+ }
70
70
  }, data.languageId, data.encodedLanguageId, data.maxTokenizationLineLength)));
71
71
  }
72
72
  $acceptModelChanged(controllerId, e) {
@@ -16,7 +16,16 @@ import { OffsetRange } from '@codingame/monaco-vscode-api/vscode/vs/editor/commo
16
16
  import { observableValue } from '@codingame/monaco-vscode-api/vscode/vs/base/common/observableInternal/observables/observableValue';
17
17
 
18
18
  class TextMateWorkerTokenizer extends MirrorTextModel {
19
- constructor(uri, lines, eol, versionId, _host, _languageId, _encodedLanguageId, maxTokenizationLineLength) {
19
+ constructor(
20
+ uri,
21
+ lines,
22
+ eol,
23
+ versionId,
24
+ _host,
25
+ _languageId,
26
+ _encodedLanguageId,
27
+ maxTokenizationLineLength
28
+ ) {
20
29
  super(uri, lines, eol, versionId);
21
30
  this._host = _host;
22
31
  this._languageId = _languageId;
@@ -72,8 +81,7 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
72
81
  false
73
82
  )), Disposable.None, this._maxTokenizationLineLength));
74
83
  this._tokenizerWithStateStore = ( new TokenizerWithStateStore(this._lines.length, tokenizationSupport));
75
- }
76
- else {
84
+ } else {
77
85
  this._tokenizerWithStateStore = null;
78
86
  }
79
87
  this._tokenize();
@@ -83,7 +91,9 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
83
91
  return;
84
92
  }
85
93
  if (!this._diffStateStacksRefEqFn) {
86
- const { diffStateStacksRefEq } = await import('vscode-textmate').then(module => module.default ?? module);
94
+ const {
95
+ diffStateStacksRefEq
96
+ } = await import('@codingame/monaco-vscode-api/_virtual/main').then(function (n) { return n.main; });
87
97
  this._diffStateStacksRefEqFn = diffStateStacksRefEq;
88
98
  }
89
99
  const startTime = ( new Date()).getTime();
@@ -103,8 +113,7 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
103
113
  if (this._tokenizerWithStateStore.store.setEndState(lineToTokenize.lineNumber, r.endState)) {
104
114
  const delta = this._diffStateStacksRefEqFn(lineToTokenize.startState, r.endState);
105
115
  stateDeltaBuilder.setState(lineToTokenize.lineNumber, delta);
106
- }
107
- else {
116
+ } else {
108
117
  stateDeltaBuilder.setState(lineToTokenize.lineNumber, null);
109
118
  }
110
119
  LineTokens.convertToEndOffset(r.tokens, text.length);
@@ -121,7 +130,12 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
121
130
  const fontUpdate = AnnotationsUpdate.create(fontTokensUpdate);
122
131
  const serializedFontUpdate = fontUpdate.serialize(serializeFontTokenOptions());
123
132
  const stateDeltas = stateDeltaBuilder.getStateDeltas();
124
- this._host.setTokensAndStates(this._versionId, tokenBuilder.serialize(), serializedFontUpdate, stateDeltas);
133
+ this._host.setTokensAndStates(
134
+ this._versionId,
135
+ tokenBuilder.serialize(),
136
+ serializedFontUpdate,
137
+ stateDeltas
138
+ );
125
139
  const deltaMs = ( new Date()).getTime() - startTime;
126
140
  if (deltaMs > 20) {
127
141
  setTimeout0(() => this._tokenize());
@@ -148,8 +162,8 @@ class TextMateWorkerTokenizer extends MirrorTextModel {
148
162
  )),
149
163
  annotation: {
150
164
  fontFamily: fontInfo.fontFamily ?? undefined,
151
- fontSize: fontInfo.fontSize ?? undefined,
152
- lineHeight: fontInfo.lineHeight ?? undefined
165
+ fontSizeMultiplier: fontInfo.fontSizeMultiplier ?? undefined,
166
+ lineHeightMultiplier: fontInfo.lineHeightMultiplier ?? undefined
153
167
  }
154
168
  });
155
169
  }
@@ -169,9 +183,11 @@ class StateDeltaBuilder {
169
183
  setState(lineNumber, stackDiff) {
170
184
  if (lineNumber === this._lastStartLineNumber + 1) {
171
185
  this._stateDeltas[this._stateDeltas.length - 1].stateDeltas.push(stackDiff);
172
- }
173
- else {
174
- this._stateDeltas.push({ startLineNumber: lineNumber, stateDeltas: [stackDiff] });
186
+ } else {
187
+ this._stateDeltas.push({
188
+ startLineNumber: lineNumber,
189
+ stateDeltas: [stackDiff]
190
+ });
175
191
  }
176
192
  this._lastStartLineNumber = lineNumber;
177
193
  }
@@ -10,14 +10,18 @@ import { ITextFileService } from '@codingame/monaco-vscode-api/vscode/vs/workben
10
10
  import { StopWatch } from '@codingame/monaco-vscode-api/vscode/vs/base/common/stopwatch';
11
11
 
12
12
  let TextMateTokenizationInstantiator = class TextMateTokenizationInstantiator {
13
- static { this.ID = 'workbench.contrib.textMateTokenizationInstantiator'; }
14
- constructor(_textMateTokenizationService) { }
13
+ static {
14
+ this.ID = "workbench.contrib.textMateTokenizationInstantiator";
15
+ }
16
+ constructor(_textMateTokenizationService) {}
15
17
  };
16
- TextMateTokenizationInstantiator = ( __decorate([
17
- ( __param(0, ITextMateTokenizationService))
18
- ], TextMateTokenizationInstantiator));
19
- registerWorkbenchContribution2(TextMateTokenizationInstantiator.ID, TextMateTokenizationInstantiator, WorkbenchPhase.BlockRestore);
20
- CommandsRegistry.registerCommand('_workbench.colorizeTextMateTokens', async (accessor, resource) => {
18
+ TextMateTokenizationInstantiator = ( __decorate([( __param(0, ITextMateTokenizationService))], TextMateTokenizationInstantiator));
19
+ registerWorkbenchContribution2(
20
+ TextMateTokenizationInstantiator.ID,
21
+ TextMateTokenizationInstantiator,
22
+ WorkbenchPhase.BlockRestore
23
+ );
24
+ CommandsRegistry.registerCommand("_workbench.colorizeTextMateTokens", async (accessor, resource) => {
21
25
  const textModelService = accessor.get(ITextFileService);
22
26
  const textModel = resource ? (await textModelService.files.resolve(resource)).textEditorModel : undefined;
23
27
  if (!textModel) {
@@ -33,5 +37,7 @@ CommandsRegistry.registerCommand('_workbench.colorizeTextMateTokens', async (acc
33
37
  state = tokenizer.tokenizeEncoded(textModel.getLineContent(i), true, state).endState;
34
38
  }
35
39
  stopwatch.stop();
36
- return { tokenizeTime: stopwatch.elapsed() };
40
+ return {
41
+ tokenizeTime: stopwatch.elapsed()
42
+ };
37
43
  });
@@ -10,7 +10,7 @@ import { ITelemetryService } from "@codingame/monaco-vscode-api/vscode/vs/platfo
10
10
  import { IWorkbenchEnvironmentService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/services/environment/common/environmentService.service";
11
11
  import { ITextMateTokenizationService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature.service";
12
12
  import { IWorkbenchThemeService } from "@codingame/monaco-vscode-api/vscode/vs/workbench/services/themes/common/workbenchThemeService.service";
13
- import type { IGrammar } from "vscode-textmate";
13
+ import type { IGrammar } from "@codingame/monaco-vscode-api/_virtual/main";
14
14
  export declare class TextMateTokenizationFeature extends Disposable implements ITextMateTokenizationService {
15
15
  private readonly _languageService;
16
16
  private readonly _themeService;