@codingame/monaco-vscode-textmate-service-override 11.1.2 → 12.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (33) hide show
  1. package/index.d.ts +4 -2
  2. package/index.js +34 -1
  3. package/package.json +21 -10
  4. package/vscode/src/vs/workbench/services/textMate/browser/arrayOperation.d.ts +28 -0
  5. package/vscode/src/vs/workbench/services/textMate/browser/arrayOperation.js +1 -0
  6. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.d.ts +27 -0
  7. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/textMateWorkerTokenizerController.js +1 -0
  8. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.d.ts +38 -0
  9. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/threadedBackgroundTokenizerFactory.js +3 -2
  10. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.d.ts +53 -0
  11. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.worker.js +1 -0
  12. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.workerMain.d.ts +1 -0
  13. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.workerMain.js +2 -1
  14. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerHost.d.ts +11 -0
  15. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerHost.js +2 -0
  16. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.d.ts +28 -0
  17. package/vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateWorkerTokenizer.js +1 -0
  18. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.d.ts +53 -0
  19. package/vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js +50 -51
  20. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.d.ts +24 -0
  21. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/textMateTokenizationSupport.js +1 -0
  22. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.d.ts +16 -0
  23. package/vscode/src/vs/workbench/services/textMate/browser/tokenizationSupport/tokenizationSupportWithLineLimit.js +1 -0
  24. package/vscode/src/vs/workbench/services/textMate/common/TMGrammarFactory.d.ts +32 -0
  25. package/vscode/src/vs/workbench/services/textMate/common/TMGrammarFactory.js +1 -0
  26. package/vscode/src/vs/workbench/services/textMate/common/TMScopeRegistry.d.ts +26 -0
  27. package/vscode/src/vs/workbench/services/textMate/common/TMScopeRegistry.js +2 -0
  28. package/vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.d.ts +3 -0
  29. package/vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.js +35 -34
  30. package/worker.js +2 -1
  31. package/workers/textmate.worker.d.ts +1 -0
  32. package/textmate.js +0 -33
  33. package/workers/textmate.worker.js +0 -1
package/index.d.ts CHANGED
@@ -1,2 +1,4 @@
1
- export * from 'vscode/service-override/textmate';
2
- export { default } from 'vscode/service-override/textmate';
1
+ import { type IEditorOverrideServices } from "vscode/vscode/vs/editor/standalone/browser/standaloneServices";
2
+ import { ITextMateTokenizationService } from "vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature.service";
3
+ export default function getServiceOverride(): IEditorOverrideServices;
4
+ export { ITextMateTokenizationService };
package/index.js CHANGED
@@ -1,2 +1,35 @@
1
- export { default } from './textmate.js';
1
+
2
+ import { StandaloneServices } from 'vscode/vscode/vs/editor/standalone/browser/standaloneServices';
3
+ import { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature.service';
2
4
  export { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature.service';
5
+ import { SyncDescriptor } from 'vscode/vscode/vs/platform/instantiation/common/descriptors';
6
+ import { TextMateTokenizationFeature } from './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js';
7
+ import { LifecyclePhase } from 'vscode/vscode/vs/workbench/services/lifecycle/common/lifecycle';
8
+ import { ILifecycleService } from 'vscode/vscode/vs/workbench/services/lifecycle/common/lifecycle.service';
9
+ import getServiceOverride$1 from '@codingame/monaco-vscode-files-service-override';
10
+ import { registerServiceInitializeParticipant } from 'vscode/lifecycle';
11
+ import { registerAssets } from 'vscode/assets';
12
+ import './vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.js';
13
+ import '@codingame/monaco-vscode-3dd5b442-bfbc-5b1d-8f4a-7567530c75f5-common/vscode/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens';
14
+
15
+ const _onigWasm = new URL('vscode-oniguruma/release/onig.wasm', import.meta.url).href;
16
+ registerAssets({
17
+ 'vscode-oniguruma/../onig.wasm': _onigWasm,
18
+ 'vs/../../node_modules/vscode-oniguruma/release/onig.wasm': _onigWasm
19
+ });
20
+ registerServiceInitializeParticipant(async (accessor) => {
21
+ void accessor
22
+ .get(ILifecycleService)
23
+ .when(LifecyclePhase.Ready)
24
+ .then(() => {
25
+ StandaloneServices.get(ITextMateTokenizationService);
26
+ });
27
+ });
28
+ function getServiceOverride() {
29
+ return {
30
+ ...getServiceOverride$1(),
31
+ [ITextMateTokenizationService.toString()]: new SyncDescriptor(TextMateTokenizationFeature, [], false)
32
+ };
33
+ }
34
+
35
+ export { getServiceOverride as default };
package/package.json CHANGED
@@ -1,6 +1,8 @@
1
1
  {
2
2
  "name": "@codingame/monaco-vscode-textmate-service-override",
3
- "version": "11.1.2",
3
+ "version": "12.0.1",
4
+ "private": false,
5
+ "description": "VSCode public API plugged on the monaco editor - textmate service-override",
4
6
  "keywords": [],
5
7
  "author": {
6
8
  "name": "CodinGame",
@@ -12,8 +14,18 @@
12
14
  "url": "git+ssh://git@github.com/CodinGame/monaco-vscode-api.git"
13
15
  },
14
16
  "type": "module",
15
- "private": false,
16
- "description": "VSCode public API plugged on the monaco editor - textmate service-override",
17
+ "dependencies": {
18
+ "@codingame/monaco-vscode-33833ac7-3af3-5e9d-8fb9-11838d852c59-common": "12.0.1",
19
+ "@codingame/monaco-vscode-3dd5b442-bfbc-5b1d-8f4a-7567530c75f5-common": "12.0.1",
20
+ "@codingame/monaco-vscode-695440c8-a687-5594-b476-bbc7b36bafe9-common": "12.0.1",
21
+ "@codingame/monaco-vscode-cb805e61-7d70-50d9-b937-bf6481ea0c17-common": "12.0.1",
22
+ "@codingame/monaco-vscode-files-service-override": "12.0.1",
23
+ "vscode": "npm:@codingame/monaco-vscode-api@12.0.1",
24
+ "vscode-oniguruma": "1.7.0",
25
+ "vscode-textmate": "9.1.0"
26
+ },
27
+ "peerDependencies": {},
28
+ "peerDependenciesMeta": {},
17
29
  "main": "index.js",
18
30
  "module": "index.js",
19
31
  "types": "index.d.ts",
@@ -22,16 +34,15 @@
22
34
  "default": "./index.js"
23
35
  },
24
36
  "./vscode/*": {
25
- "default": "./vscode/src/*.js"
37
+ "default": "./vscode/src/*.js",
38
+ "types": "./vscode/src/*.d.ts"
26
39
  },
27
40
  "./worker": {
28
41
  "default": "./worker.js"
42
+ },
43
+ "./*": {
44
+ "default": "./*.js",
45
+ "types": "./*.d.ts"
29
46
  }
30
- },
31
- "dependencies": {
32
- "vscode": "npm:@codingame/monaco-vscode-api@11.1.2",
33
- "vscode-oniguruma": "1.7.0",
34
- "vscode-textmate": "9.1.0",
35
- "@codingame/monaco-vscode-files-service-override": "11.1.2"
36
47
  }
37
48
  }
@@ -0,0 +1,28 @@
1
+ export declare class ArrayEdit {
2
+ readonly edits: readonly SingleArrayEdit[];
3
+ constructor(edits: readonly SingleArrayEdit[]);
4
+ applyToArray(array: any[]): void;
5
+ }
6
+ export declare class SingleArrayEdit {
7
+ readonly offset: number;
8
+ readonly length: number;
9
+ readonly newLength: number;
10
+ constructor(offset: number, length: number, newLength: number);
11
+ toString(): string;
12
+ }
13
+ export interface IIndexTransformer {
14
+ transform(index: number): number | undefined;
15
+ }
16
+ export declare class MonotonousIndexTransformer implements IIndexTransformer {
17
+ private readonly transformation;
18
+ static fromMany(transformations: ArrayEdit[]): IIndexTransformer;
19
+ private idx;
20
+ private offset;
21
+ constructor(transformation: ArrayEdit);
22
+ transform(index: number): number | undefined;
23
+ }
24
+ export declare class CombinedIndexTransformer implements IIndexTransformer {
25
+ private readonly transformers;
26
+ constructor(transformers: IIndexTransformer[]);
27
+ transform(index: number): number | undefined;
28
+ }
@@ -1,3 +1,4 @@
1
+
1
2
  import { compareBy, numberComparator } from 'vscode/vscode/vs/base/common/arrays';
2
3
 
3
4
  class ArrayEdit {
@@ -0,0 +1,27 @@
1
+ import { Disposable } from "vscode/vscode/vs/base/common/lifecycle";
2
+ import { IObservable } from "vscode/vscode/vs/base/common/observable";
3
+ import { Proxied } from "vscode/vscode/vs/base/common/worker/simpleWorker";
4
+ import { IBackgroundTokenizationStore, ILanguageIdCodec } from "vscode/vscode/vs/editor/common/languages";
5
+ import { ITextModel } from "vscode/vscode/vs/editor/common/model";
6
+ import { IConfigurationService } from "vscode/vscode/vs/platform/configuration/common/configuration.service";
7
+ import type { StateDeltas, TextMateTokenizationWorker } from "./worker/textMateTokenizationWorker.worker.js";
8
+ export declare class TextMateWorkerTokenizerController extends Disposable {
9
+ private readonly _model;
10
+ private readonly _worker;
11
+ private readonly _languageIdCodec;
12
+ private readonly _backgroundTokenizationStore;
13
+ private readonly _configurationService;
14
+ private readonly _maxTokenizationLineLength;
15
+ private static _id;
16
+ readonly controllerId: number;
17
+ private readonly _pendingChanges;
18
+ private readonly _states;
19
+ private readonly _loggingEnabled;
20
+ private _applyStateStackDiffFn?;
21
+ private _initialState?;
22
+ constructor(_model: ITextModel, _worker: Proxied<TextMateTokenizationWorker>, _languageIdCodec: ILanguageIdCodec, _backgroundTokenizationStore: IBackgroundTokenizationStore, _configurationService: IConfigurationService, _maxTokenizationLineLength: IObservable<number>);
23
+ dispose(): void;
24
+ requestTokens(startLineNumber: number, endLineNumberExclusive: number): void;
25
+ setTokensAndStates(controllerId: number, versionId: number, rawTokens: ArrayBuffer, stateDeltas: StateDeltas[]): Promise<void>;
26
+ private get _shouldLog();
27
+ }
@@ -1,3 +1,4 @@
1
+
1
2
  import { Disposable } from 'vscode/vscode/vs/base/common/lifecycle';
2
3
  import 'vscode/vscode/vs/base/common/arrays';
3
4
  import 'vscode/vscode/vs/base/common/event';
@@ -0,0 +1,38 @@
1
+ import { IDisposable } from "vscode/vscode/vs/base/common/lifecycle";
2
+ import { IObservable } from "vscode/vscode/vs/base/common/observable";
3
+ import { IBackgroundTokenizationStore, IBackgroundTokenizer } from "vscode/vscode/vs/editor/common/languages";
4
+ import { ILanguageService } from "vscode/vscode/vs/editor/common/languages/language";
5
+ import { ITextModel } from "vscode/vscode/vs/editor/common/model";
6
+ import { IConfigurationService } from "vscode/vscode/vs/platform/configuration/common/configuration.service";
7
+ import { IEnvironmentService } from "vscode/vscode/vs/platform/environment/common/environment.service";
8
+ import { IExtensionResourceLoaderService } from "vscode/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader.service";
9
+ import { INotificationService } from "vscode/vscode/vs/platform/notification/common/notification.service";
10
+ import { ITelemetryService } from "vscode/vscode/vs/platform/telemetry/common/telemetry.service";
11
+ import { IValidGrammarDefinition } from "../../common/TMScopeRegistry.js";
12
+ import type { IRawTheme } from "vscode-textmate";
13
+ export declare class ThreadedBackgroundTokenizerFactory implements IDisposable {
14
+ private readonly _reportTokenizationTime;
15
+ private readonly _shouldTokenizeAsync;
16
+ private readonly _extensionResourceLoaderService;
17
+ private readonly _configurationService;
18
+ private readonly _languageService;
19
+ private readonly _environmentService;
20
+ private readonly _notificationService;
21
+ private readonly _telemetryService;
22
+ private static _reportedMismatchingTokens;
23
+ private _workerProxyPromise;
24
+ private _worker;
25
+ private _workerProxy;
26
+ private readonly _workerTokenizerControllers;
27
+ private _currentTheme;
28
+ private _currentTokenColorMap;
29
+ private _grammarDefinitions;
30
+ constructor(_reportTokenizationTime: (timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean) => void, _shouldTokenizeAsync: () => boolean, _extensionResourceLoaderService: IExtensionResourceLoaderService, _configurationService: IConfigurationService, _languageService: ILanguageService, _environmentService: IEnvironmentService, _notificationService: INotificationService, _telemetryService: ITelemetryService);
31
+ dispose(): void;
32
+ createBackgroundTokenizer(textModel: ITextModel, tokenStore: IBackgroundTokenizationStore, maxTokenizationLineLength: IObservable<number>): IBackgroundTokenizer | undefined;
33
+ setGrammarDefinitions(grammarDefinitions: IValidGrammarDefinition[]): void;
34
+ acceptTheme(theme: IRawTheme, colorMap: string[]): void;
35
+ private _getWorkerProxy;
36
+ private _createWorkerProxy;
37
+ private _disposeWorker;
38
+ }
@@ -1,5 +1,6 @@
1
- import { __decorate, __param } from 'vscode/external/tslib/tslib.es6.js';
2
- import 'vscode/vscode/vs/amdX';
1
+
2
+ import { __decorate, __param } from 'vscode/external/tslib/tslib.es6';
3
+ import '@codingame/monaco-vscode-cb805e61-7d70-50d9-b937-bf6481ea0c17-common/vscode/vs/amdX';
3
4
  import { DisposableStore, toDisposable } from 'vscode/vscode/vs/base/common/lifecycle';
4
5
  import { nodeModulesPath, FileAccess } from 'vscode/vscode/vs/base/common/network';
5
6
  import 'vscode/vscode/vs/base/common/platform';
@@ -0,0 +1,53 @@
1
+ import { UriComponents } from "vscode/vscode/vs/base/common/uri";
2
+ import { LanguageId } from "vscode/vscode/vs/editor/common/encodedTokenAttributes";
3
+ import { IModelChangedEvent } from "vscode/vscode/vs/editor/common/model/mirrorTextModel";
4
+ import { IValidEmbeddedLanguagesMap, IValidTokenTypeMap } from "../../../common/TMScopeRegistry.js";
5
+ import type { IRawTheme, StackDiff } from "vscode-textmate";
6
+ import { IRequestHandler, IWorkerServer } from "vscode/vscode/vs/base/common/worker/simpleWorker";
7
+ export declare function create(workerServer: IWorkerServer): TextMateTokenizationWorker;
8
+ export interface ICreateData {
9
+ grammarDefinitions: IValidGrammarDefinitionDTO[];
10
+ onigurumaWASMUri: string;
11
+ }
12
+ export interface IValidGrammarDefinitionDTO {
13
+ location: UriComponents;
14
+ language?: string;
15
+ scopeName: string;
16
+ embeddedLanguages: IValidEmbeddedLanguagesMap;
17
+ tokenTypes: IValidTokenTypeMap;
18
+ injectTo?: string[];
19
+ balancedBracketSelectors: string[];
20
+ unbalancedBracketSelectors: string[];
21
+ sourceExtensionId?: string;
22
+ }
23
+ export interface StateDeltas {
24
+ startLineNumber: number;
25
+ stateDeltas: (StackDiff | null)[];
26
+ }
27
+ export declare class TextMateTokenizationWorker implements IRequestHandler {
28
+ _requestHandlerBrand: any;
29
+ private readonly _host;
30
+ private readonly _models;
31
+ private readonly _grammarCache;
32
+ private _grammarFactory;
33
+ constructor(workerServer: IWorkerServer);
34
+ $init(_createData: ICreateData): Promise<void>;
35
+ private _loadTMGrammarFactory;
36
+ $acceptNewModel(data: IRawModelData): void;
37
+ $acceptModelChanged(controllerId: number, e: IModelChangedEvent): void;
38
+ $retokenize(controllerId: number, startLineNumber: number, endLineNumberExclusive: number): void;
39
+ $acceptModelLanguageChanged(controllerId: number, newLanguageId: string, newEncodedLanguageId: LanguageId): void;
40
+ $acceptRemovedModel(controllerId: number): void;
41
+ $acceptTheme(theme: IRawTheme, colorMap: string[]): Promise<void>;
42
+ $acceptMaxTokenizationLineLength(controllerId: number, value: number): void;
43
+ }
44
+ export interface IRawModelData {
45
+ uri: UriComponents;
46
+ versionId: number;
47
+ lines: string[];
48
+ EOL: string;
49
+ languageId: string;
50
+ encodedLanguageId: LanguageId;
51
+ maxTokenizationLineLength: number;
52
+ controllerId: number;
53
+ }
@@ -1,3 +1,4 @@
1
+
1
2
  import { URI } from 'vscode/vscode/vs/base/common/uri';
2
3
  import { TMGrammarFactory } from '../../../common/TMGrammarFactory.js';
3
4
  import { TextMateWorkerTokenizer } from './textMateWorkerTokenizer.js';
@@ -1,4 +1,5 @@
1
+
1
2
  import { create } from './textMateTokenizationWorker.worker.js';
2
- import { bootstrapSimpleWorker } from 'vscode/vscode/vs/base/common/worker/simpleWorkerBootstrap';
3
+ import { bootstrapSimpleWorker } from '@codingame/monaco-vscode-695440c8-a687-5594-b476-bbc7b36bafe9-common/vscode/vs/base/common/worker/simpleWorkerBootstrap';
3
4
 
4
5
  bootstrapSimpleWorker(create);
@@ -0,0 +1,11 @@
1
+ import { UriComponents } from "vscode/vscode/vs/base/common/uri";
2
+ import { IWorkerServer, IWorkerClient } from "vscode/vscode/vs/base/common/worker/simpleWorker";
3
+ import { StateDeltas } from "./textMateTokenizationWorker.worker.js";
4
+ export declare abstract class TextMateWorkerHost {
5
+ static CHANNEL_NAME: string;
6
+ static getChannel(workerServer: IWorkerServer): TextMateWorkerHost;
7
+ static setChannel(workerClient: IWorkerClient<any>, obj: TextMateWorkerHost): void;
8
+ abstract $readFile(_resource: UriComponents): Promise<string>;
9
+ abstract $setTokensAndStates(controllerId: number, versionId: number, tokens: Uint8Array, lineEndStateDeltas: StateDeltas[]): Promise<void>;
10
+ abstract $reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void;
11
+ }
@@ -1,3 +1,5 @@
1
+
2
+
1
3
  class TextMateWorkerHost {
2
4
  static { this.CHANNEL_NAME = 'textMateWorkerHost'; }
3
5
  static getChannel(workerServer) {
@@ -0,0 +1,28 @@
1
+ import { URI } from "vscode/vscode/vs/base/common/uri";
2
+ import { LanguageId } from "vscode/vscode/vs/editor/common/encodedTokenAttributes";
3
+ import { IModelChangedEvent, MirrorTextModel } from "vscode/vscode/vs/editor/common/model/mirrorTextModel";
4
+ import { ICreateGrammarResult } from "../../../common/TMGrammarFactory.js";
5
+ import { StateDeltas } from "./textMateTokenizationWorker.worker.js";
6
+ export interface TextMateModelTokenizerHost {
7
+ getOrCreateGrammar(languageId: string, encodedLanguageId: LanguageId): Promise<ICreateGrammarResult | null>;
8
+ setTokensAndStates(versionId: number, tokens: Uint8Array, stateDeltas: StateDeltas[]): void;
9
+ reportTokenizationTime(timeMs: number, languageId: string, sourceExtensionId: string | undefined, lineLength: number, isRandomSample: boolean): void;
10
+ }
11
+ export declare class TextMateWorkerTokenizer extends MirrorTextModel {
12
+ private readonly _host;
13
+ private _languageId;
14
+ private _encodedLanguageId;
15
+ private _tokenizerWithStateStore;
16
+ private _isDisposed;
17
+ private readonly _maxTokenizationLineLength;
18
+ private _diffStateStacksRefEqFn?;
19
+ private readonly _tokenizeDebouncer;
20
+ constructor(uri: URI, lines: string[], eol: string, versionId: number, _host: TextMateModelTokenizerHost, _languageId: string, _encodedLanguageId: LanguageId, maxTokenizationLineLength: number);
21
+ dispose(): void;
22
+ onLanguageId(languageId: string, encodedLanguageId: LanguageId): void;
23
+ onEvents(e: IModelChangedEvent): void;
24
+ acceptMaxTokenizationLineLength(maxTokenizationLineLength: number): void;
25
+ retokenize(startLineNumber: number, endLineNumberExclusive: number): void;
26
+ private _resetTokenization;
27
+ private _tokenize;
28
+ }
@@ -1,3 +1,4 @@
1
+
1
2
  import { RunOnceScheduler } from 'vscode/vscode/vs/base/common/async';
2
3
  import { observableValue } from 'vscode/vscode/vs/base/common/observableInternal/base';
3
4
  import 'vscode/vscode/vs/base/common/arrays';
@@ -0,0 +1,53 @@
1
+ import { Disposable } from "vscode/vscode/vs/base/common/lifecycle";
2
+ import { ILanguageService } from "vscode/vscode/vs/editor/common/languages/language";
3
+ import { IConfigurationService } from "vscode/vscode/vs/platform/configuration/common/configuration.service";
4
+ import { IExtensionResourceLoaderService } from "vscode/vscode/vs/platform/extensionResourceLoader/common/extensionResourceLoader.service";
5
+ import { IInstantiationService } from "vscode/vscode/vs/platform/instantiation/common/instantiation";
6
+ import { ILogService } from "vscode/vscode/vs/platform/log/common/log.service";
7
+ import { INotificationService } from "vscode/vscode/vs/platform/notification/common/notification.service";
8
+ import { IProgressService } from "vscode/vscode/vs/platform/progress/common/progress.service";
9
+ import { ITelemetryService } from "vscode/vscode/vs/platform/telemetry/common/telemetry.service";
10
+ import { IWorkbenchEnvironmentService } from "vscode/vscode/vs/workbench/services/environment/common/environmentService.service";
11
+ import { ITextMateTokenizationService } from "vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature.service";
12
+ import { IWorkbenchThemeService } from "@codingame/monaco-vscode-3dd5b442-bfbc-5b1d-8f4a-7567530c75f5-common/vscode/vs/workbench/services/themes/common/workbenchThemeService.service";
13
+ import type { IGrammar } from "vscode-textmate";
14
+ export declare class TextMateTokenizationFeature extends Disposable implements ITextMateTokenizationService {
15
+ private readonly _languageService;
16
+ private readonly _themeService;
17
+ private readonly _extensionResourceLoaderService;
18
+ private readonly _notificationService;
19
+ private readonly _logService;
20
+ private readonly _configurationService;
21
+ private readonly _progressService;
22
+ private readonly _environmentService;
23
+ private readonly _instantiationService;
24
+ private readonly _telemetryService;
25
+ private static reportTokenizationTimeCounter;
26
+ _serviceBrand: undefined;
27
+ private readonly _styleElement;
28
+ private readonly _createdModes;
29
+ private readonly _encounteredLanguages;
30
+ private _debugMode;
31
+ private _debugModePrintFunc;
32
+ private _grammarDefinitions;
33
+ private _grammarFactory;
34
+ private readonly _tokenizersRegistrations;
35
+ private _currentTheme;
36
+ private _currentTokenColorMap;
37
+ private readonly _threadedBackgroundTokenizerFactory;
38
+ constructor(_languageService: ILanguageService, _themeService: IWorkbenchThemeService, _extensionResourceLoaderService: IExtensionResourceLoaderService, _notificationService: INotificationService, _logService: ILogService, _configurationService: IConfigurationService, _progressService: IProgressService, _environmentService: IWorkbenchEnvironmentService, _instantiationService: IInstantiationService, _telemetryService: ITelemetryService);
39
+ private getAsyncTokenizationEnabled;
40
+ private getAsyncTokenizationVerification;
41
+ private _handleGrammarsExtPoint;
42
+ private _validateGrammarDefinition;
43
+ startDebugMode(printFn: (str: string) => void, onStop: () => void): void;
44
+ private _canCreateGrammarFactory;
45
+ private _getOrCreateGrammarFactory;
46
+ private _createTokenizationSupport;
47
+ private _updateTheme;
48
+ createTokenizer(languageId: string): Promise<IGrammar | null>;
49
+ private _vscodeOniguruma;
50
+ private _getVSCodeOniguruma;
51
+ private _loadVSCodeOnigurumaWASM;
52
+ private _reportTokenizationTime;
53
+ }
@@ -1,6 +1,7 @@
1
- import { __decorate, __param } from 'vscode/external/tslib/tslib.es6.js';
2
- import { resolveAmdNodeModulePath } from 'vscode/vscode/vs/amdX';
3
- import { createStyleSheet } from 'vscode/vscode/vs/base/browser/dom';
1
+
2
+ import { __decorate, __param } from 'vscode/external/tslib/tslib.es6';
3
+ import { createStyleSheet } from 'vscode/vscode/vs/base/browser/domStylesheets';
4
+ import { resolveAmdNodeModulePath } from '@codingame/monaco-vscode-cb805e61-7d70-50d9-b937-bf6481ea0c17-common/vscode/vs/amdX';
4
5
  import { equals } from 'vscode/vscode/vs/base/common/arrays';
5
6
  import { Color } from 'vscode/vscode/vs/base/common/color';
6
7
  import { onUnexpectedError } from 'vscode/vscode/vs/base/common/errors';
@@ -14,6 +15,7 @@ import { observableFromEvent } from 'vscode/vscode/vs/base/common/observableInte
14
15
  import { isWeb } from 'vscode/vscode/vs/base/common/platform';
15
16
  import { joinPath, isEqualOrParent } from 'vscode/vscode/vs/base/common/resources';
16
17
  import { isObject } from 'vscode/vscode/vs/base/common/types';
18
+ import { StandardTokenType } from 'vscode/vscode/vs/editor/common/encodedTokenAttributes';
17
19
  import { LazyTokenizationSupport, TokenizationRegistry } from 'vscode/vscode/vs/editor/common/languages';
18
20
  import { ILanguageService } from 'vscode/vscode/vs/editor/common/languages/language';
19
21
  import { generateTokensCSSForColorMap } from 'vscode/vscode/vs/editor/common/languages/supports/tokenization';
@@ -23,6 +25,7 @@ import { IExtensionResourceLoaderService } from 'vscode/vscode/vs/platform/exten
23
25
  import { IInstantiationService } from 'vscode/vscode/vs/platform/instantiation/common/instantiation';
24
26
  import { ILogService } from 'vscode/vscode/vs/platform/log/common/log.service';
25
27
  import { INotificationService } from 'vscode/vscode/vs/platform/notification/common/notification.service';
28
+ import { ProgressLocation } from 'vscode/vscode/vs/platform/progress/common/progress';
26
29
  import { IProgressService } from 'vscode/vscode/vs/platform/progress/common/progress.service';
27
30
  import { ITelemetryService } from 'vscode/vscode/vs/platform/telemetry/common/telemetry.service';
28
31
  import { IWorkbenchEnvironmentService } from 'vscode/vscode/vs/workbench/services/environment/common/environmentService.service';
@@ -30,8 +33,8 @@ import { TextMateTokenizationSupport } from './tokenizationSupport/textMateToken
30
33
  import { TokenizationSupportWithLineLimit } from './tokenizationSupport/tokenizationSupportWithLineLimit.js';
31
34
  import { ThreadedBackgroundTokenizerFactory } from './backgroundTokenization/threadedBackgroundTokenizerFactory.js';
32
35
  import { TMGrammarFactory, missingTMGrammarErrorMessage } from '../common/TMGrammarFactory.js';
33
- import { grammarsExtPoint } from 'vscode/vscode/vs/workbench/services/textMate/common/TMGrammars';
34
- import { IWorkbenchThemeService } from 'vscode/vscode/vs/workbench/services/themes/common/workbenchThemeService.service';
36
+ import { grammarsExtPoint } from '@codingame/monaco-vscode-33833ac7-3af3-5e9d-8fb9-11838d852c59-common/vscode/vs/workbench/services/textMate/common/TMGrammars';
37
+ import { IWorkbenchThemeService } from '@codingame/monaco-vscode-3dd5b442-bfbc-5b1d-8f4a-7567530c75f5-common/vscode/vs/workbench/services/themes/common/workbenchThemeService.service';
35
38
 
36
39
  var TextMateTokenizationFeature_1;
37
40
  let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disposable {
@@ -55,7 +58,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
55
58
  this._debugModePrintFunc = () => { };
56
59
  this._grammarDefinitions = null;
57
60
  this._grammarFactory = null;
58
- this._tokenizersRegistrations = ( (new DisposableStore()));
61
+ this._tokenizersRegistrations = ( new DisposableStore());
59
62
  this._currentTheme = null;
60
63
  this._currentTokenColorMap = null;
61
64
  this._threadedBackgroundTokenizerFactory = this._instantiationService.createInstance(ThreadedBackgroundTokenizerFactory, (timeMs, languageId, sourceExtensionId, lineLength, isRandomSample) => this._reportTokenizationTime(timeMs, languageId, sourceExtensionId, lineLength, true, isRandomSample), () => this.getAsyncTokenizationEnabled());
@@ -92,7 +95,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
92
95
  if (validatedGrammar) {
93
96
  this._grammarDefinitions.push(validatedGrammar);
94
97
  if (validatedGrammar.language) {
95
- const lazyTokenizationSupport = ( (new LazyTokenizationSupport(() => this._createTokenizationSupport(validatedGrammar.language))));
98
+ const lazyTokenizationSupport = ( new LazyTokenizationSupport(() => this._createTokenizationSupport(validatedGrammar.language)));
96
99
  this._tokenizersRegistrations.add(lazyTokenizationSupport);
97
100
  this._tokenizersRegistrations.add(TokenizationRegistry.registerFactory(validatedGrammar.language, lazyTokenizationSupport));
98
101
  }
@@ -111,7 +114,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
111
114
  const grammarLocation = joinPath(extension.description.extensionLocation, grammar.path);
112
115
  const embeddedLanguages = Object.create(null);
113
116
  if (grammar.embeddedLanguages) {
114
- const scopes = ( (Object.keys(grammar.embeddedLanguages)));
117
+ const scopes = ( Object.keys(grammar.embeddedLanguages));
115
118
  for (let i = 0, len = scopes.length; i < len; i++) {
116
119
  const scope = scopes[i];
117
120
  const language = grammar.embeddedLanguages[scope];
@@ -125,18 +128,18 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
125
128
  }
126
129
  const tokenTypes = Object.create(null);
127
130
  if (grammar.tokenTypes) {
128
- const scopes = ( (Object.keys(grammar.tokenTypes)));
131
+ const scopes = ( Object.keys(grammar.tokenTypes));
129
132
  for (const scope of scopes) {
130
133
  const tokenType = grammar.tokenTypes[scope];
131
134
  switch (tokenType) {
132
135
  case 'string':
133
- tokenTypes[scope] = 2 ;
136
+ tokenTypes[scope] = StandardTokenType.String;
134
137
  break;
135
138
  case 'other':
136
- tokenTypes[scope] = 0 ;
139
+ tokenTypes[scope] = StandardTokenType.Other;
137
140
  break;
138
141
  case 'comment':
139
- tokenTypes[scope] = 1 ;
142
+ tokenTypes[scope] = StandardTokenType.Comment;
140
143
  break;
141
144
  }
142
145
  }
@@ -165,27 +168,25 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
165
168
  }
166
169
  startDebugMode(printFn, onStop) {
167
170
  if (this._debugMode) {
168
- this._notificationService.error(( localize(3084, "Already Logging.")));
171
+ this._notificationService.error(( localize(11738, "Already Logging.")));
169
172
  return;
170
173
  }
171
174
  this._debugModePrintFunc = printFn;
172
175
  this._debugMode = true;
173
176
  if (this._debugMode) {
174
177
  this._progressService.withProgress({
175
- location: 15 ,
176
- buttons: [( localize(3085, "Stop"))]
178
+ location: ProgressLocation.Notification,
179
+ buttons: [( localize(11739, "Stop"))]
177
180
  }, (progress) => {
178
181
  progress.report({
179
- message: ( localize(3086, "Preparing to log TM Grammar parsing. Press Stop when finished."))
182
+ message: ( localize(11740, "Preparing to log TM Grammar parsing. Press Stop when finished."))
180
183
  });
181
184
  return this._getVSCodeOniguruma().then((vscodeOniguruma) => {
182
185
  vscodeOniguruma.setDefaultDebugCall(true);
183
186
  progress.report({
184
- message: ( localize(3087, "Now logging TM Grammar parsing. Press Stop when finished."))
187
+ message: ( localize(11741, "Now logging TM Grammar parsing. Press Stop when finished."))
185
188
  });
186
- return (
187
- (new Promise((resolve, reject) => { }))
188
- );
189
+ return ( new Promise((resolve, reject) => { }));
189
190
  });
190
191
  }, (choice) => {
191
192
  this._getVSCodeOniguruma().then((vscodeOniguruma) => {
@@ -212,11 +213,11 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
212
213
  if (this._grammarFactory) {
213
214
  return this._grammarFactory;
214
215
  }
215
- this._grammarFactory = ( (new TMGrammarFactory({
216
+ this._grammarFactory = ( new TMGrammarFactory({
216
217
  logTrace: (msg) => this._logService.trace(msg),
217
218
  logError: (msg, err) => this._logService.error(msg, err),
218
219
  readFile: (resource) => this._extensionResourceLoaderService.readExtensionResource(resource)
219
- }, this._grammarDefinitions || [], vscodeTextmate, onigLib)));
220
+ }, this._grammarDefinitions || [], vscodeTextmate, onigLib));
220
221
  this._updateTheme(this._themeService.getColorTheme(), true);
221
222
  return this._grammarFactory;
222
223
  }
@@ -229,7 +230,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
229
230
  }
230
231
  try {
231
232
  const grammarFactory = await this._getOrCreateGrammarFactory();
232
- if (!( (grammarFactory.has(languageId)))) {
233
+ if (!( grammarFactory.has(languageId))) {
233
234
  return null;
234
235
  }
235
236
  const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
@@ -238,7 +239,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
238
239
  return null;
239
240
  }
240
241
  const maxTokenizationLineLength = observableConfigValue('editor.maxTokenizationLineLength', languageId, -1, this._configurationService);
241
- const tokenization = ( (new TextMateTokenizationSupport(
242
+ const tokenization = ( new TextMateTokenizationSupport(
242
243
  r.grammar,
243
244
  r.initialState,
244
245
  r.containsEmbeddedLanguages,
@@ -248,7 +249,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
248
249
  this._reportTokenizationTime(timeMs, languageId, r.sourceExtensionId, lineLength, false, isRandomSample);
249
250
  },
250
251
  true
251
- )));
252
+ ));
252
253
  const disposable = tokenization.onDidEncounterLanguage((encodedLanguageId) => {
253
254
  if (!this._encounteredLanguages[encodedLanguageId]) {
254
255
  const languageId = this._languageService.languageIdCodec.decodeLanguageId(encodedLanguageId);
@@ -256,9 +257,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
256
257
  this._languageService.requestBasicLanguageFeatures(languageId);
257
258
  }
258
259
  });
259
- return (
260
- (new TokenizationSupportWithLineLimit(encodedLanguageId, tokenization, disposable, maxTokenizationLineLength))
261
- );
260
+ return ( new TokenizationSupportWithLineLimit(encodedLanguageId, tokenization, disposable, maxTokenizationLineLength));
262
261
  }
263
262
  catch (err) {
264
263
  if (err.message && err.message === missingTMGrammarErrorMessage) {
@@ -289,7 +288,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
289
288
  return null;
290
289
  }
291
290
  const grammarFactory = await this._getOrCreateGrammarFactory();
292
- if (!( (grammarFactory.has(languageId)))) {
291
+ if (!( grammarFactory.has(languageId))) {
293
292
  return null;
294
293
  }
295
294
  const encodedLanguageId = this._languageService.languageIdCodec.encodeLanguageId(languageId);
@@ -317,7 +316,7 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
317
316
  return await response.arrayBuffer();
318
317
  }
319
318
  else {
320
- const response = await fetch(( (( (FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`))).toString(true))));
319
+ const response = await fetch(( ( FileAccess.asBrowserUri(`${nodeModulesPath}/vscode-oniguruma/release/onig.wasm`)).toString(true)));
321
320
  return response;
322
321
  }
323
322
  }
@@ -343,22 +342,22 @@ let TextMateTokenizationFeature = class TextMateTokenizationFeature extends Disp
343
342
  });
344
343
  }
345
344
  };
346
- TextMateTokenizationFeature = TextMateTokenizationFeature_1 = ( (__decorate([
347
- ( (__param(0, ILanguageService))),
348
- ( (__param(1, IWorkbenchThemeService))),
349
- ( (__param(2, IExtensionResourceLoaderService))),
350
- ( (__param(3, INotificationService))),
351
- ( (__param(4, ILogService))),
352
- ( (__param(5, IConfigurationService))),
353
- ( (__param(6, IProgressService))),
354
- ( (__param(7, IWorkbenchEnvironmentService))),
355
- ( (__param(8, IInstantiationService))),
356
- ( (__param(9, ITelemetryService)))
357
- ], TextMateTokenizationFeature)));
345
+ TextMateTokenizationFeature = TextMateTokenizationFeature_1 = ( __decorate([
346
+ ( __param(0, ILanguageService)),
347
+ ( __param(1, IWorkbenchThemeService)),
348
+ ( __param(2, IExtensionResourceLoaderService)),
349
+ ( __param(3, INotificationService)),
350
+ ( __param(4, ILogService)),
351
+ ( __param(5, IConfigurationService)),
352
+ ( __param(6, IProgressService)),
353
+ ( __param(7, IWorkbenchEnvironmentService)),
354
+ ( __param(8, IInstantiationService)),
355
+ ( __param(9, ITelemetryService))
356
+ ], TextMateTokenizationFeature));
358
357
  function toColorMap(colorMap) {
359
358
  const result = [null];
360
359
  for (let i = 1, len = colorMap.length; i < len; i++) {
361
- result[i] = ( (Color.fromHex(colorMap[i])));
360
+ result[i] = ( Color.fromHex(colorMap[i]));
362
361
  }
363
362
  return result;
364
363
  }
@@ -388,7 +387,7 @@ function equalsTokenRules(a, b) {
388
387
  function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _languageService) {
389
388
  if (syntax.language && ((typeof syntax.language !== 'string') || !_languageService.isRegisteredLanguageId(syntax.language))) {
390
389
  collector.error(( localize(
391
- 3088,
390
+ 11742,
392
391
  "Unknown language in `contributes.{0}.language`. Provided value: {1}",
393
392
  grammarsExtPoint.name,
394
393
  String(syntax.language)
@@ -397,7 +396,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
397
396
  }
398
397
  if (!syntax.scopeName || (typeof syntax.scopeName !== 'string')) {
399
398
  collector.error(( localize(
400
- 3089,
399
+ 11743,
401
400
  "Expected string in `contributes.{0}.scopeName`. Provided value: {1}",
402
401
  grammarsExtPoint.name,
403
402
  String(syntax.scopeName)
@@ -406,16 +405,16 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
406
405
  }
407
406
  if (!syntax.path || (typeof syntax.path !== 'string')) {
408
407
  collector.error(( localize(
409
- 3090,
408
+ 11744,
410
409
  "Expected string in `contributes.{0}.path`. Provided value: {1}",
411
410
  grammarsExtPoint.name,
412
411
  String(syntax.path)
413
412
  )));
414
413
  return false;
415
414
  }
416
- if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( (syntax.injectTo.some(scope => typeof scope !== 'string'))))) {
415
+ if (syntax.injectTo && (!Array.isArray(syntax.injectTo) || ( syntax.injectTo.some(scope => typeof scope !== 'string')))) {
417
416
  collector.error(( localize(
418
- 3091,
417
+ 11745,
419
418
  "Invalid value in `contributes.{0}.injectTo`. Must be an array of language scope names. Provided value: {1}",
420
419
  grammarsExtPoint.name,
421
420
  JSON.stringify(syntax.injectTo)
@@ -424,7 +423,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
424
423
  }
425
424
  if (syntax.embeddedLanguages && !isObject(syntax.embeddedLanguages)) {
426
425
  collector.error(( localize(
427
- 3092,
426
+ 11746,
428
427
  "Invalid value in `contributes.{0}.embeddedLanguages`. Must be an object map from scope name to language. Provided value: {1}",
429
428
  grammarsExtPoint.name,
430
429
  JSON.stringify(syntax.embeddedLanguages)
@@ -433,7 +432,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
433
432
  }
434
433
  if (syntax.tokenTypes && !isObject(syntax.tokenTypes)) {
435
434
  collector.error(( localize(
436
- 3093,
435
+ 11747,
437
436
  "Invalid value in `contributes.{0}.tokenTypes`. Must be an object map from scope name to token type. Provided value: {1}",
438
437
  grammarsExtPoint.name,
439
438
  JSON.stringify(syntax.tokenTypes)
@@ -443,7 +442,7 @@ function validateGrammarExtensionPoint(extensionLocation, syntax, collector, _la
443
442
  const grammarLocation = joinPath(extensionLocation, syntax.path);
444
443
  if (!isEqualOrParent(grammarLocation, extensionLocation)) {
445
444
  collector.warn(( localize(
446
- 3094,
445
+ 11748,
447
446
  "Expected `contributes.{0}.path` ({1}) to be included inside extension's folder ({2}). This might make the extension non-portable.",
448
447
  grammarsExtPoint.name,
449
448
  grammarLocation.path,
@@ -0,0 +1,24 @@
1
+ import { Event } from "vscode/vscode/vs/base/common/event";
2
+ import { Disposable } from "vscode/vscode/vs/base/common/lifecycle";
3
+ import { LanguageId } from "vscode/vscode/vs/editor/common/encodedTokenAttributes";
4
+ import { EncodedTokenizationResult, IBackgroundTokenizationStore, IBackgroundTokenizer, IState, ITokenizationSupport, TokenizationResult } from "vscode/vscode/vs/editor/common/languages";
5
+ import { ITextModel } from "vscode/vscode/vs/editor/common/model";
6
+ import type { IGrammar, StateStack } from "vscode-textmate";
7
+ export declare class TextMateTokenizationSupport extends Disposable implements ITokenizationSupport {
8
+ private readonly _grammar;
9
+ private readonly _initialState;
10
+ private readonly _containsEmbeddedLanguages;
11
+ private readonly _createBackgroundTokenizer;
12
+ private readonly _backgroundTokenizerShouldOnlyVerifyTokens;
13
+ private readonly _reportTokenizationTime;
14
+ private readonly _reportSlowTokenization;
15
+ private readonly _seenLanguages;
16
+ private readonly _onDidEncounterLanguage;
17
+ readonly onDidEncounterLanguage: Event<LanguageId>;
18
+ constructor(_grammar: IGrammar, _initialState: StateStack, _containsEmbeddedLanguages: boolean, _createBackgroundTokenizer: ((textModel: ITextModel, tokenStore: IBackgroundTokenizationStore) => IBackgroundTokenizer | undefined) | undefined, _backgroundTokenizerShouldOnlyVerifyTokens: () => boolean, _reportTokenizationTime: (timeMs: number, lineLength: number, isRandomSample: boolean) => void, _reportSlowTokenization: boolean);
19
+ get backgroundTokenizerShouldOnlyVerifyTokens(): boolean | undefined;
20
+ getInitialState(): IState;
21
+ tokenize(line: string, hasEOL: boolean, state: IState): TokenizationResult;
22
+ createBackgroundTokenizer(textModel: ITextModel, store: IBackgroundTokenizationStore): IBackgroundTokenizer | undefined;
23
+ tokenizeEncoded(line: string, hasEOL: boolean, state: StateStack): EncodedTokenizationResult;
24
+ }
@@ -1,3 +1,4 @@
1
+
1
2
  import { Emitter } from 'vscode/vscode/vs/base/common/event';
2
3
  import { Disposable } from 'vscode/vscode/vs/base/common/lifecycle';
3
4
  import { StopWatch } from 'vscode/vscode/vs/base/common/stopwatch';
@@ -0,0 +1,16 @@
1
+ import { LanguageId } from "vscode/vscode/vs/editor/common/encodedTokenAttributes";
2
+ import { EncodedTokenizationResult, IBackgroundTokenizationStore, IBackgroundTokenizer, IState, ITokenizationSupport, TokenizationResult } from "vscode/vscode/vs/editor/common/languages";
3
+ import { ITextModel } from "vscode/vscode/vs/editor/common/model";
4
+ import { Disposable, IDisposable } from "vscode/vscode/vs/base/common/lifecycle";
5
+ import { IObservable } from "vscode/vscode/vs/base/common/observable";
6
+ export declare class TokenizationSupportWithLineLimit extends Disposable implements ITokenizationSupport {
7
+ private readonly _encodedLanguageId;
8
+ private readonly _actual;
9
+ private readonly _maxTokenizationLineLength;
10
+ get backgroundTokenizerShouldOnlyVerifyTokens(): boolean | undefined;
11
+ constructor(_encodedLanguageId: LanguageId, _actual: ITokenizationSupport, disposable: IDisposable, _maxTokenizationLineLength: IObservable<number>);
12
+ getInitialState(): IState;
13
+ tokenize(line: string, hasEOL: boolean, state: IState): TokenizationResult;
14
+ tokenizeEncoded(line: string, hasEOL: boolean, state: IState): EncodedTokenizationResult;
15
+ createBackgroundTokenizer(textModel: ITextModel, store: IBackgroundTokenizationStore): IBackgroundTokenizer | undefined;
16
+ }
@@ -1,3 +1,4 @@
1
+
1
2
  import { nullTokenizeEncoded } from 'vscode/vscode/vs/editor/common/languages/nullTokenize';
2
3
  import { Disposable } from 'vscode/vscode/vs/base/common/lifecycle';
3
4
  import 'vscode/vscode/vs/base/common/arrays';
@@ -0,0 +1,32 @@
1
+ import { Disposable } from "vscode/vscode/vs/base/common/lifecycle";
2
+ import { URI } from "vscode/vscode/vs/base/common/uri";
3
+ import { IValidGrammarDefinition } from "./TMScopeRegistry.js";
4
+ import type { IGrammar, IOnigLib, IRawTheme, StateStack } from "vscode-textmate";
5
+ interface ITMGrammarFactoryHost {
6
+ logTrace(msg: string): void;
7
+ logError(msg: string, err: any): void;
8
+ readFile(resource: URI): Promise<string>;
9
+ }
10
+ export interface ICreateGrammarResult {
11
+ languageId: string;
12
+ grammar: IGrammar | null;
13
+ initialState: StateStack;
14
+ containsEmbeddedLanguages: boolean;
15
+ sourceExtensionId?: string;
16
+ }
17
+ export declare const missingTMGrammarErrorMessage = "No TM Grammar registered for this language.";
18
+ export declare class TMGrammarFactory extends Disposable {
19
+ private readonly _host;
20
+ private readonly _initialState;
21
+ private readonly _scopeRegistry;
22
+ private readonly _injections;
23
+ private readonly _injectedEmbeddedLanguages;
24
+ private readonly _languageToScope;
25
+ private readonly _grammarRegistry;
26
+ constructor(host: ITMGrammarFactoryHost, grammarDefinitions: IValidGrammarDefinition[], vscodeTextmate: typeof import("vscode-textmate"), onigLib: Promise<IOnigLib>);
27
+ has(languageId: string): boolean;
28
+ setTheme(theme: IRawTheme, colorMap: string[]): void;
29
+ getColorMap(): string[];
30
+ createGrammar(languageId: string, encodedLanguageId: number): Promise<ICreateGrammarResult>;
31
+ }
32
+ export {};
@@ -1,3 +1,4 @@
1
+
1
2
  import { Disposable } from 'vscode/vscode/vs/base/common/lifecycle';
2
3
  import { TMScopeRegistry } from './TMScopeRegistry.js';
3
4
 
@@ -0,0 +1,26 @@
1
+ import { URI } from "vscode/vscode/vs/base/common/uri";
2
+ import { LanguageId, StandardTokenType } from "vscode/vscode/vs/editor/common/encodedTokenAttributes";
3
+ export interface IValidGrammarDefinition {
4
+ location: URI;
5
+ language?: string;
6
+ scopeName: string;
7
+ embeddedLanguages: IValidEmbeddedLanguagesMap;
8
+ tokenTypes: IValidTokenTypeMap;
9
+ injectTo?: string[];
10
+ balancedBracketSelectors: string[];
11
+ unbalancedBracketSelectors: string[];
12
+ sourceExtensionId?: string;
13
+ }
14
+ export interface IValidTokenTypeMap {
15
+ [selector: string]: StandardTokenType;
16
+ }
17
+ export interface IValidEmbeddedLanguagesMap {
18
+ [scopeName: string]: LanguageId;
19
+ }
20
+ export declare class TMScopeRegistry {
21
+ private _scopeNameToLanguageRegistration;
22
+ constructor();
23
+ reset(): void;
24
+ register(def: IValidGrammarDefinition): void;
25
+ getGrammarDefinition(scopeName: string): IValidGrammarDefinition | null;
26
+ }
@@ -1,3 +1,5 @@
1
+
2
+
1
3
  class TMScopeRegistry {
2
4
  constructor() {
3
5
  this._scopeNameToLanguageRegistration = Object.create(null);
@@ -0,0 +1,3 @@
1
+ export declare class TokenClassificationExtensionPoints {
2
+ constructor();
3
+ }
@@ -1,34 +1,35 @@
1
- import { __decorate, __param } from 'vscode/external/tslib/tslib.es6.js';
1
+
2
+ import { __decorate, __param } from 'vscode/external/tslib/tslib.es6';
2
3
  import { localize } from 'vscode/vscode/vs/nls';
3
4
  import { ExtensionsRegistry } from 'vscode/vscode/vs/workbench/services/extensions/common/extensionsRegistry';
4
- import { getTokenClassificationRegistry, typeAndModifierIdPattern } from 'vscode/vscode/vs/platform/theme/common/tokenClassificationRegistry';
5
- import { registerWorkbenchContribution2 } from 'vscode/vscode/vs/workbench/common/contributions';
5
+ import { getTokenClassificationRegistry, typeAndModifierIdPattern } from '@codingame/monaco-vscode-3dd5b442-bfbc-5b1d-8f4a-7567530c75f5-common/vscode/vs/platform/theme/common/tokenClassificationRegistry';
6
+ import { registerWorkbenchContribution2, WorkbenchPhase } from 'vscode/vscode/vs/workbench/common/contributions';
6
7
  import { IInstantiationService } from 'vscode/vscode/vs/platform/instantiation/common/instantiation';
7
8
 
8
9
  const tokenClassificationRegistry = getTokenClassificationRegistry();
9
10
  const tokenTypeExtPoint = ExtensionsRegistry.registerExtensionPoint({
10
11
  extensionPoint: 'semanticTokenTypes',
11
12
  jsonSchema: {
12
- description: ( localize(3095, 'Contributes semantic token types.')),
13
+ description: ( localize(11949, 'Contributes semantic token types.')),
13
14
  type: 'array',
14
15
  items: {
15
16
  type: 'object',
16
17
  properties: {
17
18
  id: {
18
19
  type: 'string',
19
- description: ( localize(3096, 'The identifier of the semantic token type')),
20
+ description: ( localize(11950, 'The identifier of the semantic token type')),
20
21
  pattern: typeAndModifierIdPattern,
21
- patternErrorMessage: ( localize(3097, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*')),
22
+ patternErrorMessage: ( localize(11951, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*')),
22
23
  },
23
24
  superType: {
24
25
  type: 'string',
25
- description: ( localize(3098, 'The super type of the semantic token type')),
26
+ description: ( localize(11952, 'The super type of the semantic token type')),
26
27
  pattern: typeAndModifierIdPattern,
27
- patternErrorMessage: ( localize(3099, 'Super types should be in the form letterOrDigit[_-letterOrDigit]*')),
28
+ patternErrorMessage: ( localize(11953, 'Super types should be in the form letterOrDigit[_-letterOrDigit]*')),
28
29
  },
29
30
  description: {
30
31
  type: 'string',
31
- description: ( localize(3100, 'The description of the semantic token type')),
32
+ description: ( localize(11954, 'The description of the semantic token type')),
32
33
  }
33
34
  }
34
35
  }
@@ -37,20 +38,20 @@ const tokenTypeExtPoint = ExtensionsRegistry.registerExtensionPoint({
37
38
  const tokenModifierExtPoint = ExtensionsRegistry.registerExtensionPoint({
38
39
  extensionPoint: 'semanticTokenModifiers',
39
40
  jsonSchema: {
40
- description: ( localize(3101, 'Contributes semantic token modifiers.')),
41
+ description: ( localize(11955, 'Contributes semantic token modifiers.')),
41
42
  type: 'array',
42
43
  items: {
43
44
  type: 'object',
44
45
  properties: {
45
46
  id: {
46
47
  type: 'string',
47
- description: ( localize(3102, 'The identifier of the semantic token modifier')),
48
+ description: ( localize(11956, 'The identifier of the semantic token modifier')),
48
49
  pattern: typeAndModifierIdPattern,
49
- patternErrorMessage: ( localize(3103, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'))
50
+ patternErrorMessage: ( localize(11957, 'Identifiers should be in the form letterOrDigit[_-letterOrDigit]*'))
50
51
  },
51
52
  description: {
52
53
  type: 'string',
53
- description: ( localize(3104, 'The description of the semantic token modifier'))
54
+ description: ( localize(11958, 'The description of the semantic token modifier'))
54
55
  }
55
56
  }
56
57
  }
@@ -59,18 +60,18 @@ const tokenModifierExtPoint = ExtensionsRegistry.registerExtensionPoint({
59
60
  const tokenStyleDefaultsExtPoint = ExtensionsRegistry.registerExtensionPoint({
60
61
  extensionPoint: 'semanticTokenScopes',
61
62
  jsonSchema: {
62
- description: ( localize(3105, 'Contributes semantic token scope maps.')),
63
+ description: ( localize(11959, 'Contributes semantic token scope maps.')),
63
64
  type: 'array',
64
65
  items: {
65
66
  type: 'object',
66
67
  properties: {
67
68
  language: {
68
- description: ( localize(3106, 'Lists the languge for which the defaults are.')),
69
+ description: ( localize(11960, 'Lists the languge for which the defaults are.')),
69
70
  type: 'string'
70
71
  },
71
72
  scopes: {
72
73
  description: ( localize(
73
- 3107,
74
+ 11961,
74
75
  'Maps a semantic token (described by semantic token selector) to one or more textMate scopes used to represent that token.'
75
76
  )),
76
77
  type: 'object',
@@ -90,7 +91,7 @@ class TokenClassificationExtensionPoints {
90
91
  function validateTypeOrModifier(contribution, extensionPoint, collector) {
91
92
  if (typeof contribution.id !== 'string' || contribution.id.length === 0) {
92
93
  collector.error(( localize(
93
- 3108,
94
+ 11962,
94
95
  "'configuration.{0}.id' must be defined and can not be empty",
95
96
  extensionPoint
96
97
  )));
@@ -98,7 +99,7 @@ class TokenClassificationExtensionPoints {
98
99
  }
99
100
  if (!contribution.id.match(typeAndModifierIdPattern)) {
100
101
  collector.error(( localize(
101
- 3109,
102
+ 11963,
102
103
  "'configuration.{0}.id' must follow the pattern letterOrDigit[-_letterOrDigit]*",
103
104
  extensionPoint
104
105
  )));
@@ -107,7 +108,7 @@ class TokenClassificationExtensionPoints {
107
108
  const superType = contribution.superType;
108
109
  if (superType && !superType.match(typeAndModifierIdPattern)) {
109
110
  collector.error(( localize(
110
- 3110,
111
+ 11964,
111
112
  "'configuration.{0}.superType' must follow the pattern letterOrDigit[-_letterOrDigit]*",
112
113
  extensionPoint
113
114
  )));
@@ -115,7 +116,7 @@ class TokenClassificationExtensionPoints {
115
116
  }
116
117
  if (typeof contribution.description !== 'string' || contribution.id.length === 0) {
117
118
  collector.error(( localize(
118
- 3111,
119
+ 11965,
119
120
  "'configuration.{0}.description' must be defined and can not be empty",
120
121
  extensionPoint
121
122
  )));
@@ -128,7 +129,7 @@ class TokenClassificationExtensionPoints {
128
129
  const extensionValue = extension.value;
129
130
  const collector = extension.collector;
130
131
  if (!extensionValue || !Array.isArray(extensionValue)) {
131
- collector.error(( localize(3112, "'configuration.semanticTokenType' must be an array")));
132
+ collector.error(( localize(11966, "'configuration.semanticTokenType' must be an array")));
132
133
  return;
133
134
  }
134
135
  for (const contribution of extensionValue) {
@@ -149,7 +150,7 @@ class TokenClassificationExtensionPoints {
149
150
  const extensionValue = extension.value;
150
151
  const collector = extension.collector;
151
152
  if (!extensionValue || !Array.isArray(extensionValue)) {
152
- collector.error(( localize(3113, "'configuration.semanticTokenModifier' must be an array")));
153
+ collector.error(( localize(11967, "'configuration.semanticTokenModifier' must be an array")));
153
154
  return;
154
155
  }
155
156
  for (const contribution of extensionValue) {
@@ -170,37 +171,37 @@ class TokenClassificationExtensionPoints {
170
171
  const extensionValue = extension.value;
171
172
  const collector = extension.collector;
172
173
  if (!extensionValue || !Array.isArray(extensionValue)) {
173
- collector.error(( localize(3114, "'configuration.semanticTokenScopes' must be an array")));
174
+ collector.error(( localize(11968, "'configuration.semanticTokenScopes' must be an array")));
174
175
  return;
175
176
  }
176
177
  for (const contribution of extensionValue) {
177
178
  if (contribution.language && typeof contribution.language !== 'string') {
178
- collector.error(( localize(3115, "'configuration.semanticTokenScopes.language' must be a string")));
179
+ collector.error(( localize(11969, "'configuration.semanticTokenScopes.language' must be a string")));
179
180
  continue;
180
181
  }
181
182
  if (!contribution.scopes || typeof contribution.scopes !== 'object') {
182
183
  collector.error(( localize(
183
- 3116,
184
+ 11970,
184
185
  "'configuration.semanticTokenScopes.scopes' must be defined as an object"
185
186
  )));
186
187
  continue;
187
188
  }
188
189
  for (const selectorString in contribution.scopes) {
189
190
  const tmScopes = contribution.scopes[selectorString];
190
- if (!Array.isArray(tmScopes) || ( (tmScopes.some(l => typeof l !== 'string')))) {
191
+ if (!Array.isArray(tmScopes) || ( tmScopes.some(l => typeof l !== 'string'))) {
191
192
  collector.error(( localize(
192
- 3117,
193
+ 11971,
193
194
  "'configuration.semanticTokenScopes.scopes' values must be an array of strings"
194
195
  )));
195
196
  continue;
196
197
  }
197
198
  try {
198
199
  const selector = tokenClassificationRegistry.parseTokenSelector(selectorString, contribution.language);
199
- tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe: ( (tmScopes.map(s => s.split(' ')))) });
200
+ tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe: ( tmScopes.map(s => s.split(' '))) });
200
201
  }
201
202
  catch (e) {
202
203
  collector.error(( localize(
203
- 3118,
204
+ 11972,
204
205
  "configuration.semanticTokenScopes.scopes': Problems parsing selector {0}.",
205
206
  selectorString
206
207
  )));
@@ -215,7 +216,7 @@ class TokenClassificationExtensionPoints {
215
216
  const tmScopes = contribution.scopes[selectorString];
216
217
  try {
217
218
  const selector = tokenClassificationRegistry.parseTokenSelector(selectorString, contribution.language);
218
- tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe: ( (tmScopes.map(s => s.split(' ')))) });
219
+ tokenClassificationRegistry.registerTokenStyleDefault(selector, { scopesToProbe: ( tmScopes.map(s => s.split(' '))) });
219
220
  }
220
221
  catch (e) {
221
222
  }
@@ -232,9 +233,9 @@ let TokenClassificationExtensionPointWorkbenchContribution = class TokenClassifi
232
233
  this.instantiationService.createInstance(TokenClassificationExtensionPoints);
233
234
  }
234
235
  };
235
- TokenClassificationExtensionPointWorkbenchContribution = ( (__decorate([
236
- ( (__param(0, IInstantiationService)))
237
- ], TokenClassificationExtensionPointWorkbenchContribution)));
238
- registerWorkbenchContribution2(TokenClassificationExtensionPointWorkbenchContribution.ID, TokenClassificationExtensionPointWorkbenchContribution, 1 );
236
+ TokenClassificationExtensionPointWorkbenchContribution = ( __decorate([
237
+ ( __param(0, IInstantiationService))
238
+ ], TokenClassificationExtensionPointWorkbenchContribution));
239
+ registerWorkbenchContribution2(TokenClassificationExtensionPointWorkbenchContribution.ID, TokenClassificationExtensionPointWorkbenchContribution, WorkbenchPhase.BlockStartup);
239
240
 
240
241
  export { TokenClassificationExtensionPoints };
package/worker.js CHANGED
@@ -1 +1,2 @@
1
- import './workers/textmate.worker.js';
1
+
2
+ import './vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.workerMain.js';
@@ -0,0 +1 @@
1
+
package/textmate.js DELETED
@@ -1,33 +0,0 @@
1
- import { StandaloneServices } from 'vscode/vscode/vs/editor/standalone/browser/standaloneServices';
2
- import { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature.service';
3
- export { ITextMateTokenizationService } from 'vscode/vscode/vs/workbench/services/textMate/browser/textMateTokenizationFeature.service';
4
- import { SyncDescriptor } from 'vscode/vscode/vs/platform/instantiation/common/descriptors';
5
- import { TextMateTokenizationFeature } from './vscode/src/vs/workbench/services/textMate/browser/textMateTokenizationFeatureImpl.js';
6
- import { ILifecycleService } from 'vscode/vscode/vs/workbench/services/lifecycle/common/lifecycle.service';
7
- import getServiceOverride$1 from '@codingame/monaco-vscode-files-service-override';
8
- import { registerServiceInitializeParticipant } from 'vscode/lifecycle';
9
- import { registerAssets } from 'vscode/assets';
10
- import './vscode/src/vs/workbench/services/themes/common/tokenClassificationExtensionPoint.js';
11
- import 'vscode/vscode/vs/workbench/contrib/codeEditor/browser/inspectEditorTokens/inspectEditorTokens';
12
-
13
- const _onigWasm = new URL('vscode-oniguruma/release/onig.wasm', import.meta.url).href;
14
- registerAssets({
15
- 'vscode-oniguruma/../onig.wasm': _onigWasm,
16
- 'vs/../../node_modules/vscode-oniguruma/release/onig.wasm': _onigWasm
17
- });
18
- registerServiceInitializeParticipant(async (accessor) => {
19
- void accessor
20
- .get(ILifecycleService)
21
- .when(2 )
22
- .then(() => {
23
- StandaloneServices.get(ITextMateTokenizationService);
24
- });
25
- });
26
- function getServiceOverride() {
27
- return {
28
- ...getServiceOverride$1(),
29
- [( ITextMateTokenizationService.toString())]: new SyncDescriptor(TextMateTokenizationFeature, [], false)
30
- };
31
- }
32
-
33
- export { getServiceOverride as default };
@@ -1 +0,0 @@
1
- import '../vscode/src/vs/workbench/services/textMate/browser/backgroundTokenization/worker/textMateTokenizationWorker.workerMain.js';