@codingame/monaco-vscode-language-detection-worker-service-override 1.82.4 → 1.82.5-next.0

package/external/tslib/tslib.es6.js

@@ -0,0 +1,11 @@
+function __decorate(decorators, target, key, desc) {
+    var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
+    if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
+    else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
+    return c > 3 && r && Object.defineProperty(target, key, r), r;
+}
+function __param(paramIndex, decorator) {
+    return function (target, key) { decorator(target, key, paramIndex); }
+}
+
+export { __decorate, __param };

package/index.d.ts

@@ -1 +1 @@
-export { default } from 'vscode/service-override/languageDetectionWorker'
+export { default } from './languageDetectionWorker.js';

package/index.js

@@ -1 +1 @@
-export { default } from 'vscode/service-override/languageDetectionWorker'
+export { default } from './languageDetectionWorker.js';

package/languageDetectionWorker.d.ts

@@ -0,0 +1,5 @@
+import { IEditorOverrideServices } from 'vscode/vscode/vs/editor/standalone/browser/standaloneServices';
+
+declare function getServiceOverride(): IEditorOverrideServices;
+
+export { getServiceOverride as default };

package/languageDetectionWorker.js

@@ -0,0 +1,16 @@
+import { SyncDescriptor } from 'monaco-editor/esm/vs/platform/instantiation/common/descriptors.js';
+import { LanguageDetectionService } from './vscode/src/vs/workbench/services/languageDetection/browser/languageDetectionWorkerServiceImpl.js';
+import { ILanguageDetectionService } from 'vscode/vscode/vs/workbench/services/languageDetection/common/languageDetectionWorkerService';
+import { registerAssets } from 'vscode/assets';
+
+registerAssets({
+    'vs/../../node_modules/@vscode/vscode-languagedetection/model/model.json': new URL('@vscode/vscode-languagedetection/model/model.json', import.meta.url).href,
+    'vs/../../node_modules/@vscode/vscode-languagedetection/model/group1-shard1of1.bin': new URL('@vscode/vscode-languagedetection/model/group1-shard1of1.bin', import.meta.url).href
+});
+function getServiceOverride() {
+    return {
+        [( ILanguageDetectionService.toString())]: new SyncDescriptor(LanguageDetectionService, [], false)
+    };
+}
+
+export { getServiceOverride as default };

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@codingame/monaco-vscode-language-detection-worker-service-override",
-  "version": "1.82.4",
+  "version": "1.82.5-next.0",
   "keywords": [],
   "author": {
     "name": "CodinGame",
@@ -18,7 +18,7 @@
   "module": "index.js",
   "types": "index.d.ts",
   "dependencies": {
-    "vscode": "npm:@codingame/monaco-vscode-api@1.82.4",
+    "vscode": "npm:@codingame/monaco-vscode-api@1.82.5-next.0",
     "@vscode/vscode-languagedetection": "^1.0.22",
     "monaco-editor": "0.43.0"
   },

package/vscode/src/vs/workbench/services/languageDetection/browser/languageDetectionSimpleWorker.js

@@ -0,0 +1,175 @@
+import { ModelOperations } from '@vscode/vscode-languagedetection';
+import { StopWatch } from 'monaco-editor/esm/vs/base/common/stopwatch.js';
+import { EditorSimpleWorker } from 'monaco-editor/esm/vs/editor/common/services/editorSimpleWorker.js';
+
+function create(host) {
+    return ( new LanguageDetectionSimpleWorker(host, null));
+}
+class LanguageDetectionSimpleWorker extends EditorSimpleWorker {
+    constructor() {
+        super(...arguments);
+        this._loadFailed = false;
+        this.modelIdToCoreId = ( new Map());
+    }
+    static { this.expectedRelativeConfidence = 0.2; }
+    static { this.positiveConfidenceCorrectionBucket1 = 0.05; }
+    static { this.positiveConfidenceCorrectionBucket2 = 0.025; }
+    static { this.negativeConfidenceCorrection = 0.5; }
+    async detectLanguage(uri, langBiases, preferHistory, supportedLangs) {
+        const languages = [];
+        const confidences = [];
+        const stopWatch = ( new StopWatch());
+        const documentTextSample = this.getTextForDetection(uri);
+        if (!documentTextSample) {
+            return;
+        }
+        const neuralResolver = async () => {
+            for await (const language of this.detectLanguagesImpl(documentTextSample)) {
+                if (!( this.modelIdToCoreId.has(language.languageId))) {
+                    this.modelIdToCoreId.set(language.languageId, await this._host.fhr('getLanguageId', [language.languageId]));
+                }
+                const coreId = this.modelIdToCoreId.get(language.languageId);
+                if (coreId && (!supportedLangs?.length || supportedLangs.includes(coreId))) {
+                    languages.push(coreId);
+                    confidences.push(language.confidence);
+                }
+            }
+            stopWatch.stop();
+            if (languages.length) {
+                this._host.fhr('sendTelemetryEvent', [languages, confidences, stopWatch.elapsed()]);
+                return languages[0];
+            }
+            return undefined;
+        };
+        const neural = await neuralResolver();
+        if (neural) {
+            return neural;
+        }
+        return undefined;
+    }
+    getTextForDetection(uri) {
+        const editorModel = this._getModel(uri);
+        if (!editorModel) {
+            return;
+        }
+        const end = editorModel.positionAt(10000);
+        const content = editorModel.getValueInRange({
+            startColumn: 1,
+            startLineNumber: 1,
+            endColumn: end.column,
+            endLineNumber: end.lineNumber
+        });
+        return content;
+    }
+    async getModelOperations() {
+        if (this._modelOperations) {
+            return this._modelOperations;
+        }
+        this._modelOperations = ( new ModelOperations({
+            modelJsonLoaderFunc: async () => {
+                const response = await fetch(await this._host.fhr('getModelJsonUri', []));
+                try {
+                    const modelJSON = await response.json();
+                    return modelJSON;
+                }
+                catch (e) {
+                    const message = `Failed to parse model JSON.`;
+                    throw new Error(message);
+                }
+            },
+            weightsLoaderFunc: async () => {
+                const response = await fetch(await this._host.fhr('getWeightsUri', []));
+                const buffer = await response.arrayBuffer();
+                return buffer;
+            }
+        }));
+        return this._modelOperations;
+    }
+    adjustLanguageConfidence(modelResult) {
+        switch (modelResult.languageId) {
+            case 'js':
+            case 'html':
+            case 'json':
+            case 'ts':
+            case 'css':
+            case 'py':
+            case 'xml':
+            case 'php':
+                modelResult.confidence += LanguageDetectionSimpleWorker.positiveConfidenceCorrectionBucket1;
+                break;
+            case 'cpp':
+            case 'sh':
+            case 'java':
+            case 'cs':
+            case 'c':
+                modelResult.confidence += LanguageDetectionSimpleWorker.positiveConfidenceCorrectionBucket2;
+                break;
+            case 'bat':
+            case 'ini':
+            case 'makefile':
+            case 'sql':
+            case 'csv':
+            case 'toml':
+                modelResult.confidence -= LanguageDetectionSimpleWorker.negativeConfidenceCorrection;
+                break;
+        }
+        return modelResult;
+    }
+    async *detectLanguagesImpl(content) {
+        if (this._loadFailed) {
+            return;
+        }
+        let modelOperations;
+        try {
+            modelOperations = await this.getModelOperations();
+        }
+        catch (e) {
+            console.log(e);
+            this._loadFailed = true;
+            return;
+        }
+        let modelResults;
+        try {
+            modelResults = await modelOperations.runModel(content);
+        }
+        catch (e) {
+            console.warn(e);
+        }
+        if (!modelResults
+            || modelResults.length === 0
+            || modelResults[0].confidence < LanguageDetectionSimpleWorker.expectedRelativeConfidence) {
+            return;
+        }
+        const firstModelResult = this.adjustLanguageConfidence(modelResults[0]);
+        if (firstModelResult.confidence < LanguageDetectionSimpleWorker.expectedRelativeConfidence) {
+            return;
+        }
+        const possibleLanguages = [firstModelResult];
+        for (let current of modelResults) {
+            if (current === firstModelResult) {
+                continue;
+            }
+            current = this.adjustLanguageConfidence(current);
+            const currentHighest = possibleLanguages[possibleLanguages.length - 1];
+            if (currentHighest.confidence - current.confidence >= LanguageDetectionSimpleWorker.expectedRelativeConfidence) {
+                while (possibleLanguages.length) {
+                    yield possibleLanguages.shift();
+                }
+                if (current.confidence > LanguageDetectionSimpleWorker.expectedRelativeConfidence) {
+                    possibleLanguages.push(current);
+                    continue;
+                }
+                return;
+            }
+            else {
+                if (current.confidence > LanguageDetectionSimpleWorker.expectedRelativeConfidence) {
+                    possibleLanguages.push(current);
+                    continue;
+                }
+                return;
+            }
+        }
+    }
+}
+
+export { LanguageDetectionSimpleWorker, create };

package/vscode/src/vs/workbench/services/languageDetection/browser/languageDetectionWorkerServiceImpl.js

@@ -0,0 +1,261 @@
+import { __decorate, __param } from '../../../../../../../external/tslib/tslib.es6.js';
+import { Disposable } from 'monaco-editor/esm/vs/base/common/lifecycle.js';
+import { LanguageDetectionStatsId } from 'vscode/vscode/vs/workbench/services/languageDetection/common/languageDetectionWorkerService';
+import { FileAccess, Schemas, nodeModulesPath, nodeModulesAsarPath } from 'monaco-editor/esm/vs/base/common/network.js';
+import { IWorkbenchEnvironmentService } from 'vscode/vscode/vs/workbench/services/environment/common/environmentService';
+import { IConfigurationService } from 'monaco-editor/esm/vs/platform/configuration/common/configuration.js';
+import { ILanguageService } from 'monaco-editor/esm/vs/editor/common/languages/language.js';
+import { URI } from 'monaco-editor/esm/vs/base/common/uri.js';
+import { isWeb } from 'monaco-editor/esm/vs/base/common/platform.js';
+import { IModelService } from 'monaco-editor/esm/vs/editor/common/services/model.js';
+import { SimpleWorkerClient } from 'monaco-editor/esm/vs/base/common/worker/simpleWorker.js';
+import { ITelemetryService } from 'monaco-editor/esm/vs/platform/telemetry/common/telemetry.js';
+import { EditorWorkerClient, EditorWorkerHost } from 'monaco-editor/esm/vs/editor/browser/services/editorWorkerService.js';
+import { ILanguageConfigurationService } from 'monaco-editor/esm/vs/editor/common/languages/languageConfigurationRegistry.js';
+import { IDiagnosticsService } from 'vscode/vscode/vs/platform/diagnostics/common/diagnostics';
+import { IWorkspaceContextService } from 'monaco-editor/esm/vs/platform/workspace/common/workspace.js';
+import { IEditorService } from 'vscode/vscode/vs/workbench/services/editor/common/editorService';
+import { IStorageService } from 'monaco-editor/esm/vs/platform/storage/common/storage.js';
+import { LRUCache } from 'monaco-editor/esm/vs/base/common/map.js';
+import { ILogService } from 'monaco-editor/esm/vs/platform/log/common/log.js';
+
+var LanguageDetectionService_1;
+const TOP_LANG_COUNTS = 12;
+const regexpModuleLocation = `${nodeModulesPath}/vscode-regexp-languagedetection`;
+const regexpModuleLocationAsar = `${nodeModulesAsarPath}/vscode-regexp-languagedetection`;
+const moduleLocation = `${nodeModulesPath}/@vscode/vscode-languagedetection`;
+const moduleLocationAsar = `${nodeModulesAsarPath}/@vscode/vscode-languagedetection`;
+let LanguageDetectionService = class LanguageDetectionService extends Disposable {
+    static { LanguageDetectionService_1 = this; }
+    static { this.enablementSettingKey = 'workbench.editor.languageDetection'; }
+    static { this.historyBasedEnablementConfig = 'workbench.editor.historyBasedLanguageDetection'; }
+    static { this.preferHistoryConfig = 'workbench.editor.preferHistoryBasedLanguageDetection'; }
+    static { this.workspaceOpenedLanguagesStorageKey = 'workbench.editor.languageDetectionOpenedLanguages.workspace'; }
+    static { this.globalOpenedLanguagesStorageKey = 'workbench.editor.languageDetectionOpenedLanguages.global'; }
+    constructor(_environmentService, languageService, _configurationService, _diagnosticsService, _workspaceContextService, modelService, _editorService, telemetryService, storageService, _logService, languageConfigurationService) {
+        super();
+        this._environmentService = _environmentService;
+        this._configurationService = _configurationService;
+        this._diagnosticsService = _diagnosticsService;
+        this._workspaceContextService = _workspaceContextService;
+        this._editorService = _editorService;
+        this._logService = _logService;
+        this.hasResolvedWorkspaceLanguageIds = false;
+        this.workspaceLanguageIds = ( new Set());
+        this.sessionOpenedLanguageIds = ( new Set());
+        this.historicalGlobalOpenedLanguageIds = ( new LRUCache(TOP_LANG_COUNTS));
+        this.historicalWorkspaceOpenedLanguageIds = ( new LRUCache(TOP_LANG_COUNTS));
+        this.dirtyBiases = true;
+        this.langBiases = {};
+        this._languageDetectionWorkerClient = ( new LanguageDetectionWorkerClient(
+            modelService,
+            languageService,
+            telemetryService,
+            this._environmentService.isBuilt && !isWeb
+                ? ( ( FileAccess.asBrowserUri(`${moduleLocationAsar}/dist/lib/index.js`)).toString(true))
+                : ( ( FileAccess.asBrowserUri(`${moduleLocation}/dist/lib/index.js`)).toString(true)),
+            this._environmentService.isBuilt && !isWeb
+                ? ( ( FileAccess.asBrowserUri(`${moduleLocationAsar}/model/model.json`)).toString(true))
+                : ( ( FileAccess.asBrowserUri(`${moduleLocation}/model/model.json`)).toString(true)),
+            this._environmentService.isBuilt && !isWeb
+                ? ( ( FileAccess.asBrowserUri(`${moduleLocationAsar}/model/group1-shard1of1.bin`)).toString(true))
+                : ( ( FileAccess.asBrowserUri(`${moduleLocation}/model/group1-shard1of1.bin`)).toString(true)),
+            this._environmentService.isBuilt && !isWeb
+                ? ( ( FileAccess.asBrowserUri(`${regexpModuleLocationAsar}/dist/index.js`)).toString(true))
+                : ( ( FileAccess.asBrowserUri(`${regexpModuleLocation}/dist/index.js`)).toString(true)),
+            languageConfigurationService
+        ));
+        this.initEditorOpenedListeners(storageService);
+    }
+    async resolveWorkspaceLanguageIds() {
+        if (this.hasResolvedWorkspaceLanguageIds) {
+            return;
+        }
+        this.hasResolvedWorkspaceLanguageIds = true;
+        const fileExtensions = await this._diagnosticsService.getWorkspaceFileExtensions(this._workspaceContextService.getWorkspace());
+        let count = 0;
+        for (const ext of fileExtensions.extensions) {
+            const langId = this._languageDetectionWorkerClient.getLanguageId(ext);
+            if (langId && count < TOP_LANG_COUNTS) {
+                this.workspaceLanguageIds.add(langId);
+                count++;
+                if (count > TOP_LANG_COUNTS) {
+                    break;
+                }
+            }
+        }
+        this.dirtyBiases = true;
+    }
+    isEnabledForLanguage(languageId) {
+        return !!languageId && this._configurationService.getValue(LanguageDetectionService_1.enablementSettingKey, { overrideIdentifier: languageId });
+    }
+    getLanguageBiases() {
+        if (!this.dirtyBiases) {
+            return this.langBiases;
+        }
+        const biases = {};
+        this.sessionOpenedLanguageIds.forEach(lang => biases[lang] = (biases[lang] ?? 0) + 7);
+        this.workspaceLanguageIds.forEach(lang => biases[lang] = (biases[lang] ?? 0) + 5);
+        [...( this.historicalWorkspaceOpenedLanguageIds.keys())].forEach(lang => biases[lang] = (biases[lang] ?? 0) + 3);
+        [...( this.historicalGlobalOpenedLanguageIds.keys())].forEach(lang => biases[lang] = (biases[lang] ?? 0) + 1);
+        this._logService.trace('Session Languages:', JSON.stringify([...this.sessionOpenedLanguageIds]));
+        this._logService.trace('Workspace Languages:', JSON.stringify([...this.workspaceLanguageIds]));
+        this._logService.trace('Historical Workspace Opened Languages:', JSON.stringify([...( this.historicalWorkspaceOpenedLanguageIds.keys())]));
+        this._logService.trace('Historical Globally Opened Languages:', JSON.stringify([...( this.historicalGlobalOpenedLanguageIds.keys())]));
+        this._logService.trace('Computed Language Detection Biases:', JSON.stringify(biases));
+        this.dirtyBiases = false;
+        this.langBiases = biases;
+        return biases;
+    }
+    async detectLanguage(resource, supportedLangs) {
+        const useHistory = this._configurationService.getValue(LanguageDetectionService_1.historyBasedEnablementConfig);
+        const preferHistory = this._configurationService.getValue(LanguageDetectionService_1.preferHistoryConfig);
+        if (useHistory) {
+            await this.resolveWorkspaceLanguageIds();
+        }
+        const biases = useHistory ? this.getLanguageBiases() : undefined;
+        return this._languageDetectionWorkerClient.detectLanguage(resource, biases, preferHistory, supportedLangs);
+    }
+    initEditorOpenedListeners(storageService) {
+        try {
+            const globalLangHistoryData = JSON.parse(storageService.get(LanguageDetectionService_1.globalOpenedLanguagesStorageKey, 0 , '[]'));
+            this.historicalGlobalOpenedLanguageIds.fromJSON(globalLangHistoryData);
+        }
+        catch (e) {
+            console.error(e);
+        }
+        try {
+            const workspaceLangHistoryData = JSON.parse(storageService.get(LanguageDetectionService_1.workspaceOpenedLanguagesStorageKey, 1 , '[]'));
+            this.historicalWorkspaceOpenedLanguageIds.fromJSON(workspaceLangHistoryData);
+        }
+        catch (e) {
+            console.error(e);
+        }
+        this._register(this._editorService.onDidActiveEditorChange(() => {
+            const activeLanguage = this._editorService.activeTextEditorLanguageId;
+            if (activeLanguage && this._editorService.activeEditor?.resource?.scheme !== Schemas.untitled) {
+                this.sessionOpenedLanguageIds.add(activeLanguage);
+                this.historicalGlobalOpenedLanguageIds.set(activeLanguage, true);
+                this.historicalWorkspaceOpenedLanguageIds.set(activeLanguage, true);
+                storageService.store(LanguageDetectionService_1.globalOpenedLanguagesStorageKey, JSON.stringify(this.historicalGlobalOpenedLanguageIds.toJSON()), 0 , 1 );
+                storageService.store(LanguageDetectionService_1.workspaceOpenedLanguagesStorageKey, JSON.stringify(this.historicalWorkspaceOpenedLanguageIds.toJSON()), 1 , 1 );
+                this.dirtyBiases = true;
+            }
+        }));
+    }
+};
+LanguageDetectionService = LanguageDetectionService_1 = ( __decorate([
+    ( __param(0, IWorkbenchEnvironmentService)),
+    ( __param(1, ILanguageService)),
+    ( __param(2, IConfigurationService)),
+    ( __param(3, IDiagnosticsService)),
+    ( __param(4, IWorkspaceContextService)),
+    ( __param(5, IModelService)),
+    ( __param(6, IEditorService)),
+    ( __param(7, ITelemetryService)),
+    ( __param(8, IStorageService)),
+    ( __param(9, ILogService)),
+    ( __param(10, ILanguageConfigurationService))
+], LanguageDetectionService));
+class LanguageDetectionWorkerClient extends EditorWorkerClient {
+    constructor(modelService, _languageService, _telemetryService, _indexJsUri, _modelJsonUri, _weightsUri, _regexpModelUri, languageConfigurationService) {
+        super(modelService, true, 'languageDetectionWorkerService', languageConfigurationService);
+        this._languageService = _languageService;
+        this._telemetryService = _telemetryService;
+        this._indexJsUri = _indexJsUri;
+        this._modelJsonUri = _modelJsonUri;
+        this._weightsUri = _weightsUri;
+        this._regexpModelUri = _regexpModelUri;
+    }
+    _getOrCreateLanguageDetectionWorker() {
+        if (this.workerPromise) {
+            return this.workerPromise;
+        }
+        this.workerPromise = ( new Promise((resolve, reject) => {
+            resolve(this._register(( new SimpleWorkerClient(
+                this._workerFactory,
+                'vs/workbench/services/languageDetection/browser/languageDetectionSimpleWorker',
+                ( new EditorWorkerHost(this))
+            ))));
+        }));
+        return this.workerPromise;
+    }
+    _guessLanguageIdByUri(uri) {
+        const guess = this._languageService.guessLanguageIdByFilepathOrFirstLine(uri);
+        if (guess && guess !== 'unknown') {
+            return guess;
+        }
+        return undefined;
+    }
+    async _getProxy() {
+        return (await this._getOrCreateLanguageDetectionWorker()).getProxyObject();
+    }
+    async fhr(method, args) {
+        switch (method) {
+            case 'getIndexJsUri':
+                return this.getIndexJsUri();
+            case 'getModelJsonUri':
+                return this.getModelJsonUri();
+            case 'getWeightsUri':
+                return this.getWeightsUri();
+            case 'getRegexpModelUri':
+                return this.getRegexpModelUri();
+            case 'getLanguageId':
+                return this.getLanguageId(args[0]);
+            case 'sendTelemetryEvent':
+                return this.sendTelemetryEvent(args[0], args[1], args[2]);
+            default:
+                return super.fhr(method, args);
+        }
+    }
+    async getIndexJsUri() {
+        return this._indexJsUri;
+    }
+    getLanguageId(languageIdOrExt) {
+        if (!languageIdOrExt) {
+            return undefined;
+        }
+        if (this._languageService.isRegisteredLanguageId(languageIdOrExt)) {
+            return languageIdOrExt;
+        }
+        const guessed = this._guessLanguageIdByUri(URI.file(`file.${languageIdOrExt}`));
+        if (!guessed || guessed === 'unknown') {
+            return undefined;
+        }
+        return guessed;
+    }
+    async getModelJsonUri() {
+        return this._modelJsonUri;
+    }
+    async getWeightsUri() {
+        return this._weightsUri;
+    }
+    async getRegexpModelUri() {
+        return this._regexpModelUri;
+    }
+    async sendTelemetryEvent(languages, confidences, timeSpent) {
+        this._telemetryService.publicLog2(LanguageDetectionStatsId, {
+            languages: languages.join(','),
+            confidences: confidences.join(','),
+            timeSpent
+        });
+    }
+    async detectLanguage(resource, langBiases, preferHistory, supportedLangs) {
+        const startTime = Date.now();
+        const quickGuess = this._guessLanguageIdByUri(resource);
+        if (quickGuess) {
+            return quickGuess;
+        }
+        await this._withSyncedResources([resource]);
+        const modelId = await (await this._getProxy()).detectLanguage(( resource.toString()), langBiases, preferHistory, supportedLangs);
+        const languageId = this.getLanguageId(modelId);
+        const LanguageDetectionStatsId = 'automaticlanguagedetection.perf';
+        this._telemetryService.publicLog2(LanguageDetectionStatsId, {
+            timeSpent: Date.now() - startTime,
+            detection: languageId || 'unknown',
+        });
+        return languageId;
+    }
+}
+
+export { LanguageDetectionService, LanguageDetectionWorkerClient };

package/worker.js

@@ -1 +1 @@
-import 'vscode/workers/languageDetection.worker'
+import './workers/languageDetection.worker.js';

package/workers/languageDetection.worker.js

@@ -0,0 +1,9 @@
+import { create } from '../vscode/src/vs/workbench/services/languageDetection/browser/languageDetectionSimpleWorker.js';
+import { SimpleWorkerServer } from 'monaco-editor/esm/vs/base/common/worker/simpleWorker.js';
+
+const simpleWorker = new SimpleWorkerServer((msg) => {
+    globalThis.postMessage(msg);
+}, create);
+globalThis.onmessage = (e) => {
+    simpleWorker.onmessage(e.data);
+};