@theia/plugin-ext 1.63.0-next.52 → 1.63.0

package/src/main/browser/workspace-main.ts

@@ -13,7 +13,11 @@
 //
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
 // *****************************************************************************
-
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+// some code was copied and modified from https://github.com/Microsoft/vscode/blob/main/src/vs/workbench/api/browser/mainThreadWorkspace.ts
 import * as theia from '@theia/plugin';
 import { interfaces, injectable } from '@theia/core/shared/inversify';
 import { WorkspaceExt, StorageExt, MAIN_RPC_CONTEXT, WorkspaceMain, WorkspaceFolderPickOptionsMain, FindFilesOptions } from '../../common/plugin-api-rpc';
@@ -32,6 +36,9 @@ import { SearchInWorkspaceService } from '@theia/search-in-workspace/lib/browser
 import { FileStat } from '@theia/filesystem/lib/common/files';
 import { MonacoQuickInputService } from '@theia/monaco/lib/browser/monaco-quick-input-service';
 import { RequestService } from '@theia/core/shared/@theia/request';
+import { UTF16be, UTF16le, UTF8, UTF8_with_bom } from '@theia/core/lib/common/encodings';
+import { EncodingRegistry } from '@theia/core/lib/browser/encoding-registry';
+import { PreferenceService } from '@theia/core/lib/browser/preferences/preference-service';
 
 export class WorkspaceMainImpl implements WorkspaceMain, Disposable {
 
@@ -59,8 +66,12 @@ export class WorkspaceMainImpl implements WorkspaceMain, Disposable {
 
     private workspaceTrustService: WorkspaceTrustService;
 
+    private encodingRegistry: EncodingRegistry;
+
     private fsPreferences: FileSystemPreferences;
 
+    private preferenceService: PreferenceService;
+
     protected readonly toDispose = new DisposableCollection();
 
     protected workspaceSearch: Set<number> = new Set<number>();
@@ -79,7 +90,9 @@ export class WorkspaceMainImpl implements WorkspaceMain, Disposable {
         this.workspaceService = container.get(WorkspaceService);
         this.canonicalUriService = container.get(CanonicalUriService);
         this.workspaceTrustService = container.get(WorkspaceTrustService);
+        this.encodingRegistry = container.get(EncodingRegistry);
         this.fsPreferences = container.get(FileSystemPreferences);
+        this.preferenceService = container.get(PreferenceService);
 
         this.processWorkspaceFoldersChanged(this.workspaceService.tryGetRoots().map(root => root.resource.toString()));
         this.toDispose.push(this.workspaceService.onWorkspaceChanged(roots => {
@@ -318,6 +331,58 @@ export class WorkspaceMainImpl implements WorkspaceMain, Disposable {
         const canonicalUri = await this.canonicalUriService.provideCanonicalUri(new URI(uri), targetScheme, token);
         return isUndefined(canonicalUri) ? undefined : canonicalUri.toString();
     }
+
+    async $resolveDecoding(resource: UriComponents | undefined, options?: { encoding?: string }): Promise<{ preferredEncoding: string; guessEncoding: boolean }> {
+        const preferredEncoding = await this.getPreferredReadEncoding(resource, options);
+
+        return {
+            preferredEncoding,
+            guessEncoding: this.preferenceService.get('files.autoGuessEncoding', false, resource ? resource.toString() : undefined)
+        };
+    }
+
+    async $resolveEncoding(resource: UriComponents | undefined, options?: { encoding?: string }): Promise<{ encoding: string; hasBOM: boolean }> {
+        let encoding: string;
+        if (resource) {
+            encoding = await this.encodingRegistry.getEncodingForResource(URI.fromComponents(resource), options?.encoding);
+        } else {
+            encoding = options?.encoding ?? UTF8;
+        }
+        // see https://github.com/microsoft/vscode/blob/118f9ecd71a8f101b71ae19e3bf44802aa173209/src/vs/workbench/services/textfile/browser/textFileService.ts#L806
+        const hasBOM = encoding === UTF16be || encoding === UTF16le || encoding === UTF8_with_bom;
+
+        return { encoding, hasBOM };
+    }
+
+    async $getValidEncoding(uri: UriComponents | undefined, detectedEncoding: string | undefined, options: { encoding: string } | undefined): Promise<string> {
+        return this.getPreferredReadEncoding(uri, options, detectedEncoding);
+    }
+
+    async getPreferredReadEncoding(uri: UriComponents | undefined, options?: { encoding?: string }, detectedEncoding?: string): Promise<string> {
+        let preferredEncoding: string | undefined;
+
+        // either encoding is passed as an option
+        // or we have a detected encoding,
+        // or we are looking in the preferences
+        // or we default at UTF8
+        if (options?.encoding) {
+            if (detectedEncoding === UTF8_with_bom && options.encoding === UTF8) {
+                preferredEncoding = UTF8_with_bom; // indicate the file has BOM if we are to resolve with UTF 8
+            } else {
+                preferredEncoding = options.encoding; // give passed in encoding highest priority
+            }
+        } else if (typeof detectedEncoding === 'string') {
+            preferredEncoding = detectedEncoding;
+        }
+        let encoding;
+        if (uri) {
+            encoding = await this.encodingRegistry.getEncodingForResource(URI.fromComponents(uri), preferredEncoding);
+        } else {
+            encoding = preferredEncoding ?? UTF8;
+        }
+
+        return encoding;
+    }
 }
 
 /**

package/src/plugin/document-data.ts

@@ -34,13 +34,15 @@ export class DocumentDataExt {
 
     private disposed = false;
     private dirty: boolean;
+    private encoding: string;
     private _document: theia.TextDocument;
     private textLines = new Array<theia.TextLine>();
     private lineStarts: PrefixSumComputer | undefined;
 
     constructor(private proxy: DocumentsMain, private uri: URI, private lines: string[], private eol: string,
-        private languageId: string, private versionId: number, isDirty: boolean) {
+        private languageId: string, private versionId: number, isDirty: boolean, encoding: string) {
         this.dirty = isDirty;
+        this.encoding = encoding;
     }
 
     dispose(): void {
@@ -74,6 +76,10 @@ export class DocumentDataExt {
         ok(!this.disposed);
         this.languageId = langId;
     }
+    acceptEncoding(encoding: string): void {
+        ok(!this.disposed);
+        this.encoding = encoding;
+    }
     get document(): theia.TextDocument {
         if (!this._document) {
             const that = this;
@@ -85,6 +91,7 @@ export class DocumentDataExt {
                 get version(): number { return that.versionId; },
                 get isClosed(): boolean { return that.disposed; },
                 get isDirty(): boolean { return that.dirty; },
+                get encoding(): string { return that.encoding; },
                 save(): Promise<boolean> { return that.save(); },
                 getText(range?): string { return range ? that.getTextInRange(range) : that.getText(); },
                 get eol(): theia.EndOfLine { return that.eol === '\n' ? EndOfLine.LF : EndOfLine.CRLF; },

package/src/plugin/documents.ts

@@ -167,6 +167,20 @@ export class DocumentsExtImpl implements DocumentsExt {
             reason: undefined,
         });
     }
+    $acceptEncodingChanged(strUrl: UriComponents, encoding: string): void {
+        const uri = URI.revive(strUrl);
+        const uriString = uri.toString();
+        const data = this.editorsAndDocuments.getDocument(uriString);
+        if (!data) {
+            throw new Error('unknown document: ' + uriString);
+        }
+        data.acceptEncoding(encoding);
+        this._onDidChangeDocument.fire({
+            document: data.document,
+            contentChanges: [],
+            reason: undefined,
+        });
+    }
     $acceptModelChanged(strUrl: UriComponents, e: ModelChangedEvent, isDirty: boolean): void {
         const uri = URI.revive(strUrl);
         const uriString = uri.toString();
@@ -238,13 +252,16 @@ export class DocumentsExtImpl implements DocumentsExt {
         }
     }
 
-    async openDocument(uri: URI): Promise<DocumentDataExt | undefined> {
+    async openDocument(uri: URI, options?: { language?: string; content?: string; encoding?: string }): Promise<DocumentDataExt | undefined> {
+        // If we have the document cached and no encoding options are provided,
+        // we should just return current document
         const cached = this.editorsAndDocuments.getDocument(uri.toString());
         if (cached) {
-            return cached;
+            if (!options?.encoding || options.encoding === cached.document.encoding) {
+                return cached;
+            }
         }
-
-        await this.proxy.$tryOpenDocument(uri);
+        await this.proxy.$tryOpenDocument(uri, options?.encoding);
         return this.editorsAndDocuments.getDocument(uri.toString());
     }
 
@@ -272,7 +289,7 @@ export class DocumentsExtImpl implements DocumentsExt {
         return this.editorsAndDocuments.getDocument(uri.toString());
     }
 
-    async createDocumentData(options?: { language?: string; content?: string }): Promise<URI> {
+    async createDocumentData(options?: { language?: string; content?: string, encoding?: string }): Promise<URI> {
         return this.proxy.$tryCreateDocument(options).then(data => URI.revive(data));
     }
 

package/src/plugin/editors-and-documents.ts

@@ -77,7 +77,8 @@ export class EditorsAndDocumentsExtImpl implements EditorsAndDocumentsExt {
                     data.EOL,
                     data.modeId,
                     data.versionId,
-                    data.isDirty
+                    data.isDirty,
+                    data.encoding
                 );
                 this.documents.set(resource.toString(), documentData);
                 addedDocuments.push(documentData);

package/src/plugin/notebook/notebook-document.ts

@@ -57,7 +57,8 @@ export class Cell {
             uri: cell.uri,
             isDirty: false,
             versionId: 1,
-            modeId: cell.language
+            modeId: cell.language,
+            encoding: 'utf8' // see https://github.com/microsoft/vscode/blob/118f9ecd71a8f101b71ae19e3bf44802aa173209/src/vs/workbench/api/common/extHostNotebookDocument.ts#L44
         };
     }
 

package/src/plugin/plugin-context.ts

@@ -785,24 +785,37 @@ export function createAPIFactory(
             onDidChangeConfiguration(listener, thisArgs?, disposables?): theia.Disposable {
                 return preferenceRegistryExt.onDidChangeConfiguration(listener, thisArgs, disposables);
             },
-            async openTextDocument(uriOrFileNameOrOptions?: theia.Uri | string | { language?: string; content?: string; }): Promise<theia.TextDocument | undefined> {
-                const options = uriOrFileNameOrOptions as { language?: string; content?: string; };
-
+            decode(content: Uint8Array, options?: { uri?: theia.Uri; encoding?: string }) {
+                return workspaceExt.decode(content, options);
+            },
+            encode(content: string, options?: { uri?: theia.Uri; encoding?: string }) {
+                return workspaceExt.encode(content, options);
+            },
+            async openTextDocument(
+                uriOrPathOrOptions?: theia.Uri | string | { language?: string; content?: string; encoding?: string },
+                options?: { readonly encoding?: string }
+            ): Promise<theia.TextDocument | undefined> {
                 let uri: URI;
-                if (typeof uriOrFileNameOrOptions === 'string') {
-                    uri = URI.file(uriOrFileNameOrOptions);
-
-                } else if (uriOrFileNameOrOptions instanceof URI) {
-                    uri = uriOrFileNameOrOptions;
-
-                } else if (!options || typeof options === 'object') {
-                    uri = await documents.createDocumentData(options);
-
+                let documentOptions: { language?: string; content?: string; encoding?: string } | undefined;
+
+                if (typeof uriOrPathOrOptions === 'string') {
+                    // It's a file path
+                    uri = URI.file(uriOrPathOrOptions);
+                    documentOptions = options;
+                } else if (URI.isUri(uriOrPathOrOptions)) {
+                    // It's a URI
+                    uri = uriOrPathOrOptions;
+                    documentOptions = options;
+                } else if (!uriOrPathOrOptions || typeof uriOrPathOrOptions === 'object') {
+                    // It's options for creating a new document
+                    documentOptions = uriOrPathOrOptions as { language?: string; content?: string; encoding?: string };
+                    uri = await documents.createDocumentData(documentOptions);
                 } else {
-                    return Promise.reject(new Error('illegal argument - uriOrFileNameOrOptions'));
+                    return Promise.reject(new Error('illegal argument - uriOrPathOrOptions'));
                 }
 
-                const data = await documents.openDocument(uri);
+                // If we have options with encoding from any source, we need to pass them to openDocument
+                const data = await documents.openDocument(uri, documentOptions);
                 return data && data.document;
             },
             async openNotebookDocument(uriOrType: theia.Uri | string, content?: NotebookData): Promise<theia.NotebookDocument | undefined> {

package/src/plugin/workspace.ts

@@ -38,13 +38,16 @@ import { EditorsAndDocumentsExtImpl } from './editors-and-documents';
 import { Disposable, URI } from './types-impl';
 import { normalize } from '@theia/core/lib/common/paths';
 import { relative } from '../common/paths-util';
-import { Schemes } from '../common/uri-components';
+import { Schemes, UriComponents } from '../common/uri-components';
 import { toWorkspaceFolder } from './type-converters';
 import { MessageRegistryExt } from './message-registry';
 import * as Converter from './type-converters';
 import { FileStat } from '@theia/filesystem/lib/common/files';
 import { isUndefinedOrNull, isUndefined } from '../common/types';
 import { PluginLogger } from './logger';
+import { consumeStream } from '@theia/core/lib/common/stream';
+import { EncodingService } from '@theia/core/lib/common/encoding-service';
+import { BinaryBuffer, BinaryBufferReadableStream } from '@theia/core/lib/common/buffer';
 
 @injectable()
 export class WorkspaceExtImpl implements WorkspaceExt {
@@ -61,6 +64,9 @@ export class WorkspaceExtImpl implements WorkspaceExt {
     private proxy: WorkspaceMain;
     private logger: PluginLogger;
 
+    @inject(EncodingService)
+    protected encodingService: EncodingService;
+
     private workspaceFoldersChangedEmitter = new Emitter<theia.WorkspaceFoldersChangeEvent>();
     public readonly onDidChangeWorkspaceFolders: Event<theia.WorkspaceFoldersChangeEvent> = this.workspaceFoldersChangedEmitter.event;
 
@@ -508,4 +514,36 @@ export class WorkspaceExtImpl implements WorkspaceExt {
     $registerEditSessionIdentityProvider(scheme: string, provider: theia.EditSessionIdentityProvider): theia.Disposable {
         return Disposable.NULL;
     }
+
+    async decode(content: Uint8Array, args?: { uri?: theia.Uri; encoding?: string }): Promise<string> {
+        const [uri, opts] = this.asEncodeDecodeParameters(args);
+        const { preferredEncoding, guessEncoding } = await this.proxy.$resolveDecoding(uri, opts);
+        const decodeStreamOptions = {
+            guessEncoding,
+            overwriteEncoding: (detectedEncoding: string | undefined) => {
+                if (detectedEncoding === null || detectedEncoding === preferredEncoding) {
+                    return Promise.resolve(preferredEncoding);
+                }
+
+                return this.proxy.$getValidEncoding(uri, detectedEncoding, opts);
+            }
+        };
+        const stream = (await this.encodingService.decodeStream(BinaryBufferReadableStream.fromBuffer(BinaryBuffer.wrap(content)), decodeStreamOptions)).stream;
+        return consumeStream(stream, s => s.join(''));
+
+    }
+
+    async encode(content: string, args?: { uri?: theia.Uri; encoding?: string }): Promise<Uint8Array> {
+        const [uri, options] = this.asEncodeDecodeParameters(args);
+        const { encoding, hasBOM } = await this.proxy.$resolveEncoding(uri, options);
+        const buffer = this.encodingService.encode(content, { encoding, hasBOM });
+        return buffer.buffer;
+    }
+
+    private asEncodeDecodeParameters(opts?: { uri?: theia.Uri; encoding?: string }): [UriComponents | undefined, { encoding: string } | undefined] {
+        const uri = Converter.isUriComponents(opts?.uri) ? opts.uri : undefined;
+        const encoding = typeof opts?.encoding === 'string' ? opts.encoding : undefined;
+
+        return [uri, encoding ? { encoding } : undefined];
+    }
 }