@theia/remote 1.43.0

package/src/electron-node/setup/remote-copy-contribution.ts

@@ -0,0 +1,90 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { ApplicationPackage } from '@theia/core/shared/@theia/application-package';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { glob as globCallback } from 'glob';
+import { promisify } from 'util';
+import * as path from 'path';
+import { MaybePromise } from '@theia/core';
+
+const promiseGlob = promisify(globCallback);
+
+export const RemoteCopyContribution = Symbol('RemoteCopyContribution');
+
+export interface RemoteCopyContribution {
+    copy(registry: RemoteCopyRegistry): MaybePromise<void>
+}
+
+export interface RemoteCopyOptions {
+    /**
+     * The mode that the file should be set to once copied to the remote.
+     *
+     * Only relevant for POSIX-like systems
+     */
+    mode?: number;
+}
+
+export interface RemoteFile {
+    path: string
+    target: string
+    options?: RemoteCopyOptions;
+}
+
+@injectable()
+export class RemoteCopyRegistry {
+
+    @inject(ApplicationPackage)
+    protected readonly applicationPackage: ApplicationPackage;
+
+    protected readonly files: RemoteFile[] = [];
+
+    getFiles(): RemoteFile[] {
+        return this.files.slice();
+    }
+
+    async glob(pattern: string, target?: string): Promise<void> {
+        const projectPath = this.applicationPackage.projectPath;
+        const globResult = await promiseGlob(pattern, {
+            cwd: projectPath
+        });
+        const relativeFiles = globResult.map(file => path.relative(projectPath, file));
+        for (const file of relativeFiles) {
+            const targetFile = this.withTarget(file, target);
+            this.files.push({
+                path: file,
+                target: targetFile
+            });
+        }
+    }
+
+    file(file: string, target?: string, options?: RemoteCopyOptions): void {
+        const targetFile = this.withTarget(file, target);
+        this.files.push({
+            path: file,
+            target: targetFile,
+            options
+        });
+    }
+
+    async directory(dir: string, target?: string): Promise<void> {
+        return this.glob(dir + '/**', target);
+    }
+
+    protected withTarget(file: string, target?: string): string {
+        return target ? path.join(target, file) : file;
+    }
+}

package/src/electron-node/setup/remote-copy-service.ts

@@ -0,0 +1,114 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as archiver from 'archiver';
+import * as path from 'path';
+import * as fs from 'fs';
+import * as os from 'os';
+import { ApplicationPackage } from '@theia/core/shared/@theia/application-package';
+import { inject, injectable, named } from '@theia/core/shared/inversify';
+import { RemoteConnection, RemotePlatform } from '../remote-types';
+import { RemoteNativeDependencyService } from './remote-native-dependency-service';
+import { ContributionProvider } from '@theia/core';
+import { RemoteCopyContribution, RemoteCopyRegistry, RemoteFile } from './remote-copy-contribution';
+
+@injectable()
+export class RemoteCopyService {
+
+    @inject(ApplicationPackage)
+    protected readonly applicationPackage: ApplicationPackage;
+
+    @inject(RemoteCopyRegistry)
+    protected readonly copyRegistry: RemoteCopyRegistry;
+
+    @inject(RemoteNativeDependencyService)
+    protected readonly nativeDependencyService: RemoteNativeDependencyService;
+
+    @inject(ContributionProvider) @named(RemoteCopyContribution)
+    protected readonly copyContributions: ContributionProvider<RemoteCopyContribution>;
+
+    protected initialized = false;
+
+    async copyToRemote(remote: RemoteConnection, remotePlatform: RemotePlatform, destination: string): Promise<void> {
+        const zipName = path.basename(destination);
+        const projectPath = this.applicationPackage.projectPath;
+        const tempDir = await this.getTempDir();
+        const zipPath = path.join(tempDir, zipName);
+        const files = await this.getFiles(remotePlatform, tempDir);
+        // We stream to a file here and then copy it because it is faster
+        // Copying files via sftp is 4x times faster compared to readable streams
+        const stream = fs.createWriteStream(zipPath);
+        const archive = archiver('tar', {
+            gzip: true
+        });
+        archive.pipe(stream);
+        for (const file of files) {
+            const filePath = path.isAbsolute(file.path)
+                ? file.path
+                : path.join(projectPath, file.path);
+
+            archive.file(filePath, {
+                name: file.target,
+                mode: file.options?.mode
+            });
+        }
+        await archive.finalize();
+        await remote.copy(zipPath, destination);
+        await fs.promises.rm(tempDir, {
+            recursive: true,
+            force: true
+        });
+    }
+
+    protected async getFiles(remotePlatform: RemotePlatform, tempDir: string): Promise<RemoteFile[]> {
+        const [localFiles, nativeDependencies] = await Promise.all([
+            this.loadCopyContributions(),
+            this.loadNativeDependencies(remotePlatform, tempDir)
+        ]);
+        return [...localFiles, ...nativeDependencies];
+    }
+
+    protected async loadCopyContributions(): Promise<RemoteFile[]> {
+        if (this.initialized) {
+            return this.copyRegistry.getFiles();
+        }
+        await Promise.all(this.copyContributions.getContributions()
+            .map(copyContribution => copyContribution.copy(this.copyRegistry)));
+        this.initialized = true;
+        return this.copyRegistry.getFiles();
+    }
+
+    protected async loadNativeDependencies(remotePlatform: RemotePlatform, tempDir: string): Promise<RemoteFile[]> {
+        const dependencyFiles = await this.nativeDependencyService.downloadDependencies(remotePlatform, tempDir);
+        return dependencyFiles.map(file => ({
+            path: file.path,
+            target: file.target,
+            options: {
+                mode: file.mode
+            }
+        }));
+    }
+
+    protected async getTempDir(): Promise<string> {
+        const dir = path.join(os.tmpdir(), 'theia-remote-');
+        const tempDir = await fs.promises.mkdtemp(dir);
+        return tempDir;
+    }
+
+    protected async getRemoteDownloadLocation(): Promise<string | undefined> {
+        return undefined;
+    }
+}

package/src/electron-node/setup/remote-native-dependency-contribution.ts

@@ -0,0 +1,63 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { isObject } from '@theia/core';
+import { RequestOptions } from '@theia/core/shared/@theia/request';
+import { RemotePlatform } from '../remote-types';
+
+export interface FileDependencyResult {
+    path: string;
+    mode?: number;
+}
+
+export type DependencyDownload = FileDependencyDownload | DirectoryDependencyDownload;
+
+export interface FileDependencyDownload {
+    file: FileDependencyResult
+    buffer: Buffer
+}
+
+export namespace FileDependencyResult {
+    export function is(item: unknown): item is FileDependencyDownload {
+        return isObject(item) && 'buffer' in item && 'file' in item;
+    }
+}
+
+export interface DirectoryDependencyDownload {
+    archive: 'tar' | 'zip' | 'tgz'
+    buffer: Buffer
+}
+
+export namespace DirectoryDependencyDownload {
+    export function is(item: unknown): item is DirectoryDependencyDownload {
+        return isObject(item) && 'buffer' in item && 'archive' in item;
+    }
+}
+
+export interface DownloadOptions {
+    remotePlatform: RemotePlatform
+    theiaVersion: string;
+    download: (requestInfo: string | RequestOptions) => Promise<Buffer>
+}
+
+export const RemoteNativeDependencyContribution = Symbol('RemoteNativeDependencyContribution');
+
+/**
+ * contribution used for downloading prebuild native dependency when connecting to a remote machine with a different system
+ */
+export interface RemoteNativeDependencyContribution {
+    download(options: DownloadOptions): Promise<DependencyDownload>;
+}

package/src/electron-node/setup/remote-native-dependency-service.ts

@@ -0,0 +1,111 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { ContributionProvider, THEIA_VERSION } from '@theia/core';
+import { inject, injectable, named } from '@theia/core/shared/inversify';
+import { RequestContext, RequestService, RequestOptions } from '@theia/core/shared/@theia/request';
+import * as decompress from 'decompress';
+import * as path from 'path';
+import * as fs from 'fs/promises';
+import { DependencyDownload, DirectoryDependencyDownload, RemoteNativeDependencyContribution } from './remote-native-dependency-contribution';
+import { RemotePlatform } from '../remote-types';
+
+const decompressTar = require('decompress-tar');
+const decompressTargz = require('decompress-targz');
+const decompressUnzip = require('decompress-unzip');
+
+export const DEFAULT_HTTP_OPTIONS = {
+    method: 'GET',
+    headers: {
+        Accept: 'application/octet-stream'
+    },
+};
+
+export interface NativeDependencyFile {
+    path: string;
+    target: string;
+    mode?: number;
+}
+
+@injectable()
+export class RemoteNativeDependencyService {
+
+    @inject(ContributionProvider) @named(RemoteNativeDependencyContribution)
+    protected nativeDependencyContributions: ContributionProvider<RemoteNativeDependencyContribution>;
+
+    @inject(RequestService)
+    protected requestService: RequestService;
+
+    async downloadDependencies(remotePlatform: RemotePlatform, directory: string): Promise<NativeDependencyFile[]> {
+        const contributionResults = await Promise.all(this.nativeDependencyContributions.getContributions()
+            .map(async contribution => {
+                const result = await contribution.download({
+                    remotePlatform,
+                    theiaVersion: THEIA_VERSION,
+                    download: requestInfo => this.downloadDependency(requestInfo)
+                });
+                const dependency = await this.storeDependency(result, directory);
+                return dependency;
+            }));
+        return contributionResults.flat();
+    }
+
+    protected async downloadDependency(downloadURI: string | RequestOptions): Promise<Buffer> {
+        const options = typeof downloadURI === 'string'
+            ? { url: downloadURI, ...DEFAULT_HTTP_OPTIONS }
+            : { ...DEFAULT_HTTP_OPTIONS, ...downloadURI };
+        const req = await this.requestService.request(options);
+        if (RequestContext.isSuccess(req)) {
+            return Buffer.from(req.buffer);
+        } else {
+            throw new Error('Server error while downloading native dependency from: ' + options.url);
+        }
+    }
+
+    protected async storeDependency(dependency: DependencyDownload, directory: string): Promise<NativeDependencyFile[]> {
+        if (DirectoryDependencyDownload.is(dependency)) {
+            const archiveBuffer = dependency.buffer;
+            const plugins: unknown[] = [];
+            if (dependency.archive === 'tar') {
+                plugins.push(decompressTar());
+            } else if (dependency.archive === 'tgz') {
+                plugins.push(decompressTargz());
+            } else if (dependency.archive === 'zip') {
+                plugins.push(decompressUnzip());
+            }
+            const files = await decompress(archiveBuffer, directory, { plugins });
+            const result: NativeDependencyFile[] = await Promise.all(files.map(async file => {
+                const localPath = path.join(directory, file.path);
+                return {
+                    path: localPath,
+                    target: file.path,
+                    mode: file.mode
+                };
+            }));
+            return result;
+        } else {
+            const fileName = path.basename(dependency.file.path);
+            const localPath = path.join(directory, fileName);
+            await fs.writeFile(localPath, dependency.buffer);
+            return [{
+                path: localPath,
+                target: dependency.file.path,
+                mode: dependency.file.mode
+            }];
+        }
+    }
+
+}

package/src/electron-node/setup/remote-node-setup-service.ts

@@ -0,0 +1,123 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import * as path from 'path';
+import * as fs from '@theia/core/shared/fs-extra';
+import * as os from 'os';
+
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { RequestService } from '@theia/core/shared/@theia/request';
+import { RemoteSetupScriptService } from './remote-setup-script-service';
+import { RemotePlatform } from '../remote-types';
+import { OS } from '@theia/core';
+
+/**
+ * The current node version that Theia recommends.
+ *
+ * Native dependencies are compiled against this version.
+ */
+export const REMOTE_NODE_VERSION = '18.17.0';
+
+@injectable()
+export class RemoteNodeSetupService {
+
+    @inject(RequestService)
+    protected readonly requestService: RequestService;
+
+    @inject(RemoteSetupScriptService)
+    protected readonly scriptService: RemoteSetupScriptService;
+
+    getNodeDirectoryName(platform: RemotePlatform): string {
+        return `node-v${REMOTE_NODE_VERSION}-${this.getPlatformName(platform)}-${platform.arch}`;
+    }
+
+    protected getPlatformName(platform: RemotePlatform): string {
+        let platformId: string;
+        if (platform.os === OS.Type.Windows) {
+            platformId = 'win';
+        } else if (platform.os === OS.Type.OSX) {
+            platformId = 'darwin';
+        } else {
+            platformId = 'linux';
+        }
+        return platformId;
+    }
+
+    protected validatePlatform(platform: RemotePlatform): void {
+        if (platform.os === OS.Type.Windows && !platform.arch.match(/^x(64|86)$/)) {
+            this.throwPlatformError(platform, 'x64 and x86');
+        } else if (platform.os === OS.Type.Linux && !platform.arch.match(/^(x64|armv7l|arm64)$/)) {
+            this.throwPlatformError(platform, 'x64, armv7l and arm64');
+        } else if (platform.os === OS.Type.OSX && !platform.arch.match(/^(x64|arm64)$/)) {
+            this.throwPlatformError(platform, 'x64 and arm64');
+        }
+    }
+
+    protected throwPlatformError(platform: RemotePlatform, supportedArch: string): never {
+        throw new Error(`Invalid architecture for ${platform.os}: '${platform.arch}'. Only ${supportedArch} are supported.`);
+    }
+
+    protected getNodeFileExtension(platform: RemotePlatform): string {
+        let fileExtension: string;
+        if (platform.os === OS.Type.Windows) {
+            fileExtension = 'zip';
+        } else if (platform.os === OS.Type.OSX) {
+            fileExtension = 'tar.gz';
+        } else {
+            fileExtension = 'tar.xz';
+        }
+        return fileExtension;
+    }
+
+    getNodeFileName(platform: RemotePlatform): string {
+        return `${this.getNodeDirectoryName(platform)}.${this.getNodeFileExtension(platform)}`;
+    }
+
+    async downloadNode(platform: RemotePlatform, downloadTemplate?: string): Promise<string> {
+        this.validatePlatform(platform);
+        const fileName = this.getNodeFileName(platform);
+        const tmpdir = os.tmpdir();
+        const localPath = path.join(tmpdir, fileName);
+        if (!await fs.pathExists(localPath)) {
+            const downloadPath = this.getDownloadPath(platform, downloadTemplate);
+            const downloadResult = await this.requestService.request({
+                url: downloadPath
+            });
+            await fs.writeFile(localPath, downloadResult.buffer);
+        }
+        return localPath;
+    }
+
+    generateDownloadScript(platform: RemotePlatform, targetPath: string, downloadTemplate?: string): string {
+        this.validatePlatform(platform);
+        const fileName = this.getNodeFileName(platform);
+        const downloadPath = this.getDownloadPath(platform, downloadTemplate);
+        const zipPath = this.scriptService.joinPath(platform, targetPath, fileName);
+        const download = this.scriptService.downloadFile(platform, downloadPath, zipPath);
+        const unzip = this.scriptService.unzip(platform, zipPath, targetPath);
+        return this.scriptService.joinScript(platform, download, unzip);
+    }
+
+    protected getDownloadPath(platform: RemotePlatform, downloadTemplate?: string): string {
+        const template = downloadTemplate || 'https://nodejs.org/dist/v{version}/node-v{version}-{os}-{arch}.{ext}';
+        const downloadPath = template
+            .replace(/{version}/g, REMOTE_NODE_VERSION)
+            .replace(/{os}/g, this.getPlatformName(platform))
+            .replace(/{arch}/g, platform.arch)
+            .replace(/{ext}/g, this.getNodeFileExtension(platform));
+        return downloadPath;
+    }
+}

package/src/electron-node/setup/remote-setup-script-service.ts

@@ -0,0 +1,146 @@
+// *****************************************************************************
+// Copyright (C) 2023 TypeFox and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0-only WITH Classpath-exception-2.0
+// *****************************************************************************
+
+import { OS } from '@theia/core';
+import { inject, injectable } from '@theia/core/shared/inversify';
+import { RemotePlatform } from '../remote-types';
+
+export interface RemoteScriptStrategy {
+    exec(): string;
+    downloadFile(url: string, output: string): string;
+    unzip(file: string, directory: string): string;
+    mkdir(path: string): string;
+    home(): string;
+    joinPath(...segments: string[]): string;
+    joinScript(...segments: string[]): string;
+}
+
+@injectable()
+export class RemoteWindowsScriptStrategy implements RemoteScriptStrategy {
+
+    home(): string {
+        return 'PowerShell -Command $HOME';
+    }
+
+    exec(): string {
+        return 'PowerShell -Command';
+    }
+
+    downloadFile(url: string, output: string): string {
+        return `PowerShell -Command Invoke-WebRequest -Uri "${url}" -OutFile ${output}`;
+    }
+
+    unzip(file: string, directory: string): string {
+        return `tar -xf "${file}" -C "${directory}"`;
+    }
+
+    mkdir(path: string): string {
+        return `PowerShell -Command New-Item -Force -itemType Directory -Path "${path}"`;
+    }
+
+    joinPath(...segments: string[]): string {
+        return segments.join('\\');
+    }
+
+    joinScript(...segments: string[]): string {
+        return segments.join('\r\n');
+    }
+}
+
+@injectable()
+export class RemotePosixScriptStrategy implements RemoteScriptStrategy {
+
+    home(): string {
+        return 'eval echo ~';
+    }
+
+    exec(): string {
+        return 'sh -c';
+    }
+
+    downloadFile(url: string, output: string): string {
+        return `
+if [ "$(command -v wget)" ]; then
+    echo "Downloading using wget"
+    wget -O "${output}" "${url}"
+elif [ "$(command -v curl)" ]; then
+    echo "Downloading using curl"
+    curl "${url}" --output "${output}"
+else
+    echo "Failed to find wget or curl."
+    exit 1
+fi
+`.trim();
+    }
+
+    unzip(file: string, directory: string): string {
+        return `tar -xf "${file}" -C "${directory}"`;
+    }
+
+    mkdir(path: string): string {
+        return `mkdir -p "${path}"`;
+    }
+
+    joinPath(...segments: string[]): string {
+        return segments.join('/');
+    }
+
+    joinScript(...segments: string[]): string {
+        return segments.join('\n');
+    }
+}
+
+@injectable()
+export class RemoteSetupScriptService {
+
+    @inject(RemoteWindowsScriptStrategy)
+    protected windowsStrategy: RemoteWindowsScriptStrategy;
+
+    @inject(RemotePosixScriptStrategy)
+    protected posixStrategy: RemotePosixScriptStrategy;
+
+    protected getStrategy(platform: RemotePlatform): RemoteScriptStrategy {
+        return platform.os === OS.Type.Windows ? this.windowsStrategy : this.posixStrategy;
+    }
+
+    home(platform: RemotePlatform): string {
+        return this.getStrategy(platform).home();
+    }
+
+    exec(platform: RemotePlatform): string {
+        return this.getStrategy(platform).exec();
+    }
+
+    downloadFile(platform: RemotePlatform, url: string, output: string): string {
+        return this.getStrategy(platform).downloadFile(url, output);
+    }
+
+    unzip(platform: RemotePlatform, file: string, directory: string): string {
+        return this.getStrategy(platform).unzip(file, directory);
+    }
+
+    mkdir(platform: RemotePlatform, path: string): string {
+        return this.getStrategy(platform).mkdir(path);
+    }
+
+    joinPath(platform: RemotePlatform, ...segments: string[]): string {
+        return this.getStrategy(platform).joinPath(...segments);
+    }
+
+    joinScript(platform: RemotePlatform, ...segments: string[]): string {
+        return this.getStrategy(platform).joinScript(...segments);
+    }
+}