@theia/core 1.26.0 → 1.27.0-next.10

package/src/common/message-rpc/rpc-protocol.ts

@@ -0,0 +1,217 @@
+// *****************************************************************************
+// Copyright (C) 2021 Red Hat, Inc. and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+/* eslint-disable @typescript-eslint/no-explicit-any */
+
+import { CancellationToken, CancellationTokenSource } from '../cancellation';
+import { DisposableCollection } from '../disposable';
+import { Emitter, Event } from '../event';
+import { Deferred } from '../promise-util';
+import { Channel } from './channel';
+import { RpcMessage, RpcMessageDecoder, RpcMessageEncoder, RpcMessageType } from './rpc-message-encoder';
+import { Uint8ArrayWriteBuffer } from './uint8-array-message-buffer';
+
+/**
+ * Handles request messages received by the {@link RpcServer}.
+ */
+export type RequestHandler = (method: string, args: any[]) => Promise<any>;
+
+/**
+ * Initialization options for a {@link RpcProtocol}.
+ */
+export interface RpcProtocolOptions {
+    /**
+     * The message encoder that should be used. If `undefined` the default {@link RpcMessageEncoder} will be used.
+     */
+    encoder?: RpcMessageEncoder,
+    /**
+     * The message decoder that should be used. If `undefined` the default {@link RpcMessageDecoder} will be used.
+     */
+    decoder?: RpcMessageDecoder
+}
+
+/**
+ * Establish a bi-directional RPC protocol on top of a given channel. Bi-directional means to send
+ * sends requests and notifications to the remote side as well as receiving requests and notifications from the remote side.
+ * Clients can get a promise for a remote request result that will be either resolved or
+ * rejected depending on the success of the request. Keeps track of outstanding requests and matches replies to the appropriate request
+ * Currently, there is no timeout handling for long running requests implemented.
+ */
+export class RpcProtocol {
+    static readonly CANCELLATION_TOKEN_KEY = 'add.cancellation.token';
+
+    protected readonly pendingRequests: Map<number, Deferred<any>> = new Map();
+
+    protected nextMessageId: number = 0;
+
+    protected readonly encoder: RpcMessageEncoder;
+    protected readonly decoder: RpcMessageDecoder;
+
+    protected readonly onNotificationEmitter: Emitter<{ method: string; args: any[]; }> = new Emitter();
+    protected readonly cancellationTokenSources = new Map<number, CancellationTokenSource>();
+
+    get onNotification(): Event<{ method: string; args: any[]; }> {
+        return this.onNotificationEmitter.event;
+    }
+
+    protected toDispose = new DisposableCollection();
+
+    constructor(public readonly channel: Channel, public readonly requestHandler: RequestHandler, options: RpcProtocolOptions = {}) {
+        this.encoder = options.encoder ?? new RpcMessageEncoder();
+        this.decoder = options.decoder ?? new RpcMessageDecoder();
+        this.toDispose.push(this.onNotificationEmitter);
+        this.toDispose.push(channel.onMessage(readBuffer => this.handleMessage(this.decoder.parse(readBuffer()))));
+        channel.onClose(() => this.toDispose.dispose());
+    }
+
+    handleMessage(message: RpcMessage): void {
+        switch (message.type) {
+            case RpcMessageType.Cancel: {
+                this.handleCancel(message.id);
+                break;
+            }
+            case RpcMessageType.Request: {
+                this.handleRequest(message.id, message.method, message.args);
+                break;
+            }
+            case RpcMessageType.Notification: {
+                this.handleNotify(message.id, message.method, message.args);
+                break;
+            }
+            case RpcMessageType.Reply: {
+                this.handleReply(message.id, message.res);
+                break;
+            }
+            case RpcMessageType.ReplyErr: {
+                this.handleReplyErr(message.id, message.err);
+                break;
+            }
+        }
+    }
+
+    protected handleReply(id: number, value: any): void {
+        const replyHandler = this.pendingRequests.get(id);
+        if (replyHandler) {
+            this.pendingRequests.delete(id);
+            replyHandler.resolve(value);
+        } else {
+            throw new Error(`No reply handler for reply with id: ${id}`);
+        }
+    }
+
+    protected handleReplyErr(id: number, error: any): void {
+        try {
+            const replyHandler = this.pendingRequests.get(id);
+            if (replyHandler) {
+                this.pendingRequests.delete(id);
+                replyHandler.reject(error);
+            } else {
+                throw new Error(`No reply handler for error reply with id: ${id}`);
+            }
+        } catch (err) {
+            throw err;
+        }
+    }
+
+    sendRequest<T>(method: string, args: any[]): Promise<T> {
+        const id = this.nextMessageId++;
+        const reply = new Deferred<T>();
+
+        // The last element of the request args might be a cancellation token. As these tokens are not serializable we have to remove it from the
+        // args array and the `CANCELLATION_TOKEN_KEY` string instead.
+        const cancellationToken: CancellationToken | undefined = args.length && CancellationToken.is(args[args.length - 1]) ? args.pop() : undefined;
+        if (cancellationToken && cancellationToken.isCancellationRequested) {
+            return Promise.reject(this.cancelError());
+        }
+
+        if (cancellationToken) {
+            args.push(RpcProtocol.CANCELLATION_TOKEN_KEY);
+            cancellationToken.onCancellationRequested(() => {
+                this.sendCancel(id);
+                this.pendingRequests.get(id)?.reject(this.cancelError());
+            }
+            );
+        }
+        this.pendingRequests.set(id, reply);
+
+        const output = this.channel.getWriteBuffer();
+        this.encoder.request(output, id, method, args);
+        output.commit();
+        return reply.promise;
+    }
+
+    sendNotification(method: string, args: any[]): void {
+        const output = this.channel.getWriteBuffer();
+        this.encoder.notification(output, this.nextMessageId++, method, args);
+        output.commit();
+    }
+
+    sendCancel(requestId: number): void {
+        const output = this.channel.getWriteBuffer();
+        this.encoder.cancel(output, requestId);
+        output.commit();
+    }
+
+    cancelError(): Error {
+        const error = new Error('"Request has already been canceled by the sender"');
+        error.name = 'Cancel';
+        return error;
+    }
+
+    protected handleCancel(id: number): void {
+        const cancellationTokenSource = this.cancellationTokenSources.get(id);
+        if (cancellationTokenSource) {
+            this.cancellationTokenSources.delete(id);
+            cancellationTokenSource.cancel();
+        }
+    }
+
+    protected async handleRequest(id: number, method: string, args: any[]): Promise<void> {
+        const output = this.channel.getWriteBuffer();
+
+        // Check if the last argument of the received args is the key for indicating that a cancellation token should be used
+        // If so remove the key from the args and create a new cancellation token.
+        const addToken = args.length && args[args.length - 1] === RpcProtocol.CANCELLATION_TOKEN_KEY ? args.pop() : false;
+        if (addToken) {
+            const tokenSource = new CancellationTokenSource();
+            this.cancellationTokenSources.set(id, tokenSource);
+            args.push(tokenSource.token);
+        }
+
+        try {
+            const result = await this.requestHandler(method, args);
+            this.cancellationTokenSources.delete(id);
+            this.encoder.replyOK(output, id, result);
+            output.commit();
+        } catch (err) {
+            // In case of an error the output buffer might already contains parts of an message.
+            // => Dispose the current buffer and retrieve a new, clean one for writing the response error.
+            if (output instanceof Uint8ArrayWriteBuffer) {
+                output.dispose();
+            }
+            const errorOutput = this.channel.getWriteBuffer();
+            this.cancellationTokenSources.delete(id);
+            this.encoder.replyErr(errorOutput, id, err);
+            errorOutput.commit();
+        }
+    }
+
+    protected async handleNotify(id: number, method: string, args: any[]): Promise<void> {
+        if (this.toDispose.disposed) {
+            return;
+        }
+        this.onNotificationEmitter.fire({ method, args });
+    }
+}

package/src/common/message-rpc/uint8-array-message-buffer.spec.ts

@@ -0,0 +1,41 @@
+// *****************************************************************************
+// Copyright (C) 2021 Red Hat, Inc. and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+import { expect } from 'chai';
+import { Uint8ArrayReadBuffer, Uint8ArrayWriteBuffer } from './uint8-array-message-buffer';
+
+describe('array message buffer tests', () => {
+    it('basic read write test', () => {
+        const buffer = new Uint8Array(1024);
+        const writer = new Uint8ArrayWriteBuffer(buffer);
+
+        writer.writeUint8(8);
+        writer.writeUint32(10000);
+        writer.writeBytes(new Uint8Array([1, 2, 3, 4]));
+        writer.writeString('this is a string');
+        writer.writeString('another string');
+        writer.commit();
+
+        const written = writer.getCurrentContents();
+
+        const reader = new Uint8ArrayReadBuffer(written);
+
+        expect(reader.readUint8()).equal(8);
+        expect(reader.readUint32()).equal(10000);
+        expect(reader.readBytes()).deep.equal(new Uint8Array([1, 2, 3, 4]));
+        expect(reader.readString()).equal('this is a string');
+        expect(reader.readString()).equal('another string');
+    });
+});

package/src/common/message-rpc/uint8-array-message-buffer.ts

@@ -0,0 +1,206 @@
+// *****************************************************************************
+// Copyright (C) 2021 Red Hat, Inc. and others.
+//
+// This program and the accompanying materials are made available under the
+// terms of the Eclipse Public License v. 2.0 which is available at
+// http://www.eclipse.org/legal/epl-2.0.
+//
+// This Source Code may also be made available under the following Secondary
+// Licenses when the conditions for such availability set forth in the Eclipse
+// Public License v. 2.0 are satisfied: GNU General Public License, version 2
+// with the GNU Classpath Exception which is available at
+// https://www.gnu.org/software/classpath/license.html.
+//
+// SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
+// *****************************************************************************
+import { Disposable } from '../disposable';
+import { Emitter, Event } from '../event';
+import { ReadBuffer, WriteBuffer } from './message-buffer';
+
+/**
+ * The default {@link WriteBuffer} implementation. Uses a {@link Uint8Array} for buffering.
+ * The {@link Uint8ArrayWriteBuffer.onCommit} hook can be used to rect to on-commit events.
+ * After the {@link Uint8ArrayWriteBuffer.commit} method has been called the buffer is disposed
+ * and can no longer be used for writing data. If the writer buffer is no longer needed but the message
+ * has not been committed yet it has to be disposed manually.
+ */
+export class Uint8ArrayWriteBuffer implements WriteBuffer, Disposable {
+
+    private encoder = new TextEncoder();
+    private msg: DataView;
+    private isDisposed = false;
+    private offset: number;
+
+    constructor(private buffer: Uint8Array = new Uint8Array(1024), writePosition: number = 0) {
+        this.offset = buffer.byteOffset + writePosition;
+        this.msg = new DataView(buffer.buffer);
+    }
+
+    ensureCapacity(value: number): this {
+        let newLength = this.buffer.byteLength;
+        while (newLength < this.offset + value) {
+            newLength *= 2;
+        }
+        if (newLength !== this.buffer.byteLength) {
+            const newBuffer = new Uint8Array(newLength);
+            newBuffer.set(this.buffer);
+            this.buffer = newBuffer;
+            this.msg = new DataView(this.buffer.buffer);
+        }
+        return this;
+    }
+
+    writeLength(length: number): this {
+        if (length < 0 || (length % 1) !== 0) {
+            throw new Error(`Could not write the given length value. '${length}' is not an integer >= 0`);
+        }
+        if (length < 127) {
+            this.writeUint8(length);
+        } else {
+            this.writeUint8(128 + (length & 127));
+            this.writeLength(length >> 7);
+        }
+        return this;
+    }
+
+    writeNumber(value: number): this {
+        this.ensureCapacity(8);
+        this.msg.setFloat64(this.offset, value);
+        this.offset += 8;
+        return this;
+    }
+
+    writeUint8(value: number): this {
+        this.ensureCapacity(1);
+        this.buffer[this.offset++] = value;
+        return this;
+    }
+
+    writeUint16(value: number): this {
+        this.ensureCapacity(2);
+        this.msg.setUint16(this.offset, value);
+        this.offset += 2;
+        return this;
+    }
+
+    writeUint32(value: number): this {
+        this.ensureCapacity(4);
+        this.msg.setUint32(this.offset, value);
+        this.offset += 4;
+        return this;
+    }
+
+    writeString(value: string): this {
+        this.ensureCapacity(4 * value.length);
+        const result = this.encoder.encodeInto(value, this.buffer.subarray(this.offset + 4));
+        this.msg.setUint32(this.offset, result.written!);
+        this.offset += 4 + result.written!;
+        return this;
+    }
+
+    writeBytes(value: Uint8Array): this {
+        this.writeLength(value.byteLength);
+        this.ensureCapacity(value.length);
+        this.buffer.set(value, this.offset);
+        this.offset += value.length;
+        return this;
+    }
+
+    private onCommitEmitter = new Emitter<Uint8Array>();
+    get onCommit(): Event<Uint8Array> {
+        return this.onCommitEmitter.event;
+    }
+
+    commit(): void {
+        if (this.isDisposed) {
+            throw new Error("Could not invoke 'commit'. The WriteBuffer is already disposed.");
+        }
+        this.onCommitEmitter.fire(this.getCurrentContents());
+        this.dispose();
+    }
+
+    getCurrentContents(): Uint8Array {
+        return this.buffer.slice(this.buffer.byteOffset, this.offset);
+    }
+
+    dispose(): void {
+        if (!this.isDisposed) {
+            this.onCommitEmitter.dispose();
+            this.isDisposed = true;
+        }
+    }
+
+}
+/**
+ * The default {@link ReadBuffer} implementation. Uses a {@link Uint8Array} for buffering.
+ * Is for single message read. A message can only be read once.
+ */
+export class Uint8ArrayReadBuffer implements ReadBuffer {
+    private offset: number = 0;
+    private msg: DataView;
+    private decoder = new TextDecoder();
+
+    constructor(private readonly buffer: Uint8Array, readPosition = 0) {
+        this.offset = buffer.byteOffset + readPosition;
+        this.msg = new DataView(buffer.buffer);
+    }
+
+    readUint8(): number {
+        return this.msg.getUint8(this.offset++);
+    }
+
+    readUint16(): number {
+        const result = this.msg.getUint16(this.offset);
+        this.offset += 2;
+        return result;
+    }
+
+    readUint32(): number {
+        const result = this.msg.getUint32(this.offset);
+        this.offset += 4;
+        return result;
+    }
+
+    readLength(): number {
+        let shift = 0;
+        let byte = this.readUint8();
+        let value = (byte & 127) << shift;
+        while (byte > 127) {
+            shift += 7;
+            byte = this.readUint8();
+            value = value + ((byte & 127) << shift);
+        }
+        return value;
+    }
+
+    readNumber(): number {
+        const result = this.msg.getFloat64(this.offset);
+        this.offset += 8;
+        return result;
+    }
+
+    readString(): string {
+        const len = this.readUint32();
+        const sliceOffset = this.offset - this.buffer.byteOffset;
+        const result = this.decodeString(this.buffer.slice(sliceOffset, sliceOffset + len));
+        this.offset += len;
+        return result;
+    }
+
+    private decodeString(buf: Uint8Array): string {
+        return this.decoder.decode(buf);
+    }
+
+    readBytes(): Uint8Array {
+        const length = this.readLength();
+        const sliceOffset = this.offset - this.buffer.byteOffset;
+        const result = this.buffer.slice(sliceOffset, sliceOffset + length);
+        this.offset += length;
+        return result;
+    }
+
+    sliceAtReadPosition(): ReadBuffer {
+        const sliceOffset = this.offset - this.buffer.byteOffset;
+        return new Uint8ArrayReadBuffer(this.buffer, sliceOffset);
+    }
+}

package/src/common/messaging/abstract-connection-provider.ts

@@ -15,11 +15,10 @@
 // *****************************************************************************
 
 import { injectable, interfaces } from 'inversify';
-import { ConsoleLogger, createWebSocketConnection, Logger } from 'vscode-ws-jsonrpc';
 import { Emitter, Event } from '../event';
 import { ConnectionHandler } from './handler';
 import { JsonRpcProxy, JsonRpcProxyFactory } from './proxy-factory';
-import { WebSocketChannel } from './web-socket-channel';
+import { Channel, ChannelMultiplexer } from '../message-rpc/channel';
 
 /**
  * Factor common logic according to `ElectronIpcConnectionProvider` and
@@ -45,9 +44,6 @@ export abstract class AbstractConnectionProvider<AbstractOptions extends object>
         throw new Error('abstract');
     }
 
-    protected channelIdSeq = 0;
-    protected readonly channels = new Map<number, WebSocketChannel>();
-
     protected readonly onIncomingMessageActivityEmitter: Emitter<void> = new Emitter();
     get onIncomingMessageActivity(): Event<void> {
         return this.onIncomingMessageActivityEmitter.event;
@@ -75,50 +71,45 @@ export abstract class AbstractConnectionProvider<AbstractOptions extends object>
         return factory.createProxy();
     }
 
+    protected channelMultiPlexer?: ChannelMultiplexer;
+
+    // A set of channel opening functions that are executed if the backend reconnects to restore the
+    // the channels that were open before the disconnect occurred.
+    protected reconnectChannelOpeners: Array<() => Promise<void>> = [];
+
+    protected initializeMultiplexer(): void {
+        const mainChannel = this.createMainChannel();
+        mainChannel.onMessage(() => this.onIncomingMessageActivityEmitter.fire());
+        this.channelMultiPlexer = new ChannelMultiplexer(mainChannel);
+    }
+
     /**
      * Install a connection handler for the given path.
      */
     listen(handler: ConnectionHandler, options?: AbstractOptions): void {
         this.openChannel(handler.path, channel => {
-            const connection = createWebSocketConnection(channel, this.createLogger());
-            connection.onDispose(() => channel.close());
-            handler.onConnection(connection);
+            handler.onConnection(channel);
         }, options);
     }
 
-    openChannel(path: string, handler: (channel: WebSocketChannel) => void, options?: AbstractOptions): void {
-        const id = this.channelIdSeq++;
-        const channel = this.createChannel(id);
-        this.channels.set(id, channel);
-        channel.onClose(() => {
-            if (this.channels.delete(channel.id)) {
-                const { reconnecting } = { reconnecting: true, ...options };
-                if (reconnecting) {
-                    this.openChannel(path, handler, options);
-                }
-            } else {
-                console.error('The ws channel does not exist', channel.id);
+    async openChannel(path: string, handler: (channel: Channel) => void, options?: AbstractOptions): Promise<void> {
+        if (!this.channelMultiPlexer) {
+            throw new Error('The channel multiplexer has not been initialized yet!');
+        }
+        const newChannel = await this.channelMultiPlexer.open(path);
+        newChannel.onClose(() => {
+            const { reconnecting } = { reconnecting: true, ...options };
+            if (reconnecting) {
+                this.reconnectChannelOpeners.push(() => this.openChannel(path, handler, options));
             }
         });
-        channel.onOpen(() => handler(channel));
-        channel.open(path);
-    }
-
-    protected abstract createChannel(id: number): WebSocketChannel;
 
-    protected handleIncomingRawMessage(data: string): void {
-        const message: WebSocketChannel.Message = JSON.parse(data);
-        const channel = this.channels.get(message.id);
-        if (channel) {
-            channel.handleMessage(message);
-        } else {
-            console.error('The ws channel does not exist', message.id);
-        }
-        this.onIncomingMessageActivityEmitter.fire(undefined);
+        handler(newChannel);
     }
 
-    protected createLogger(): Logger {
-        return new ConsoleLogger();
-    }
+    /**
+     * Create the main connection that is used for multiplexing all service channels.
+     */
+    protected abstract createMainChannel(): Channel;
 
 }

package/src/common/messaging/connection-error-handler.ts

@@ -14,7 +14,6 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import { Message } from 'vscode-ws-jsonrpc';
 import { ILogger } from '../../common';
 
 export interface ResolvedConnectionErrorHandlerOptions {
@@ -51,7 +50,7 @@ export class ConnectionErrorHandler {
         };
     }
 
-    shouldStop(error: Error, message?: Message, count?: number): boolean {
+    shouldStop(error: Error, count?: number): boolean {
         return !count || count > this.options.maxErrors;
     }
 

package/src/common/messaging/handler.ts

@@ -14,11 +14,11 @@
 // SPDX-License-Identifier: EPL-2.0 OR GPL-2.0 WITH Classpath-exception-2.0
 // *****************************************************************************
 
-import { MessageConnection } from 'vscode-ws-jsonrpc';
+import { Channel } from '../message-rpc/channel';
 
 export const ConnectionHandler = Symbol('ConnectionHandler');
 
 export interface ConnectionHandler {
     readonly path: string;
-    onConnection(connection: MessageConnection): void;
+    onConnection(connection: Channel): void;
 }

package/src/common/messaging/proxy-factory.spec.ts

@@ -15,21 +15,11 @@
 // *****************************************************************************
 
 import * as chai from 'chai';
-import { ConsoleLogger } from '../../node/messaging/logger';
 import { JsonRpcProxyFactory, JsonRpcProxy } from './proxy-factory';
-import { createMessageConnection } from 'vscode-jsonrpc/lib/main';
-import * as stream from 'stream';
+import { ChannelPipe } from '../message-rpc/channel.spec';
 
 const expect = chai.expect;
 
-class NoTransform extends stream.Transform {
-
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
-    override _transform(chunk: any, encoding: string, callback: Function): void {
-        callback(undefined, chunk);
-    }
-}
-
 class TestServer {
     requests: string[] = [];
     doStuff(arg: string): Promise<string> {
@@ -102,15 +92,12 @@ function getSetup(): {
     const server = new TestServer();
 
     const serverProxyFactory = new JsonRpcProxyFactory<TestServer>(client);
-    const client2server = new NoTransform();
-    const server2client = new NoTransform();
-    const serverConnection = createMessageConnection(server2client, client2server, new ConsoleLogger());
-    serverProxyFactory.listen(serverConnection);
+    const pipe = new ChannelPipe();
+    serverProxyFactory.listen(pipe.right);
     const serverProxy = serverProxyFactory.createProxy();
 
     const clientProxyFactory = new JsonRpcProxyFactory<TestClient>(server);
-    const clientConnection = createMessageConnection(client2server, server2client, new ConsoleLogger());
-    clientProxyFactory.listen(clientConnection);
+    clientProxyFactory.listen(pipe.left);
     const clientProxy = clientProxyFactory.createProxy();
     return {
         client,