@gitpod/gitpod-protocol 0.1.5-sefftinge-create-feature-flag-for-4354.0 → 0.1.5-server-vscode-oauth2.11

package/src/messaging/browser/connection.ts

@@ -5,10 +5,13 @@
  * You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
  */
 
-import { listen as doListen, Logger, ConsoleLogger } from "vscode-ws-jsonrpc";
+import { Logger, ConsoleLogger, toSocket, IWebSocket } from "vscode-ws-jsonrpc";
+import { createMessageConnection } from "vscode-jsonrpc";
+import { AbstractMessageWriter } from "vscode-jsonrpc/lib/messageWriter";
+import { AbstractMessageReader } from "vscode-jsonrpc/lib/messageReader";
 import { JsonRpcProxyFactory, JsonRpcProxy } from "../proxy-factory";
-import { ConnectionHandler } from "../handler";
-import ReconnectingWebSocket from 'reconnecting-websocket';
+import { ConnectionEventHandler, ConnectionHandler } from "../handler";
+import ReconnectingWebSocket, { Event } from 'reconnecting-websocket';
 
 export interface WebSocketOptions {
     onerror?: (event: Event) => void;
@@ -29,7 +32,10 @@ export class WebSocketConnectionProvider {
         const startListening = (path: string) => {
             const socket = this.listen({
                 path,
-                onConnection: c => factory.listen(c)
+                onConnection: c => factory.listen(c),
+            }, {
+                onTransportDidClose: () => factory.fireConnectionClosed(),
+                onTransportDidOpen: () => factory.fireConnectionOpened(),
             },
                 options
             );
@@ -49,7 +55,7 @@ export class WebSocketConnectionProvider {
     /**
      * Install a connection handler for the given path.
      */
-    listen(handler: ConnectionHandler, options?: WebSocketOptions): WebSocket {
+    listen(handler: ConnectionHandler, eventHandler: ConnectionEventHandler, options?: WebSocketOptions): WebSocket {
         const url = handler.path;
         const webSocket = this.createWebSocket(url);
 
@@ -64,11 +70,12 @@ export class WebSocketConnectionProvider {
                 logger.error(JSON.stringify(error));
             });
         }
-        doListen({
-            webSocket,
-            onConnection: connection => handler.onConnection(connection),
-            logger
-        });
+        doListen(
+            webSocket as any as ReconnectingWebSocket,
+            handler,
+            eventHandler,
+            logger,
+        );
         return webSocket;
     }
 
@@ -91,3 +98,162 @@ export class WebSocketConnectionProvider {
     }
 
 }
+
+// The following was extracted from vscode-ws-jsonrpc to make these changes:
+//  - switch from WebSocket to ReconnectingWebSocket
+//  - webSocket.onopen: making sure it's only ever called once so we're re-using MessageConnection
+//  - WebSocketMessageWriter: buffer and re-try messages instead of throwing an error immidiately
+//  - WebSocketMessageReader: don't close MessageConnection on 'socket.onclose'
+function doListen(resocket: ReconnectingWebSocket, handler: ConnectionHandler, eventHandler: ConnectionEventHandler, logger: Logger) {
+    resocket.addEventListener("close", () => eventHandler.onTransportDidClose());
+
+    let alreadyOpened = false;
+    resocket.onopen = () => {
+        // trigerr "open" every time we re-open the underlying websocket
+        eventHandler.onTransportDidOpen();
+
+        // make sure we're only ever creating one MessageConnection, irregardless of how many times we have to re-open the underlying (reconnecting) websocket
+        if (alreadyOpened) {
+            return;
+        }
+        alreadyOpened = true;
+
+        const connection = createWebSocketConnection(resocket, logger);
+        handler.onConnection(connection);
+    };
+}
+
+function createWebSocketConnection(resocket: ReconnectingWebSocket, logger: Logger) {
+    const socket = toSocket(resocket as any);
+    const messageReader = new NonClosingWebSocketMessageReader(socket);
+    const messageWriter = new BufferingWebSocketMessageWriter(resocket, logger);
+    const connection = createMessageConnection(messageReader, messageWriter, logger);
+    connection.onClose(() => connection.dispose());
+    return connection;
+}
+
+/**
+ * This takes vscode-ws-jsonrpc/lib/socket/writer/WebSocketMessageWriter and adds a buffer
+ */
+class BufferingWebSocketMessageWriter extends AbstractMessageWriter {
+    protected readonly socket: ReconnectingWebSocket;
+    protected readonly logger: Logger;
+    protected errorCount: number = 0;
+
+    protected buffer: any[] = [];
+
+    constructor(socket: ReconnectingWebSocket, logger: Logger) {
+        super();
+        this.socket = socket;
+        this.logger = logger;
+
+        socket.addEventListener("open", (event: Event) => this.flushBuffer());
+    }
+
+    write(msg: any) {
+        if (this.socket.readyState !== ReconnectingWebSocket.OPEN) {
+            this.bufferMsg(msg);
+            return;
+        }
+
+        try {
+            const content = JSON.stringify(msg);
+            this.socket.send(content);
+        } catch (e) {
+            this.errorCount++;
+            this.fireError(e, msg, this.errorCount);
+
+            this.bufferMsg(msg);
+        }
+    }
+
+    protected flushBuffer() {
+        if (this.buffer.length === 0) {
+            return
+        }
+
+        const buffer = [...this.buffer];
+        this.buffer = [];
+        for (const msg of buffer) {
+            this.write(msg);
+        }
+        //this.logger.info(`flushed buffer (${this.buffer.length})`)
+    }
+
+    protected bufferMsg(msg: any) {
+        this.buffer.push(msg);
+        //this.logger.info(`buffered message (${this.buffer.length})`);
+    }
+}
+
+
+/**
+ * This takes vscode-ws-jsonrpc/lib/socket/reader/WebSocketMessageReader and removes the "onClose -> fireClose" connection
+ */
+class NonClosingWebSocketMessageReader extends AbstractMessageReader {
+    protected readonly socket: IWebSocket;
+    protected readonly events: any[] = [];
+    protected state: 'initial' | 'listening' | 'closed' = 'initial';
+    protected callback: (message: any) => void = () => {};
+
+    constructor(socket: IWebSocket) {
+        super();
+        this.socket = socket;
+        this.socket.onMessage(message => this.readMessage(message));
+        this.socket.onError(error => this.fireError(error));
+        this.socket.onClose((code, reason) => {
+            if (code !== 1000) {
+                const error = {
+                    name: '' + code,
+                    message: `Error during socket reconnect: code = ${code}, reason = ${reason}`
+                };
+                this.fireError(error);
+            }
+            // this.fireClose();        // <-- reason for this class to be copied over
+        });
+    }
+    listen(callback: (message: any) => void) {
+        if (this.state === 'initial') {
+            this.state = 'listening';
+            this.callback = callback;
+            while (this.events.length !== 0) {
+                const event = this.events.pop();
+                if (event.message) {
+                    this.readMessage(event.message);
+                }
+                else if (event.error) {
+                    this.fireError(event.error);
+                }
+                else {
+                    this.fireClose();
+                }
+            }
+        }
+    }
+    readMessage(message: any) {
+        if (this.state === 'initial') {
+            this.events.splice(0, 0, { message });
+        }
+        else if (this.state === 'listening') {
+            const data = JSON.parse(message);
+            this.callback(data);
+        }
+    }
+    fireError(error: any) {
+        if (this.state === 'initial') {
+            this.events.splice(0, 0, { error });
+        }
+        else if (this.state === 'listening') {
+            super.fireError(error);
+        }
+    }
+    fireClose() {
+        if (this.state === 'initial') {
+            this.events.splice(0, 0, {});
+        }
+        else if (this.state === 'listening') {
+            super.fireClose();
+        }
+        this.state = 'closed';
+    }
+}

package/src/messaging/client-call-metrics.ts

@@ -0,0 +1,97 @@
+/**
+ * Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { injectable } from 'inversify';
+import * as prometheusClient from 'prom-client';
+
+type GrpcMethodType = 'unary' | 'client_stream' | 'server_stream' | 'bidi_stream';
+export interface IGrpcCallMetricsLabels {
+	service: string,
+	method: string,
+	type: GrpcMethodType,
+}
+
+export interface IGrpcCallMetricsLabelsWithCode extends IGrpcCallMetricsLabels {
+	code: string
+}
+
+export const IClientCallMetrics = Symbol("IClientCallMetrics");
+
+export interface IClientCallMetrics {
+	started(labels: IGrpcCallMetricsLabels) : void;
+    sent(labels: IGrpcCallMetricsLabels) : void;
+    received(labels: IGrpcCallMetricsLabels) : void;
+    handled(labels: IGrpcCallMetricsLabelsWithCode) : void;
+}
+
+@injectable()
+export class PrometheusClientCallMetrics implements IClientCallMetrics {
+
+	readonly startedCounter: prometheusClient.Counter<string>;
+	readonly sentCounter: prometheusClient.Counter<string>;
+	readonly receivedCounter: prometheusClient.Counter<string>;
+	readonly handledCounter: prometheusClient.Counter<string>;
+
+	constructor() {
+		this.startedCounter = new prometheusClient.Counter({
+			name: 'grpc_client_started_total',
+			help: 'Total number of RPCs started on the client.',
+			labelNames: ['grpc_service', 'grpc_method', 'grpc_type'],
+			registers: [prometheusClient.register]
+		});
+		this.sentCounter = new prometheusClient.Counter({
+			name: 'grpc_client_msg_sent_total',
+			help: ' Total number of gRPC stream messages sent by the client.',
+			labelNames: ['grpc_service', 'grpc_method', 'grpc_type'],
+			registers: [prometheusClient.register]
+		});
+		this.receivedCounter = new prometheusClient.Counter({
+			name: 'grpc_client_msg_received_total',
+			help: 'Total number of RPC stream messages received by the client.',
+			labelNames: ['grpc_service', 'grpc_method', 'grpc_type'],
+			registers: [prometheusClient.register]
+		});
+		this.handledCounter = new prometheusClient.Counter({
+			name: 'grpc_client_handled_total',
+			help: 'Total number of RPCs completed by the client, regardless of success or failure.',
+			labelNames: ['grpc_service', 'grpc_method', 'grpc_type', 'grpc_code'],
+			registers: [prometheusClient.register]
+		});
+	}
+
+	started(labels: IGrpcCallMetricsLabels): void {
+		this.startedCounter.inc({
+			grpc_service: labels.service,
+			grpc_method: labels.method,
+			grpc_type: labels.type
+		});
+	}
+
+	sent(labels: IGrpcCallMetricsLabels): void {
+		this.sentCounter.inc({
+			grpc_service: labels.service,
+			grpc_method: labels.method,
+			grpc_type: labels.type
+		});
+	}
+
+	received(labels: IGrpcCallMetricsLabels): void {
+		this.receivedCounter.inc({
+			grpc_service: labels.service,
+			grpc_method: labels.method,
+			grpc_type: labels.type
+		});
+	}
+
+	handled(labels: IGrpcCallMetricsLabelsWithCode): void {
+		this.handledCounter.inc({
+			grpc_service: labels.service,
+			grpc_method: labels.method,
+			grpc_type: labels.type,
+			grpc_code: labels.code
+		});
+	}
+}

package/src/messaging/error.ts

@@ -34,6 +34,9 @@ export namespace ErrorCodes {
     // 460 Context Parse Error (custom status code)
     export const CONTEXT_PARSE_ERROR = 460;
 
+    // 461 Invalid gitpod yml
+    export const INVALID_GITPOD_YML = 461;
+
     // 450 Payment error
     export const PAYMENT_ERROR = 450;
 

package/src/messaging/handler.ts

@@ -13,3 +13,15 @@ export interface ConnectionHandler {
     readonly path: string;
     onConnection(connection: MessageConnection, session?: object): void;
 }
+
+export interface ConnectionEventHandler {
+    /**
+     * Called when the transport underpinning the connection got closed
+     */
+     onTransportDidClose(): void;
+
+     /**
+      * Called when the transport underpinning the connection is (re-)opened
+      */
+     onTransportDidOpen(): void;
+}

package/src/messaging/node/connection.ts

@@ -65,13 +65,13 @@ export function toIWebSocket(webSocket: ws) {
         send: content => {
             if (webSocket.readyState !== ws.OPEN) {
                 if (sendsAfterOpen++ > 3) {
-                    log.error(`Repeated try to send on closed web socket (readyState was ${webSocket.readyState})`, { ws });
+                    //log.debug(`Repeated try to send on closed web socket (readyState was ${webSocket.readyState})`, { ws });
                 }
                 return;
             }
             webSocket.send(content, err => {
                 if (err) {
-                    log.error('Error in ws.send()', err, { ws });
+                    log.error('error in ws.send()', err, { ws });
                 }
             })
         },

package/src/messaging/proxy-factory.ts

@@ -105,13 +105,19 @@ export class JsonRpcProxyFactory<T extends object> implements ProxyHandler<T> {
             this.connectionPromiseResolve = resolve
         );
         this.connectionPromise.then(connection => {
-            connection.onClose(() =>
-                this.onDidCloseConnectionEmitter.fire(undefined)
-            );
-            this.onDidOpenConnectionEmitter.fire(undefined);
+            connection.onClose(() => this.fireConnectionClosed());
+            this.fireConnectionOpened();
         });
     }
 
+    fireConnectionClosed() {
+        this.onDidCloseConnectionEmitter.fire(undefined)
+    }
+
+    fireConnectionOpened() {
+        this.onDidOpenConnectionEmitter.fire(undefined);
+    }
+
     /**
      * Connect a MessageConnection to the factory.
      *
@@ -158,7 +164,9 @@ export class JsonRpcProxyFactory<T extends object> implements ProxyHandler<T> {
      * methods calls.
      */
     protected onNotification(method: string, ...args: any[]): void {
-        this.target[method](...args);
+        if (this.target[method]) {
+            this.target[method](...args);
+        }
     }
 
     /**

package/src/permission.ts

@@ -16,7 +16,6 @@ export const Permissions = {
     "admin-api": undefined,
     "ide-settings": undefined,
     "new-workspace-cluster": undefined,
-    "teams-and-projects": undefined,
 };
 export type PermissionName = keyof (typeof Permissions);
 export const Roles = {"devops": undefined, "viewer": undefined, "admin": undefined };

package/src/protocol.ts

@@ -6,6 +6,7 @@
 
 import { WorkspaceInstance, PortVisibility } from "./workspace-instance";
 import { RoleOrPermission } from "./permission";
+import { Project } from "./teams-projects-protocol";
 
 export interface UserInfo {
     name?: string
@@ -27,8 +28,6 @@ export interface User {
 
     identities: Identity[]
 
-    allowsMarketingCommunication: boolean;
-
     /**
      * Whether the user has been blocked to use our service, because of TOS violation for example.
      * Optional for backwards compatibility.
@@ -103,10 +102,14 @@ export interface AdditionalUserData {
     // key is the name of the OAuth client i.e. local app, string the iso date when it was approved
     // TODO(rl): provide a management UX to allow rescinding of approval
     oauthClientsApproved?: { [key: string]: string }
+    // to remember GH Orgs the user installed/updated the GH App for
+    knownGitHubOrgs?: string[];
 }
 
 export interface EmailNotificationSettings {
-    disallowTransactionalEmails?: boolean;
+    allowsChangelogMail?: boolean;
+    allowsDevXMail?: boolean;
+    allowsOnboardingMail?: boolean;
 }
 
 export type IDESettings = {
@@ -392,6 +395,7 @@ export interface Workspace {
     contextURL: string;
     description: string;
     ownerId: string;
+    projectId?: string;
     context: WorkspaceContext;
     config: WorkspaceConfig;
 
@@ -559,11 +563,13 @@ export interface WorkspaceConfig {
      * Where the config object originates from.
      *
      * repo - from the repository
+     * project-db - from the "Project" stored in the database
      * definitly-gp - from github.com/gitpod-io/definitely-gp
      * derived - computed based on analyzing the repository
+     * additional-content - config comes from additional content, usually provided through the project's configuration
      * default - our static catch-all default config
      */
-    _origin?: 'repo' | 'definitely-gp' | 'derived' | 'default';
+    _origin?: 'repo' | 'project-db' | 'definitely-gp' | 'derived' | 'additional-content' | 'default';
 
     /**
      * Set of automatically infered feature flags. That's not something the user can set, but
@@ -593,16 +599,13 @@ export namespace GithubAppPrebuildConfig {
 
 export type WorkspaceImageSource = WorkspaceImageSourceDocker | WorkspaceImageSourceReference;
 export interface WorkspaceImageSourceDocker {
-    // TODO: clean this up. We should have the commit and an ImageSource in here, not duplicate the whole thing again.
-    //       We have a ton of those objects in the database, thus cleaning this up means lengthy DB migrations. Yuck.
-    dockerFileHash: string
-    dockerFileSource: Commit
     dockerFilePath: string
+    dockerFileHash: string
+    dockerFileSource?: Commit
 }
 export namespace WorkspaceImageSourceDocker {
     export function is(obj: object): obj is WorkspaceImageSourceDocker {
         return 'dockerFileHash' in obj
-            && 'dockerFileSource' in obj
             && 'dockerFilePath' in obj;
     }
 }
@@ -631,6 +634,8 @@ export type PrebuiltWorkspaceState
 export interface PrebuiltWorkspace {
     id: string;
     cloneURL: string;
+    branch?: string;
+    projectId?: string;
     commit: string;
     buildWorkspaceId: string;
     creationTime: string;
@@ -700,7 +705,7 @@ export interface TaskConfig {
     init?: string;
     prebuild?: string;
     command?: string;
-    env?: { [env: string]: string };
+    env?: { [env: string]: any };
     openIn?: 'bottom' | 'main' | 'left' | 'right';
     openMode?: 'split-top' | 'split-left' | 'split-right' | 'split-bottom' | 'tab-before' | 'tab-after';
 }
@@ -800,6 +805,29 @@ export namespace WithPrebuild {
     }
 }
 
+/**
+ * WithDefaultConfig contexts disable the download of the gitpod.yml from the repository
+ * and fall back to the built-in configuration.
+ */
+export interface WithDefaultConfig {
+    withDefaultConfig: true;
+}
+
+export namespace WithDefaultConfig {
+    export function is(context: any): context is WithDefaultConfig {
+        return context
+            && 'withDefaultConfig' in context
+            && context.withDefaultConfig;
+    }
+
+    export function mark(ctx: WorkspaceContext): WorkspaceContext & WithDefaultConfig {
+        return {
+            ...ctx,
+            withDefaultConfig: true
+        }
+    }
+}
+
 export interface SnapshotContext extends WorkspaceContext, WithSnapshot {
     snapshotId: string;
 }
@@ -815,6 +843,8 @@ export namespace SnapshotContext {
 export interface StartPrebuildContext extends WorkspaceContext {
     actual: WorkspaceContext;
     commitHistory?: string[];
+    project?: Project;
+    branch?: string;
 }
 
 export namespace StartPrebuildContext {
@@ -884,6 +914,25 @@ export interface Commit {
     refType?: RefType
 }
 
+export interface AdditionalContentContext extends WorkspaceContext {
+
+    /**
+     * utf-8 encoded contents that will be copied on top of the workspace's filesystem
+     */
+    additionalFiles: {[filePath: string]: string};
+
+}
+
+export namespace AdditionalContentContext {
+    export function is(ctx: any): ctx is AdditionalContentContext {
+        return 'additionalFiles' in ctx;
+    }
+
+    export function hasDockerConfig(ctx: any, config: WorkspaceConfig): boolean {
+        return is(ctx) && ImageConfigFile.is(config.image) && !!ctx.additionalFiles[config.image.file];
+    }
+}
+
 export interface CommitContext extends WorkspaceContext, Commit {
     /** @deprecated Moved to .repository.cloneUrl, left here for backwards-compatibility for old workspace contextes in the DB */
     cloneUrl?: string
@@ -959,6 +1008,19 @@ export interface Repository {
         parent: Repository
     }
 }
+export interface Branch {
+    name: string;
+    commit: CommitInfo;
+    htmlUrl: string;
+}
+
+export interface CommitInfo {
+    author: string;
+    sha: string;
+    commitMessage: string;
+    authorAvatarUrl?: string;
+    authorDate?: string;
+}
 
 export namespace Repository {
     export function fullRepoName(repo: Repository): string {
@@ -1001,6 +1063,7 @@ export interface WorkspaceCreationResult {
     runningWorkspacePrebuild?: {
         prebuildID: string
         workspaceID: string
+        instanceID: string
         starting: RunningWorkspacePrebuildStarting
         sameCluster: boolean
     }
@@ -1177,21 +1240,3 @@ export interface Terms {
     readonly content: string;
     readonly formElements?: object;
 }
-
-export interface Project {
-    id: string;
-    cloneUrl: string;
-    teamId: string;
-    appInstallationId: string;
-    creationTime: string;
-    /** This is a flag that triggers the HARD DELETION of this entity */
-    deleted?: boolean;
-}
-
-export interface Team {
-    id: string;
-    name: string;
-    creationTime: string;
-    /** This is a flag that triggers the HARD DELETION of this entity */
-    deleted?: boolean;
-}

package/src/team-subscription-protocol.ts

@@ -4,7 +4,6 @@
  * See License-AGPL.txt in the project root for license information.
  */
 
-import { Without } from "./util/without";
 import uuidv4 = require("uuid/v4");
 import { Subscription } from "./accounting-protocol";
 
@@ -22,7 +21,7 @@ export interface TeamSubscription {
 }
 
 export namespace TeamSubscription {
-    export const create = (ts: Without<TeamSubscription, 'id'>): TeamSubscription => {
+    export const create = (ts: Omit<TeamSubscription, 'id'>): TeamSubscription => {
         const withId = ts as TeamSubscription;
         withId.id = uuidv4();
         return withId;
@@ -49,7 +48,7 @@ export type TeamSubscriptionSlotAssigned = TeamSubscriptionSlot & TeamSubscripti
 export type TeamSubscriptionSlotState = 'unassigned' | 'assigned' | 'deactivated' | 'cancelled';
 
 export namespace TeamSubscriptionSlot {
-    export const create = (ts: Without<TeamSubscriptionSlot, 'id'>): TeamSubscriptionSlot => {
+    export const create = (ts: Omit<TeamSubscriptionSlot, 'id'>): TeamSubscriptionSlot => {
         const withId = ts as TeamSubscriptionSlot;
         withId.id = uuidv4();
         return withId;
@@ -83,6 +82,9 @@ export namespace TeamSubscriptionSlot {
         }
 
     }
+    export const isActive = (slot: TeamSubscriptionSlot): boolean => {
+        return !slot.cancellationDate;
+    }
 }
 
 /**