@gitpod/gitpod-protocol 0.1.5-v0.6.0-beta4 → 0.1.5-vn-6525.75

package/src/util/repeat.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { Disposable } from "..";
+import { log } from "./logging";
+
+/**
+ * This intends to be a drop-in replacement for 'setInterval' implemented with a 'setTimeout' chain
+ * to ensure we're not creating more timeouts than we can process.
+ * @param op
+ * @param everyMilliseconds
+ * @returns
+ */
+export function repeat(op: () => Promise<void> | void, everyMilliseconds: number): Disposable {
+    let timer: NodeJS.Timeout | undefined = undefined;
+    let stopped = false;
+    const repeated = () => {
+        if (stopped) {
+            // in case we missed the clearTimeout i 'await'
+            return;
+        }
+
+        timer = setTimeout(async () => {
+            try {
+                await op();
+            } catch (err) {
+                // catch error here to
+                log.error(err);
+            }
+
+            repeated(); // chain ourselves - after the 'await'
+        }, everyMilliseconds);
+    };
+    repeated();
+
+    return Disposable.create(() => {
+        stopped = true;
+        if (timer) {
+            clearTimeout(timer);
+        }
+    });
+}

package/src/util/semaphore.ts

@@ -4,20 +4,19 @@
  * See License-AGPL.txt in the project root for license information.
  */
 
-
 export class Semaphore {
     protected queue: (() => void)[] = [];
     protected used: number;
 
-    constructor(protected readonly capacity: number) { 
-        if(capacity < 1) {
+    constructor(protected readonly capacity: number) {
+        if (capacity < 1) {
             throw new Error("Capacity cannot be less than 1");
         }
     }
 
     public release() {
-        if(this.used == 0) return;
-        
+        if (this.used == 0) return;
+
         const queued = this.queue.shift();
         if (queued) {
             queued();
@@ -28,7 +27,7 @@ export class Semaphore {
 
     public async acquire(): Promise<void> {
         this.used++;
-        if(this.used <= this.capacity) {
+        if (this.used <= this.capacity) {
             return Promise.resolve();
         }
 
@@ -36,5 +35,4 @@ export class Semaphore {
             this.queue.push(rs);
         });
     }
-
-}
+}

package/src/util/skip-if.ts

@@ -4,7 +4,6 @@
  * See License-AGPL.txt in the project root for license information.
  */
 
-
 /**
  * The subset of actually available fields and methods which are not exported but we care about
  */
@@ -18,16 +17,20 @@ interface TestSuiteContext extends Mocha.ISuiteCallbackContext {
  * @param doSkip A function which takes a TestSuite and decides if it should be skipped
  */
 export function skipIf(doSkip: (suite: TestSuiteContext) => boolean): MochaTypeScript.SuiteTrait {
-    const trait: MochaTypeScript.SuiteTrait = function(this: Mocha.ISuiteCallbackContext, ctx: Mocha.ISuiteCallbackContext, ctor: Function): void {
-        const suite = ctx as any as TestSuiteContext;  // No idea why those fields are not exported in the types
+    const trait: MochaTypeScript.SuiteTrait = function (
+        this: Mocha.ISuiteCallbackContext,
+        ctx: Mocha.ISuiteCallbackContext,
+        ctor: Function,
+    ): void {
+        const suite = ctx as any as TestSuiteContext; // No idea why those fields are not exported in the types
         const skip = doSkip(suite);
-        suite.beforeEach(function(this: Mocha.IHookCallbackContext) {
+        suite.beforeEach(function (this: Mocha.IHookCallbackContext) {
             if (skip) {
                 this.skip();
             }
-        })
+        });
     };
-    
+
     // Mark as "trait": mimics the behavior of https://github.com/testdeck/testdeck/blob/9d2dd6a458c2c86c945f6f2999b8278b7528a7a7/index.ts#L433
     (trait as any)["__mts_isTrait"] = true;
     return trait;
@@ -45,4 +48,4 @@ export function skipIfEnvVarNotSet(name: string): MochaTypeScript.SuiteTrait {
         }
         return skip;
     });
-}
+}

package/src/util/timeutil.spec.ts

@@ -4,40 +4,39 @@
  * See License-AGPL.txt in the project root for license information.
  */
 
-import * as chai from 'chai';
+import * as chai from "chai";
 const expect = chai.expect;
-import { suite, test } from 'mocha-typescript';
-import { oneMonthLater } from './timeutil';
+import { suite, test } from "mocha-typescript";
+import { oneMonthLater } from "./timeutil";
 
 @suite()
 export class TimeutilSpec {
-
     @test
     testTimeutil() {
         // targeting a 1st, 1th of Jan => 1st of Feb
-        this.isOneMonthLater(new Date(2000, 0, 1), 1, new Date(2000, 1, 1))
+        this.isOneMonthLater(new Date(2000, 0, 1), 1, new Date(2000, 1, 1));
 
         // targeting a 31th, 30th of Apr => 31st of May
-        this.isOneMonthLater(new Date(2000, 3, 30), 31, new Date(2000, 4, 31))
+        this.isOneMonthLater(new Date(2000, 3, 30), 31, new Date(2000, 4, 31));
 
         // targeting a 31th, 31th of Mar => 30th of Apr
-        this.isOneMonthLater(new Date(2000, 2, 31), 31, new Date(2000, 3, 30))
+        this.isOneMonthLater(new Date(2000, 2, 31), 31, new Date(2000, 3, 30));
 
         // targeting a 30th, 30th of Mar => 30th of Apr
-        this.isOneMonthLater(new Date(2000, 2, 30), 30, new Date(2000, 3, 30))
+        this.isOneMonthLater(new Date(2000, 2, 30), 30, new Date(2000, 3, 30));
 
         // next year
-        this.isOneMonthLater(new Date(2000, 11, 1), 1, new Date(2001, 0, 1))
-        this.isOneMonthLater(new Date(2000, 11, 31), 31, new Date(2001, 0, 31))
+        this.isOneMonthLater(new Date(2000, 11, 1), 1, new Date(2001, 0, 1));
+        this.isOneMonthLater(new Date(2000, 11, 31), 31, new Date(2001, 0, 31));
 
         // Feb
-        this.isOneMonthLater(new Date(2001, 0, 31), 31, new Date(2001, 1, 28))
+        this.isOneMonthLater(new Date(2001, 0, 31), 31, new Date(2001, 1, 28));
         // Feb leap year
-        this.isOneMonthLater(new Date(2000, 0, 31), 31, new Date(2000, 1, 29))
+        this.isOneMonthLater(new Date(2000, 0, 31), 31, new Date(2000, 1, 29));
     }
 
     isOneMonthLater(from: Date, day: number, expectation: Date) {
-        const later = oneMonthLater(from.toISOString(), day)
-        expect(later, `expected ${later} to be equal ${expectation}`).to.be.equal(expectation.toISOString())
+        const later = oneMonthLater(from.toISOString(), day);
+        expect(later, `expected ${later} to be equal ${expectation}`).to.be.equal(expectation.toISOString());
     }
-}
+}

package/src/util/timeutil.ts

@@ -6,12 +6,12 @@
 
 /**
  * Returns the <code>day</code>th of the next month from <code>formDate</code>.
- * If the next month does not have a <code>day</code>th, the last day of that 
+ * If the next month does not have a <code>day</code>th, the last day of that
  * month is taken.
  * The time is copied from <code>fromDate</code>.
- * 
- * @param fromDate 
- * @param day 
+ *
+ * @param fromDate
+ * @param day
  */
 export function oneMonthLater(fromDate: string, day?: number): string {
     const later = new Date(fromDate);
@@ -24,28 +24,32 @@ export function oneMonthLater(fromDate: string, day?: number): string {
     }
     return later.toISOString();
 }
-export const yearsLater = (fromDate: string, years: number): string => liftDate1(fromDate, (d) => {
-    d.setUTCFullYear(d.getUTCFullYear() + years);
-    return d.toISOString();
-});
+export const yearsLater = (fromDate: string, years: number): string =>
+    liftDate1(fromDate, (d) => {
+        d.setUTCFullYear(d.getUTCFullYear() + years);
+        return d.toISOString();
+    });
 
 // tslint:disable-next-line:no-shadowed-variable
-export const addMillis = (d1: string, millis: number) => liftDate1(d1, (d1) => new Date(d1.getTime() + millis).toISOString());
-export const durationInHours = (d1: string, d2: string) => liftDate(d1, d2, (d1, d2) => millisecondsToHours(d1.getTime() - d2.getTime()));
+export const addMillis = (d1: string, millis: number) =>
+    liftDate1(d1, (d1) => new Date(d1.getTime() + millis).toISOString());
+export const durationInHours = (d1: string, d2: string) =>
+    liftDate(d1, d2, (d1, d2) => millisecondsToHours(d1.getTime() - d2.getTime()));
 export const durationInMillis = (d1: string, d2: string) => liftDate(d1, d2, (d1, d2) => d1.getTime() - d2.getTime());
 // tslint:disable-next-line:no-shadowed-variable
-export const isDateGreaterOrEqual = (d1: string, d2: string): boolean => liftDate(d1, d2, (d1, d2) => d1.getTime() >= d2.getTime());
+export const isDateGreaterOrEqual = (d1: string, d2: string): boolean =>
+    liftDate(d1, d2, (d1, d2) => d1.getTime() >= d2.getTime());
 export const isDateSmallerOrEqual = (d1: string, d2: string | undefined) => !d2 || d1 <= d2;
 export const isDateSmaller = (d1: string, d2: string | undefined) => !d2 || d1 < d2;
-export const oldest = (d1: string, d2: string): string => d1 > d2 ? d1 : d2;
-export const earliest = (d1: string, d2: string): string => d1 < d2 ? d1 : d2;
+export const oldest = (d1: string, d2: string): string => (d1 > d2 ? d1 : d2);
+export const earliest = (d1: string, d2: string): string => (d1 < d2 ? d1 : d2);
 export const orderAsc = (d1: string, d2: string): number => liftDate(d1, d2, (d1, d2) => d1.getTime() - d2.getTime());
 export const liftDate1 = <T>(d1: string, f: (d1: Date) => T): T => f(new Date(d1));
 export const liftDate = <T>(d1: string, d2: string, f: (d1: Date, d2: Date) => T): T => f(new Date(d1), new Date(d2));
 
 export function hoursLater(date: string, hours: number): string {
-    const result = new Date(date)
-    result.setHours(result.getHours() + hours)
+    const result = new Date(date);
+    result.setHours(result.getHours() + hours);
     return result.toISOString();
 }
 

package/src/util/tracing.spec.ts

@@ -0,0 +1,88 @@
+/**
+ * Copyright (c) 2021 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License-AGPL.txt in the project root for license information.
+ */
+
+import { suite, test } from "mocha-typescript";
+import * as chai from "chai";
+import { TraceContext } from "./tracing";
+import { MockTracer } from "opentracing";
+
+const expect = chai.expect;
+
+@suite
+class TestTracing {
+    @test public async testTracingContext_addNestedTags() {
+        const tracer = new MockTracer();
+        const span = tracer.startSpan("testTracingContext_addNestedTags");
+        TraceContext.addNestedTags(
+            { span },
+            {
+                rpc: {
+                    system: "jsonrpc",
+                    jsonrpc: {
+                        version: "1.0",
+                        method: "test",
+                        parameters: ["abc", "def"],
+                    },
+                },
+            },
+        );
+
+        const mockSpan = tracer.report().spans[0];
+        expect(mockSpan.tags()).to.deep.equal({
+            "rpc.system": "jsonrpc",
+            "rpc.jsonrpc.version": "1.0",
+            "rpc.jsonrpc.method": "test",
+            "rpc.jsonrpc.parameters.0": "abc",
+            "rpc.jsonrpc.parameters.1": "def",
+        });
+    }
+
+    @test public async testTracingContext_addNestedTags_null() {
+        const tracer = new MockTracer();
+        const span = tracer.startSpan("testTracingContext_addNestedTags_null");
+        TraceContext.addNestedTags(
+            { span },
+            {
+                someShape: {
+                    thisIsNull: null,
+                    thisIsUndefined: undefined,
+                },
+            },
+        );
+
+        const mockSpan = tracer.report().spans[0];
+        expect(mockSpan.tags()).to.deep.equal({
+            "someShape.thisIsNull": null,
+            "someShape.thisIsUndefined": undefined,
+        });
+    }
+
+    @test public async testTracingContext_addJsonRPCParameters() {
+        const tracer = new MockTracer();
+        const span = tracer.startSpan("testTracingContext_addJsonRPCParameters");
+        const ctx = { span };
+        TraceContext.addJsonRPCParameters(ctx, {
+            one: "one",
+            two: {
+                name: "two",
+                some: "shape",
+                containing: "PII",
+            },
+            three: "three",
+        });
+
+        const mockSpan = tracer.report().spans[0];
+        expect(mockSpan.tags()).to.deep.equal({
+            "rpc.jsonrpc.parameters.one": "one",
+            "rpc.jsonrpc.parameters.two.containing": "PII",
+            "rpc.jsonrpc.parameters.two.name": "two",
+            "rpc.jsonrpc.parameters.two.some": "shape",
+            "rpc.jsonrpc.parameters.three": "three",
+            "rpc.system": "jsonrpc",
+        });
+    }
+}
+module.exports = new TestTracing();

package/src/util/tracing.ts

@@ -4,48 +4,194 @@
  * See License-AGPL.txt in the project root for license information.
  */
 
-
-import * as opentracing from 'opentracing';
-import { TracingConfig, initTracerFromEnv, Sampler, SamplingDecision } from 'jaeger-client';
-import { initGlobalTracer } from 'opentracing';
-import { injectable } from 'inversify';
+import * as opentracing from "opentracing";
+import { TracingConfig, initTracerFromEnv } from "jaeger-client";
+import { Sampler, SamplingDecision } from "./jaeger-client-types";
+import { initGlobalTracer } from "opentracing";
+import { injectable } from "inversify";
+import { ResponseError } from "vscode-jsonrpc";
+import { log, LogContext } from "./logging";
 
 export interface TraceContext {
-    span?: opentracing.Span
+    span?: opentracing.Span;
 }
+export type TraceContextWithSpan = TraceContext & {
+    span: opentracing.Span;
+};
 
 export namespace TraceContext {
-    export function startSpan(operation: string, ctx: TraceContext): opentracing.Span {
-        const options: opentracing.SpanOptions = {
-            childOf: ctx.span
+    export function startSpan(operation: string, parentCtx?: TraceContext): opentracing.Span {
+        const options: opentracing.SpanOptions = {};
+        if (parentCtx && parentCtx.span && !!parentCtx.span.context().toSpanId()) {
+            options.childOf = parentCtx.span;
         }
+        // TODO(gpl) references lead to a huge amount of errors in prod logs. Avoid those until we have time to figure out how to fix it.
+        // if (referencedSpans) {
+        //     // note: allthough followsForm's type says it takes 'opentracing.Span | opentracing.SpanContext', it only works with SpanContext (typing mismatch)
+        //     // note2: we need to filter out debug spans (spanId === "")
+        //     options.references = referencedSpans.filter(s => s !== undefined)
+        //         .filter(s => !!s!.context().toSpanId())
+        //         .map(s => followsFrom(s!.context()));
+        // }
+
         return opentracing.globalTracer().startSpan(operation, options);
     }
 
-    export function startAsyncSpan(operation: string, ctx: TraceContext): opentracing.Span {
-        const options: opentracing.SpanOptions = {};
-        if (!!ctx.span) {
-            options.references = [opentracing.followsFrom(ctx.span.context())];
+    export function childContext(operation: string, parentCtx: TraceContext): TraceContextWithSpan {
+        const span = startSpan(operation, parentCtx);
+        return { span };
+    }
+
+    export function withSpan(operation: string, callback: (ctx: TraceContext) => void, ctx?: TraceContext): void {
+        // if we don't have a parent span, don't create a trace here as those <trace-without-root-spans> are not useful.
+        if (!ctx || !ctx.span || !ctx.span.context()) {
+            callback({});
+            return;
+        }
+
+        const span = TraceContext.startSpan(operation, ctx);
+        try {
+            callback({ span });
+        } catch (e) {
+            TraceContext.setError({ span }, e);
+            throw e;
+        } finally {
+            span.finish();
+        }
+    }
+
+    export function setError(ctx: TraceContext, err: Error) {
+        if (!ctx.span) {
+            return;
+        }
+
+        TraceContext.addNestedTags(ctx, {
+            error: {
+                message: err.message,
+                stacktrace: err.stack,
+            },
+        });
+        ctx.span.setTag("error", true);
+    }
+
+    export function setJsonRPCMetadata(ctx: TraceContext, method?: string) {
+        if (!ctx.span) {
+            return;
+        }
+
+        const tags: { [key: string]: any } = {
+            rpc: {
+                system: "jsonrpc",
+                //  version,
+            },
+        };
+        if (method) {
+            tags.rpc.method = method;
+        }
+        addNestedTags(ctx, tags);
+    }
+
+    export function setJsonRPCError(
+        ctx: TraceContext,
+        method: string,
+        err: ResponseError<any>,
+        withStatusCode: boolean = false,
+    ) {
+        if (!ctx.span) {
+            return;
+        }
+        // not use setError bc this is (most likely) a working operation
+
+        setJsonRPCMetadata(ctx, method);
+        // https://github.com/open-telemetry/opentelemetry-specification/blob/main/specification/trace/semantic_conventions/rpc.md#json-rpc
+        addNestedTags(ctx, {
+            rpc: {
+                jsonrpc: {
+                    error_code: err.code,
+                    error_message: err.message,
+                },
+            },
+        });
+
+        // the field "status_code" is used by honeycomb to derive insights like success rate, etc. Defaults to "0".
+        if (withStatusCode) {
+            ctx.span.setTag("status_code", err.code);
         }
-        return opentracing.globalTracer().startSpan(operation, options);
     }
 
-    export function logError(ctx: TraceContext, err: Error) {
+    export function addJsonRPCParameters(ctx: TraceContext, params: { [key: string]: any }) {
         if (!ctx.span) {
             return;
         }
 
-        ctx.span.log({
-            "error": err.message,
-            "stacktrace": err.stack
-        })
-        ctx.span.setTag("error", true)
+        setJsonRPCMetadata(ctx);
+        addNestedTags(ctx, {
+            rpc: {
+                jsonrpc: {
+                    parameters: params,
+                },
+            },
+        });
+    }
+
+    /**
+     * Does what one would expect from `span.addTags`: Calls `span.addTag` for all keys in map, recursively for objects.
+     * Example:
+     * ```
+     * TraceContext.addNestedTags(ctx, {
+     *    rpc: {
+     *       system: "jsonrpc",
+     *       jsonrpc: {
+     *          version: "1.0",
+     *          method: "test",
+     *          parameters: ["abc", "def"],
+     *       },
+     *    },
+     * });
+     * ```
+     * gives
+     * rpc.system = "jsonrpc"
+     * rpc.jsonrpc.version = "1.0"
+     * rpc.jsonrpc.method = "test"
+     * rpc.jsonrpc.parameters.0 = "abc"
+     * rpc.jsonrpc.parameters.1 = "def"
+     * @param ctx
+     * @param keyValueMap
+     * @returns
+     */
+    export function addNestedTags(ctx: TraceContext, keyValueMap: { [key: string]: any }, _namespace?: string) {
+        if (!ctx.span) {
+            return;
+        }
+        const namespace = _namespace ? `${_namespace}.` : "";
+
+        try {
+            for (const k of Object.keys(keyValueMap)) {
+                const v = keyValueMap[k];
+                if (v instanceof Object) {
+                    addNestedTags(ctx, v, `${namespace}${k}`);
+                } else {
+                    ctx.span.setTag(`${namespace}${k}`, v);
+                }
+            }
+        } catch (err) {
+            // general resilience against odd shapes/parameters
+            log.error("Tracing.addNestedTags", err, { namespace });
+        }
+    }
+
+    export function setOWI(ctx: TraceContext, owi: LogContext) {
+        if (!ctx.span) {
+            return;
+        }
+        addNestedTags(ctx, {
+            context: owi,
+        });
     }
 }
 
 @injectable()
 export class TracingManager {
-
     public setup(serviceName: string, opts?: CustomTracerOpts) {
         initGlobalTracer(this.getTracerForService(serviceName, opts));
     }
@@ -54,13 +200,18 @@ export class TracingManager {
         const config: TracingConfig = {
             disable: false,
             reporter: {
-                logSpans: false
+                logSpans: false,
             },
-            serviceName
-        }
+            serviceName,
+        };
         const t = initTracerFromEnv(config, {
-            logger: console
+            logger: console,
+            tags: {
+                "service.build.commit": process.env.GITPOD_BUILD_GIT_COMMIT,
+                "service.build.version": process.env.GITPOD_BUILD_VERSION,
+            },
         });
+
         if (opts) {
             if (opts.perOpSampling) {
                 (t as any)._sampler = new PerOperationSampler((t as any)._sampler, opts.perOpSampling);
@@ -68,24 +219,22 @@ export class TracingManager {
         }
         return t;
     }
-
 }
 
 export interface CustomTracerOpts {
-    perOpSampling?: PerOperationSampling
+    perOpSampling?: PerOperationSampling;
 }
 
-
 // has to conform to https://github.com/jaegertracing/jaeger-client-node/blob/0042b1c0a0796bb655eb93e77ff76ab5e94c2bb6/src/_flow/sampler-thrift.js#L32
 export interface PerOperationSampling {
-    [key: string]: boolean
+    [key: string]: boolean;
 }
 
 export class PerOperationSampler implements Sampler {
     constructor(protected readonly fallback: Sampler, protected readonly strategies: PerOperationSampling) {}
 
     name(): string {
-        return 'PerOperationSampler';
+        return "PerOperationSampler";
     }
 
     toString(): string {
@@ -125,10 +274,32 @@ export class PerOperationSampler implements Sampler {
         return false; // TODO equal should be removed
     }
 
-    close(callback: ()=>void): void {
+    close(callback: () => void): void {
         // all nested samplers are of simple types, so we do not need to Close them
         if (callback) {
             callback();
         }
     }
 }
+
+// Augment interfaces with an leading parameter "TraceContext" on every method
+type IsValidArg<T> = T extends object ? (keyof T extends never ? false : true) : true;
+type AddTraceContext<T> = T extends (a: infer A, b: infer B, c: infer C, d: infer D, e: infer E, f: infer F) => infer R
+    ? IsValidArg<F> extends true
+        ? (ctx: TraceContextWithSpan, a: A, b: B, c: C, d: D, e: E, f: F) => R
+        : IsValidArg<E> extends true
+        ? (ctx: TraceContextWithSpan, a: A, b: B, c: C, d: D, e: E) => R
+        : IsValidArg<D> extends true
+        ? (ctx: TraceContextWithSpan, a: A, b: B, c: C, d: D) => R
+        : IsValidArg<C> extends true
+        ? (ctx: TraceContextWithSpan, a: A, b: B, c: C) => R
+        : IsValidArg<B> extends true
+        ? (ctx: TraceContextWithSpan, a: A, b: B) => R
+        : IsValidArg<A> extends true
+        ? (ctx: TraceContextWithSpan, a: A) => R
+        : (ctx: TraceContextWithSpan) => Promise<R>
+    : never;
+
+export type InterfaceWithTraceContext<T> = {
+    [P in keyof T]: AddTraceContext<T[P]>;
+};

package/src/util/workspace-port-authentication.ts

@@ -5,16 +5,13 @@
  */
 
 /**
-* These cookies are set in the Theia frontend. This pattern is relied upon in:
-*  - proxy:
-*    - to filter it out on port locations
-*    - to forward it to the server for authentication
-*  - server:
-*    - to authenticate access to port locations
-*/
-export const worspacePortAuthCookieName = function(host: string, workspaceId: string): string {
-    return host
-        .replace(/https?/, '')
-        .replace(/[\W_]+/g, "_")
-        + `_ws_${workspaceId}_port_auth_`;
-};
+ * These cookies are set in the Theia frontend. This pattern is relied upon in:
+ *  - proxy:
+ *    - to filter it out on port locations
+ *    - to forward it to the server for authentication
+ *  - server:
+ *    - to authenticate access to port locations
+ */
+export const worspacePortAuthCookieName = function (host: string, workspaceId: string): string {
+    return host.replace(/https?/, "").replace(/[\W_]+/g, "_") + `_ws_${workspaceId}_port_auth_`;
+};