canopycms 0.0.68-int.98 → 0.1.0

package/dist/worker/gateway-egress.d.ts

@@ -0,0 +1,43 @@
+/**
+ * Scrub `token` from everything that leaves the gateway from now on: raw, and as the extraheader
+ * carries it, `base64("x-access-token:" + token)`, raw and URL-encoded. Shared by every gateway in
+ * the process, so a token one resolved is scrubbed from all of them.
+ */
+export declare function registerLiveSecret(token: string): void;
+/**
+ * Scrub `value` from everything that leaves the gateway from now on, raw and URL-encoded: a secret
+ * that is not a GitHub token (the auth provider's key). It shares {@link registerLiveSecret}'s
+ * store and cap, so a rotated key's predecessor stays scrubbed until the cap drops it, as a
+ * token's does.
+ */
+export declare function registerSecretText(value: string): void;
+/**
+ * Scrub each line of a PEM key's body from everything that leaves the gateway from now on, so the
+ * scrub survives JSON or log escaping, which split a PEM at its newlines. Lines split on real
+ * line breaks and on the literal `\n`, `\r\n` or `\r` (a key pasted into a single-line config
+ * field); the BEGIN/END lines and lines shorter than the secret floor are skipped. A key given as
+ * one base64 line (a base64-wrapped PEM) is one entry, and the PEM it wraps is registered too.
+ */
+export declare function registerPrivateKey(pem: string): void;
+/** `text` without any live token in any registered form, or anything `redactCredentials` finds. */
+export declare function redactEgress(text: string): string;
+/**
+ * An answer with every string in it (a Map's keys included) scrubbed of live tokens, never by
+ * pattern: the same object when nothing in it changed, else a copy. Plain data only: anything
+ * else (a stream) is returned as it is.
+ */
+export declare function redactValue<T>(value: T): T;
+/**
+ * `err` scrubbed for leaving the gateway: its message, stack and `stderr`, its `cause` chain, and
+ * an Octokit error's `response` and `request`. Scrubbed in place, so its class and the fields the
+ * task runner classifies on (`status`, `code`, `kind`) survive; a field that cannot be set (a
+ * DOMException's message) makes it a plain Error of the same name instead. Only fields that hold a secret are
+ * touched. A thrown string is scrubbed; anything else is returned as it is.
+ */
+export declare function redactError(err: unknown, depth?: number): unknown;
+/** The gateway's `workerLog`: one scrubbed, escaped line. */
+export declare function gatewayLog(...args: unknown[]): void;
+/** The gateway's `workerLogWarn`: one scrubbed, escaped line. */
+export declare function gatewayLogWarn(...args: unknown[]): void;
+/** The gateway's `workerLogError`: one scrubbed, escaped line. */
+export declare function gatewayLogError(...args: unknown[]): void;

package/dist/worker/gateway-egress.js

@@ -0,0 +1,209 @@
+/**
+ * What leaves the GitHub gateway: every error it throws, every answer it returns and every line it
+ * logs goes through here, `github-service.ts`'s and Octokit's lines for it included (they are
+ * given {@link gatewayLogWarn} and {@link gatewayLogError}), except `openBundle`'s errors, which
+ * hold no secret. Each is scrubbed of every live token the gateway has resolved, in every form
+ * git's extraheader gives it, and of the auth provider's key and the App's private key, as exact
+ * strings. Errors and log lines are then scrubbed by
+ * `redactCredentials`'s patterns too; answers are not, because they are data the worker acts on,
+ * and a pattern would rewrite a GitHub branch named `chore/github_pat_rotation`. The
+ * exact scrub comes first and cannot be left to the patterns: one can stop part-way through a
+ * form beside another (at the `%` of a URL-encoded one) and leave its tail unmatchable. Each log
+ * line also has its non-printable characters escaped, since request fields (branch names, pull
+ * request text) and git's output (commit subjects) are attacker-chosen. The gateway process's own
+ * modules (secret reads, the termination watch) and `@octokit/auth-app` log through `workerLog`
+ * or the console, unscrubbed: none of their lines carries a secret's value.
+ */
+import { getErrorMessage, redactCredentials } from '../utils/error.js';
+import { escapeForLog } from '../utils/escape-text.js';
+import { workerLog, workerLogError, workerLogWarn } from './log.js';
+/** How many tokens are scrubbed: the current one and those it replaced. */
+const MAX_LIVE_SECRETS = 32;
+/** Shorter values are never scrubbed as exact strings: they would hit ordinary text. */
+const MIN_SECRET_LENGTH = 8;
+/** Each live token's forms, longest first; the most recently registered token last. */
+const liveSecrets = new Map();
+/**
+ * Scrub `token` from everything that leaves the gateway from now on: raw, and as the extraheader
+ * carries it, `base64("x-access-token:" + token)`, raw and URL-encoded. Shared by every gateway in
+ * the process, so a token one resolved is scrubbed from all of them.
+ */
+export function registerLiveSecret(token) {
+    if (token.length < MIN_SECRET_LENGTH)
+        return;
+    const basic = Buffer.from(`x-access-token:${token}`).toString('base64');
+    liveSecrets.delete(token);
+    liveSecrets.set(token, [...new Set([token, basic, encodeURIComponent(basic)])].sort((a, b) => b.length - a.length));
+    for (const oldest of liveSecrets.keys()) {
+        if (liveSecrets.size <= MAX_LIVE_SECRETS)
+            break;
+        liveSecrets.delete(oldest);
+    }
+}
+/**
+ * Scrub `value` from everything that leaves the gateway from now on, raw and URL-encoded: a secret
+ * that is not a GitHub token (the auth provider's key). It shares {@link registerLiveSecret}'s
+ * store and cap, so a rotated key's predecessor stays scrubbed until the cap drops it, as a
+ * token's does.
+ */
+export function registerSecretText(value) {
+    if (value.length < MIN_SECRET_LENGTH)
+        return;
+    liveSecrets.delete(value);
+    liveSecrets.set(value, [...new Set([value, encodeURIComponent(value)])].sort((a, b) => b.length - a.length));
+    for (const oldest of liveSecrets.keys()) {
+        if (liveSecrets.size <= MAX_LIVE_SECRETS)
+            break;
+        liveSecrets.delete(oldest);
+    }
+}
+/** How many private-key lines are scrubbed; the oldest are dropped past it. */
+const MAX_PRIVATE_KEY_LINES = 1024;
+/**
+ * The lines of every registered private key, in their own set: a PEM is about 27 lines and the
+ * token store holds {@link MAX_LIVE_SECRETS}, so sharing it would evict live tokens.
+ */
+const privateKeyLines = new Set();
+/** `privateKeyLines`, longest first; rebuilt when a key is registered. */
+let privateKeyLinesLongestFirst = [];
+/**
+ * Scrub each line of a PEM key's body from everything that leaves the gateway from now on, so the
+ * scrub survives JSON or log escaping, which split a PEM at its newlines. Lines split on real
+ * line breaks and on the literal `\n`, `\r\n` or `\r` (a key pasted into a single-line config
+ * field); the BEGIN/END lines and lines shorter than the secret floor are skipped. A key given as
+ * one base64 line (a base64-wrapped PEM) is one entry, and the PEM it wraps is registered too.
+ */
+export function registerPrivateKey(pem) {
+    for (const raw of pem.split(/\r?\n|\r|(?:\\r)?\\n|\\r/)) {
+        const line = raw.trim();
+        if (line.length < MIN_SECRET_LENGTH || line.startsWith('-----'))
+            continue;
+        privateKeyLines.delete(line);
+        privateKeyLines.add(line);
+        // A base64-wrapped PEM: the key inside it is the one a decoding consumer holds.
+        const decoded = Buffer.from(line, 'base64').toString('utf8');
+        if (decoded.includes('-----BEGIN'))
+            registerPrivateKey(decoded);
+    }
+    for (const oldest of privateKeyLines) {
+        if (privateKeyLines.size <= MAX_PRIVATE_KEY_LINES)
+            break;
+        privateKeyLines.delete(oldest);
+    }
+    privateKeyLinesLongestFirst = [...privateKeyLines].sort((a, b) => b.length - a.length);
+}
+/** `text` without any live token or registered secret in any registered form. */
+function redactLiveSecrets(text) {
+    let result = text;
+    for (const forms of liveSecrets.values()) {
+        for (const form of forms)
+            result = result.split(form).join('***');
+    }
+    for (const line of privateKeyLinesLongestFirst)
+        result = result.split(line).join('***');
+    return result;
+}
+/** `text` without any live token in any registered form, or anything `redactCredentials` finds. */
+export function redactEgress(text) {
+    return redactCredentials(redactLiveSecrets(text));
+}
+/** The deepest a returned value is walked; anything deeper passes as it is. */
+const MAX_DEPTH = 8;
+/**
+ * An answer with every string in it (a Map's keys included) scrubbed of live tokens, never by
+ * pattern: the same object when nothing in it changed, else a copy. Plain data only: anything
+ * else (a stream) is returned as it is.
+ */
+export function redactValue(value) {
+    return redactAny(value, 0, redactLiveSecrets);
+}
+function redactAny(value, depth, scrub) {
+    if (typeof value === 'string')
+        return scrub(value);
+    if (depth >= MAX_DEPTH || typeof value !== 'object' || value === null)
+        return value;
+    if (Array.isArray(value)) {
+        const items = value.map((item) => redactAny(item, depth + 1, scrub));
+        return items.every((item, i) => item === value[i]) ? value : items;
+    }
+    if (value instanceof Map) {
+        const before = [...value];
+        const after = before.map(([key, item]) => [
+            redactAny(key, depth + 1, scrub),
+            redactAny(item, depth + 1, scrub),
+        ]);
+        return after.every(([key, item], i) => key === before[i][0] && item === before[i][1])
+            ? value
+            : new Map(after);
+    }
+    if (Object.getPrototypeOf(value) !== Object.prototype)
+        return value;
+    const entries = Object.entries(value);
+    const redacted = entries.map(([key, item]) => [key, redactAny(item, depth + 1, scrub)]);
+    return redacted.every(([, item], i) => item === entries[i][1])
+        ? value
+        : Object.fromEntries(redacted);
+}
+/**
+ * `err` scrubbed for leaving the gateway: its message, stack and `stderr`, its `cause` chain, and
+ * an Octokit error's `response` and `request`. Scrubbed in place, so its class and the fields the
+ * task runner classifies on (`status`, `code`, `kind`) survive; a field that cannot be set (a
+ * DOMException's message) makes it a plain Error of the same name instead. Only fields that hold a secret are
+ * touched. A thrown string is scrubbed; anything else is returned as it is.
+ */
+export function redactError(err, depth = 0) {
+    if (typeof err === 'string')
+        return redactEgress(err);
+    if (!(err instanceof Error) || depth >= MAX_DEPTH)
+        return err;
+    const fields = err;
+    const updates = [];
+    // `stderr`: a GitProcessError (worker/git-process.ts) carries git's own output beside its message.
+    for (const key of ['message', 'stack', 'stderr']) {
+        const value = fields[key];
+        if (typeof value === 'string' && redactEgress(value) !== value) {
+            updates.push([key, redactEgress(value)]);
+        }
+    }
+    for (const key of ['response', 'request']) {
+        const redacted = redactAny(fields[key], 0, redactEgress);
+        if (redacted !== fields[key])
+            updates.push([key, redacted]);
+    }
+    if (fields.cause !== undefined) {
+        const cause = redactError(fields.cause, depth + 1);
+        if (cause !== fields.cause)
+            updates.push(['cause', cause]);
+    }
+    try {
+        for (const [key, value] of updates) {
+            Object.defineProperty(fields, key, {
+                value,
+                writable: true,
+                configurable: true,
+                enumerable: Object.prototype.propertyIsEnumerable.call(fields, key),
+            });
+        }
+        return err;
+    }
+    catch {
+        const replacement = new Error(redactEgress(err.message));
+        replacement.name = err.name;
+        return replacement;
+    }
+}
+function logLine(args) {
+    return escapeForLog(redactEgress(args.map((arg) => (typeof arg === 'string' ? arg : getErrorMessage(arg))).join(' ')));
+}
+/** The gateway's `workerLog`: one scrubbed, escaped line. */
+export function gatewayLog(...args) {
+    workerLog(logLine(args));
+}
+/** The gateway's `workerLogWarn`: one scrubbed, escaped line. */
+export function gatewayLogWarn(...args) {
+    workerLogWarn(logLine(args));
+}
+/** The gateway's `workerLogError`: one scrubbed, escaped line. */
+export function gatewayLogError(...args) {
+    workerLogError(logLine(args));
+}

package/dist/worker/gateway-instance.d.ts

@@ -0,0 +1,42 @@
+import type { GatewayTerminationNotice } from './github-gateway.js';
+export interface GatewayInstanceOptions {
+    /**
+     * Resolves once the instance is being terminated. Started at the first `termination()` or
+     * `completeLifecycle()` call, once; a rejection is logged and the next call starts it again.
+     */
+    watchTermination?: () => Promise<GatewayTerminationNotice>;
+    /** Completes the Auto Scaling lifecycle action. Never rejects by contract: the provider logs its own failure. */
+    completeLifecycle?: () => Promise<void>;
+    /** Test seam: one `termination()` poll's wait (default {@link TERMINATION_POLL_WAIT_MS}). */
+    terminationPollWaitMs?: number;
+}
+/**
+ * The gateway's two operations on the instance itself, `termination` and `completeLifecycle`
+ * (see {@link GitHubGateway}). They need only the instance role, never the GitHub credential, so
+ * a gateway that failed to boot serves them too (canopycms-cdk `worker/gateway-run.ts`).
+ */
+export declare class GatewayInstance {
+    private readonly options;
+    /** Whether the termination watch is running: started by a poll, and again by one after it failed. */
+    private terminationWatching;
+    /** The notice this gateway's own termination watch saw; null until it sees one. */
+    private terminationNotice;
+    /**
+     * The polls waiting for a notice, each removed when its poll ends. A set, not reactions on one
+     * long-lived promise: those would pile up, one per poll, for the instance's whole life.
+     */
+    private readonly terminationWaiters;
+    /** Whether the gateway's own watch has seen an Auto Scaling notice: what `completeLifecycle` requires. */
+    private sawAutoScalingNotice;
+    /** The one `completeLifecycle` provider call, which every call answers. */
+    private lifecycleCompletion;
+    constructor(options: GatewayInstanceOptions);
+    termination(signal?: AbortSignal): Promise<GatewayTerminationNotice | null>;
+    completeLifecycle(signal?: AbortSignal): Promise<void>;
+    /** Start the termination watch unless it runs or has seen a notice; a failure is logged, and the next poll starts it again. */
+    private startTerminationWatch;
+    /** The watch's notice, or null after `waitMs`; rejects with `signal`'s reason when it aborts first. */
+    private waitForTerminationNotice;
+}
+/** `promise`, unless `signal` aborts first, which ends only this wait. */
+export declare function untilAborted<T>(promise: Promise<T>, signal?: AbortSignal): Promise<T>;

package/dist/worker/gateway-instance.js

@@ -0,0 +1,129 @@
+import { getErrorMessage } from '../utils/error.js';
+import { gatewayLogError, gatewayLogWarn, redactEgress, redactError, redactValue, } from './gateway-egress.js';
+/** How long one `termination()` poll waits for a notice before answering that none came. */
+const TERMINATION_POLL_WAIT_MS = 25_000;
+/**
+ * How long `completeLifecycle` watches for a notice it has not seen: one read of the instance's
+ * metadata, which a terminating instance answers at once.
+ */
+const LIFECYCLE_NOTICE_WAIT_MS = 10_000;
+/**
+ * The gateway's two operations on the instance itself, `termination` and `completeLifecycle`
+ * (see {@link GitHubGateway}). They need only the instance role, never the GitHub credential, so
+ * a gateway that failed to boot serves them too (canopycms-cdk `worker/gateway-run.ts`).
+ */
+export class GatewayInstance {
+    constructor(options) {
+        this.options = options;
+        /** Whether the termination watch is running: started by a poll, and again by one after it failed. */
+        this.terminationWatching = false;
+        /** The notice this gateway's own termination watch saw; null until it sees one. */
+        this.terminationNotice = null;
+        /**
+         * The polls waiting for a notice, each removed when its poll ends. A set, not reactions on one
+         * long-lived promise: those would pile up, one per poll, for the instance's whole life.
+         */
+        this.terminationWaiters = new Set();
+        /** Whether the gateway's own watch has seen an Auto Scaling notice: what `completeLifecycle` requires. */
+        this.sawAutoScalingNotice = false;
+        /** The one `completeLifecycle` provider call, which every call answers. */
+        this.lifecycleCompletion = null;
+    }
+    async termination(signal) {
+        if (this.terminationNotice !== null)
+            return this.terminationNotice;
+        this.startTerminationWatch();
+        // A failed watch leaves the poll waiting its whole period, so a client polling in a loop is paced.
+        return this.waitForTerminationNotice(this.options.terminationPollWaitMs ?? TERMINATION_POLL_WAIT_MS, signal);
+    }
+    async completeLifecycle(signal) {
+        const complete = this.options.completeLifecycle;
+        if (complete === undefined) {
+            gatewayLogWarn('Nothing completes the Auto Scaling lifecycle action: no provider is configured');
+            return;
+        }
+        // Only an Auto Scaling notice this gateway's own watch observed opens it: a worker cannot be
+        // trusted to say the instance is terminating, and completing the action ends the instance. A
+        // gateway restarted after the worker had its notice has not watched yet, so it watches now,
+        // for a bounded wait, before it refuses.
+        if (!this.sawAutoScalingNotice) {
+            this.startTerminationWatch();
+            await this.waitForTerminationNotice(Math.min(this.options.terminationPollWaitMs ?? TERMINATION_POLL_WAIT_MS, LIFECYCLE_NOTICE_WAIT_MS), signal);
+        }
+        if (!this.sawAutoScalingNotice) {
+            throw new Error('The GitHub gateway has seen no Auto Scaling termination, so it will not complete the lifecycle action');
+        }
+        // `signal` ends only this caller's wait: a started CompleteLifecycleAction is not taken back.
+        try {
+            await untilAborted((this.lifecycleCompletion ??= complete()), signal);
+        }
+        catch (err) {
+            throw redactError(err);
+        }
+    }
+    /** Start the termination watch unless it runs or has seen a notice; a failure is logged, and the next poll starts it again. */
+    startTerminationWatch() {
+        const watchTermination = this.options.watchTermination;
+        if (watchTermination === undefined || this.terminationWatching)
+            return;
+        if (this.terminationNotice !== null)
+            return;
+        this.terminationWatching = true;
+        void (async () => watchTermination())().then((notice) => {
+            this.terminationWatching = false;
+            const scrubbed = redactValue(notice);
+            this.terminationNotice = scrubbed;
+            if (notice.kind === 'auto-scaling')
+                this.sawAutoScalingNotice = true;
+            for (const waiter of [...this.terminationWaiters])
+                waiter(scrubbed);
+        }, (err) => {
+            this.terminationWatching = false;
+            gatewayLogError('The termination watch failed:', redactEgress(getErrorMessage(err)));
+        });
+    }
+    /** The watch's notice, or null after `waitMs`; rejects with `signal`'s reason when it aborts first. */
+    waitForTerminationNotice(waitMs, signal) {
+        if (this.terminationNotice !== null)
+            return Promise.resolve(this.terminationNotice);
+        signal?.throwIfAborted();
+        return new Promise((resolve, reject) => {
+            const end = () => {
+                clearTimeout(timer);
+                this.terminationWaiters.delete(onNotice);
+                signal?.removeEventListener('abort', onAbort);
+            };
+            const onNotice = (notice) => {
+                end();
+                resolve(notice);
+            };
+            const onAbort = () => {
+                end();
+                reject(signal?.reason);
+            };
+            const timer = setTimeout(() => {
+                end();
+                resolve(null);
+            }, waitMs);
+            this.terminationWaiters.add(onNotice);
+            signal?.addEventListener('abort', onAbort, { once: true });
+        });
+    }
+}
+/** `promise`, unless `signal` aborts first, which ends only this wait. */
+export async function untilAborted(promise, signal) {
+    if (signal === undefined)
+        return promise;
+    signal.throwIfAborted();
+    let onAbort = () => undefined;
+    const aborted = new Promise((_, reject) => {
+        onAbort = () => reject(signal.reason);
+        signal.addEventListener('abort', onAbort, { once: true });
+    });
+    try {
+        return await Promise.race([promise, aborted]);
+    }
+    finally {
+        signal.removeEventListener('abort', onAbort);
+    }
+}

package/dist/worker/git-process.d.ts

@@ -0,0 +1,81 @@
+import { Readable } from 'node:stream';
+/**
+ * git with its stdin or stdout as a stream, which simple-git cannot do: a bundle's way into
+ * `bundle unbundle -` and `fetch`, out of `bundle create -`, and `update-ref --stdin`'s
+ * transactions. The caller supplies the whole environment, config and working directory, so the
+ * pins of the repository it runs in apply here exactly as they do through simple-git
+ * (worker/shared-repo-git.ts, worker/github-mirror.ts).
+ *
+ * Bounded three ways: while git runs, killed after {@link GitProcessOptions.inactivityTimeoutMs}
+ * with no output and no input consumed, unless output it wrote is still waiting for the reader
+ * (git is then blocked on the reader, and the reader's own bounds end that wait; once git has
+ * exited 0, output still unread is the reader's to take, however slowly); killed when `signal`
+ * aborts; and holding at most {@link STDERR_CAP_BYTES} of stderr and `maxStdoutBytes` of captured stdout, so a process fed
+ * hostile bytes cannot make this one buffer without limit. A stop reaches everything git spawned:
+ * git runs in its own process group.
+ */
+export interface GitProcessOptions {
+    /** The process's working directory, absolute. */
+    cwd: string;
+    /** The process's entire environment. */
+    env: Record<string, string>;
+    /** `-c` settings, given ahead of the subcommand. */
+    config?: readonly string[];
+    /** What git reads on stdin; without it stdin is closed at once. */
+    stdin?: Readable | string;
+    /**
+     * While git runs, killed when nothing is written to stdout or stderr, and nothing consumed from
+     * stdin, for this long. Not while what it wrote waits unread for the reader, nor after it exits
+     * 0: what it wrote is then the reader's.
+     */
+    inactivityTimeoutMs: number;
+    /**
+     * Killed when git has run this long, output or not: the bound for a command that writes nothing
+     * while it works (`repack`, `commit-graph write`), which an inactivity bound would kill.
+     */
+    deadlineMs?: number;
+    signal?: AbortSignal;
+    /** Told of each sign of life the inactivity bound counts: output written, input consumed. */
+    onActivity?: () => void;
+}
+/**
+ * git exited non-zero, by a signal, or was killed here; `message` is its stderr when it wrote any,
+ * after its stdout when `runGit` was asked for that (`stdoutInFailure`).
+ * @internal Exported for tests.
+ */
+export declare class GitProcessError extends Error {
+    readonly exitCode: number | null;
+    readonly stderr: string;
+    constructor(message: string, exitCode: number | null, stderr: string);
+}
+/**
+ * The most stderr kept, from the start: git's diagnosis comes first.
+ * @internal Exported for tests.
+ */
+export declare const STDERR_CAP_BYTES: number;
+/** A git process with its stdout as a stream; see {@link spawnGit}. */
+export interface GitProcess {
+    readonly stdout: Readable;
+    /**
+     * Settles once git has exited and its output is read or released; resolves only when git exited
+     * 0 on its own, its input source did not fail, and no stop came first.
+     */
+    readonly exited: Promise<void>;
+    /** Stop git now; `exited` then rejects. */
+    kill(reason: string): void;
+}
+/**
+ * Start `git <config> <args>` with stdout as a stream the caller must read or {@link GitProcess.kill}:
+ * git blocks on a full pipe, for as long as the reader holds back.
+ */
+export declare function spawnGit(args: readonly string[], options: GitProcessOptions): GitProcess;
+/**
+ * Run `git <config> <args>` to completion and resolve with its stdout, refusing more than
+ * `maxStdoutBytes` of it (default 64 MiB). With `stdoutInFailure`, a failure's message is what git
+ * printed to stdout and then its own message, as simple-git words one: `push --porcelain` prints
+ * the rejected ref's status, which callers classify, on stdout.
+ */
+export declare function runGit(args: readonly string[], options: GitProcessOptions & {
+    maxStdoutBytes?: number;
+    stdoutInFailure?: boolean;
+}): Promise<string>;