canopycms 0.0.68-int.98 → 0.1.0

package/dist/worker/github-mirror.d.ts

@@ -0,0 +1,364 @@
+import type { Readable } from 'node:stream';
+import { type SimpleGit } from 'simple-git';
+import { type BundleHeader } from './bundle-header.js';
+import { BranchPolicy, GatewayRefusalError } from './branch-policy.js';
+/**
+ * The worker's private bare mirror of the GitHub repository, in a state directory the CMS Lambda
+ * cannot reach. It and the quarantines beside it are the only repositories in which git ever runs
+ * with the GitHub credential, and the state directory the only place the GitHub gateway runs git
+ * at all. Every gateway git names a repository inside it, by `GIT_DIR` and working directory
+ * alike, so no config or hook the Lambda writes is ever read by it; Lambda-written objects reach
+ * it only inside a pushed bundle, checked in a quarantine. The credential reaches git only as a
+ * per-command config file ({@link withCredentialConfig}).
+ *
+ * Objects cross the boundary as git bundles, never by a fetch or push against `remote.git`:
+ * - **out:** a fetch bundles what the caller's `have` lacks into `outgoing/`, one pending bundle
+ *   at a time ({@link GitHubMirror.openBundle});
+ * - **in:** a pushed bundle is written to `incoming/`, its header read in node
+ *   (worker/bundle-header.ts), and fetched, object-checked, into a fresh quarantine repository
+ *   that borrows the mirror's objects through `objects/info/alternates`. The push to GitHub goes
+ *   out from there. Pushed objects therefore never enter the mirror, whose objects come only
+ *   from GitHub; the quarantine is deleted whole.
+ *
+ * `refs/heads/*` mirrors GitHub as of the last fetch. A cache: deleting it costs one fetch of the
+ * whole repository from GitHub.
+ *
+ * One process owns it, and {@link GitHubMirror.exclusive} runs one session at a time, so a fetch's
+ * `--prune` never meets a push half way, and the mirror's heads cannot move between a push's
+ * reachability test and the push. It sweeps what a killed git left, rebuilds itself from GitHub
+ * when it would lie, and runs maintenance beside sessions under a mutex of its own
+ * (worker/github-mirror-upkeep.ts, whose module doc holds the lock order).
+ *
+ * Every transfer runs with `--progress`: the timeout is inactivity, and a quiet fetch of a whole
+ * repository would otherwise be killed for its size. The connectivity walk after a fetch still
+ * prints nothing (safeinsights/canopycms#901).
+ */
+export declare class GitHubMirror {
+    private readonly stateDirectory;
+    private readonly timeoutMs;
+    private readonly maintenanceIntervalMs;
+    private readonly bundleRetentionMs;
+    readonly gitDir: string;
+    private readonly incomingDir;
+    private readonly outgoingDir;
+    private tail;
+    private ingestTail;
+    private warnedSize;
+    private ready;
+    private swept;
+    private transferDirs;
+    private pendingBundle;
+    private readonly maintenance;
+    private lastMaintenanceAt;
+    private corruption;
+    /**
+     * Changes whenever the mirror stops holding what the last GitHub fetch put there: it was
+     * created anew, found corrupt, or a rebuild replaced it. A GitHub fetch is reused only within
+     * the generation it completed in.
+     */
+    generation: number;
+    private rebuildFailedAt;
+    /** A sweep the maintenance mutex prevented, done at the next session start that can take it. */
+    private sweepOwed;
+    private readonly activityListeners;
+    /**
+     * `maintenanceIntervalMs` (default {@link MAINTENANCE_INTERVAL_MS}) also delays the first run:
+     * a mirror just fetched is in few packs. `bundleRetentionMs` (default
+     * {@link BUNDLE_RETENTION_MS}) is how long a bundle no one opens is kept.
+     */
+    constructor(stateDirectory: string, timeoutMs: number, maintenanceIntervalMs?: number, bundleRetentionMs?: number);
+    /**
+     * Run `fn` with the mirror to itself, after every session started before it. The first session
+     * of a gateway first clears what a killed one left ({@link sweepAtStart}), and any session first
+     * does a sweep that is owed ({@link sweepIfOwed}). Creates the mirror if it is missing, and
+     * checks it again (for corruption too) after any session that failed. Deletes any credential
+     * file a killed or failed cleanup left: no session is running, so none is in use.
+     */
+    exclusive<T>(fn: (session: MirrorSession) => Promise<T>): Promise<T>;
+    /**
+     * Call `listener` at each sign of life from a session's git (output, input taken) until the
+     * returned function is called: what a caller waiting on the mirror sees of progress, whoever's
+     * session it is.
+     */
+    watchActivity(listener: () => void): () => void;
+    private readonly activity;
+    /**
+     * Create the state directory if it is missing, and refuse one this process does not own outright
+     * or another user can write: the gateway's boot check, whose failure no later session could clear.
+     */
+    assertStateDirectory(): Promise<void>;
+    /** Create the mirror now rather than at its first use. */
+    ensure(): Promise<void>;
+    /**
+     * Clear what a killed gateway left, before any git runs: finish or undo an interrupted rebuild,
+     * then every lock and temporary in the mirror. Nothing in this process writes it yet:
+     * maintenance starts only after a session. A git a stopped dev worker left running is not
+     * covered (safeinsights/canopycms#907).
+     */
+    private sweepAtStart;
+    /**
+     * Create the mirror, or recreate one that is not a readable bare repository, and record whether
+     * an existing one is corrupt ({@link findCorruption}), which the next GitHub fetch's session
+     * repairs ({@link heal}); until then the generation change keeps every caller off it.
+     */
+    private create;
+    /**
+     * Refuse a session that cannot repair the mirror while it is corrupt: one without the credential
+     * a rebuild fetches with.
+     */
+    assertUsable(): void;
+    /**
+     * Inside a GitHub fetch's session, rebuild a mirror {@link create} found corrupt, as a plain
+     * call (the session is not re-entrant), waiting for maintenance to finish: the lock order is session, then the
+     * maintenance mutex. The replacement is fetched whole from GitHub into `github.git.rebuild`, and
+     * swapped in only once complete: the mirror is renamed to `.old`, `.rebuild` into its place, and
+     * `.old` deleted ({@link recoverRebuild} finishes or undoes a swap a crash interrupted). No
+     * quarantine is live while the session is held, so moving the objects it would borrow is safe.
+     * After a failed rebuild, sessions answer transient for {@link REBUILD_BACKOFF_MS} rather than
+     * fetch the whole repository again each.
+     */
+    heal(session: MirrorSession, credential: GitHubCredential): Promise<void>;
+    /**
+     * The GitHub fetch, inside a session. A failure sweeps the mirror's locks and temporaries
+     * ({@link sweepUnlessMaintained}), so it leaves no pack of its own behind; when git failed on
+     * one of the mirror's own leftovers ({@link isOwnLockFailure}) and the sweep ran, the fetch is
+     * tried once more. While maintenance runs, which holds such files live, nothing is swept, the
+     * sweep is owed, and a leftover's failure is transient. Never waits for the mutex, so a push
+     * never waits behind a repack.
+     */
+    fetchFromGitHub(session: MirrorSession, credential: GitHubCredential, signal?: AbortSignal): Promise<void>;
+    /**
+     * Inside a session, sweep the mirror's locks and temporaries ({@link sweepLocksAndTemporaries}),
+     * logging what went under `label`, unless maintenance holds the mutex: then the sweep is owed.
+     * Returns whether it swept.
+     */
+    private sweepUnlessMaintained;
+    /** At a session's start, the sweep a held mutex prevented, if it can take the mutex now. */
+    private sweepIfOwed;
+    /**
+     * After a completed GitHub fetch's session: warn once if the mirror has grown past
+     * {@link MIRROR_SIZE_WARN_KIB}, and start maintenance, detached, at most once per maintenance
+     * interval, unless the mutex is held (a rebuild, or maintenance itself).
+     */
+    afterFetch(): Promise<void>;
+    /**
+     * One maintenance run, holding the mutex; it never takes a session. A failed run owes a sweep
+     * of what it left, and the next run waits a whole interval from this one's end.
+     */
+    private runMaintenance;
+    private warnIfLarge;
+    /**
+     * Empty `incoming/` and `outgoing/` of what an earlier gateway left, once per gateway: after
+     * that, everything in them belongs to a transfer of this one's, which removes its own, so a
+     * later call only re-creates one that has gone (the state directory removed under it).
+     */
+    private ensureTransferDirs;
+    /**
+     * git for `check-ref-format`, run in the state directory like every other gateway git. It reads
+     * the repository's config, as any git does, and its refs only to expand `@{-N}`, which
+     * `assertPlainBranchName` refuses.
+     */
+    checkGit(): Promise<SimpleGit>;
+    /** Where a session writes a new outgoing bundle: a fresh name, never a caller's. */
+    outgoingBundlePath(): Promise<string>;
+    /**
+     * Make `file` the one pending outgoing bundle, or none, deleting the one it replaces (never an
+     * opened one), so a loop of fetches never fills the state directory, and deleting it after the
+     * bundle retention if no one opens it. Returns the id that opens it: a key into this object,
+     * never a path.
+     */
+    setPendingBundle(file: string | null): Promise<string | null>;
+    /**
+     * The pending bundle `id` names, readable once: it is no longer pending once opened, so a later
+     * fetch never deletes it, and its file is deleted when the stream closes.
+     * Throws {@link GitHubTransientError} for an id that is not the pending bundle's.
+     */
+    openBundle(id: string): Promise<Readable>;
+    /**
+     * Receive a pushed bundle into a directory of its own under `incoming/`, refusing it as it
+     * arrives (header, size), run `fn` with it, then delete the directory. One at a time, so pushed
+     * bundles hold at most one cap's worth of disk, plus about as much again for the quarantine's
+     * copy of the pack.
+     */
+    ingest<T>(body: Readable, expected: {
+        sha: string;
+        branch: string;
+    }, maxBytes: number, fn: (bundle: ReceivedBundle) => Promise<T>): Promise<T>;
+}
+/** A pushed bundle, received and its header read: {@link GitHubMirror.ingest}. */
+export interface ReceivedBundle {
+    /** The directory it and its quarantine live in, deleted after the push. */
+    readonly dir: string;
+    readonly file: string;
+    readonly header: BundleHeader;
+}
+/**
+ * A push or fetch the gateway could not complete for a reason a later attempt can clear: GitHub
+ * unreachable, a bundle that did not arrive whole or valid, a prerequisite GitHub no longer has,
+ * a bundle already gone. The task retries it. The code says where it failed, not whose doing it
+ * was: a `github-fetch-failed` or a `mirror-corrupt` can each follow a GitHub-bound git that
+ * failed, and only that arms a credential re-read (`LocalGitHubGateway.credentialed`).
+ */
+export declare class GitHubTransientError extends Error {
+    readonly code: 'github-fetch-failed' | 'bundle-rejected' | 'prerequisite-gone' | 'bundle-gone' | 'mirror-corrupt' | 'objects-still-needed' | 'ref-kept-moving' | 'default-branch-unread';
+    constructor(code: 'github-fetch-failed' | 'bundle-rejected' | 'prerequisite-gone' | 'bundle-gone' | 'mirror-corrupt' | 'objects-still-needed' | 'ref-kept-moving' | 'default-branch-unread', message: string);
+}
+/**
+ * What a GitHub-bound git command authenticates with: the GitHub fetch, the `ls-remote` that reads
+ * GitHub's default branch, and the push to GitHub.
+ */
+export interface GitHubCredential {
+    /** GitHub's bare https URL, or a test's stand-in for it. Never carries the token. */
+    readonly url: string;
+    readonly token: string;
+    /** Told of each GitHub-bound command that fails, except on a lock of the mirror's own. */
+    onFailure(): void;
+}
+/**
+ * Whether git's own output, not a `remote:` line relayed from GitHub, says a lock file or
+ * `packed-refs.new` was already there: the mirror's own housekeeping holds it, or a killed git
+ * left it, which says nothing about the credential.
+ * @internal Exported for tests.
+ */
+export declare function isOwnLockFailure(message: string): boolean;
+/**
+ * A push the worker never makes: to a name that is not a plain branch name, to a protected branch
+ * (`BranchPolicy`), to a settings branch under a lease, or of an object that is not a commit.
+ */
+export declare class RefusedPushError extends GatewayRefusalError {
+    constructor(branch: string, reason: 'invalid' | 'protected' | 'settings' | 'not-a-commit', sha?: string);
+}
+/**
+ * Throw {@link RefusedPushError} (or, for another `operation`, {@link GatewayRefusalError}) unless
+ * `branch` is a plain branch name: one `branchNameProblem` passes (with `shape` false, only its
+ * length cap and no leading `-`) and `git check-ref-format --branch` accepts unchanged. That rules out a `:` (git would read what
+ * follows it in a refspec as another destination), `..`, `^`, `~`, `@{`, spaces and a leading
+ * `-`, so `refs/heads/<branch>` is exactly the ref a push names. `check-ref-format --branch` takes
+ * exactly one argument and refuses `--end-of-options`, which is why it alone goes without one.
+ * `git` is the caller's instance: the gateway's runs in its state directory.
+ */
+export declare function assertPlainBranchName(branch: string, git?: SimpleGit, operation?: 'push' | 'delete' | 'pull-request', 
+/** False for a name the gateway only passes on, never acts on: git's rules, its length, no leading `-`. */
+shape?: boolean): Promise<void>;
+/** What one {@link GitHubMirror.exclusive} call may do. */
+export declare class MirrorSession {
+    private readonly gitDir;
+    private readonly stateDirectory;
+    private readonly timeoutMs;
+    private readonly onActivity?;
+    constructor(gitDir: string, stateDirectory: string, timeoutMs: number, onActivity?: (() => void) | undefined);
+    /** simple-git in `repo`: the mirror, or a quarantine of this session's. */
+    private git;
+    /** git with `stdin` fed to it, in `repo` under `config` (the mirror's pins by default). */
+    private run;
+    /**
+     * Run one GitHub-bound command in `repo`, `args` naming `credential.url` where git expects the
+     * remote. The credential reaches it only through {@link withCredentialConfig}'s file, named by
+     * `GIT_CONFIG_GLOBAL`, so it is in no argv and in no process's environment. No trace variable
+     * reaches this git: trace2 can print config values. Its failure is reported to the credential
+     * unless {@link isOwnLockFailure}.
+     *
+     * It runs through worker/git-process.ts, in a process group of its own, so an abort or a kill
+     * reaches what git spawned for the transfer too (`git-remote-https`, `pack-objects`); simple-git
+     * would signal git alone. Besides the inactivity bound, curl's low-speed limit ends a transfer
+     * that trickles: {@link LOW_SPEED_BYTES_PER_SECOND} for as long as the inactivity bound.
+     */
+    private githubBound;
+    /**
+     * Bring `refs/heads/*` of `repo` (the mirror, or a rebuild's replacement) to exactly what GitHub
+     * holds. Sessions fetch through {@link GitHubMirror.fetchFromGitHub}, which cleans up after it.
+     */
+    fetchFromGitHub(credential: GitHubCredential, signal?: AbortSignal, repo?: string): Promise<void>;
+    /** Every branch GitHub had at the last fetch, mapped to its tip. */
+    heads(): Promise<Map<string, string>>;
+    /**
+     * Bundle every branch into `file`, less what `have` already holds: each ID the mirror has as a
+     * commit is excluded with its history, and any other ID is ignored. False, with no file, when
+     * that leaves nothing to send. Each ID is validated before it reaches here, since rev-list's
+     * `--stdin` honours a pseudo-option such as `--all`.
+     */
+    createBundle(have: readonly string[], file: string, signal?: AbortSignal): Promise<boolean>;
+    /**
+     * Each of `ids` mapped to its object type in `repo`, or to `missing`, from one
+     * `cat-file --batch-check`, which prints `<id> missing` for an absent object where `cat-file -t`
+     * shares exit 128 with every other failure. Every ID must already be validated.
+     */
+    objectTypes(ids: readonly string[], repo?: string): Promise<Map<string, string>>;
+    /**
+     * Whether a GitHub branch contained the commit `sha` at the last fetch. Only for a commit:
+     * `--contains` exits 129 on a missing or non-commit object.
+     */
+    reachableFromHeads(sha: string): Promise<boolean>;
+    /** Of `prerequisites`, the commits a GitHub branch contained at the last fetch. */
+    usablePrerequisites(prerequisites: readonly string[]): Promise<string[]>;
+    /**
+     * Whether every one of `commits` is reachable from one of `tips`, in ONE walk: rev-list fed the
+     * commits and a `^<tip>` per tip lists what the tips do not reach, so empty output means all
+     * are reached (several `--contains` would mean "any", not "all"). Only for commits: rev-list
+     * passes a tree or blob ID silently.
+     */
+    private allReachable;
+    /**
+     * The branch GitHub's HEAD names, or null for a repository with none (empty, or HEAD unborn). A
+     * failure to reach GitHub throws, so a push that cannot learn it does not happen.
+     */
+    githubDefaultBranch(credential: GitHubCredential, signal?: AbortSignal): Promise<string | null>;
+    /**
+     * Take a received bundle into a fresh quarantine repository beside it, returning the
+     * quarantine's path, with `sha` there as a commit. In order:
+     * 1. each prerequisite must be a commit the mirror holds, reachable from one of `heads` (the
+     *    branches GitHub had at this session's fetch): one `cat-file --batch-check`, then one walk;
+     * 2. the quarantine borrows the mirror's objects through `objects/info/alternates`, with the
+     *    mirror's pins except `core.alternateRefsCommand` ({@link quarantinePins});
+     * 3. `fetch` of the bundle's one head into a staging ref, checking every object, by a refspec
+     *    built from the validated branch, never from the bundle;
+     * 4. the staging ref must be `sha`, and `sha` a commit.
+     * A refusal is {@link GitHubTransientError} (`bundle-rejected`, or `prerequisite-gone` for a
+     * prerequisite GitHub no longer has), except a non-commit `sha`, which is
+     * {@link RefusedPushError}. Nothing here writes the mirror.
+     */
+    quarantine(bundle: ReceivedBundle, expected: {
+        sha: string;
+        branch: string;
+    }, heads: ReadonlyMap<string, string>, signal?: AbortSignal): Promise<string>;
+    /**
+     * Push exactly `sha` from `repo` (the mirror, or a quarantine holding it) to GitHub's
+     * `refs/heads/<branch>`. `lease` is the commit GitHub must still be at for the push to replace
+     * it (`--force-with-lease`); without one the push is an ordinary fast-forward. Resolves with
+     * what the push did to GitHub's ref.
+     *
+     * Validation and `policy` are here, at the one place every GitHub push (plain or under a lease)
+     * goes through, so no caller can skip them. When `policy` needs GitHub's default branch read
+     * afresh, it is read here, in this session, from the repository the push goes to. A bundle's quarantine fetch
+     * names `branch` first, so the gateway validates it before the ingest too (`pushOnce`).
+     */
+    pushToGitHub(credential: GitHubCredential, branch: string, sha: string, options: {
+        lease?: string;
+        signal?: AbortSignal;
+        repo?: string;
+        policy: BranchPolicy;
+        /** Called once the push is allowed, just before git pushes. */
+        onPushing?: () => void;
+    }): Promise<GitHubRefUpdate>;
+}
+/**
+ * What a push did to GitHub's `refs/heads/<branch>`. `from` is git's abbreviation of the old tip,
+ * null for a new branch; `to` is the full SHA pushed.
+ */
+export type GitHubRefUpdate = {
+    moved: false;
+} | {
+    moved: true;
+    from: string | null;
+    to: string;
+};
+/**
+ * Read `refs/heads/<branch>`'s line from `git push --porcelain`'s stdout: `=` up to date, ` ` a
+ * fast-forward (`old..new`), `+` a forced update (`old...new`), `*` a new branch. A rejected ref
+ * (`!`) makes git exit non-zero before this runs. Output with no such line throws, rather than
+ * report a push this did not see.
+ * @internal Exported for tests.
+ */
+export declare function parsePushStatus(stdout: string, branch: string, sha: string): GitHubRefUpdate;
+/** A full SHA-1 or SHA-256 object ID, so nothing else can reach a refspec or rev-list's stdin. */
+export declare function isObjectId(value: string): boolean;