canopycms 0.0.68-int.98 → 0.1.0

package/dist/worker/git-process.js

@@ -0,0 +1,234 @@
+import { spawn } from 'node:child_process';
+import { Readable, Transform, Writable } from 'node:stream';
+import { pipeline } from 'node:stream/promises';
+import { getErrorMessage } from '../utils/error.js';
+/**
+ * git exited non-zero, by a signal, or was killed here; `message` is its stderr when it wrote any,
+ * after its stdout when `runGit` was asked for that (`stdoutInFailure`).
+ * @internal Exported for tests.
+ */
+export class GitProcessError extends Error {
+    constructor(message, exitCode, stderr) {
+        super(message);
+        this.exitCode = exitCode;
+        this.stderr = stderr;
+        this.name = 'GitProcessError';
+    }
+}
+/**
+ * The most stderr kept, from the start: git's diagnosis comes first.
+ * @internal Exported for tests.
+ */
+export const STDERR_CAP_BYTES = 64 * 1024;
+/** How long a process asked to stop has before it is killed outright. */
+const KILL_GRACE_MS = 2_000;
+/**
+ * Start `git <config> <args>` with stdout as a stream the caller must read or {@link GitProcess.kill}:
+ * git blocks on a full pipe, for as long as the reader holds back.
+ */
+export function spawnGit(args, options) {
+    const child = spawn('git', [...(options.config ?? []).flatMap((c) => ['-c', c]), ...args], {
+        cwd: options.cwd,
+        // An app's own ProcessEnv augmentation (Next.js requires NODE_ENV) says nothing about git's.
+        env: options.env,
+        stdio: ['pipe', 'pipe', 'pipe'],
+        // Its own process group, so a stop reaches what git spawned too (bundle create's
+        // pack-objects), which would otherwise outlive it holding the pipes open.
+        detached: process.platform !== 'win32',
+    });
+    const { stdout } = child;
+    if (stdout === null)
+        throw new Error('git was spawned without a stdout pipe');
+    const command = args.find((arg) => !arg.startsWith('-')) ?? 'git';
+    let stderr = '';
+    let stderrBytes = 0;
+    let killedFor = null;
+    let closed = false;
+    let exitedCleanly = false;
+    let inactivity;
+    const idle = () => {
+        // Output git wrote is waiting here for the reader, so git is blocked on the reader, not stuck:
+        // the reader's own bounds end that wait. With nothing unread, or a reader that has gone
+        // (a destroyed stream can keep its count until it closes), silence is git's.
+        if (stdout.readableLength > 0 && !stdout.destroyed) {
+            inactivity = setTimeout(idle, options.inactivityTimeoutMs);
+            return;
+        }
+        stop(`git ${command} wrote nothing for ${options.inactivityTimeoutMs}ms`);
+    };
+    const touch = () => {
+        options.onActivity?.();
+        clearTimeout(inactivity);
+        // Once git has exited 0, output still unread is the reader's to take, however slowly.
+        if (exitedCleanly)
+            return;
+        inactivity = setTimeout(idle, options.inactivityTimeoutMs);
+    };
+    // A stop counts until 'close', not until git exits: git can exit with its last output still
+    // unread, and that output holds 'close' back until the stream is destroyed.
+    const stop = (reason) => {
+        if (closed || killedFor !== null)
+            return;
+        // git's own failure, when it came first, stays the reason.
+        if (child.exitCode === 0 || (child.exitCode === null && child.signalCode === null)) {
+            killedFor = reason;
+        }
+        // After git exits too: what it spawned can outlive it, holding stderr open.
+        terminate(child);
+        stdout.destroy();
+    };
+    child.on('exit', (code) => {
+        if (code !== 0)
+            return;
+        exitedCleanly = true;
+        clearTimeout(inactivity);
+        clearTimeout(deadline);
+    });
+    child.stderr?.on('data', (chunk) => {
+        touch();
+        if (stderrBytes < STDERR_CAP_BYTES) {
+            stderr += chunk.subarray(0, STDERR_CAP_BYTES - stderrBytes).toString('utf8');
+        }
+        stderrBytes += chunk.length;
+    });
+    stdout.on('data', touch);
+    const deadline = options.deadlineMs === undefined
+        ? undefined
+        : setTimeout(() => stop(`git ${command} ran longer than ${options.deadlineMs}ms`), options.deadlineMs);
+    const onAbort = () => stop(`git ${command} was aborted`);
+    if (options.signal?.aborted)
+        onAbort();
+    options.signal?.addEventListener('abort', onAbort, { once: true });
+    const input = feedStdin(child, options.stdin, touch, (err) => stop(`git ${command}'s input failed: ${getErrorMessage(err)}`));
+    // Read only when git exits 0; a kill reports the failure itself.
+    input.catch(() => undefined);
+    touch();
+    const exited = new Promise((resolve, reject) => {
+        child.on('error', (err) => {
+            // Never started (git missing, a bad cwd): no 'close' is coming.
+            if (child.pid === undefined) {
+                clearTimeout(inactivity);
+                clearTimeout(deadline);
+                options.signal?.removeEventListener('abort', onAbort);
+                reject(new GitProcessError(`git ${command} could not run: ${err.message}`, null, ''));
+            }
+        });
+        child.on('close', (code, signal) => {
+            closed = true;
+            clearTimeout(inactivity);
+            clearTimeout(deadline);
+            options.signal?.removeEventListener('abort', onAbort);
+            const written = stderrBytes > STDERR_CAP_BYTES ? `${stderr}\n[stderr truncated]` : stderr;
+            if (killedFor === null && code === 0) {
+                // A failed source stream is a failure even when git made do with what it got.
+                input.then(resolve, (err) => reject(new GitProcessError(`git ${command}'s input failed: ${getErrorMessage(err)}`, 0, written)));
+                return;
+            }
+            const why = killedFor ??
+                (written.trim() ||
+                    (code === null
+                        ? `git ${command} was killed by ${signal ?? 'a signal'}`
+                        : `git ${command} exited with code ${code}`));
+            reject(new GitProcessError(why, code, written));
+        });
+    });
+    // The caller may never await it after a kill it chose.
+    exited.catch(() => undefined);
+    return { stdout, exited, kill: stop };
+}
+/**
+ * Run `git <config> <args>` to completion and resolve with its stdout, refusing more than
+ * `maxStdoutBytes` of it (default 64 MiB). With `stdoutInFailure`, a failure's message is what git
+ * printed to stdout and then its own message, as simple-git words one: `push --porcelain` prints
+ * the rejected ref's status, which callers classify, on stdout.
+ */
+export async function runGit(args, options) {
+    const max = options.maxStdoutBytes ?? 64 * 1024 * 1024;
+    const proc = spawnGit(args, options);
+    const chunks = [];
+    let total = 0;
+    const collect = new Writable({
+        write(chunk, _encoding, callback) {
+            total += chunk.length;
+            if (total > max) {
+                proc.kill(`git printed more than ${max} bytes`);
+                callback();
+                return;
+            }
+            chunks.push(chunk);
+            callback();
+        },
+    });
+    try {
+        await Promise.all([pipeline(proc.stdout, collect).catch(() => undefined), proc.exited]);
+    }
+    catch (err) {
+        if (options.stdoutInFailure && err instanceof GitProcessError && chunks.length > 0) {
+            throw new GitProcessError(`${Buffer.concat(chunks).toString('utf8')}${err.message}`, err.exitCode, err.stderr);
+        }
+        throw err;
+    }
+    return Buffer.concat(chunks).toString('utf8');
+}
+/**
+ * Ask git and everything it spawned to stop, then kill them if git's pipes have not closed within
+ * {@link KILL_GRACE_MS}. While they are open some member of the group holds them, so its id cannot
+ * have been reused.
+ */
+function terminate(child) {
+    signalGroup(child, 'SIGTERM');
+    const timer = setTimeout(() => signalGroup(child, 'SIGKILL'), KILL_GRACE_MS);
+    timer.unref();
+    child.once('close', () => clearTimeout(timer));
+}
+/** Signal git's process group, or git alone where it has none of its own (Windows). */
+function signalGroup(child, signal) {
+    if (child.pid !== undefined && process.platform !== 'win32') {
+        try {
+            process.kill(-child.pid, signal);
+            return;
+        }
+        catch {
+            // The group is gone already.
+        }
+    }
+    child.kill(signal);
+}
+/**
+ * Write `source` to git's stdin, resolving when it is all written or git has closed its stdin,
+ * and rejecting if the SOURCE fails. git closing its stdin early (it exited, or needs no more) is
+ * not this stream's failure: git's own exit reports it, and the source is then released unread.
+ */
+function feedStdin(child, source, touch, onSourceError) {
+    const sink = child.stdin;
+    if (sink === null)
+        return Promise.resolve();
+    // EPIPE when git exits before reading everything.
+    sink.on('error', () => undefined);
+    if (source === undefined) {
+        sink.end();
+        return Promise.resolve();
+    }
+    const readable = typeof source === 'string' ? Readable.from([Buffer.from(source)]) : source;
+    const counted = new Transform({
+        transform(chunk, _encoding, callback) {
+            touch();
+            callback(null, chunk);
+        },
+    });
+    return new Promise((resolve, reject) => {
+        readable.once('error', (err) => {
+            // A source that failed part way must not leave git waiting for the rest of it.
+            onSourceError(err);
+            reject(err);
+        });
+        sink.once('close', () => {
+            if (!readable.readableEnded)
+                readable.destroy();
+            resolve();
+        });
+        // Not stream.pipeline, which would destroy the source with git's EPIPE and report it as the
+        // source's own failure.
+        readable.pipe(counted).pipe(sink);
+    });
+}

package/dist/worker/git-sync.d.ts

@@ -7,30 +7,38 @@ import type { WorkerContext } from './worker-context.js';
  * slower of the worker's two poll loops (default 5 minutes, against the task
  * queue's 5 seconds).
  *
- * One cycle, in order: fetch every GitHub branch into the tracking namespace,
- * bring `refs/heads/*` toward it non-destructively (`reconcileTrackedBranches`),
- * push this deployment's own settings branch, fast-forward the base branch's
- * workspace, rebase every branch that is behind it (rebase.ts), then sweep old
- * tasks and expired trashed branch directories.
+ * One cycle, in order: repair what killed provisioning and deletes left under
+ * the branches root (`repairBranchDirResidue`), move sparse clones to a changed
+ * content root's cone (sparse-cone.ts), repack `remote.git` when it needs it
+ * (remote-git-maintenance.ts), fetch every GitHub branch into the tracking namespace,
+ * bring `refs/heads/*` toward GitHub's branches without touching editor work
+ * (`reconcileTrackedBranches`, which holds the base branch while schema-gate.ts
+ * says the serving editor lacks a schema the incoming content names), push this
+ * deployment's own settings
+ * branch, fast-forward the base branch's workspace, rebase every branch that is
+ * behind it (rebase.ts), then sweep old tasks and expired trashed branch
+ * directories.
  *
  * One ordering is load-bearing and nothing enforces it: `runRebaseCycle` MUST
  * follow `reconcileTrackedBranches`. Branch clones fetch the base tip from
  * `remote.git`, and `reconcileTrackedBranches` is what advances
- * `remote.git`'s `refs/heads/*` toward what the fetch above put in the tracking
- * namespace; reorder them and every branch rebases onto the PREVIOUS cycle's
+ * `remote.git`'s `refs/heads/*` toward the tips the fetch above reported;
+ * reorder them and every branch rebases onto the PREVIOUS cycle's
  * base tip -- not corrupting, but silently a cycle behind.
- * (`pushSettingsBranches` consuming `trackedNames` is a DATA dependency its
+ * (`pushSettingsBranches` consuming the fetch's tips is a DATA dependency its
  * signature already enforces; `refreshBaseBranchWorkspace` and the two sweeps
  * are order-independent.)
  *
  * The whole cycle is wrapped so both outcomes record a worker-status.json
  * snapshot, since an operator's only view of this loop is the admin panel.
  */
-export type GitSyncContext = Pick<WorkerContext, 'baseBranch' | 'sanitizedBaseBranch' | 'contentBranchesPath' | 'remoteGitPath' | 'taskDir' | 'taskTimeoutMs' | 'log' | 'buildGitHubUrl' | 'ensureSettingsBranch' | 'ensureStatusReport' | 'isRunning'> & RebaseContext;
+export type GitSyncContext = Pick<WorkerContext, 'baseBranch' | 'sanitizedBaseBranch' | 'contentBranchesPath' | 'remoteGitPath' | 'schemaHoldMaxMs' | 'taskDir' | 'taskTimeoutMs' | 'log' | 'github' | 'ensureSettingsBranch' | 'ensureStatusReport' | 'warnedDivergences' | 'isRunning' | 'isDraining' | 'shutdownSignal'> & RebaseContext;
 /**
  * Push THIS deployment's own settings branch (`ensureSettingsBranch()`) from
- * remote.git to GitHub. Non-fatal: a no-op push for an up-to-date branch just
- * succeeds quietly.
+ * remote.git to GitHub. Non-fatal. Skipped when `trackedTips`, GitHub's tips
+ * as fetched this cycle, already has the local head, so an unchanged branch
+ * costs no GitHub round trip; every push that does happen still passes the
+ * refusal checks in `MirrorSession.pushToGitHub`.
  *
  * Narrowed to that ONE branch, never every local `canopycms-settings-*`:
  * `reconcileTrackedBranches` creates local heads for branches that exist on
@@ -38,15 +46,24 @@ export type GitSyncContext = Pick<WorkerContext, 'baseBranch' | 'sanitizedBaseBr
  * can legitimately show up as a local head here. Pushing it would be this
  * deployment shipping settings state it does not own.
  */
-export declare function pushSettingsBranches(ctx: GitSyncContext, git: ReturnType<typeof simpleGit>, trackedNames: ReadonlySet<string>): Promise<void>;
+export declare function pushSettingsBranches(ctx: GitSyncContext, git: ReturnType<typeof simpleGit>, trackedTips: ReadonlyMap<string, string>): Promise<void>;
 export declare function syncGit(ctx: GitSyncContext): Promise<void>;
+/**
+ * Repair what killed provisioning and deletes left under the branches root: quarantine residue
+ * at a branch's name (branch-provisioning.ts's `classifyFinalDir`) once it has been quiet for
+ * {@link ORPHAN_YOUTH_THRESHOLD_MS}, then sweep stale `.prov-*`, `.deleting-*` and `.repair-*`
+ * siblings and settings leftovers. Each directory is best-effort, like [SYNC-M2]. The provisioning
+ * lock is taken zero-retry, so a live provisioner or admin action wins and the directory is
+ * revisited next cycle.
+ * @internal Exported for tests; `syncGit` runs it.
+ */
+export declare function repairBranchDirResidue(ctx: Pick<GitSyncContext, 'contentBranchesPath' | 'remoteGitPath'>, now?: number): Promise<void>;
 /**
  * [C1] Remove `.trash-*` branch directories (created by the admin purge action,
- * api/admin-branch-health.ts) whose name-embedded stamp is older than
- * {@link TRASH_RETENTION_MS}. A name that doesn't match `.trash-{dirName}-
- * {STAMP}`, or whose stamp fails to parse, is left alone and logged once per
- * cycle: purge is the only writer of this naming scheme, so an unparseable name
- * is unexpected and worth a human looking rather than a silent skip.
+ * api/admin-branch-health.ts, and by residue quarantine, branch-provisioning.ts)
+ * whose name-embedded stamp is older than {@link TRASH_RETENTION_MS}. A name
+ * whose trailing stamp fails to parse is left alone and logged once per cycle:
+ * both writers end every name with one, so it is worth a human looking.
  */
 export declare function cleanupTrashedBranchDirs(ctx: Pick<GitSyncContext, 'contentBranchesPath'>): Promise<number>;
 /**