canopycms 0.12.0 → 0.13.0

package/dist/worker/github-transfer.js

@@ -28,16 +28,19 @@ const MAX_PUSH_RESTARTS = 3;
 /**
  * Fetch every GitHub branch into `remote.git`'s tracking namespace (`GITHUB_TRACKING_REF_PREFIX`),
  * which then holds exactly GitHub's branches. `have` is every tip `remote.git` holds, so the
- * bundle carries only what it lacks. Returns GitHub's branches, and what an operator must fix in
- * remote.git for the namespace to keep up, if anything (see {@link moveTrackingRefs}).
+ * bundle carries only what it lacks. `fresh` takes no GitHub fetch the gateway began before this
+ * call. Returns GitHub's branches, when the GitHub fetch they come from began, and what an operator
+ * must fix in remote.git for the namespace to keep up, if anything (see {@link moveTrackingRefs}).
  */
-export async function fetchGitHubIntoRemoteGit(ctx, signal) {
+export async function fetchGitHubIntoRemoteGit(ctx, signal, fresh = false) {
     const gitDir = ctx.remoteGitPath;
-    const { refs, bundleId } = await ctx.github().fetch({ have: await haveOf(gitDir) }, signal);
+    const { refs, bundleId, startedAt } = await ctx
+        .github()
+        .fetch({ have: await haveOf(gitDir), fresh }, signal);
     if (bundleId !== null)
         await unbundle(ctx, gitDir, bundleId, signal);
     const needsOperator = await moveTrackingRefs(ctx, gitDir, refs, signal);
-    return { refs, needsOperator };
+    return { refs, fetchedAt: startedAt, needsOperator };
 }
 /**
  * Seed the new bare repository at `gitDir`, which must not exist yet, with every GitHub branch as

package/dist/worker/history-rewrite.d.ts

@@ -46,7 +46,8 @@ export declare function readPublishedSha(ctx: Pick<HistoryRewriteContext, 'remot
  *
  * `rewrittenTip` goes in the same write as `lastGitHubTip`, so a push that lands
  * and dies before its stamp is never read as someone rewriting the branch on
- * GitHub (github-commits.ts).
+ * GitHub (github-commits.ts), and as `historyRewrittenTo`, which unlike the
+ * marker each rewrite advances: it is what the publish sends.
  *
  * Resolves whether `branchRef`'s branch.json now holds a marker, set here or kept by set-once.
  * When it does not, the caller publishes nothing: a rewrite pushed without its marker leaves
@@ -68,9 +69,20 @@ export declare function markHistoryRewritten(ctx: Pick<HistoryRewriteContext, 'c
  * in safeinsights/canopycms#902.
  */
 export declare function clearHistoryRewrittenMarker(ctx: Pick<HistoryRewriteContext, 'contentBranchesPath'>, branchPath: string, branchRef: string): Promise<void>;
+/** How {@link forcePublishToLocalRemote} ended. */
+export type RewritePublish = {
+    kind: 'published';
+} | {
+    kind: 'moved';
+} | {
+    kind: 'failed';
+    message: string;
+};
 /**
- * Publish a branch clone's rebased history into `remote.git`, replacing
- * EXACTLY `expectedSha` and nothing else.
+ * Publish `rewrittenTip`, a branch clone's rebased history, into `remote.git`,
+ * replacing EXACTLY `expectedSha` and nothing else. The tip, never the clone's
+ * branch ref, which can hold a commit made since that no one has submitted
+ * (publish-decision.ts).
  *
  * The lease is the entire safety argument: `--force-with-lease=<ref>:<sha>`
  * refuses unless `remote.git` still stands at `<sha>`, so this can only ever
@@ -79,10 +91,12 @@ export declare function clearHistoryRewrittenMarker(ctx: Pick<HistoryRewriteCont
  * carryForwardRewrittenHistory for the interleaving where that would silently
  * delete someone's direct push.
  *
- * Returns whether the push landed. A refused lease means a concurrent Lambda
- * push moved the ref; logged and retried by the self-heal pass, never thrown.
+ * Resolves `published`, `moved` for a refused lease (a concurrent Lambda push
+ * moved the ref; the next cycle judges the branch again), or `failed` with
+ * git's message.
+ * Never throws.
  */
-export declare function forcePublishToLocalRemote(ctx: Pick<HistoryRewriteContext, 'remoteGitPath' | 'taskTimeoutMs'>, branchPath: string, branchRef: string, expectedSha: string): Promise<boolean>;
+export declare function forcePublishToLocalRemote(ctx: Pick<HistoryRewriteContext, 'remoteGitPath' | 'taskTimeoutMs'>, branchPath: string, branchRef: string, expectedSha: string, rewrittenTip: string): Promise<RewritePublish>;
 /**
  * Queue the GitHub hop for a branch whose rewritten history now sits in
  * `remote.git`, so an open PR's head follows the rebase within a cycle instead
@@ -99,6 +113,18 @@ export declare function forcePublishToLocalRemote(ctx: Pick<HistoryRewriteContex
  * the policy does not predict; the tasks are idempotent.
  */
 export declare function enqueueGitHubPush(ctx: Pick<HistoryRewriteContext, 'taskDir'>, branchRef: string, gatewayPolicy: BranchPolicyDescription | undefined): Promise<void>;
+/**
+ * `settled`: `remote.git` no longer stands at the marker, or never will by this pass. `waiting`: a
+ * concurrent push moved it this cycle. `failed`: the publish cannot land; `reason` says why.
+ */
+export type PendingRewrite = {
+    kind: 'settled';
+} | {
+    kind: 'waiting';
+} | {
+    kind: 'failed';
+    reason: string;
+};
 /**
  * Complete a rewrite this worker started but did not finish: get the rebased
  * history into `remote.git` and queued for GitHub.
@@ -108,7 +134,11 @@ export declare function enqueueGitHubPush(ctx: Pick<HistoryRewriteContext, 'task
  * waiting for the next base-branch advance.
  *
  * Always leases on the MARKER, never on remote.git's current tip -- the marker
- * is the one commit we know our own rebase replaced.
+ * is the one commit we know our own rebase replaced -- and sends the recorded
+ * rebased tip, `target`, never `headSha`: a commit made in the clone since the
+ * rewrite is no one's submitted work. Resolves {@link PendingRewrite}: while
+ * `remote.git` still stands at the marker the branch takes nothing in, since a
+ * rebase would replace the rewrite along with any commit made on top of it.
  */
 export declare function reconcilePendingRewrite(ctx: HistoryRewriteContext, options: {
     branchPath: string;
@@ -116,8 +146,10 @@ export declare function reconcilePendingRewrite(ctx: HistoryRewriteContext, opti
     branchRef: string;
     headSha: string;
     marker: string;
+    /** `historyRewrittenTo`, the rebased tip the rewrite recorded. */
+    target: string | undefined;
     gatewayPolicy: BranchPolicyDescription | undefined;
-}): Promise<void>;
+}): Promise<PendingRewrite>;
 /**
  * Whether this branch carries a pending history rewrite -- the rebase loop
  * rewrote already-published history and the GitHub push has not landed yet

package/dist/worker/history-rewrite.js

@@ -49,7 +49,8 @@ export async function readPublishedSha(ctx, branchRef) {
  *
  * `rewrittenTip` goes in the same write as `lastGitHubTip`, so a push that lands
  * and dies before its stamp is never read as someone rewriting the branch on
- * GitHub (github-commits.ts).
+ * GitHub (github-commits.ts), and as `historyRewrittenTo`, which unlike the
+ * marker each rewrite advances: it is what the publish sends.
  *
  * Resolves whether `branchRef`'s branch.json now holds a marker, set here or kept by set-once.
  * When it does not, the caller publishes nothing: a rewrite pushed without its marker leaves
@@ -66,6 +67,7 @@ export async function markHistoryRewritten(ctx, branchPath, branchRef, expectedS
         return {
             branch: {
                 historyRewrittenFrom: typeof marker === 'string' ? marker : expectedSha,
+                historyRewrittenTo: rewrittenTip,
                 lastGitHubTip: rewrittenTip,
             },
         };
@@ -92,7 +94,7 @@ export async function clearHistoryRewrittenMarker(ctx, branchPath, branchRef) {
         // Explicit undefined clears the key -- save()'s merge only overwrites
         // keys present in the update (same pattern as rebaseFailure). Only a
         // marker on `branchRef`'s own branch.json is cleared, so none is ever created here.
-        await meta.saveIf({ branch: { historyRewrittenFrom: undefined } }, (current) => current?.branch.name === branchRef &&
+        await meta.saveIf({ branch: { historyRewrittenFrom: undefined, historyRewrittenTo: undefined } }, (current) => current?.branch.name === branchRef &&
             typeof current.branch.historyRewrittenFrom === 'string');
     }
     catch (err) {
@@ -100,8 +102,10 @@ export async function clearHistoryRewrittenMarker(ctx, branchPath, branchRef) {
     }
 }
 /**
- * Publish a branch clone's rebased history into `remote.git`, replacing
- * EXACTLY `expectedSha` and nothing else.
+ * Publish `rewrittenTip`, a branch clone's rebased history, into `remote.git`,
+ * replacing EXACTLY `expectedSha` and nothing else. The tip, never the clone's
+ * branch ref, which can hold a commit made since that no one has submitted
+ * (publish-decision.ts).
  *
  * The lease is the entire safety argument: `--force-with-lease=<ref>:<sha>`
  * refuses unless `remote.git` still stands at `<sha>`, so this can only ever
@@ -110,10 +114,12 @@ export async function clearHistoryRewrittenMarker(ctx, branchPath, branchRef) {
  * carryForwardRewrittenHistory for the interleaving where that would silently
  * delete someone's direct push.
  *
- * Returns whether the push landed. A refused lease means a concurrent Lambda
- * push moved the ref; logged and retried by the self-heal pass, never thrown.
+ * Resolves `published`, `moved` for a refused lease (a concurrent Lambda push
+ * moved the ref; the next cycle judges the branch again), or `failed` with
+ * git's message.
+ * Never throws.
  */
-export async function forcePublishToLocalRemote(ctx, branchPath, branchRef, expectedSha) {
+export async function forcePublishToLocalRemote(ctx, branchPath, branchRef, expectedSha, rewrittenTip) {
     // sharedRepoGit's env is gitChildEnv's, whose C-locale pin is what keeps
     // isStaleLeaseRejection below from silently becoming a no-op on a
     // non-English host.
@@ -121,24 +127,26 @@ export async function forcePublishToLocalRemote(ctx, branchPath, branchRef, expe
     try {
         await pushGit.raw([
             'push',
-            `--force-with-lease=${branchRef}:${expectedSha}`,
+            `--force-with-lease=refs/heads/${branchRef}:${expectedSha}`,
             // Otherwise remote.git's own hooks run, as the worker.
             `--receive-pack=${pinnedReceivePack()}`,
             // Real flags must precede --end-of-options; everything after it is
             // positional (see GitManager.push() for the same guard).
             '--end-of-options',
             ctx.remoteGitPath,
-            `${branchRef}:${branchRef}`,
+            `${rewrittenTip}:refs/heads/${branchRef}`,
         ]);
         workerLog(`  Published rebased ${branchRef} into remote.git`);
-        return true;
+        return { kind: 'published' };
     }
     catch (err) {
         const message = redactCredentials(getErrorMessage(err));
-        workerLogWarn(isStaleLeaseRejection(message)
-            ? `  Did not publish rebased ${branchRef} into remote.git: it moved since this cycle read it (concurrent submit?) -- retrying next cycle`
-            : `  Failed to publish rebased ${branchRef} into remote.git: ${message}`);
-        return false;
+        if (isStaleLeaseRejection(message)) {
+            workerLogWarn(`  Did not publish rebased ${branchRef} into remote.git: it moved since this cycle read it (concurrent submit?) -- retrying next cycle`);
+            return { kind: 'moved' };
+        }
+        workerLogWarn(`  Failed to publish rebased ${branchRef} into remote.git: ${message}`);
+        return { kind: 'failed', message };
     }
 }
 /**
@@ -178,6 +186,10 @@ export async function enqueueGitHubPush(ctx, branchRef, gatewayPolicy) {
         workerLogWarn(`  Failed to queue GitHub push for ${branchRef}: ${getErrorMessage(err)}`);
     }
 }
+/** Why a branch whose rewrite was recorded without its rebased tip takes nothing in. */
+const REWRITE_TIP_UNKNOWN = `this branch's rebased history was recorded without the commit to publish, so neither it nor the ` +
+    `base is taken in. A developer publishes the rewrite by hand before the branch is submitted again.`;
+const COMMIT_ID = /^[0-9a-f]{40}([0-9a-f]{24})?$/;
 /**
  * Complete a rewrite this worker started but did not finish: get the rebased
  * history into `remote.git` and queued for GitHub.
@@ -187,33 +199,49 @@ export async function enqueueGitHubPush(ctx, branchRef, gatewayPolicy) {
  * waiting for the next base-branch advance.
  *
  * Always leases on the MARKER, never on remote.git's current tip -- the marker
- * is the one commit we know our own rebase replaced.
+ * is the one commit we know our own rebase replaced -- and sends the recorded
+ * rebased tip, `target`, never `headSha`: a commit made in the clone since the
+ * rewrite is no one's submitted work. Resolves {@link PendingRewrite}: while
+ * `remote.git` still stands at the marker the branch takes nothing in, since a
+ * rebase would replace the rewrite along with any commit made on top of it.
  */
 export async function reconcilePendingRewrite(ctx, options) {
     const { branchPath, branchDir, branchRef, headSha, marker, gatewayPolicy } = options;
+    const target = options.target !== undefined && COMMIT_ID.test(options.target) ? options.target : undefined;
     const publishedSha = await readPublishedSha(ctx, branchRef);
     if (publishedSha === null) {
         workerLogWarn(`  ${branchDir}: a rewritten history is recorded but ${branchRef} is gone from remote.git -- nothing to publish`);
-        return;
+        return { kind: 'settled' };
     }
-    if (publishedSha === headSha) {
-        // remote.git already carries the rewrite; only the GitHub hop is left
-        // (a crash between the push and the queue, or a task that was lost).
+    if (publishedSha === target || publishedSha === headSha) {
+        // remote.git already carries the rewrite, or a submit since; only the
+        // GitHub hop is left (a crash between the push and the queue, or a task
+        // that was lost).
         await enqueueGitHubPush(ctx, branchRef, gatewayPolicy);
-        return;
+        return { kind: 'settled' };
     }
     if (publishedSha === marker) {
         // The publish into remote.git never landed -- a crash right after the
         // marker was written, or a lease refused by a concurrent submit.
-        if (await forcePublishToLocalRemote(ctx, branchPath, branchRef, marker)) {
-            await enqueueGitHubPush(ctx, branchRef, gatewayPolicy);
+        if (target === undefined)
+            return { kind: 'failed', reason: REWRITE_TIP_UNKNOWN };
+        const publish = await forcePublishToLocalRemote(ctx, branchPath, branchRef, marker, target);
+        if (publish.kind === 'moved')
+            return { kind: 'waiting' };
+        if (publish.kind === 'failed') {
+            return {
+                kind: 'failed',
+                reason: `this branch's rebased history could not be published into the deployment's repository: ${publish.message}`,
+            };
         }
-        return;
+        await enqueueGitHubPush(ctx, branchRef, gatewayPolicy);
+        return { kind: 'settled' };
     }
     // Neither this branch's rebased tip nor the commit its rewrite replaced:
     // something else moved remote.git. Never force over that.
     workerLogWarn(`  ${branchDir}: remote.git is at ${publishedSha} for ${branchRef}, which is neither the ` +
         `rebased tip nor the commit the rewrite replaced -- left untouched`);
+    return { kind: 'settled' };
 }
 /**
  * Whether this branch carries a pending history rewrite -- the rebase loop

package/dist/worker/publish-decision.d.ts

@@ -0,0 +1,37 @@
+import type { SimpleGit } from 'simple-git';
+/**
+ * What the worker may publish for a branch: move its head in `remote.git`, which is all a GitHub
+ * push ever sends (task-runner.ts `pushBranchToGitHub`). Work the author has saved but not
+ * submitted reaches neither until their submit pushes it, so a commit the worker builds on top of
+ * such work stays in the clone. The worker publishes in two places, and each asks here or follows
+ * the rule stated here:
+ *
+ * - a merge it built (github-commits.ts `mergeIntoBranch`): {@link decidePublish};
+ * - a rewrite of history already published (history-rewrite.ts): only the rebased tip recorded with
+ *   the rewrite, never the clone's current head, which can carry a commit made since. While that
+ *   tip is unpublished the branch takes nothing in (rebase.ts), so nothing is built on it.
+ */
+export type PublishDecision = 
+/** `remote.git` has no head for the branch: nothing of it was ever published. */
+{
+    kind: 'never-published';
+}
+/** Every commit on the branch's side is already in `remote.git`'s head. */
+ | {
+    kind: 'publish';
+}
+/** The branch's side holds commits the author has not submitted: the merge stays in the clone. */
+ | {
+    kind: 'hold';
+};
+/**
+ * Whether a merge built on `head` may be published over `published`, `remote.git`'s head: only
+ * when `remote.git` already holds all of `head`, whatever the branch's status, since a save can be
+ * committed in the moment between a submit's push and its status write. Judged from the commits
+ * every time, so nothing records a hold: the next submit pushes the clone, held merges included,
+ * as a fast-forward. The caller has checked that the clone holds `published`.
+ */
+export declare function decidePublish(git: SimpleGit, { head, published }: {
+    head: string;
+    published: string | null;
+}): Promise<PublishDecision>;

package/dist/worker/publish-decision.js

@@ -0,0 +1,16 @@
+import { commitsApart } from './branch-merge.js';
+/**
+ * Whether a merge built on `head` may be published over `published`, `remote.git`'s head: only
+ * when `remote.git` already holds all of `head`, whatever the branch's status, since a save can be
+ * committed in the moment between a submit's push and its status write. Judged from the commits
+ * every time, so nothing records a hold: the next submit pushes the clone, held merges included,
+ * as a fast-forward. The caller has checked that the clone holds `published`.
+ */
+export async function decidePublish(git, { head, published }) {
+    if (published === null)
+        return { kind: 'never-published' };
+    if (published === head)
+        return { kind: 'publish' };
+    const { onlyA } = await commitsApart(git, head, published);
+    return onlyA === 0 ? { kind: 'publish' } : { kind: 'hold' };
+}

package/dist/worker/rebase.d.ts

@@ -7,8 +7,9 @@ import type { WorkerContext } from './worker-context.js';
  * The rebase loop: the deepest leaf of the git-sync cluster, and the most
  * safety-critical part of the worker. `syncGit` (git-sync.ts) calls
  * `runRebaseCycle` once per cycle; every branch workspace behind the base
- * branch is rebased onto it, keeping the BRANCH's version of any conflicting
- * file. Three things make it the part most likely to misbehave under real EFS
+ * branch takes it in: by a rebase where git and the field merge agree there is
+ * nothing to decide, and by a merge (github-commits.ts' `mergeIntoBranch`)
+ * wherever there is. Three things make it the part most likely to misbehave under real EFS
  * and spot conditions, all documented at the point they happen:
  *
  * - it holds the [SYNC-C1] cross-host content-write lock for the whole rebase,
@@ -27,7 +28,7 @@ import type { WorkerContext } from './worker-context.js';
  * but still need their PR's resolution polled, and this loop is the only thing
  * that walks every branch workspace.
  */
-export type RebaseContext = Pick<WorkerContext, 'baseBranch' | 'sanitizedBaseBranch' | 'contentBranchesPath' | 'contentRoot' | 'remoteGitPath' | 'taskDir' | 'taskTimeoutMs' | 'github' | 'ensureStatusReport' | 'afterConflictDetectedForTesting' | 'afterRebaseCompletedForTesting' | 'ensureSettingsBranch' | 'isDraining'>;
+export type RebaseContext = Pick<WorkerContext, 'baseBranch' | 'sanitizedBaseBranch' | 'contentBranchesPath' | 'contentRoot' | 'remoteGitPath' | 'taskDir' | 'taskTimeoutMs' | 'github' | 'ensureStatusReport' | 'afterConflictDetectedForTesting' | 'afterRebaseCompletedForTesting' | 'ensureSettingsBranch' | 'isDraining' | 'requestBranchSync'>;
 /**
  * Per-cycle outcome of `rebaseActiveBranches()`, folded by `syncGit()` into the
  * worker's self-reported status (`WorkerStatusReport.lastGitSync`, see
@@ -73,16 +74,36 @@ export interface RebaseSummary {
  * next sync cycle. Resolves GitHub's answer for a pull request not merged.
  */
 export declare function pollMergeState(ctx: RebaseContext, branchDir: string, branchPath: string, metaFile: BranchMetadataFile | null): Promise<GitHubPullRequest | null>;
+/**
+ * What a request for the latest changes is answered from: the fetch of the base that the branch
+ * was brought up to. The worker's cycle takes it from its GitHub fetch; a run in the request
+ * ({@link syncBranchNow}) from its own start, its base being `remote.git`'s.
+ */
+interface BaseUpdateSource {
+    /** When that fetch began. */
+    fetchedAt: number;
+    /** The base commit it found; undefined when it reported none. */
+    baseTip?: string;
+}
 /**
  * Rebase every branch workspace that is behind the base branch, one cycle.
  *
- * The per-branch work is {@link rebaseOneBranch}; what stays here is the walk
- * over `content-branches/`, the structural pre-filter (entries that are not
- * branch workspaces at all, plus the base branch's own clone), and folding
- * each branch's outcome into the summary.
+ * The per-branch work is {@link syncBranchDir}; what stays here is the walk
+ * over `content-branches/` and folding each branch's outcome into the summary.
+ * `source` answers requests for the latest changes; the GitHub fetch's by default. `local` is a
+ * run with no GitHub (see {@link syncBranchDir}).
  *
  * Deliberately sequential rather than concurrent: each branch's rebase holds
  * that branch's cross-host content-write lock and runs a series of git
  * subprocesses against a shared filesystem.
  */
-export declare function runRebaseCycle(ctx: RebaseContext, gatewayPolicy?: BranchPolicyDescription, gitHub?: GitHubSnapshot): Promise<RebaseSummary>;
+export declare function runRebaseCycle(ctx: RebaseContext, gatewayPolicy?: BranchPolicyDescription, gitHub?: GitHubSnapshot, requested?: ReadonlySet<string>, source?: BaseUpdateSource | undefined, local?: boolean): Promise<RebaseSummary>;
+/**
+ * Bring the base into one branch now and answer its request for the latest changes, for a mode that
+ * runs no worker loop (operating-mode `baseUpdateRunsInRequest`). With no GitHub, the base is
+ * `remote.git`'s, so the answer is from this run's start. A run that answers nothing writes
+ * `failed`, since nothing else ever will: `busy` says a lock held it back, which a retry in a moment
+ * gets past. Never throws but for the metadata write of that fallback.
+ */
+export declare function syncBranchNow(ctx: RebaseContext, branchName: string): Promise<'answered' | 'busy'>;
+export {};