canopycms 0.0.67 → 0.0.68-int.101

package/dist/worker/cms-worker.d.ts

@@ -15,11 +15,11 @@ export type AuthCacheRefresher = () => Promise<void>;
  * interface for the two shapes and why an App is optional.
  */
 export interface CmsWorkerConfig extends GitHubAuthConfig {
-    /** Path to workspace root on EFS (e.g., /mnt/efs/workspace) */
+    /** Path to workspace root on EFS (e.g., /mnt/efs) */
     workspacePath: string;
-    /** GitHub owner (e.g., 'safeinsights') */
+    /** GitHub owner (e.g., 'acme') */
     githubOwner: string;
-    /** GitHub repo name (e.g., 'docs-site') */
+    /** GitHub repo name (e.g., 'site') */
     githubRepo: string;
     /** Called periodically to update the auth metadata cache on EFS. */
     refreshAuthCache?: AuthCacheRefresher;
@@ -203,6 +203,11 @@ export declare class CmsWorker {
      * survives, so a failed scrub is never indistinguishable from a clean one.
      */
     private scrubPersistedRemote;
+    /**
+     * Best-effort: without these settings remote.git still works, and a boot that
+     * cannot write its own config has a louder problem to report elsewhere.
+     */
+    private applyRemoteGitConfig;
     /**
      * Ensure the remote.git bare repo exists, cloning it from GitHub on first
      * run.

package/dist/worker/cms-worker.js

@@ -5,10 +5,12 @@ import lockfile from 'proper-lockfile';
 import { recoverOrphanedTasks, cmsTaskQueueLogger } from '../task-queue/cms-task-queue.js';
 import { createCanopyOctokit } from '../github-service.js';
 import { isTransientAuthFailure, resolveWorkerGitHubAuth, } from './github-auth.js';
+import { ensureRemoteGitConfig } from '../git-manager.js';
 import { sanitizeBranchName, RESERVED_SETTINGS_BRANCH_PREFIX } from '../paths/branch-name.js';
 import { resolveDeploymentName } from '../operating-mode/deployment-name.js';
 import { getErrorMessage, isNodeError, redactCredentials } from '../utils/error.js';
-import { writeWorkerStatus } from '../task-queue/worker-status.js';
+import { readLastFatalError, writeWorkerStatus } from '../task-queue/worker-status.js';
+import { CANOPYCMS_VERSION } from '../version.js';
 import { workerLog, workerLogWarn, workerLogError } from './log.js';
 import { executeTask, orphanRecoveryMaxAgeMs, processTaskQueue, pushBranchToGitHub, updateBranchMetadata, } from './task-runner.js';
 import { pollMergeState, runRebaseCycle } from './rebase.js';
@@ -67,7 +69,12 @@ export class CmsWorker {
     ensureStatusReport() {
         if (!this.statusReport) {
             const now = new Date().toISOString();
-            this.statusReport = { version: 1, startedAt: now, updatedAt: now };
+            this.statusReport = {
+                version: 1,
+                workerVersion: CANOPYCMS_VERSION,
+                startedAt: now,
+                updatedAt: now,
+            };
         }
         return this.statusReport;
     }
@@ -144,6 +151,21 @@ export class CmsWorker {
         workerLog('CMS Worker starting...');
         this.ensureStatusReport();
         await this.acquireLock();
+        // Replace the previous holder's status file now: the first sync can take
+        // minutes, and until then System health would report the old worker's
+        // version. Best-effort, like the startup-failure write below. The previous
+        // `lastFatalError` rides along in this snapshot only, so a crash loop keeps
+        // its alert between restarts while the first successful sync still clears it.
+        try {
+            const lastFatalError = await readLastFatalError(this.taskDir);
+            await writeWorkerStatus(this.taskDir, {
+                ...this.ensureStatusReport(),
+                ...(lastFatalError ? { lastFatalError } : {}),
+            });
+        }
+        catch (err) {
+            workerLogError('Failed to write worker status after acquiring the lock:', getErrorMessage(err));
+        }
         // Everything below runs while holding the cross-host worker lock. A failure
         // here (most notably the empty-remote guard inside ensureRemoteGit) means
         // the process is about to exit and systemd (Restart=always) will retry —
@@ -411,6 +433,18 @@ export class CmsWorker {
                 `shared storage. Refusing to continue.`);
         }
     }
+    /**
+     * Best-effort: without these settings remote.git still works, and a boot that
+     * cannot write its own config has a louder problem to report elsewhere.
+     */
+    async applyRemoteGitConfig(gitDir) {
+        try {
+            await ensureRemoteGitConfig(gitDir);
+        }
+        catch (err) {
+            workerLogWarn(`Could not apply remote.git config in ${gitDir}: ${getErrorMessage(err)}`);
+        }
+    }
     /**
      * Ensure the remote.git bare repo exists, cloning it from GitHub on first
      * run.
@@ -450,6 +484,7 @@ export class CmsWorker {
                 // deletion is the operator's call.
                 throw new Error(`remote.git at ${this.remoteGitPath} has no branch '${this.baseBranch}' (likely cloned while the GitHub repo was empty). Delete ${this.remoteGitPath} and restart the worker to re-clone.`);
             }
+            await this.applyRemoteGitConfig(this.remoteGitPath);
             return; // Already exists and has the base branch
         }
         workerLog('Initializing remote.git from GitHub...');
@@ -468,6 +503,7 @@ export class CmsWorker {
             // swallowing) if the scrub does not take.
             await this.scrubPersistedRemote(stagingPath);
             await this.verifyBaseBranchExists(stagingPath);
+            await this.applyRemoteGitConfig(stagingPath);
         }
         catch (err) {
             workerLogError(`remote.git clone failed: ${redactCredentials(getErrorMessage(err))}`);

package/dist/worker/git-sync.d.ts

@@ -1,4 +1,5 @@
 import { simpleGit } from 'simple-git';
+import type { BaseRefreshReport } from '../types.js';
 import { type RebaseContext } from './rebase.js';
 import type { WorkerContext } from './worker-context.js';
 /**
@@ -6,7 +7,10 @@ import type { WorkerContext } from './worker-context.js';
  * slower of the worker's two poll loops (default 5 minutes, against the task
  * queue's 5 seconds).
  *
- * One cycle, in order: fetch every GitHub branch into the tracking namespace,
+ * One cycle, in order: repair what killed provisioning and deletes left under
+ * the branches root (`repairBranchDirResidue`), move sparse clones to a changed
+ * content root's cone (sparse-cone.ts), repack `remote.git` when it needs it
+ * (remote-git-maintenance.ts), fetch every GitHub branch into the tracking namespace,
  * bring `refs/heads/*` toward it non-destructively (`reconcileTrackedBranches`),
  * push this deployment's own settings branch, fast-forward the base branch's
  * workspace, rebase every branch that is behind it (rebase.ts), then sweep old
@@ -14,7 +18,7 @@ import type { WorkerContext } from './worker-context.js';
  *
  * One ordering is load-bearing and nothing enforces it: `runRebaseCycle` MUST
  * follow `reconcileTrackedBranches`. Branch clones fetch the base tip from
- * `remote.git` (`origin`), and `reconcileTrackedBranches` is what advances
+ * `remote.git`, and `reconcileTrackedBranches` is what advances
  * `remote.git`'s `refs/heads/*` toward what the fetch above put in the tracking
  * namespace; reorder them and every branch rebases onto the PREVIOUS cycle's
  * base tip -- not corrupting, but silently a cycle behind.
@@ -39,27 +43,41 @@ export type GitSyncContext = Pick<WorkerContext, 'baseBranch' | 'sanitizedBaseBr
  */
 export declare function pushSettingsBranches(ctx: GitSyncContext, git: ReturnType<typeof simpleGit>, trackedNames: ReadonlySet<string>): Promise<void>;
 export declare function syncGit(ctx: GitSyncContext): Promise<void>;
+/**
+ * Repair what killed provisioning and deletes left under the branches root: quarantine residue
+ * at a branch's name (branch-provisioning.ts's `classifyFinalDir`) once it has been quiet for
+ * {@link ORPHAN_YOUTH_THRESHOLD_MS}, then sweep stale `.prov-*`, `.deleting-*` and `.repair-*`
+ * siblings and settings leftovers. Each directory is best-effort, like [SYNC-M2]. The provisioning
+ * lock is taken zero-retry, so a live provisioner or admin action wins and the directory is
+ * revisited next cycle.
+ * @internal Exported for tests; `syncGit` runs it.
+ */
+export declare function repairBranchDirResidue(ctx: Pick<GitSyncContext, 'contentBranchesPath' | 'remoteGitPath'>, now?: number): Promise<void>;
 /**
  * [C1] Remove `.trash-*` branch directories (created by the admin purge action,
- * api/admin-branch-health.ts) whose name-embedded stamp is older than
- * {@link TRASH_RETENTION_MS}. A name that doesn't match `.trash-{dirName}-
- * {STAMP}`, or whose stamp fails to parse, is left alone and logged once per
- * cycle: purge is the only writer of this naming scheme, so an unparseable name
- * is unexpected and worth a human looking rather than a silent skip.
+ * api/admin-branch-health.ts, and by residue quarantine, branch-provisioning.ts)
+ * whose name-embedded stamp is older than {@link TRASH_RETENTION_MS}. A name
+ * whose trailing stamp fails to parse is left alone and logged once per cycle:
+ * both writers end every name with one, so it is worth a human looking.
  */
 export declare function cleanupTrashedBranchDirs(ctx: Pick<GitSyncContext, 'contentBranchesPath'>): Promise<number>;
 /**
  * Fast-forward the base branch's own working-tree clone
- * (content-branches/<baseBranch>) to match origin/<baseBranch>, every sync
+ * (content-branches/<baseBranch>) to match remote.git's <baseBranch>, every sync
  * cycle, so the drift window is bounded by gitSyncInterval.
  *
  * A dedicated, explicit and LOUD step rather than a side effect of the rebase
  * loop, whose skip paths (a dirty tree, a missing .git) are silent: a wedged
  * base clone otherwise leaves no diagnosable signal, and an editor forking a
- * new branch "from base" silently gets a stale snapshot.
+ * new branch "from base" silently gets a stale snapshot. The returned outcome
+ * goes to worker-status.json for the same reason.
  *
- * ff-only on purpose: this clone must stay a linear mirror of
- * origin/<baseBranch>, so a merge that isn't a fast-forward (diverged local
+ * ff-only on purpose: this clone must stay a linear mirror of remote.git's
+ * <baseBranch>, so a merge that isn't a fast-forward (diverged local
  * history) is left untouched rather than force-resolved.
+ *
+ * Holds the provisioning lock and then, like the rebase loop, the [SYNC-C1]
+ * content-write lock, both try-only: the base branch is writable in dev, and
+ * a save racing the merge's working-tree update can be overwritten by it.
  */
-export declare function refreshBaseBranchWorkspace(ctx: GitSyncContext): Promise<void>;
+export declare function refreshBaseBranchWorkspace(ctx: GitSyncContext): Promise<BaseRefreshReport>;

package/dist/worker/git-sync.js

@@ -2,39 +2,31 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 import { simpleGit } from 'simple-git';
 import { BranchMetadataFileManager, getBranchMetadataFileManager } from '../branch-metadata.js';
+import { BRANCH_META_DIR, BRANCH_META_FILE } from '../branch-metadata-file.js';
+import { ORPHAN_YOUTH_THRESHOLD_MS } from '../branch-health.js';
+import { classifyFinalDir, parseDirStamp, quarantineResidueAt, sweepProvisioningLeftovers, } from '../branch-provisioning.js';
 import { invalidateBranchContentCaches } from '../content-index-generation.js';
-import { GITHUB_TRACKING_REF_PREFIX, gitNetworkChildEnv } from '../git-manager.js';
+import { GITHUB_TRACKING_REF_PREFIX, ensureGitExcludePattern, gitNetworkChildEnv, } from '../git-manager.js';
 import { RESERVED_SETTINGS_BRANCH_PREFIX } from '../paths/branch-name.js';
+import { tryAcquireContentWriteLock } from '../utils/content-write-lock.js';
+import { branchProvisioningLockName, tryAcquireProvisioningLock } from '../utils/provisioning-lock.js';
 import { getErrorMessage, isNodeError, redactCredentials } from '../utils/error.js';
-import { isNonFastForwardRejection } from '../utils/git.js';
+import { CANOPY_META_DIR, isCanopyInternalPath, isNonFastForwardRejection } from '../utils/git.js';
+import { MAX_REPORTED_PATHS, TRACKED_CANOPY_STATE_FIX, isUntracked, listTrackedCanopyState, restoreRetiredSchemaCache, splitByUpstreamTracking, untrackInIndex, } from './canopy-state.js';
 import { hasPendingHistoryRewrite } from './history-rewrite.js';
 import { runRebaseCycle } from './rebase.js';
 import { cleanupOldTasks } from '../task-queue/cms-task-queue.js';
 import { writeWorkerStatus } from '../task-queue/worker-status.js';
 import { workerLog, workerLogError, workerLogWarn } from './log.js';
+import { holdProvisionedWorkspace, releaseProvisionedWorkspace } from './provisioned-workspace.js';
+import { maintainRemoteGit } from './remote-git-maintenance.js';
+import { reapplySparseCones } from './sparse-cone.js';
 /**
  * [C1] Retention window for `.trash-*` branch directories left behind by the
  * admin purge action (api/admin-branch-health.ts). Matches
  * cleanupOldTasks's default task retention for consistency.
  */
 const TRASH_RETENTION_MS = 30 * 24 * 60 * 60_000;
-/** Matches `.trash-{dirName}-{STAMP}` names, capturing the trailing stamp. */
-const TRASH_DIR_STAMP_RE = /-(\d{8}T\d{6}Z)$/;
-/**
- * Parse a purge-generated `YYYYMMDDTHHMMSSZ` stamp into a Date, or null if
- * malformed. Age comes ONLY from this name-embedded stamp, never the dir's
- * own mtime -- `fs.rename` preserves the original directory's mtime, so an
- * mtime-based retention check would delete a months-stale orphan's trash on
- * the very first cleanup pass after purge.
- */
-function parseTrashStamp(stamp) {
-    const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(stamp);
-    if (!match)
-        return null;
-    const [, year, month, day, hour, minute, second] = match;
-    const date = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.000Z`);
-    return Number.isNaN(date.getTime()) ? null : date;
-}
 /**
  * Push THIS deployment's own settings branch (`ensureSettingsBranch()`) from
  * remote.git to GitHub. Non-fatal: a no-op push for an up-to-date branch just
@@ -288,6 +280,27 @@ export async function syncGit(ctx) {
     // mask the real error on a failed one. Failures rethrow, so scheduleLoop's
     // per-cycle catch stays the loud path.
     try {
+        // Best-effort and ahead of the GitHub fetch, so a GitHub outage never
+        // stalls upkeep and a failure here never costs the cycle. start() runs a
+        // cycle at once, so residue repair also runs on boot.
+        try {
+            await repairBranchDirResidue(ctx);
+        }
+        catch (err) {
+            workerLogWarn(`Branch directory residue repair failed: ${getErrorMessage(err)}`);
+        }
+        try {
+            await reapplySparseCones(ctx);
+        }
+        catch (err) {
+            workerLogWarn(`Sparse-checkout cone update failed: ${getErrorMessage(err)}`);
+        }
+        try {
+            await maintainRemoteGit(ctx.remoteGitPath);
+        }
+        catch (err) {
+            workerLogWarn(`remote.git maintenance failed: ${getErrorMessage(err)}`);
+        }
         // Direct URL (no named remote), into the GITHUB_TRACKING_REF_PREFIX
         // remote-tracking namespace rather than refs/heads/* -- see that constant's
         // doc comment for the destructive-fetch bug this avoids. Raw git, because
@@ -306,7 +319,7 @@ export async function syncGit(ctx) {
         // correctness dependency now that the fetch can't clobber refs/heads/*
         // -- this could run before or after them just as safely.
         await pushSettingsBranches(ctx, git, trackedNames);
-        await refreshBaseBranchWorkspace(ctx);
+        const baseRefresh = await refreshBaseBranchWorkspace(ctx);
         const rebaseSummary = await runRebaseCycle(ctx);
         await cleanupOldTasks(ctx.taskDir, undefined, ctx.log);
         // [C1] Sweep branch directories the admin purge action trashed more than
@@ -323,9 +336,14 @@ export async function syncGit(ctx) {
         report.lastGitSync = {
             durationMs: Date.now() - cycleStartedAt,
             rebased: rebaseSummary.rebased,
-            skippedDirty: rebaseSummary.skippedDirty,
-            skippedLocked: rebaseSummary.skippedLocked,
+            skippedDirty: baseRefresh.outcome === 'skipped-dirty'
+                ? [ctx.sanitizedBaseBranch, ...rebaseSummary.skippedDirty]
+                : rebaseSummary.skippedDirty,
+            skippedLocked: baseRefresh.outcome === 'skipped-locked'
+                ? [ctx.sanitizedBaseBranch, ...rebaseSummary.skippedLocked]
+                : rebaseSummary.skippedLocked,
             failed: rebaseSummary.failed,
+            baseRefresh,
             tracked: trackedSummary,
         };
         await writeWorkerStatus(ctx.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
@@ -343,13 +361,87 @@ export async function syncGit(ctx) {
         throw err;
     }
 }
+/** Foreign repositories already reported by this process, so each is logged once. */
+const reportedForeignDirs = new Set();
+/**
+ * Repair what killed provisioning and deletes left under the branches root: quarantine residue
+ * at a branch's name (branch-provisioning.ts's `classifyFinalDir`) once it has been quiet for
+ * {@link ORPHAN_YOUTH_THRESHOLD_MS}, then sweep stale `.prov-*`, `.deleting-*` and `.repair-*`
+ * siblings and settings leftovers. Each directory is best-effort, like [SYNC-M2]. The provisioning
+ * lock is taken zero-retry, so a live provisioner or admin action wins and the directory is
+ * revisited next cycle.
+ * @internal Exported for tests; `syncGit` runs it.
+ */
+export async function repairBranchDirResidue(ctx, now = Date.now()) {
+    let entries;
+    try {
+        entries = await fs.readdir(ctx.contentBranchesPath, { withFileTypes: true });
+    }
+    catch (err) {
+        if (isNodeError(err) && err.code === 'ENOENT')
+            return;
+        throw err;
+    }
+    for (const entry of entries) {
+        if (!entry.isDirectory() || entry.name.startsWith('.'))
+            continue;
+        try {
+            await repairOneBranchDir(ctx, entry.name, now);
+        }
+        catch (err) {
+            workerLogWarn(`  Residue check of ${entry.name} failed: ${getErrorMessage(err)}`);
+        }
+    }
+    const leftovers = await sweepProvisioningLeftovers(ctx.contentBranchesPath, path.dirname(ctx.contentBranchesPath), now);
+    for (const { name, action, detail } of leftovers) {
+        const line = `  Leftover ${name}: ${action}${detail ? ` (${detail})` : ''}`;
+        if (action === 'removed')
+            workerLog(line);
+        else if (action === 'failed')
+            workerLogWarn(line);
+        else
+            workerLogError(line);
+    }
+}
+async function repairOneBranchDir(ctx, dirName, now) {
+    const dirPath = path.join(ctx.contentBranchesPath, dirName);
+    // One stat for the common case: a live (or corrupt) branch is never residue.
+    const hasMeta = await fs.access(path.join(dirPath, BRANCH_META_DIR, BRANCH_META_FILE)).then(() => true, () => false);
+    if (hasMeta)
+        return;
+    const state = await classifyFinalDir(dirPath, ctx.remoteGitPath);
+    if (state.kind === 'foreign' && !reportedForeignDirs.has(dirPath)) {
+        reportedForeignDirs.add(dirPath);
+        workerLogWarn(`  ${dirName} holds a git repository this deployment did not create; left for an admin`);
+    }
+    if (state.kind !== 'residue')
+        return;
+    let release;
+    try {
+        release = await tryAcquireProvisioningLock(ctx.contentBranchesPath, branchProvisioningLockName(dirName));
+    }
+    catch (err) {
+        if (isNodeError(err) && err.code === 'ELOCKED')
+            return;
+        throw err;
+    }
+    try {
+        await quarantineResidueAt(ctx.contentBranchesPath, dirName, {
+            minQuietMs: ORPHAN_YOUTH_THRESHOLD_MS,
+            expectedRemoteUrl: ctx.remoteGitPath,
+            now,
+        });
+    }
+    finally {
+        await releaseProvisionedWorkspace(release, dirName);
+    }
+}
 /**
  * [C1] Remove `.trash-*` branch directories (created by the admin purge action,
- * api/admin-branch-health.ts) whose name-embedded stamp is older than
- * {@link TRASH_RETENTION_MS}. A name that doesn't match `.trash-{dirName}-
- * {STAMP}`, or whose stamp fails to parse, is left alone and logged once per
- * cycle: purge is the only writer of this naming scheme, so an unparseable name
- * is unexpected and worth a human looking rather than a silent skip.
+ * api/admin-branch-health.ts, and by residue quarantine, branch-provisioning.ts)
+ * whose name-embedded stamp is older than {@link TRASH_RETENTION_MS}. A name
+ * whose trailing stamp fails to parse is left alone and logged once per cycle:
+ * both writers end every name with one, so it is worth a human looking.
  */
 export async function cleanupTrashedBranchDirs(ctx) {
     let entries;
@@ -367,8 +459,7 @@ export async function cleanupTrashedBranchDirs(ctx) {
     for (const name of entries) {
         if (!name.startsWith('.trash-'))
             continue;
-        const stampMatch = TRASH_DIR_STAMP_RE.exec(name);
-        const stampDate = stampMatch ? parseTrashStamp(stampMatch[1]) : null;
+        const stampDate = parseDirStamp(name);
         if (!stampDate) {
             if (!loggedUnparseable) {
                 workerLog(`CanopyCMS: Skipping trash dir with unparseable stamp: ${name}`);
@@ -393,37 +484,76 @@ export async function cleanupTrashedBranchDirs(ctx) {
     }
     return removed;
 }
+/**
+ * Warned (base path + tracked file list) pairs, so a repo that tracks
+ * `.canopy-meta/` is called out once per worker process rather than every cycle;
+ * worker-status.json carries it every cycle regardless.
+ */
+const warnedTrackedCanopyState = new Set();
 /**
  * Fast-forward the base branch's own working-tree clone
- * (content-branches/<baseBranch>) to match origin/<baseBranch>, every sync
+ * (content-branches/<baseBranch>) to match remote.git's <baseBranch>, every sync
  * cycle, so the drift window is bounded by gitSyncInterval.
  *
  * A dedicated, explicit and LOUD step rather than a side effect of the rebase
  * loop, whose skip paths (a dirty tree, a missing .git) are silent: a wedged
  * base clone otherwise leaves no diagnosable signal, and an editor forking a
- * new branch "from base" silently gets a stale snapshot.
+ * new branch "from base" silently gets a stale snapshot. The returned outcome
+ * goes to worker-status.json for the same reason.
  *
- * ff-only on purpose: this clone must stay a linear mirror of
- * origin/<baseBranch>, so a merge that isn't a fast-forward (diverged local
+ * ff-only on purpose: this clone must stay a linear mirror of remote.git's
+ * <baseBranch>, so a merge that isn't a fast-forward (diverged local
  * history) is left untouched rather than force-resolved.
+ *
+ * Holds the provisioning lock and then, like the rebase loop, the [SYNC-C1]
+ * content-write lock, both try-only: the base branch is writable in dev, and
+ * a save racing the merge's working-tree update can be overwritten by it.
  */
 export async function refreshBaseBranchWorkspace(ctx) {
     // Sanitized name for the workspace directory (a base branch containing
     // e.g. '/' would otherwise stat a wrong nested path here forever).
     const basePath = path.join(ctx.contentBranchesPath, ctx.sanitizedBaseBranch);
-    const gitDir = path.join(basePath, '.git');
+    let trackedCanopyMeta;
+    let releaseProvisioning;
+    let releaseContentLock;
     try {
-        let gitDirStat;
-        try {
-            gitDirStat = await fs.stat(gitDir);
-        }
-        catch {
-            gitDirStat = null;
+        // Held to the end, so no git step below races a clone of this directory.
+        const hold = await holdProvisionedWorkspace(ctx.contentBranchesPath, ctx.sanitizedBaseBranch);
+        if (hold.kind === 'locked') {
+            workerLog(`Base branch workspace (${ctx.baseBranch}): provisioning lock held elsewhere, skipping refresh`);
+            return { outcome: 'skipped-locked' };
         }
-        if (!gitDirStat || !gitDirStat.isDirectory()) {
+        if (hold.kind === 'not-provisioned') {
             workerLog(`Base branch workspace (${ctx.baseBranch}): not yet provisioned, skipping refresh`);
-            return;
+            return { outcome: 'skipped-not-provisioned' };
         }
+        releaseProvisioning = hold.release;
+        let contentLockCompromised = false;
+        try {
+            releaseContentLock = await tryAcquireContentWriteLock(basePath, (lockErr) => {
+                contentLockCompromised = true;
+                workerLogWarn(`Base branch workspace (${ctx.baseBranch}): content-write lock compromised mid-refresh: ${getErrorMessage(lockErr)}`);
+            });
+        }
+        catch (lockErr) {
+            if (isNodeError(lockErr) && lockErr.code === 'ELOCKED') {
+                workerLog(`Base branch workspace (${ctx.baseBranch}): content write in progress, skipping refresh`);
+                return { outcome: 'skipped-locked' };
+            }
+            throw lockErr;
+        }
+        // Checked before each destructive step: a lost provisioning lock means
+        // another process may be cloning into this directory, a lost content lock
+        // that an editor save may be landing in it.
+        const lockLost = () => {
+            if (!hold.isCompromised() && !contentLockCompromised)
+                return null;
+            const lost = hold.isCompromised() ? 'provisioning' : 'content-write';
+            workerLogWarn(`Base branch workspace (${ctx.baseBranch}): ${lost} lock lost mid-refresh, stopping`);
+            return { outcome: 'skipped-locked', trackedCanopyMeta };
+        };
+        // Idempotent, and applied every cycle so any clone lacking it gets it.
+        await ensureGitExcludePattern(basePath, `${CANOPY_META_DIR}/`);
         const baseGit = simpleGit({
             baseDir: basePath,
             // Keep git non-interactive during the merge so it never blocks on an
@@ -437,37 +567,80 @@ export async function refreshBaseBranchWorkspace(ctx) {
             // Inactivity-based, so a slow-but-flowing transfer is unaffected.
             timeout: { block: ctx.taskTimeoutMs },
         });
+        const trackedState = await listTrackedCanopyState(baseGit);
+        if (trackedState.length > 0) {
+            trackedCanopyMeta = trackedState.slice(0, MAX_REPORTED_PATHS);
+            const warnKey = `${basePath}\0${trackedState.join('\0')}`;
+            if (!warnedTrackedCanopyState.has(warnKey)) {
+                warnedTrackedCanopyState.add(warnKey);
+                workerLogWarn(`Base branch (${ctx.baseBranch}) tracks canopycms state that must not be committed: ` +
+                    `${trackedState.join(', ')}. To fix, ${TRACKED_CANOPY_STATE_FIX}.`);
+            }
+        }
+        let status = await baseGit.status();
+        const lostBeforeRestore = lockLost();
+        if (lostBeforeRestore)
+            return lostBeforeRestore;
+        if (await restoreRetiredSchemaCache(baseGit, status)) {
+            workerLog(`Base branch workspace (${ctx.baseBranch}): restored the retired in-tree schema cache`);
+            status = await baseGit.status();
+        }
         // Nothing makes this clone read-only, and a direct edit here wedges every
         // editor's view of the base branch until an operator intervenes, so a dirty
-        // tree is loud, not a quiet skip. Only TRACKED changes block the refresh: a
-        // stray untracked file must not wedge the fast-forward forever, and if one
-        // would collide with incoming content the --ff-only merge below refuses on
-        // its own and that failure is already logged loudly.
-        const status = await baseGit.status();
-        const trackedDirty = status.files.filter((f) => f.index !== '?' || f.working_dir !== '?');
+        // tree is loud, not a quiet skip. Only TRACKED content changes block the
+        // refresh: canopycms's own state never does, and neither does a stray
+        // untracked file. A modified tracked file or an untracked one that would
+        // collide with incoming content makes the --ff-only merge below refuse and
+        // report `failed`; an IGNORED file, which every .canopy-meta file is, git
+        // overwrites instead.
+        const trackedDirty = status.files
+            .filter((f) => !isUntracked(f) && !isCanopyInternalPath(f.path))
+            .map((f) => f.path);
         if (trackedDirty.length > 0) {
-            workerLogError(`Base branch workspace (${ctx.baseBranch}) has uncommitted changes -- skipping refresh. Dirty files: ${trackedDirty.map((f) => f.path).join(', ')}`);
-            return;
+            workerLogError(`Base branch workspace (${ctx.baseBranch}) has uncommitted changes -- skipping refresh. Dirty files: ${trackedDirty.join(', ')}`);
+            return {
+                outcome: 'skipped-dirty',
+                dirtyFiles: trackedDirty.slice(0, MAX_REPORTED_PATHS),
+                message: `${trackedDirty.length} uncommitted tracked file(s) in the base branch workspace`,
+                trackedCanopyMeta,
+            };
         }
         // Raw (unsanitized) name from here on: these are git ref operations
-        // against origin/<baseBranch>, not filesystem paths, so they must use
+        // against remote.git's <baseBranch>, not filesystem paths, so they must use
         // the same name GitHub knows the branch by.
-        await baseGit.fetch('origin', ctx.baseBranch);
+        await baseGit.fetch(ctx.remoteGitPath, ctx.baseBranch);
         // rev-list, not status.behind, which needs an upstream tracking branch that
-        // is not guaranteed here. Against the just-fetched tip rather than
-        // origin/<base>: workspaces are cloned --single-branch (git-manager.ts), so
-        // for any other base branch origin/<base> never exists and rev-list dies
-        // with "ambiguous argument". Pin FETCH_HEAD to a SHA immediately -- it is
-        // one shared mutable file per repo, silently repointed by any other fetch.
+        // is not guaranteed here. Against the just-fetched tip: a fetch by path
+        // updates no remote-tracking ref. Pin FETCH_HEAD to a SHA immediately -- it
+        // is one shared mutable file per repo, silently repointed by any other fetch.
         const fetchedTip = (await baseGit.revparse(['FETCH_HEAD'])).trim();
         const behindCount = parseInt((await baseGit.raw(['rev-list', '--count', `HEAD..${fetchedTip}`])).trim(), 10);
         if (behindCount > 0) {
+            const lostBeforeMerge = lockLost();
+            if (lostBeforeMerge)
+                return lostBeforeMerge;
+            // Untrack, in the index only, any state the tip has stopped tracking:
+            // the merge would otherwise refuse to overwrite a modified copy, or
+            // delete a clean one from disk. Safe here and not in the rebase loop,
+            // because this clone has no commits of its own to replay. State still
+            // tracked upstream is left for the merge, which refuses where upstream
+            // changed a locally modified copy.
+            const { droppedUpstream } = await splitByUpstreamTracking(baseGit, trackedState, fetchedTip);
+            if (droppedUpstream.length > 0) {
+                await untrackInIndex(baseGit, droppedUpstream);
+                workerLog(`Base branch workspace (${ctx.baseBranch}): stopped tracking ${droppedUpstream.join(', ')}, as upstream has`);
+            }
             try {
                 await baseGit.merge(['--ff-only', fetchedTip]);
             }
             catch (err) {
                 workerLogError(`Base branch workspace (${ctx.baseBranch}) failed to fast-forward (diverged local history?): ${getErrorMessage(err)}`);
-                return;
+                return {
+                    outcome: 'failed',
+                    // [REDACT] Served to the browser by the admin panel.
+                    message: redactCredentials(`failed to fast-forward: ${getErrorMessage(err)}`),
+                    trackedCanopyMeta,
+                };
             }
             await invalidateBranchContentCaches(basePath);
         }
@@ -491,8 +664,30 @@ export async function refreshBaseBranchWorkspace(ctx) {
         workerLog(behindCount > 0
             ? `Base branch workspace (${ctx.baseBranch}): fast-forwarded ${behindCount} commit(s)`
             : `Base branch workspace (${ctx.baseBranch}): up to date`);
+        if (trackedCanopyMeta && behindCount > 0) {
+            const stillTracked = await listTrackedCanopyState(baseGit);
+            trackedCanopyMeta =
+                stillTracked.length > 0 ? stillTracked.slice(0, MAX_REPORTED_PATHS) : undefined;
+        }
+        return { outcome: behindCount > 0 ? 'refreshed' : 'up-to-date', trackedCanopyMeta };
     }
     catch (err) {
         workerLogError(`Base branch workspace (${ctx.baseBranch}) refresh failed: ${getErrorMessage(err)}`);
+        // [REDACT] Served to the browser by the admin panel.
+        return {
+            outcome: 'failed',
+            message: redactCredentials(getErrorMessage(err)),
+            trackedCanopyMeta,
+        };
+    }
+    finally {
+        if (releaseContentLock) {
+            await releaseContentLock().catch((err) => {
+                workerLogWarn(`Base branch workspace (${ctx.baseBranch}): failed to release content-write lock: ${getErrorMessage(err)}`);
+            });
+        }
+        if (releaseProvisioning) {
+            await releaseProvisionedWorkspace(releaseProvisioning, ctx.sanitizedBaseBranch);
+        }
     }
 }

package/dist/worker/history-rewrite.d.ts

@@ -72,7 +72,7 @@ export declare function clearHistoryRewrittenMarker(ctx: Pick<HistoryRewriteCont
  * Returns whether the push landed. A refused lease means a concurrent Lambda
  * push moved the ref; logged and retried by the self-heal pass, never thrown.
  */
-export declare function forcePublishToLocalRemote(ctx: Pick<HistoryRewriteContext, 'taskTimeoutMs'>, branchPath: string, branchRef: string, expectedSha: string): Promise<boolean>;
+export declare function forcePublishToLocalRemote(ctx: Pick<HistoryRewriteContext, 'remoteGitPath' | 'taskTimeoutMs'>, branchPath: string, branchRef: string, expectedSha: string): Promise<boolean>;
 /**
  * Queue the GitHub hop for a branch whose rewritten history now sits in
  * `remote.git`, so an open PR's head follows the rebase within a cycle instead

package/dist/worker/history-rewrite.js

@@ -108,7 +108,7 @@ export async function forcePublishToLocalRemote(ctx, branchPath, branchRef, expe
             // Real flags must precede --end-of-options; everything after it is
             // positional (see GitManager.push() for the same guard).
             '--end-of-options',
-            'origin',
+            ctx.remoteGitPath,
             `${branchRef}:${branchRef}`,
         ]);
         workerLog(`  Published rebased ${branchRef} into remote.git`);