canopycms 0.0.61-int.75 → 0.0.61

package/dist/worker/cms-worker.js

@@ -2,15 +2,26 @@ import fs from 'node:fs/promises';
 import path from 'node:path';
 import { simpleGit } from 'simple-git';
 import lockfile from 'proper-lockfile';
-import { dequeueTask, completeTask, failTask, retryTask, recoverOrphanedTasks, cleanupOldTasks, cmsTaskQueueLogger, } from './task-queue.js';
+import { enqueueTask, listTasks, dequeueTask, completeTask, failTask, retryTask, recoverOrphanedTasks, cleanupOldTasks, cmsTaskQueueLogger, } from './task-queue.js';
 import { createOrUpdatePullRequest, createCanopyOctokit } from '../github-service.js';
 import { getBranchMetadataFileManager, BranchMetadataFileManager, buildMergedBranchUpdate, } from '../branch-metadata.js';
 import { extractIdFromFilename } from '../content-id-index.js';
 import { invalidateBranchContentCaches } from '../content-index-generation.js';
 import { ROOT_COLLECTION_ID } from '../paths/types.js';
-import { sanitizeBranchName } from '../paths/branch-name.js';
+import { sanitizeBranchName, RESERVED_SETTINGS_BRANCH_PREFIX } from '../paths/branch-name.js';
+import { normalizeFilesystemPath } from '../paths/normalize.js';
+import { GITHUB_TRACKING_REF_PREFIX, gitChildEnv, gitNetworkChildEnv } from '../git-manager.js';
+import { resolveDeploymentName } from '../operating-mode/deployment-name.js';
 import { getErrorMessage, isNodeError, redactCredentials } from '../utils/error.js';
+import { isNonFastForwardRejection, isStaleLeaseRejection } from '../utils/git.js';
 import { writeWorkerStatus } from './worker-status.js';
+import { workerLog, workerLogWarn, workerLogError } from './log.js';
+// Re-exported so the AWS entrypoint (packages/canopycms-cdk/worker/index.ts)
+// can prefix its own startup lines through the same helpers without adding a
+// new package entrypoint - `canopycms/worker/cms-worker` already exists. Every
+// line in worker.log must carry the timestamp prefix or it gets folded into
+// the previous CloudWatch event; see ./log.ts.
+export { workerLog, workerLogWarn, workerLogError, installWorkerLogger } from './log.js';
 const DEFAULT_TASK_TIMEOUT = 60_000;
 const DEFAULT_MAX_RETRIES = 3;
 const DEFAULT_LOCK_STALE_MS = 60_000;
@@ -183,9 +194,45 @@ export class CmsWorker {
         }
         return this.statusReport;
     }
+    /**
+     * Resolve this deployment's settings branch, throwing if the infra-stamped
+     * deployment name is not a valid git ref component.
+     *
+     * Routed through the shared resolver, not a local `?? 'prod'`: it is the
+     * single definition of the env > config > mode-default precedence the Lambda
+     * already follows, and the only place the resolved value is validated.
+     * Without it the worker could silently own a different settings branch than
+     * the Lambda writing to the same workspace -- and pushSettingsBranches would
+     * then report the real branch as foreign and never push it.
+     *
+     * DEFERRED out of the constructor deliberately, and this is the whole point
+     * of the method existing. Resolving there made the throw land during `new
+     * CmsWorker(...)` (canopycms-cdk/worker/index.ts constructs before it calls
+     * start()), which is BEFORE the only code that writes `lastFatalError` --
+     * start()'s catch. The result was billed as "a loud startup exit" but was
+     * nothing of the kind: with systemd `Type=simple` + `Restart=always` and no
+     * cfn-signal, an invalid deployment name produced an invisible ~5s
+     * crash-loop that `cdk deploy` reported as success while the admin panel
+     * showed the worker as 'absent' with no fatal error to explain it. Resolving
+     * inside start()'s try block instead means the same throw is recorded to
+     * worker-status.json and surfaces in the admin panel.
+     *
+     * Lazy rather than start()-only so the value is still available to unit tests
+     * that drive pushSettingsBranches() directly without calling start() -- the
+     * same reason ensureStatusReport() above is shaped this way. Idempotent: the
+     * resolver is pure, so a later call returns the identical string.
+     */
+    ensureSettingsBranch() {
+        if (this.settingsBranchResolved === undefined) {
+            this.settingsBranchResolved =
+                this.config.settingsBranch ??
+                    `${RESERVED_SETTINGS_BRANCH_PREFIX}${resolveDeploymentName({ deploymentName: this.config.deploymentName }, 'prod')}`;
+        }
+        return this.settingsBranchResolved;
+    }
     async start() {
         this.running = true;
-        console.log('CMS Worker starting...');
+        workerLog('CMS Worker starting...');
         this.ensureStatusReport();
         // Acquire lock to prevent concurrent workers
         await this.acquireLock();
@@ -202,12 +249,24 @@ export class CmsWorker {
         // clones elsewhere) — acquiring the lock first is what already
         // serializes that.
         try {
+            // FIRST inside the try, before any I/O: an infra-stamped deployment name
+            // that is not a valid git ref component throws here, where the catch
+            // below records it to worker-status.json. Resolving it in the
+            // constructor (as this used to) put the throw outside every
+            // status-writing path -- see ensureSettingsBranch()'s doc comment.
+            this.ensureSettingsBranch();
             // Ensure remote.git exists (init bare repo if first run)
             await this.ensureRemoteGit();
-            // Recover any orphaned tasks from a previous crash
-            const recovered = await recoverOrphanedTasks(this.taskDir, undefined, this.log);
+            // Recover any orphaned tasks from a previous crash, immediately rather
+            // than waiting for the first processTaskQueue() poll (taskPollInterval,
+            // default 5s) to run it. Not the only call site any more -
+            // processTaskQueue() below now repeats this on every cycle; see its
+            // doc comment for why a boot-only call is insufficient once the
+            // worker's ASG rolls on every `cdk deploy` (CanopyCmsService's
+            // UpdatePolicy).
+            const recovered = await recoverOrphanedTasks(this.taskDir, this.orphanRecoveryMaxAgeMs(), this.log);
             if (recovered > 0) {
-                console.log(`Recovered ${recovered} orphaned task(s)`);
+                workerLog(`Recovered ${recovered} orphaned task(s)`);
             }
             // Run initial sync + cache refresh immediately
             const initialTasks = [this.syncGit()];
@@ -234,7 +293,7 @@ export class CmsWorker {
                 await writeWorkerStatus(this.taskDir, report);
             }
             catch (writeErr) {
-                console.error('Failed to write worker status on startup failure:', getErrorMessage(writeErr));
+                workerLogError('Failed to write worker status on startup failure:', getErrorMessage(writeErr));
             }
             await this.releaseLock();
             throw err;
@@ -248,11 +307,11 @@ export class CmsWorker {
         if (this.config.refreshAuthCache) {
             const cacheInterval = this.config.authCacheRefreshInterval ?? 15 * 60_000;
             this.scheduleLoop(() => this.refreshAuthCache(), cacheInterval);
-            console.log(`  Auth cache refresh: every ${cacheInterval / 1000}s`);
+            workerLog(`  Auth cache refresh: every ${cacheInterval / 1000}s`);
         }
-        console.log('CMS Worker started');
-        console.log(`  Task queue poll: every ${taskInterval / 1000}s`);
-        console.log(`  Git sync: every ${gitInterval / 1000}s`);
+        workerLog('CMS Worker started');
+        workerLog(`  Task queue poll: every ${taskInterval / 1000}s`);
+        workerLog(`  Git sync: every ${gitInterval / 1000}s`);
     }
     async stop() {
         this.running = false;
@@ -270,7 +329,7 @@ export class CmsWorker {
         ]);
         clearTimeout(drainTimer);
         await this.releaseLock();
-        console.log('CMS Worker stopped');
+        workerLog('CMS Worker stopped');
     }
     /**
      * Acquire the cross-host worker lock (DEP-C2).
@@ -307,7 +366,7 @@ export class CmsWorker {
                     // Our heartbeat could not be maintained (lock deleted or taken
                     // over). Another worker may now be consuming the queue — stop
                     // processing to preserve the single-consumer invariant.
-                    console.error('Worker lock compromised, shutting down:', getErrorMessage(err));
+                    workerLogError('Worker lock compromised, shutting down:', getErrorMessage(err));
                     this.releaseLockFn = null; // the lock is already lost; nothing to release
                     void this.stop();
                 },
@@ -344,7 +403,7 @@ export class CmsWorker {
             const timeout = setTimeout(async () => {
                 this.activeTimeouts.delete(timeout);
                 const operation = fn().catch((err) => {
-                    console.error('Worker loop error:', err instanceof Error ? err.message : err);
+                    workerLogError('Worker loop error:', err instanceof Error ? err.message : err);
                 });
                 this.activeOperations.add(operation);
                 operation.finally(() => this.activeOperations.delete(operation));
@@ -409,7 +468,7 @@ export class CmsWorker {
                 await this.verifyBaseBranchExists(this.remoteGitPath);
             }
             catch (err) {
-                console.error(`remote.git base branch verification failed: ${getErrorMessage(err)}`);
+                workerLogError(`remote.git base branch verification failed: ${getErrorMessage(err)}`);
                 // Do NOT auto-delete: an existing remote.git could hold unpushed
                 // canopycms-settings-* branches or other state worth preserving.
                 // Deletion here is the operator's call, not ours.
@@ -417,14 +476,14 @@ export class CmsWorker {
             }
             return; // Already exists and has the base branch
         }
-        console.log('Initializing remote.git from GitHub...');
+        workerLog('Initializing remote.git from GitHub...');
         const git = simpleGit();
         await git.clone(this.buildGitHubUrl(), this.remoteGitPath, ['--bare']);
         try {
             await this.verifyBaseBranchExists(this.remoteGitPath);
         }
         catch (err) {
-            console.error(`remote.git base branch verification failed after clone: ${getErrorMessage(err)}`);
+            workerLogError(`remote.git base branch verification failed after clone: ${getErrorMessage(err)}`);
             // Deleting before throwing is what makes this recoverable: the next
             // start() sees no remote.git and re-clones, instead of being stuck
             // forever behind a poisoned bare repo that fs.stat alone can't detect.
@@ -434,7 +493,20 @@ export class CmsWorker {
         // Remove the origin remote so the token doesn't persist in config
         const bareGit = simpleGit({ baseDir: this.remoteGitPath });
         await bareGit.removeRemote('origin').catch(() => { });
-        console.log('remote.git initialized');
+        workerLog('remote.git initialized');
+    }
+    /**
+     * Staleness threshold for recoverOrphanedTasks, derived from the
+     * configured task timeout rather than fixed: the safety argument for
+     * running recovery on every poll cycle is "no legitimately in-flight task
+     * can be this old, because executeTaskWithTimeout bounds every attempt by
+     * taskTimeoutMs" -- which is only true if this threshold scales with
+     * taskTimeoutMs. 2x leaves the same comfortable margin the defaults have
+     * (60s timeout vs 5min threshold); the 5-minute floor preserves the
+     * long-standing default for a replacement instance's boot window.
+     */
+    orphanRecoveryMaxAgeMs() {
+        return Math.max(5 * 60_000, this.taskTimeoutMs * 2);
     }
     /**
      * Process queued tasks from Lambda.
@@ -445,6 +517,32 @@ export class CmsWorker {
     async processTaskQueue() {
         if (!this.running)
             return;
+        // Recover tasks orphaned in processing/ on EVERY cycle, not only at boot
+        // (start()'s call above). Before this fix, recoverOrphanedTasks() ran
+        // exactly once, at start() - fine when an instance was replaced rarely
+        // (a spot interruption), but CanopyCmsService's worker ASG now rolls on
+        // every `cdk deploy` (see its UpdatePolicy), making replacement routine.
+        // A replacement instance's user-data (yum install git/unzip/nodejs/
+        // efs-utils, mount EFS) takes roughly 2-4 minutes, well under
+        // recoverOrphanedTasks()'s 5-minute default staleness threshold - so
+        // THAT ONE boot-time call would see the just-orphaned file as "too
+        // fresh" and skip it, and nothing rescanned afterward: the task (and its
+        // branch's syncStatus) would be wedged forever. Re-checking every poll
+        // means the file's age eventually crosses the threshold and it gets
+        // recovered without operator intervention.
+        //
+        // Safe to run this often: executeTaskWithTimeout() guarantees every task
+        // THIS process dequeues is completed, failed, or retried (all three
+        // remove the processing/ file) within taskTimeoutMs - and the staleness
+        // threshold is derived from taskTimeoutMs (orphanRecoveryMaxAgeMs below)
+        // precisely so that guarantee holds for ANY configured timeout, not just
+        // the 60s default: a fixed 5-minute threshold would have this call steal
+        // the worker's own still-in-flight task back to pending whenever an
+        // adopter configured taskTimeoutMs above ~5 minutes.
+        const recovered = await recoverOrphanedTasks(this.taskDir, this.orphanRecoveryMaxAgeMs(), this.log);
+        if (recovered > 0) {
+            workerLog(`Recovered ${recovered} orphaned task(s)`);
+        }
         let processed = 0;
         let task;
         while (processed < this.maxTasksPerCycle &&
@@ -456,7 +554,7 @@ export class CmsWorker {
             }
             catch (err) {
                 const message = getErrorMessage(err);
-                console.error(`Task ${task.id} (${task.action}) failed:`, message);
+                workerLogError(`Task ${task.id} (${task.action}) failed:`, message);
                 // [HIGH-1] task.error is persisted (pending/failed task JSON) and
                 // served to the browser by the admin panel's Tasks tab -- a push
                 // failure's message can embed the bot token via buildGitHubUrl().
@@ -470,12 +568,12 @@ export class CmsWorker {
                 const maxRetries = task.maxRetries ?? this.maxRetries;
                 if (!permanent && retryCount < maxRetries) {
                     await retryTask(this.taskDir, task.id, persistedMessage, this.log);
-                    console.log(`  Will retry (attempt ${retryCount + 1}/${maxRetries})`);
+                    workerLog(`  Will retry (attempt ${retryCount + 1}/${maxRetries})`);
                 }
                 else {
                     await failTask(this.taskDir, task.id, persistedMessage, this.log);
                     await this.updateBranchMetadataOnFailure(task, persistedMessage);
-                    console.error(permanent
+                    workerLogError(permanent
                         ? '  Permanently failed (non-retryable error)'
                         : `  Permanently failed after ${maxRetries} retries`);
                 }
@@ -489,7 +587,7 @@ export class CmsWorker {
         if (processed > 0) {
             const report = this.ensureStatusReport();
             report.lastTaskCycleAt = new Date().toISOString();
-            await writeWorkerStatus(this.taskDir, report).catch((writeErr) => console.error('Failed to write worker status:', getErrorMessage(writeErr)));
+            await writeWorkerStatus(this.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
         }
     }
     /**
@@ -538,7 +636,7 @@ export class CmsWorker {
                     body: optionalString(payload, 'body', ''),
                     request: { signal },
                 });
-                console.log(`Created PR #${pr.data.number} for ${branch}`);
+                workerLog(`Created PR #${pr.data.number} for ${branch}`);
                 return { prUrl: pr.data.html_url, prNumber: pr.data.number };
             }
             case 'push-and-update-pr': {
@@ -553,7 +651,7 @@ export class CmsWorker {
                     body: optionalString(payload, 'body', ''),
                     request: { signal },
                 });
-                console.log(`Updated PR #${prNumber} for ${branch}`);
+                workerLog(`Updated PR #${prNumber} for ${branch}`);
                 return { prNumber };
             }
             case 'push-and-create-or-update-pr': {
@@ -590,7 +688,7 @@ export class CmsWorker {
                     markReadyIfDraft: payload.markReadyIfDraft === true,
                     signal,
                 });
-                console.log(result.created
+                workerLog(result.created
                     ? `Created PR #${result.number} for ${branch}`
                     : `Updated existing PR #${result.number} for ${branch}`);
                 return { prUrl: result.url, prNumber: result.number };
@@ -606,7 +704,7 @@ export class CmsWorker {
                     request: { signal },
                 });
                 await this.octokit.graphql(`mutation($id: ID!) { convertPullRequestToDraft(input: { pullRequestId: $id }) { pullRequest { isDraft } } }`, { id: pr.node_id, request: { signal } });
-                console.log(`Converted PR #${draftPrNumber} to draft`);
+                workerLog(`Converted PR #${draftPrNumber} to draft`);
                 return { prNumber: draftPrNumber, draft: true };
             }
             case 'close-pr': {
@@ -642,7 +740,7 @@ export class CmsWorker {
         const branch = typeof task.payload.branch === 'string' ? task.payload.branch : null;
         if (!branch)
             return;
-        const branchPath = path.join(this.contentBranchesPath, branch);
+        const branchPath = this.branchWorkspacePath(branch);
         try {
             await fs.stat(branchPath);
         }
@@ -654,6 +752,12 @@ export class CmsWorker {
             const updates = {
                 name: branch,
                 syncStatus: 'synced',
+                // save()'s merge only overwrites keys present in this update (see
+                // BranchMetadataFileManager.save()'s spread order) -- a prior
+                // syncFailureReason from an earlier failed attempt would otherwise
+                // survive forever past this successful sync. Explicitly clear it,
+                // same pattern as rebaseFailure's PR-W2 M2 clearing elsewhere.
+                syncFailureReason: undefined,
             };
             if (result.prUrl)
                 updates.pullRequestUrl = result.prUrl;
@@ -675,18 +779,20 @@ export class CmsWorker {
             await meta.save({ branch: updates });
         }
         catch (err) {
-            console.error(`Failed to update metadata for ${branch}:`, err instanceof Error ? err.message : err);
+            workerLogError(`Failed to update metadata for ${branch}:`, err instanceof Error ? err.message : err);
         }
     }
     /**
      * Update branch metadata after permanent task failure.
-     * Sets syncStatus to 'sync-failed' with error details.
+     * Sets syncStatus to 'sync-failed' and records `error` (already redacted
+     * by the caller, processTaskQueue -- see [HIGH-1] there) as
+     * syncFailureReason, so the editor can show WHY, not just that it failed.
      */
-    async updateBranchMetadataOnFailure(task, _error) {
+    async updateBranchMetadataOnFailure(task, error) {
         const branch = typeof task.payload.branch === 'string' ? task.payload.branch : null;
         if (!branch)
             return;
-        const branchPath = path.join(this.contentBranchesPath, branch);
+        const branchPath = this.branchWorkspacePath(branch);
         try {
             await fs.stat(branchPath);
         }
@@ -695,15 +801,222 @@ export class CmsWorker {
         }
         try {
             const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
-            await meta.save({ branch: { name: branch, syncStatus: 'sync-failed' } });
+            await meta.save({
+                branch: { name: branch, syncStatus: 'sync-failed', syncFailureReason: error },
+            });
         }
         catch (err) {
-            console.error(`Failed to update failure metadata for ${branch}:`, err instanceof Error ? err.message : err);
+            workerLogError(`Failed to update failure metadata for ${branch}:`, err instanceof Error ? err.message : err);
         }
     }
     buildGitHubUrl() {
         return `https://x-access-token:${this.config.githubToken}@github.com/${this.config.githubOwner}/${this.config.githubRepo}.git`;
     }
+    /**
+     * The workspace directory for a branch named by its GIT REF name -- the
+     * form task payloads carry (`context.branch.name`), not the directory form.
+     *
+     * These differ for any name outside `[A-Za-z0-9._-]`, `/` being the obvious
+     * one: workspaces are provisioned under `sanitizeBranchName(...)` (see
+     * paths/branch.ts's `resolveBranchPaths`), so `feature/x` lives in
+     * `feature-x`. Joining the raw name instead silently addresses a directory
+     * that does not exist -- which for the metadata writers below meant the
+     * update was quietly dropped, and for the leased push meant the
+     * history-rewrite marker read as absent and the push went out unleased,
+     * wedging exactly the branch this workstream exists to unwedge.
+     *
+     * `name` inside the metadata itself stays the raw ref name; only the path
+     * is sanitized.
+     */
+    branchWorkspacePath(branchRefName) {
+        return path.join(this.contentBranchesPath, sanitizeBranchName(branchRefName));
+    }
+    /**
+     * What `remote.git` currently holds for `branchRef`, or null when this
+     * branch was never published there (never submitted) or the ref is
+     * unreadable.
+     *
+     * Uses the same explicit `--git-dir` shape as verifyBaseBranchExists()
+     * above: reading a bare repo that way does not depend on
+     * `safe.bareRepository` being permissive, which is why prod code takes
+     * this route rather than the config override the test-only `openBareRepo`
+     * helper uses.
+     */
+    async readPublishedSha(branchRef) {
+        try {
+            const out = await simpleGit().raw([
+                '--git-dir',
+                this.remoteGitPath,
+                'rev-parse',
+                '--verify',
+                `refs/heads/${branchRef}`,
+            ]);
+            const sha = out.trim();
+            return sha.length > 0 ? sha : null;
+        }
+        catch {
+            // Absent from remote.git: this branch was never submitted, so none of
+            // its history was ever published. Not an error.
+            return null;
+        }
+    }
+    /**
+     * Record that this worker rewrote `expectedSha` out of a branch's already
+     * published history (see BranchMetadata.historyRewrittenFrom).
+     *
+     * Set-once: if a marker is already present it is LEFT ALONE. Across two
+     * rebases before any GitHub push lands, GitHub still holds the commit the
+     * FIRST rebase replaced, so advancing the marker would aim the lease at a
+     * commit GitHub never had and permanently wedge the branch.
+     *
+     * Re-reads metadata rather than trusting the caller's loop-top snapshot:
+     * the task loop runs concurrently with syncGit() (see scheduleLoop) and may
+     * have cleared the marker while this branch was rebasing.
+     */
+    async markHistoryRewritten(branchPath, branchDir, expectedSha) {
+        const current = await BranchMetadataFileManager.loadOnly(branchPath);
+        if (typeof current?.branch.historyRewrittenFrom === 'string')
+            return;
+        const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
+        await meta.save({ branch: { name: branchDir, historyRewrittenFrom: expectedSha } });
+    }
+    /**
+     * Clear the marker once GitHub is confirmed to hold the rewritten history.
+     *
+     * Best-effort only in the sense that a failure here destroys nothing: the
+     * lease still refuses anything unexpected, and the plain-push fallback only
+     * ever fast-forwards. It is NOT harmless. A marker that outlives its
+     * episode can wedge the NEXT one -- if the base advances before the stale
+     * marker is revisited, the queued push leases a commit GitHub has already
+     * moved off, falls back to a plain push of a rebased (non-ancestor)
+     * history, and fails permanently with a "something else moved it on
+     * GitHub" diagnosis that is false: we did.
+     *
+     * Tracked, with the concurrent-clear race that can drop a marker
+     * mid-arming, in
+     * .claude/future-tasks/worker-history-rewrite-marker-races.md.
+     */
+    async clearHistoryRewrittenMarker(branchPath, branchDir) {
+        try {
+            const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
+            // Explicit undefined clears the key -- save()'s merge only overwrites
+            // keys present in the update (same pattern as rebaseFailure).
+            await meta.save({ branch: { name: branchDir, historyRewrittenFrom: undefined } });
+        }
+        catch (err) {
+            workerLogWarn(`  Failed to clear history-rewrite marker for ${branchDir}: ${getErrorMessage(err)}`);
+        }
+    }
+    /**
+     * Publish a branch clone's rebased history into `remote.git`, replacing
+     * EXACTLY `expectedSha` and nothing else.
+     *
+     * The lease is the entire safety argument. `--force-with-lease=<ref>:<sha>`
+     * refuses unless `remote.git` still stands at `<sha>`, so this can only
+     * ever undo the commit our own rebase rewrote away. Callers must never pass
+     * "whatever remote.git currently holds" -- see the arming guard in
+     * rebaseActiveBranches() for the interleaving where that would silently
+     * delete a reviewer's direct push.
+     *
+     * Returns whether the push landed. A refused lease means a concurrent
+     * Lambda push moved the ref; that is logged and retried by the self-heal
+     * pass on a later cycle, never thrown.
+     */
+    async forcePublishToLocalRemote(branchPath, branchRef, expectedSha) {
+        // A dedicated instance rather than the caller's: `.env()` replaces the
+        // whole child environment, and the rebase loop's instance must keep its
+        // ambient one. gitChildEnv (not gitNetworkChildEnv) because this push
+        // targets the local bare repo -- and the locale pin is what keeps
+        // isStaleLeaseRejection below from silently becoming a no-op.
+        const pushGit = simpleGit({ baseDir: branchPath, timeout: { block: this.taskTimeoutMs } });
+        pushGit.env(gitChildEnv({}));
+        try {
+            await pushGit.raw([
+                'push',
+                `--force-with-lease=${branchRef}:${expectedSha}`,
+                // Real flags must precede --end-of-options; everything after it is
+                // positional (see GitManager.push() for the same guard).
+                '--end-of-options',
+                'origin',
+                `${branchRef}:${branchRef}`,
+            ]);
+            workerLog(`  Published rebased ${branchRef} into remote.git`);
+            return true;
+        }
+        catch (err) {
+            const message = redactCredentials(getErrorMessage(err));
+            workerLogWarn(isStaleLeaseRejection(message)
+                ? `  Did not publish rebased ${branchRef} into remote.git: it moved since this cycle read it (concurrent submit?) -- retrying next cycle`
+                : `  Failed to publish rebased ${branchRef} into remote.git: ${message}`);
+            return false;
+        }
+    }
+    /**
+     * Queue the GitHub hop for a branch whose rewritten history now sits in
+     * `remote.git`, so an open PR's head follows the rebase within a cycle
+     * instead of waiting for the editor's next submit.
+     *
+     * Deliberately NOT skipped when a marker is already set: inferring "a task
+     * must already be queued" from the marker starves this hop whenever a task
+     * was lost, failed permanently, or was never written. Duplicate push tasks
+     * are bounded by base-branch advances and are idempotent (a repeat push is
+     * a no-op once GitHub holds the tip).
+     */
+    async enqueueGitHubPush(branchRef) {
+        try {
+            // Dedupe against tasks actually in flight rather than against the
+            // marker: a branch whose GitHub push keeps failing would otherwise
+            // gain one task per sync cycle forever.
+            for (const status of ['pending', 'processing']) {
+                const inFlight = await listTasks(this.taskDir, status);
+                if (inFlight.some((t) => t.action === 'push-branch' && t.payload.branch === branchRef)) {
+                    return;
+                }
+            }
+            await enqueueTask(this.taskDir, { action: 'push-branch', payload: { branch: branchRef } });
+            workerLog(`  Queued GitHub push for ${branchRef}`);
+        }
+        catch (err) {
+            workerLogWarn(`  Failed to queue GitHub push for ${branchRef}: ${getErrorMessage(err)}`);
+        }
+    }
+    /**
+     * Complete a rewrite this worker started but did not finish: get the
+     * rebased history into `remote.git` and queued for GitHub.
+     *
+     * Runs from the rebase loop for any branch carrying a marker, whether or
+     * not it is behind base this cycle, so an interrupted publish converges
+     * without waiting for the next base-branch advance.
+     *
+     * Always leases on the MARKER, never on remote.git's current tip -- the
+     * marker is the one commit we know our own rebase replaced.
+     */
+    async reconcilePendingRewrite(options) {
+        const { branchPath, branchDir, branchRef, headSha, marker } = options;
+        const publishedSha = await this.readPublishedSha(branchRef);
+        if (publishedSha === null) {
+            workerLogWarn(`  ${branchDir}: a rewritten history is recorded but ${branchRef} is gone from remote.git -- nothing to publish`);
+            return;
+        }
+        if (publishedSha === headSha) {
+            // remote.git already carries the rewrite; only the GitHub hop is left
+            // (a crash between the push and the queue, or a task that was lost).
+            await this.enqueueGitHubPush(branchRef);
+            return;
+        }
+        if (publishedSha === marker) {
+            // The publish into remote.git never landed -- a crash right after the
+            // marker was written, or a lease refused by a concurrent submit.
+            if (await this.forcePublishToLocalRemote(branchPath, branchRef, marker)) {
+                await this.enqueueGitHubPush(branchRef);
+            }
+            return;
+        }
+        // Neither this branch's rebased tip nor the commit its rewrite replaced:
+        // something else moved remote.git. Never force over that.
+        workerLogWarn(`  ${branchDir}: remote.git is at ${publishedSha} for ${branchRef}, which is neither the ` +
+            `rebased tip nor the commit the rewrite replaced -- left untouched`);
+    }
     async pushBranchToGitHub(branch) {
         const git = simpleGit({
             baseDir: this.remoteGitPath,
@@ -712,37 +1025,384 @@ export class CmsWorker {
             // it hang past the task timeout.
             timeout: { block: this.taskTimeoutMs },
         });
-        // Pass URL directly to avoid persisting the token in remote.git/config
-        await git.push(this.buildGitHubUrl(), branch);
-        console.log(`Pushed ${branch} to GitHub`);
+        // Force stable (English) git output so isNonFastForwardRejection below
+        // can reliably match it -- git's rejection text is gettext-translated, so
+        // a non-English host would silently turn that classifier into a no-op.
+        // gitNetworkChildEnv (NOT gitChildEnv) because this call talks to GitHub:
+        // it keeps ambient HTTPS_PROXY/GIT_SSL_*/GIT_SSH_COMMAND, which
+        // gitChildEnv's local-ops allowlist deliberately drops.
+        git.env(gitNetworkChildEnv());
+        // [SYNC-H1] If the rebase loop rewrote this branch's already-published
+        // history, GitHub still holds the commit it replaced, so an ordinary
+        // push is non-fast-forward forever. Push under a lease keyed to exactly
+        // that commit: it moves GitHub off the commit we rewrote away, and
+        // refuses in every other case (including a commit someone else pushed).
+        const branchPath = this.branchWorkspacePath(branch);
+        const metaFile = await BranchMetadataFileManager.loadOnly(branchPath).catch(() => null);
+        const marker = metaFile?.branch.historyRewrittenFrom;
+        // What this push will actually send: remote.git's tip for the branch.
+        const outgoingSha = await this.readPublishedSha(branch);
+        try {
+            if (marker) {
+                await git.raw([
+                    'push',
+                    `--force-with-lease=${branch}:${marker}`,
+                    '--end-of-options',
+                    this.buildGitHubUrl(),
+                    `${branch}:${branch}`,
+                ]);
+            }
+            else {
+                // Pass URL directly to avoid persisting the token in remote.git/config
+                await git.push(this.buildGitHubUrl(), branch);
+            }
+        }
+        catch (err) {
+            const message = getErrorMessage(err);
+            // A refused lease means GitHub is not at the commit we rewrote, so the
+            // marker is stale -- which is routine, not exceptional: tasks are
+            // re-run after a crash (recoverOrphanedTasks), and the marker survives
+            // any failure to clear it. Two benign shapes reach here, and both are
+            // ordinary fast-forwards that a lease has no business blocking:
+            // GitHub already holds the rewritten history from an earlier attempt,
+            // or the branch has since moved past it.
+            //
+            // Retry PLAIN, and let git adjudicate. A non-forced push succeeds if
+            // and only if it fast-forwards, so it can never destroy anything --
+            // there is no ancestry check to get wrong here, and no extra network
+            // round trip to read GitHub's tip. Only if THAT is also rejected has
+            // the branch genuinely diverged.
+            //
+            // (Verified: git evaluates the lease only when it actually has an
+            // update to apply. An up-to-date ref with a stale lease prints
+            // "Everything up-to-date" and exits 0, so the already-landed case is
+            // usually absorbed above and never even reaches this branch.)
+            if (marker && isStaleLeaseRejection(message)) {
+                try {
+                    await git.push(this.buildGitHubUrl(), branch);
+                }
+                catch (retryErr) {
+                    const retryMessage = getErrorMessage(retryErr);
+                    if (isNonFastForwardRejection(retryMessage)) {
+                        throw new PermanentTaskError(`Push rejected for branch "${branch}": GitHub's tip is neither the commit this ` +
+                            `deployment last published nor an ancestor of what it is pushing, so the branch ` +
+                            `has genuinely diverged and nothing was overwritten. Something else moved it on ` +
+                            `GitHub -- a direct push, or another CanopyCMS deployment sharing this repository.`);
+                    }
+                    throw retryErr;
+                }
+                // The lease was refused, so GitHub is provably not at the marker: it
+                // has moved past the rewritten commit and the marker is spent.
+                await this.clearHistoryRewrittenMarker(branchPath, branch);
+                workerLog(`Pushed ${branch} to GitHub (GitHub had already moved past the rewritten commit)`);
+                return;
+            }
+            // An ordinary non-fast-forward rejection: GitHub has commits this
+            // deployment never published. Retrying the identical push can never
+            // succeed (DEP-L1's git-failure-is-transient carve-out does NOT apply
+            // here), so fail fast instead of burning the task's retry budget.
+            // Deliberately does NOT advise renaming the branch: a branch reaching
+            // this point usually has an open PR, and renaming would orphan it.
+            if (isNonFastForwardRejection(message)) {
+                throw new PermanentTaskError(`Push rejected for branch "${branch}": GitHub's tip is not what this deployment last ` +
+                    `published, so the branch has diverged and needs reconciling. Something else moved it ` +
+                    `on GitHub -- a direct push, or another CanopyCMS deployment sharing this repository.`);
+            }
+            throw err;
+        }
+        // Clear the marker only once GitHub is confirmed to hold something other
+        // than the commit we rewrote. A push that sent nothing new (remote.git
+        // still at the marker because its own publish has not landed yet) must
+        // leave the marker set -- it is the sole trigger for the self-heal pass.
+        if (marker && outgoingSha && outgoingSha !== marker) {
+            await this.clearHistoryRewrittenMarker(branchPath, branch);
+        }
+        workerLog(`Pushed ${branch} to GitHub`);
     }
     /**
-     * Push any canopycms-settings-* branches from remote.git to GitHub.
-     * Non-fatal: a no-op push for up-to-date branches just succeeds quietly.
+     * Push THIS deployment's own settings branch (`ensureSettingsBranch()`) from
+     * remote.git to GitHub. Non-fatal: a no-op push for an up-to-date branch
+     * just succeeds quietly.
+     *
+     * Deliberately narrowed to one branch — this used to push EVERY local
+     * branch matching `canopycms-settings-*`. With the tracking-namespace fetch
+     * fix (GITHUB_TRACKING_REF_PREFIX), `reconcileTrackedBranches` creates local
+     * heads for branches that exist on GitHub, so ANOTHER deployment's settings
+     * branch (sharing this same GitHub repo) can legitimately show up as a
+     * local head here too. Pushing it would be this deployment shipping
+     * settings state it doesn't own.
      */
-    async pushSettingsBranches(git) {
+    async pushSettingsBranches(git, trackedNames) {
         try {
+            // Resolved once here rather than at each use below: ensureSettingsBranch()
+            // is idempotent, but a local reads better and keeps the eight references
+            // in this method obviously talking about one value.
+            const settingsBranch = this.ensureSettingsBranch();
             const branches = await git.branch();
-            const settingsBranches = branches.all.filter((b) => b.startsWith('canopycms-settings-'));
-            for (const branch of settingsBranches) {
+            const settingsBranches = branches.all.filter((b) => b.startsWith(RESERVED_SETTINGS_BRANCH_PREFIX));
+            const foreign = settingsBranches.filter((b) => b !== settingsBranch);
+            if (foreign.length > 0) {
+                // Signal, not an error: this is exactly the "two deployments, one repo"
+                // condition this workstream exists to make visible. Never push these.
+                workerLogWarn(`Found settings branch(es) not owned by this deployment (${settingsBranch}): ` +
+                    `${foreign.join(', ')}. Another CanopyCMS deployment may share this GitHub repo. Not pushing them.`);
+            }
+            // Check the full branch list, not the `canopycms-settings-*` subset: an
+            // adopter-supplied `settingsBranch` override need not carry that prefix.
+            const ownBranchMissing = !branches.all.includes(settingsBranch);
+            // [SYNC-M3] A settings branch present in remote.git but absent from
+            // GitHub's tracking refs was pushed here LOCALLY -- only this
+            // deployment's own API can write to remote.git -- and has never
+            // reached GitHub. That is the discriminating signature: in the
+            // SUPPORTED two-deployments-one-repo case the foreign branch arrives
+            // through the GitHub fetch and therefore always has a tracking ref, so
+            // the warn above cannot tell the two apart on its own and a
+            // "owned-branch-absent" test alone would fire on every deployment that
+            // simply has not had a settings edit yet.
+            //
+            // With this deployment's own branch also missing, the API and this
+            // worker have resolved different deploymentNames, and every settings
+            // change the API commits is stranded in remote.git forever.
+            const strandedLocal = foreign.filter((b) => !trackedNames.has(b));
+            if (strandedLocal.length > 0) {
+                workerLogWarn(ownBranchMissing
+                    ? `Settings branch mismatch: this worker owns "${settingsBranch}", which does not ` +
+                        `exist in remote.git, while ${strandedLocal.join(', ')} exist(s) here and has never ` +
+                        `been pushed to GitHub. The API and this worker disagree about deploymentName, so ` +
+                        `settings changes are NOT reaching GitHub. Set CANOPYCMS_DEPLOYMENT_NAME (or ` +
+                        `settingsBranch) on this worker to match what the API resolves.`
+                    : `Settings branch(es) ${strandedLocal.join(', ')} exist in remote.git but not on ` +
+                        `GitHub, and are not owned by this deployment (${settingsBranch}) -- nothing ` +
+                        `will ever push them onward. Check that deploymentName matches across this ` +
+                        `deployment's API and worker.`);
+            }
+            if (ownBranchMissing) {
+                // Not created locally yet — nothing to push.
+                return;
+            }
+            try {
+                await git.push(this.buildGitHubUrl(), settingsBranch);
+                workerLog(`Pushed settings branch ${settingsBranch} to GitHub`);
+            }
+            catch (err) {
+                // Non-fatal: branch may already be up-to-date. This call site has no
+                // task to throw a PermanentTaskError into (unlike pushBranchToGitHub)
+                // and is deliberately not restructured to add one -- but a
+                // non-fast-forward rejection here is the highest-signal instance of
+                // this whole failure class: it means another CanopyCMS deployment's
+                // worker already pushed ITS OWN state to its own settings branch on
+                // GitHub (an actual settings-branch name collision, not just the
+                // "foreign branch found locally" case warned about above), so make
+                // that explicit instead of a generic "push failed" line.
+                const message = getErrorMessage(err);
+                if (isNonFastForwardRejection(message)) {
+                    workerLogWarn(`Settings push for ${settingsBranch} was rejected (non-fast-forward): another ` +
+                        `CanopyCMS deployment appears to own this settings branch on GitHub. Settings from ` +
+                        `that deployment will NOT be overwritten; this deployment's local settings changes ` +
+                        `were not pushed. Rename this deployment's settings branch (config.settingsBranch or ` +
+                        `deploymentName) to resolve the collision.`);
+                }
+                else {
+                    workerLogWarn(`Settings push for ${settingsBranch}:`, message);
+                }
+            }
+        }
+        catch (err) {
+            workerLogWarn('Failed to list branches for settings push:', err instanceof Error ? err.message : err);
+        }
+    }
+    /**
+     * Bring `refs/heads/*` in `remote.git` toward what was just fetched into
+     * `GITHUB_TRACKING_REF_PREFIX` -- WITHOUT ever force-rewinding or deleting
+     * a local head. This is the non-destructive replacement for what the old
+     * `+refs/heads/*:refs/heads/*` fetch refspec used to do implicitly (and
+     * destructively) as part of the fetch itself; see
+     * `GITHUB_TRACKING_REF_PREFIX`'s doc comment for the two failure modes
+     * that refspec caused.
+     *
+     * Per tracked branch:
+     * - no local `refs/heads/<name>` yet -> create it at the tracked commit
+     *   (a branch created on GitHub, or by another deployment sharing this
+     *   GitHub repo, becomes visible locally).
+     * - local is a strict ancestor of tracked (behind) -> fast-forward it.
+     * - local === tracked -> nothing to do.
+     * - tracked is a strict ancestor of local (ahead) -> LEAVE IT ALONE. This
+     *   is unpushed editor/settings work; the queued push task (or
+     *   pushSettingsBranches) ships it. This is exactly the branch state the
+     *   old refspec used to destroy.
+     * - neither is an ancestor of the other (diverged) -> LEAVE IT ALONE and
+     *   count/log it. A real collision (e.g. another deployment moved the
+     *   same branch name on GitHub); the next push attempt will be rejected
+     *   non-fast-forward, which is the correct, visible outcome -- this
+     *   method must never silently pick a winner. The one expected, benign
+     *   form of this -- our own rebase loop having published a rewrite into
+     *   remote.git with the GitHub push still queued -- is split out into the
+     *   `rewritten` bucket so the collision warning stays meaningful.
+     *
+     * Never deletes a local head: a branch removed on GitHub simply stops
+     * being tracked here; the local ref persists until removed through its
+     * own explicit path (the sync loop must not be one of them).
+     *
+     * `remote.git` is bare, so there is no worktree to invalidate by moving
+     * these refs -- unlike a non-bare repo, updating the ref that happens to
+     * be "checked out" is a non-issue here.
+     *
+     * Concurrency: `remote.git` is bare and on EFS, and the Lambda pushes into
+     * it concurrently (`GitManager.push()`'s `target:target` refspec) while
+     * this runs. Every `update-ref` below passes the expected old value (the
+     * all-zeros OID for "must not exist yet" on creation, the previously-read
+     * SHA for the fast-forward case) so a concurrent Lambda write landing in
+     * the gap between the read and the write loses the ref update instead of
+     * being silently clobbered -- the branch is simply revisited next cycle.
+     */
+    async reconcileTrackedBranches(git) {
+        const GIT_ZERO_OID = '0000000000000000000000000000000000000000';
+        const created = [];
+        const fastForwarded = [];
+        const ahead = [];
+        const diverged = [];
+        const rewritten = [];
+        // One invocation enumerates both namespaces: refs/heads/<name> (what the
+        // Lambda pushes into and branch clones read from) and
+        // GITHUB_TRACKING_REF_PREFIX<name> (GitHub's tip, just fetched above).
+        const raw = await git.raw([
+            'for-each-ref',
+            '--format=%(refname) %(objectname)',
+            'refs/heads/',
+            GITHUB_TRACKING_REF_PREFIX,
+        ]);
+        const heads = new Map();
+        const tracked = new Map();
+        for (const line of raw.split('\n')) {
+            const trimmed = line.trim();
+            if (!trimmed)
+                continue;
+            const [refname, sha] = trimmed.split(' ');
+            if (refname.startsWith('refs/heads/')) {
+                heads.set(refname.slice('refs/heads/'.length), sha);
+            }
+            else if (refname.startsWith(GITHUB_TRACKING_REF_PREFIX)) {
+                tracked.set(refname.slice(GITHUB_TRACKING_REF_PREFIX.length), sha);
+            }
+        }
+        for (const [name, trackedSha] of tracked) {
+            const localSha = heads.get(name);
+            const localRef = `refs/heads/${name}`;
+            if (!localSha) {
                 try {
-                    await git.push(this.buildGitHubUrl(), branch);
-                    console.log(`Pushed settings branch ${branch} to GitHub`);
+                    // Zero old-value asserts the ref does not already exist -- guards
+                    // against a concurrent Lambda push creating this exact branch
+                    // name between the for-each-ref read above and this update.
+                    await git.raw(['update-ref', localRef, trackedSha, GIT_ZERO_OID]);
+                    created.push(name);
                 }
                 catch (err) {
-                    // Non-fatal: branch may already be up-to-date or not yet created
-                    console.warn(`Settings push for ${branch}:`, err instanceof Error ? err.message : err);
+                    workerLogWarn(`  Tracked-branch reconcile: failed to create local ref for ${name} (concurrent update?): ${getErrorMessage(err)}`);
+                }
+                continue;
+            }
+            if (localSha === trackedSha)
+                continue; // nothing to do
+            // [SYNC-M2] Everything below is per-branch best-effort. The rev-list
+            // used to sit between the two guarded update-ref calls with no guard
+            // of its own, so a single ref pointing at a missing or partially
+            // written object -- plausible on EFS with the Lambda writing
+            // concurrently -- threw out of this loop, out of syncGit()'s try, and
+            // skipped pushSettingsBranches(), refreshBaseBranchWorkspace() and
+            // rebaseActiveBranches() entirely. Nothing self-healed, so it recurred
+            // every cycle. One unreadable ref must cost its own branch, not the
+            // whole sync cycle.
+            try {
+                // Count commits unique to each side in one call: left = commits local
+                // has that tracked doesn't (ahead), right = commits tracked has that
+                // local doesn't (behind). Exits 0 regardless of ancestry direction,
+                // unlike `merge-base --is-ancestor` (which simple-git would throw on
+                // a non-zero exit for the common "not an ancestor" case).
+                const counts = (await git.raw(['rev-list', '--left-right', '--count', `${localSha}...${trackedSha}`])).trim();
+                const [leftStr, rightStr] = counts.split(/\s+/);
+                const localAheadCount = parseInt(leftStr, 10);
+                const localBehindCount = parseInt(rightStr, 10);
+                if (!Number.isInteger(localAheadCount) || !Number.isInteger(localBehindCount)) {
+                    // Unparseable output means we could not read this branch, NOT that
+                    // it diverged: falling through to the `diverged` bucket below would
+                    // warn operators about a cross-deployment collision that never
+                    // happened (parseInt yields NaN, and NaN fails both comparisons).
+                    workerLogWarn(`  Tracked-branch reconcile: skipping ${name}: unparseable rev-list output ${JSON.stringify(counts)}`);
+                    continue;
                 }
+                if (localAheadCount === 0 && localBehindCount > 0) {
+                    try {
+                        await git.raw(['update-ref', localRef, trackedSha, localSha]);
+                        fastForwarded.push(name);
+                    }
+                    catch (err) {
+                        // Concurrent Lambda push moved the ref since the read above --
+                        // the guard did its job; this branch is simply revisited next cycle.
+                        workerLogWarn(`  Tracked-branch reconcile: failed to fast-forward ${name} (concurrent update?): ${getErrorMessage(err)}`);
+                    }
+                }
+                else if (localBehindCount === 0 && localAheadCount > 0) {
+                    // Unpushed local work -- exactly what the old destructive refspec
+                    // used to force-rewind or delete. Leave it.
+                    ahead.push(name);
+                }
+                else if (await this.hasPendingHistoryRewrite(name)) {
+                    // [SYNC-H1] Our own rebase published a rewrite into remote.git and
+                    // the GitHub push has not landed yet. Ref-level this is identical
+                    // to a collision, but it is expected and self-resolving, so it must
+                    // not fire the collision warning below.
+                    rewritten.push(name);
+                }
+                else {
+                    // Neither side is an ancestor of the other. Leave both alone.
+                    diverged.push(name);
+                }
+            }
+            catch (err) {
+                workerLogWarn(`  Tracked-branch reconcile: skipping ${name} (unreadable ref or object?): ${getErrorMessage(err)}`);
+                continue;
             }
         }
-        catch (err) {
-            console.warn('Failed to list branches for settings push:', err instanceof Error ? err.message : err);
+        if (diverged.length > 0) {
+            workerLogWarn(`  Tracked-branch reconcile: ${diverged.length} branch(es) diverged from GitHub and were left untouched: ${diverged.join(', ')}`);
+        }
+        if (rewritten.length > 0) {
+            workerLog(`  Tracked-branch reconcile: ${rewritten.length} branch(es) rebased locally with the GitHub push still pending: ${rewritten.join(', ')}`);
+        }
+        // trackedNames is returned alongside the summary (rather than folded into
+        // it) because it is a working set for pushSettingsBranches' stranded-
+        // branch check, not part of the worker-status snapshot -- it lists every
+        // branch on GitHub and would bloat worker-status.json for no reader.
+        return {
+            summary: { created, fastForwarded, ahead, diverged, rewritten },
+            trackedNames: new Set(tracked.keys()),
+        };
+    }
+    /**
+     * Whether this branch carries a pending history rewrite -- the rebase loop
+     * rewrote already-published history and the GitHub push has not landed yet
+     * (see BranchMetadata.historyRewrittenFrom).
+     *
+     * `branchName` is a git ref name; branch workspaces are directories named
+     * with the sanitized form, hence the conversion. Best-effort: a settings
+     * branch (no workspace at all), a missing directory or an unreadable
+     * branch.json all mean "no known rewrite", which is the conservative
+     * answer -- it keeps the branch in the louder `diverged` bucket.
+     */
+    async hasPendingHistoryRewrite(branchName) {
+        try {
+            const branchPath = path.join(this.contentBranchesPath, sanitizeBranchName(branchName));
+            const metaFile = await BranchMetadataFileManager.loadOnly(branchPath);
+            return typeof metaFile?.branch.historyRewrittenFrom === 'string';
+        }
+        catch {
+            return false;
         }
     }
     async syncGit() {
         if (!this.running)
             return;
-        console.log('Syncing git...');
+        workerLog('Syncing git...');
         const cycleStartedAt = Date.now();
         const git = simpleGit({
             baseDir: this.remoteGitPath,
@@ -751,6 +1411,11 @@ export class CmsWorker {
             // is inactivity-based, so a slow-but-flowing transfer is unaffected.
             timeout: { block: this.taskTimeoutMs },
         });
+        // This instance also drives pushSettingsBranches(git) below, whose
+        // classification of a rejected settings-branch push (see that method)
+        // needs the same stable-English guarantee as pushBranchToGitHub. Network
+        // env for the same reason: the fetch and push here reach GitHub.
+        git.env(gitNetworkChildEnv());
         // PR-W1: the whole cycle is wrapped so both outcomes -- success and
         // hard failure (e.g. the fetch throwing against a poisoned remote.git)
         // -- record a worker-status.json snapshot. The status write itself is
@@ -759,14 +1424,28 @@ export class CmsWorker {
         // on a failed one. On failure we rethrow so scheduleLoop's existing
         // per-cycle catch stays the loud path.
         try {
-            // Fetch all branches from GitHub using direct URL (no named remote)
-            // We use raw git commands since simple-git's fetch() with a URL
-            // doesn't support --prune directly
-            await git.raw(['fetch', this.buildGitHubUrl(), '--prune', '+refs/heads/*:refs/heads/*']);
-            console.log('Fetched from GitHub');
+            // Fetch all branches from GitHub using direct URL (no named remote),
+            // into the GITHUB_TRACKING_REF_PREFIX remote-tracking namespace rather
+            // than refs/heads/* directly -- see that constant's doc comment for
+            // the destructive-fetch bug this avoids. We use raw git commands since
+            // simple-git's fetch() with a URL doesn't support --prune directly.
+            await git.raw([
+                'fetch',
+                this.buildGitHubUrl(),
+                '--prune',
+                `+refs/heads/*:${GITHUB_TRACKING_REF_PREFIX}*`,
+            ]);
+            workerLog('Fetched from GitHub');
+            // Bring refs/heads/* toward what was just fetched, WITHOUT ever
+            // force-rewinding or deleting a local head -- see
+            // reconcileTrackedBranches()'s doc comment.
+            const { summary: trackedSummary, trackedNames } = await this.reconcileTrackedBranches(git);
             // Push settings branches to GitHub (belt-and-suspenders for task queue).
             // Ensures settings reach GitHub even if a task queue entry is lost.
-            await this.pushSettingsBranches(git);
+            // Ordering relative to the fetch/reconcile above is no longer a
+            // correctness dependency now that the fetch can't clobber refs/heads/*
+            // -- this could run before or after them just as safely.
+            await this.pushSettingsBranches(git, trackedNames);
             await this.refreshBaseBranchWorkspace();
             const rebaseSummary = await this.rebaseActiveBranches();
             // Periodically clean up old completed/failed tasks
@@ -777,7 +1456,7 @@ export class CmsWorker {
             // place actual removal happens.
             const trashRemoved = await this.cleanupTrashedBranchDirs();
             if (trashRemoved > 0) {
-                console.log(`Removed ${trashRemoved} expired trashed branch dir(s)`);
+                workerLog(`Removed ${trashRemoved} expired trashed branch dir(s)`);
             }
             const report = this.ensureStatusReport();
             report.lastGitSyncAt = new Date().toISOString();
@@ -787,8 +1466,9 @@ export class CmsWorker {
                 rebased: rebaseSummary.rebased,
                 skippedDirty: rebaseSummary.skippedDirty,
                 failed: rebaseSummary.failed,
+                tracked: trackedSummary,
             };
-            await writeWorkerStatus(this.taskDir, report).catch((writeErr) => console.error('Failed to write worker status:', getErrorMessage(writeErr)));
+            await writeWorkerStatus(this.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
         }
         catch (err) {
             const report = this.ensureStatusReport();
@@ -799,7 +1479,7 @@ export class CmsWorker {
                 message: redactCredentials(getErrorMessage(err)),
                 at: new Date().toISOString(),
             };
-            await writeWorkerStatus(this.taskDir, report).catch((writeErr) => console.error('Failed to write worker status:', getErrorMessage(writeErr)));
+            await writeWorkerStatus(this.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
             throw err;
         }
     }
@@ -833,7 +1513,7 @@ export class CmsWorker {
             const stampDate = stampMatch ? parseTrashStamp(stampMatch[1]) : null;
             if (!stampDate) {
                 if (!loggedUnparseable) {
-                    console.log(`CanopyCMS: Skipping trash dir with unparseable stamp: ${name}`);
+                    workerLog(`CanopyCMS: Skipping trash dir with unparseable stamp: ${name}`);
                     loggedUnparseable = true;
                 }
                 continue;
@@ -850,7 +1530,7 @@ export class CmsWorker {
                 removed++;
             }
             catch (err) {
-                console.error(`CanopyCMS: Failed to remove trashed branch dir ${name}:`, getErrorMessage(err));
+                workerLogError(`CanopyCMS: Failed to remove trashed branch dir ${name}:`, getErrorMessage(err));
             }
         }
         return removed;
@@ -890,7 +1570,7 @@ export class CmsWorker {
                 gitDirStat = null;
             }
             if (!gitDirStat || !gitDirStat.isDirectory()) {
-                console.log(`Base branch workspace (${this.baseBranch}): not yet provisioned, skipping refresh`);
+                workerLog(`Base branch workspace (${this.baseBranch}): not yet provisioned, skipping refresh`);
                 return;
             }
             const baseGit = simpleGit({
@@ -921,7 +1601,7 @@ export class CmsWorker {
             const status = await baseGit.status();
             const trackedDirty = status.files.filter((f) => f.index !== '?' || f.working_dir !== '?');
             if (trackedDirty.length > 0) {
-                console.error(`Base branch workspace (${this.baseBranch}) has uncommitted changes -- skipping refresh. Dirty files: ${trackedDirty.map((f) => f.path).join(', ')}`);
+                workerLogError(`Base branch workspace (${this.baseBranch}) has uncommitted changes -- skipping refresh. Dirty files: ${trackedDirty.map((f) => f.path).join(', ')}`);
                 return;
             }
             // Raw (unsanitized) name from here on: these are git ref operations
@@ -945,7 +1625,7 @@ export class CmsWorker {
                     await baseGit.merge(['--ff-only', fetchedTip]);
                 }
                 catch (err) {
-                    console.error(`Base branch workspace (${this.baseBranch}) failed to fast-forward (diverged local history?): ${getErrorMessage(err)}`);
+                    workerLogError(`Base branch workspace (${this.baseBranch}) failed to fast-forward (diverged local history?): ${getErrorMessage(err)}`);
                     return;
                 }
                 await invalidateBranchContentCaches(basePath);
@@ -968,12 +1648,12 @@ export class CmsWorker {
                 });
             }
             // One concise per-cycle line -- the diagnostic for the next live deploy.
-            console.log(behindCount > 0
+            workerLog(behindCount > 0
                 ? `Base branch workspace (${this.baseBranch}): fast-forwarded ${behindCount} commit(s)`
                 : `Base branch workspace (${this.baseBranch}): up to date`);
         }
         catch (err) {
-            console.error(`Base branch workspace (${this.baseBranch}) refresh failed: ${getErrorMessage(err)}`);
+            workerLogError(`Base branch workspace (${this.baseBranch}) refresh failed: ${getErrorMessage(err)}`);
         }
     }
     /**
@@ -1008,7 +1688,7 @@ export class CmsWorker {
                 await meta.save({
                     branch: buildMergedBranchUpdate(branchDir, data.merged_at ? new Date(data.merged_at) : undefined),
                 });
-                console.log(`  PR #${prNumber} for ${branchDir} is merged -> archived`);
+                workerLog(`  PR #${prNumber} for ${branchDir} is merged -> archived`);
                 return;
             }
             const newState = data.state === 'closed' ? 'closed' : 'open';
@@ -1020,11 +1700,11 @@ export class CmsWorker {
                 return;
             const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
             await meta.save({ branch: { name: branchDir, pullRequestState: newState } });
-            console.log(`  PR #${prNumber} for ${branchDir}: pullRequestState -> ${newState}`);
+            workerLog(`  PR #${prNumber} for ${branchDir}: pullRequestState -> ${newState}`);
         }
         catch (err) {
             // Non-fatal: transient GitHub/network errors are retried next cycle.
-            console.warn(`  Failed to poll PR #${prNumber} for ${branchDir}: ${getErrorMessage(err)}`);
+            workerLogWarn(`  Failed to poll PR #${prNumber} for ${branchDir}: ${getErrorMessage(err)}`);
         }
         finally {
             clearTimeout(timer);
@@ -1082,7 +1762,7 @@ export class CmsWorker {
             // against the same corrupt file would just throw again) as well as
             // any other load/save failure -- recording is best-effort
             // observability, never allowed to abort the branch loop.
-            console.warn(`  Failed to record rebase failure for ${branchDir}: ${getErrorMessage(err)}`);
+            workerLogWarn(`  Failed to record rebase failure for ${branchDir}: ${getErrorMessage(err)}`);
         }
     }
     async rebaseActiveBranches() {
@@ -1113,12 +1793,12 @@ export class CmsWorker {
             try {
                 const stat = await fs.stat(gitDir);
                 if (!stat.isDirectory()) {
-                    console.log(`  Skipping ${branchDir}: .git is not a directory`);
+                    workerLog(`  Skipping ${branchDir}: .git is not a directory`);
                     continue;
                 }
             }
             catch {
-                console.log(`  Skipping ${branchDir}: no .git directory (not a branch workspace)`);
+                workerLog(`  Skipping ${branchDir}: no .git directory (not a branch workspace)`);
                 continue;
             }
             // The base branch's own clone is refreshed ff-only by
@@ -1128,7 +1808,7 @@ export class CmsWorker {
             // metadata on it. Compare sanitized-vs-sanitized: branchDir is a
             // filesystem name (already sanitized), this.baseBranch is raw.
             if (branchDir === this.sanitizedBaseBranch) {
-                console.log(`  Skipping ${branchDir}: base branch (refreshed separately)`);
+                workerLog(`  Skipping ${branchDir}: base branch (refreshed separately)`);
                 continue;
             }
             try {
@@ -1142,12 +1822,12 @@ export class CmsWorker {
                 // - archived: already merged, no reason to rebase and no PR left to
                 //   poll (avoid the wasted API call).
                 if (branchStatus === 'submitted' || branchStatus === 'approved') {
-                    console.log(`  Skipping ${branchDir} (${branchStatus})`);
+                    workerLog(`  Skipping ${branchDir} (${branchStatus})`);
                     await this.pollMergeState(branchDir, branchPath, metaFile);
                     continue;
                 }
                 if (branchStatus === 'archived') {
-                    console.log(`  Skipping ${branchDir} (${branchStatus})`);
+                    workerLog(`  Skipping ${branchDir} (${branchStatus})`);
                     continue;
                 }
                 const branchGit = simpleGit({
@@ -1165,10 +1845,33 @@ export class CmsWorker {
                 // the catch block will abort safely — the branch stays behind and retries next cycle.
                 const dirtyCheck = await branchGit.status();
                 if (dirtyCheck.files.length > 0) {
-                    console.log(`  Skipping ${branchDir}: has uncommitted changes`);
+                    workerLog(`  Skipping ${branchDir}: has uncommitted changes`);
                     skippedDirty.push(branchDir);
                     continue;
                 }
+                // The clone's own ref name: branchDir is the sanitized DIRECTORY
+                // name and need not match it. A literal 'HEAD' means a detached
+                // clone (e.g. a crashed rebase left one behind) -- nothing below can
+                // safely name a ref then, so every publish path stays disarmed,
+                // which is the safe direction.
+                const branchRef = (await branchGit.revparse(['--abbrev-ref', 'HEAD'])).trim();
+                const canPublish = branchRef.length > 0 && branchRef !== 'HEAD';
+                // [SYNC-H1] Self-heal: finish an interrupted publish even when this
+                // branch is not behind base. Every crash window in the arming
+                // sequence below leaves the marker set with the work unfinished, and
+                // this is what completes it -- without it, one lost lease race would
+                // strand the branch until the base branch happened to advance again.
+                // Gated on the loop-top snapshot, so unmarked branches (nearly all
+                // of them, every cycle) cost nothing extra.
+                if (canPublish && metaFile?.branch.historyRewrittenFrom) {
+                    await this.reconcilePendingRewrite({
+                        branchPath,
+                        branchDir,
+                        branchRef,
+                        headSha: (await branchGit.revparse(['HEAD'])).trim(),
+                        marker: metaFile.branch.historyRewrittenFrom,
+                    });
+                }
                 await branchGit.fetch('origin', this.baseBranch);
                 // Use rev-list instead of status.behind — status.behind only works when the
                 // branch has an upstream tracking branch configured, which isn't guaranteed
@@ -1216,7 +1919,13 @@ export class CmsWorker {
                     });
                     continue;
                 }
-                console.log(`Rebasing ${branchDir} (${behindCount} commits behind)...`);
+                workerLog(`Rebasing ${branchDir} (${behindCount} commits behind)...`);
+                // Read BOTH sides before rewriting anything: the arming guard after
+                // the rebase compares what remote.git published against what this
+                // clone is about to rebase away. Reading them afterwards would be
+                // useless -- the clone's pre-rebase tip is exactly what disappears.
+                const preRebaseHead = (await branchGit.revparse(['HEAD'])).trim();
+                const publishedSha = canPublish ? await this.readPublishedSha(branchRef) : null;
                 // Resolve-and-continue loop: keep branch version for conflicting files, then continue
                 // Non-conflicting files get main's changes; conflicting files keep branch version.
                 const conflictedFiles = [];
@@ -1267,7 +1976,7 @@ export class CmsWorker {
                                 // the rebase didn't complete so we can't determine the true conflict
                                 // state. Previous metadata (possibly stale) is preserved until the
                                 // next successful rebase cycle corrects it.
-                                console.warn(`  Unexpected rebase error in ${branchDir}: ${msg || 'Unknown error'}`);
+                                workerLogWarn(`  Unexpected rebase error in ${branchDir}: ${msg || 'Unknown error'}`);
                                 failureReason = msg || 'Unknown error';
                                 await branchGit.rebase(['--abort']).catch(() => { });
                                 break;
@@ -1280,7 +1989,7 @@ export class CmsWorker {
                     // error" break above -- MAX_ROUNDS exhaustion is a distinct exit
                     // path with no error message of its own, so the warn text must
                     // not conflate the two.
-                    console.warn(failureReason !== undefined
+                    workerLogWarn(failureReason !== undefined
                         ? `  Rebase of ${branchDir} aborted due to unexpected error: ${failureReason}`
                         : `  Rebase of ${branchDir} did not complete within ${MAX_ROUNDS} rounds, aborting`);
                     await branchGit.rebase(['--abort']).catch(() => { });
@@ -1311,8 +2020,26 @@ export class CmsWorker {
                 // Entry files have IDs in their filename (e.g., "post.slug.a1b2c3d4e5f6.mdx").
                 // .collection.json files have no ID themselves (extractIdFromFilename returns null
                 // for dot-prefixed files), so we extract the ID from the parent directory instead.
-                // The root content directory (e.g., "content/") has no embedded ID, so we use
-                // ROOT_COLLECTION_ID as a sentinel — but only for the configured contentRoot.
+                // The root content directory (e.g., "content/", or "cms/content/" for a
+                // multi-segment contentRoot) has no embedded ID, so we use ROOT_COLLECTION_ID
+                // as a sentinel — but only for the configured contentRoot.
+                //
+                // Two different notions of "parent" are needed below, and conflating them
+                // reproduces the exact bug this comparison guards against (see
+                // schema-store.ts's contentRootName doc comment for the same shape elsewhere):
+                //  - `parentDir` (a basename) recovers a SUB-collection's own embedded ID
+                //    (e.g. "posts.cNbR5xFm2Kpd" -> "cNbR5xFm2Kpd") — correct as a basename,
+                //    since a collection directory carries its ID in its own name, one path
+                //    segment.
+                //  - `parentPath` (the full relative parent path, normalized) is what must be
+                //    compared against `this.contentRoot`, because `contentRoot` is documented
+                //    (config/helpers.ts) as allowed to span multiple segments (e.g.
+                //    "cms/content"). Comparing a basename ("content") against that full value
+                //    is always false, which silently drops the root collection's conflict.
+                //    Git reports POSIX-style paths and the configured value may be authored
+                //    with either separator, so both sides go through normalizeFilesystemPath
+                //    before comparing.
+                const normalizedContentRoot = normalizeFilesystemPath(this.contentRoot);
                 const conflictIds = [...new Set(conflictedFiles)]
                     .map((f) => {
                     const fileId = extractIdFromFilename(path.basename(f));
@@ -1322,9 +2049,10 @@ export class CmsWorker {
                     const dirId = extractIdFromFilename(parentDir);
                     if (dirId)
                         return dirId;
-                    // Only assign ROOT_COLLECTION_ID when the parent matches the configured
-                    // content root (e.g., "content"). Other unrecognized paths are filtered out.
-                    if (path.basename(f) === '.collection.json' && parentDir === this.contentRoot) {
+                    // Only assign ROOT_COLLECTION_ID when the file's parent directory IS the
+                    // configured content root. Other unrecognized paths are filtered out.
+                    const parentPath = normalizeFilesystemPath(path.dirname(f));
+                    if (path.basename(f) === '.collection.json' && parentPath === normalizedContentRoot) {
                         return ROOT_COLLECTION_ID;
                     }
                     return null;
@@ -1332,7 +2060,7 @@ export class CmsWorker {
                     .filter((id) => id !== null);
                 const conflictIdsDeduped = [...new Set(conflictIds)];
                 const hadConflicts = conflictIdsDeduped.length > 0;
-                console.log(hadConflicts
+                workerLog(hadConflicts
                     ? `  Rebased ${branchDir} (kept branch version for ${conflictIdsDeduped.length} conflicting file(s))`
                     : `  Rebased ${branchDir} successfully`);
                 await meta.save({
@@ -1350,10 +2078,51 @@ export class CmsWorker {
                 // in the summary. Branches already up to date `continue`d above and
                 // are deliberately not listed here.
                 rebased.push(branchDir);
+                // [SYNC-H1] The rebase just rewrote this clone's history. If that
+                // history was already published, nothing else will ever reconcile
+                // remote.git (and GitHub) with it -- the editor's next submit would
+                // simply be rejected non-fast-forward. Carry the rewrite forward.
+                if (publishedSha !== null) {
+                    if (publishedSha === preRebaseHead) {
+                        // ARMING GUARD. remote.git holds EXACTLY what this clone just
+                        // rebased away and nothing more, so a lease keyed to it can only
+                        // undo our own rewrite.
+                        //
+                        // The inequality case below is not defensive padding: branch
+                        // clones never fetch their own branch (GitManager's clone is
+                        // --single-branch and checkoutBranch only checks out an existing
+                        // local branch), while reconcileTrackedBranches fast-forwards
+                        // remote.git to GitHub's tip. So after a reviewer pushes a fixup
+                        // straight to the PR branch, remote.git legitimately holds a
+                        // commit this clone has never seen. Leasing on "whatever
+                        // remote.git currently holds" would be SATISFIED there and would
+                        // delete that fixup from remote.git and then from GitHub,
+                        // silently. Keying the lease to the pre-rebase tip turns that
+                        // case into the visible divergence it should be.
+                        //
+                        // Order matters: mark, then push, then queue. A crash after any
+                        // step leaves the marker set with the work unfinished, which
+                        // reconcilePendingRewrite() completes on a later cycle. Pushing
+                        // first would leave remote.git rewritten, GitHub stale and
+                        // nothing recorded -- unrecoverable, and landing on exactly the
+                        // false "another deployment" diagnosis this change removes.
+                        await this.markHistoryRewritten(branchPath, branchDir, publishedSha);
+                        if (await this.forcePublishToLocalRemote(branchPath, branchRef, publishedSha)) {
+                            await this.enqueueGitHubPush(branchRef);
+                        }
+                    }
+                    else {
+                        const divergence = `rebased locally, but remote.git holds ${publishedSha} for ${branchRef}, which this ` +
+                            `clone never had (a direct push to the branch?). Left untouched -- reconcile it ` +
+                            `before submitting again.`;
+                        workerLogWarn(`  ${branchDir}: ${divergence}`);
+                        await this.recordRebaseFailure(branchPath, branchDir, divergence);
+                    }
+                }
             }
             catch (err) {
                 const message = err instanceof Error ? err.message : 'Unknown error';
-                console.warn(`  Failed to sync ${branchDir}: ${message}`);
+                workerLogWarn(`  Failed to sync ${branchDir}: ${message}`);
                 // [HIGH-1] Same rationale as the `if (!completed)` push site above --
                 // this catches fetch/rev-list/unexpected errors, whose message can
                 // embed the bot token.
@@ -1370,14 +2139,14 @@ export class CmsWorker {
     async refreshAuthCache() {
         if (!this.running || !this.config.refreshAuthCache)
             return;
-        console.log('Refreshing auth cache...');
+        workerLog('Refreshing auth cache...');
         try {
             await this.config.refreshAuthCache();
-            console.log('Auth cache refreshed');
+            workerLog('Auth cache refreshed');
         }
         catch (err) {
             const message = err instanceof Error ? err.message : 'Unknown error';
-            console.error('Failed to refresh auth cache:', message);
+            workerLogError('Failed to refresh auth cache:', message);
         }
     }
 }