canopycms 0.0.59 → 0.0.61-int.74

package/dist/worker/cms-worker.js

@@ -4,14 +4,39 @@ import { simpleGit } from 'simple-git';
 import lockfile from 'proper-lockfile';
 import { dequeueTask, completeTask, failTask, retryTask, recoverOrphanedTasks, cleanupOldTasks, cmsTaskQueueLogger, } from './task-queue.js';
 import { createOrUpdatePullRequest, createCanopyOctokit } from '../github-service.js';
-import { getBranchMetadataFileManager, BranchMetadataFileManager } from '../branch-metadata.js';
+import { getBranchMetadataFileManager, BranchMetadataFileManager, buildMergedBranchUpdate, } from '../branch-metadata.js';
 import { extractIdFromFilename } from '../content-id-index.js';
 import { invalidateBranchContentCaches } from '../content-index-generation.js';
 import { ROOT_COLLECTION_ID } from '../paths/types.js';
-import { getErrorMessage, isNodeError } from '../utils/error.js';
+import { sanitizeBranchName } from '../paths/branch-name.js';
+import { getErrorMessage, isNodeError, redactCredentials } from '../utils/error.js';
+import { writeWorkerStatus } from './worker-status.js';
 const DEFAULT_TASK_TIMEOUT = 60_000;
 const DEFAULT_MAX_RETRIES = 3;
 const DEFAULT_LOCK_STALE_MS = 60_000;
+/**
+ * [C1] Retention window for `.trash-*` branch directories left behind by the
+ * admin purge action (api/admin-branch-health.ts). Matches
+ * cleanupOldTasks's default task retention for consistency.
+ */
+const TRASH_RETENTION_MS = 30 * 24 * 60 * 60_000;
+/** Matches `.trash-{dirName}-{STAMP}` names, capturing the trailing stamp. */
+const TRASH_DIR_STAMP_RE = /-(\d{8}T\d{6}Z)$/;
+/**
+ * Parse a purge-generated `YYYYMMDDTHHMMSSZ` stamp into a Date, or null if
+ * malformed. Age comes ONLY from this name-embedded stamp, never the dir's
+ * own mtime -- `fs.rename` preserves the original directory's mtime, so an
+ * mtime-based retention check would delete a months-stale orphan's trash on
+ * the very first cleanup pass after purge.
+ */
+function parseTrashStamp(stamp) {
+    const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(stamp);
+    if (!match)
+        return null;
+    const [, year, month, day, hour, minute, second] = match;
+    const date = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.000Z`);
+    return Number.isNaN(date.getTime()) ? null : date;
+}
 /**
  * An error inherent to the task itself (malformed payload, unknown action):
  * retrying can never succeed, so the task should fail fast instead of
@@ -135,6 +160,7 @@ export class CmsWorker {
         this.remoteGitPath = path.join(config.workspacePath, 'remote.git');
         this.contentBranchesPath = path.join(config.workspacePath, 'content-branches');
         this.baseBranch = config.baseBranch ?? 'main';
+        this.sanitizedBaseBranch = sanitizeBranchName(this.baseBranch);
         this.maxTasksPerCycle = config.maxTasksPerCycle ?? 10;
         this.taskTimeoutMs = config.taskTimeoutMs ?? DEFAULT_TASK_TIMEOUT;
         this.maxRetries = config.maxRetries ?? DEFAULT_MAX_RETRIES;
@@ -142,9 +168,25 @@ export class CmsWorker {
         this.lockStaleMs = config.lockStaleMs ?? DEFAULT_LOCK_STALE_MS;
         this.contentRoot = config.contentRoot ?? 'content';
     }
+    /**
+     * Lazily get (and initialize if necessary) this worker's self-reported
+     * status object (PR-W1). Normally set once, up front, at the top of
+     * start(). The lazy fallback here covers two cases: (1) something in
+     * start() reaching a status-write point before that normal init runs
+     * (defensive -- see start()'s catch), and (2) unit tests that exercise
+     * syncGit()/processTaskQueue() directly without calling start() first.
+     */
+    ensureStatusReport() {
+        if (!this.statusReport) {
+            const now = new Date().toISOString();
+            this.statusReport = { version: 1, startedAt: now, updatedAt: now };
+        }
+        return this.statusReport;
+    }
     async start() {
         this.running = true;
         console.log('CMS Worker starting...');
+        this.ensureStatusReport();
         // Acquire lock to prevent concurrent workers
         await this.acquireLock();
         // Everything below runs while holding the cross-host worker lock. A
@@ -175,6 +217,25 @@ export class CmsWorker {
             await Promise.allSettled(initialTasks);
         }
         catch (err) {
+            // PR-W1: surface a startup failure (e.g. the empty-remote guard's
+            // poisoned remote.git) to the admin panel via worker-status.json,
+            // not only journald/CloudWatch. Best-effort and BEFORE releaseLock():
+            // a status-write failure must never block releasing the lock.
+            const report = this.ensureStatusReport();
+            report.lastFatalError = {
+                // [HIGH-1] Persisted to worker-status.json and served to the
+                // browser by the admin panel -- must never carry the bot token
+                // that a poisoned/failed git URL (buildGitHubUrl()) can embed.
+                message: redactCredentials(getErrorMessage(err)),
+                at: new Date().toISOString(),
+                phase: 'startup',
+            };
+            try {
+                await writeWorkerStatus(this.taskDir, report);
+            }
+            catch (writeErr) {
+                console.error('Failed to write worker status on startup failure:', getErrorMessage(writeErr));
+            }
             await this.releaseLock();
             throw err;
         }
@@ -396,6 +457,11 @@ export class CmsWorker {
             catch (err) {
                 const message = getErrorMessage(err);
                 console.error(`Task ${task.id} (${task.action}) failed:`, message);
+                // [HIGH-1] task.error is persisted (pending/failed task JSON) and
+                // served to the browser by the admin panel's Tasks tab -- a push
+                // failure's message can embed the bot token via buildGitHubUrl().
+                // Console output above stays raw (journald/CloudWatch is trusted).
+                const persistedMessage = redactCredentials(message);
                 // DEP-L1: only transient failures (network, 429/5xx, timeouts) are
                 // worth retrying; permanent ones (malformed payload, other 4xx) would
                 // just burn the retry budget on an identical doomed request.
@@ -403,12 +469,12 @@ export class CmsWorker {
                 const retryCount = task.retryCount ?? 0;
                 const maxRetries = task.maxRetries ?? this.maxRetries;
                 if (!permanent && retryCount < maxRetries) {
-                    await retryTask(this.taskDir, task.id, message, this.log);
+                    await retryTask(this.taskDir, task.id, persistedMessage, this.log);
                     console.log(`  Will retry (attempt ${retryCount + 1}/${maxRetries})`);
                 }
                 else {
-                    await failTask(this.taskDir, task.id, message, this.log);
-                    await this.updateBranchMetadataOnFailure(task, message);
+                    await failTask(this.taskDir, task.id, persistedMessage, this.log);
+                    await this.updateBranchMetadataOnFailure(task, persistedMessage);
                     console.error(permanent
                         ? '  Permanently failed (non-retryable error)'
                         : `  Permanently failed after ${maxRetries} retries`);
@@ -416,6 +482,15 @@ export class CmsWorker {
             }
             processed++;
         }
+        // PR-W1: only stamp/write when work actually happened this poll --
+        // otherwise every idle 5s poll would hit worker-status.json, an EFS
+        // write treadmill for no signal (liveness is already covered by the
+        // lock heartbeat; see api/admin.ts's classifyWorkerLiveness).
+        if (processed > 0) {
+            const report = this.ensureStatusReport();
+            report.lastTaskCycleAt = new Date().toISOString();
+            await writeWorkerStatus(this.taskDir, report).catch((writeErr) => console.error('Failed to write worker status:', getErrorMessage(writeErr)));
+        }
     }
     /**
      * Execute a task bounded by taskTimeoutMs (DEP-H1). Two layers:
@@ -491,13 +566,23 @@ export class CmsWorker {
                 // direct-API path) so the list->tiebreak->update/create logic and
                 // the draft->ready conversion live in one place.
                 const branch = requireString(payload, 'branch');
+                const base = optionalString(payload, 'baseBranch', this.baseBranch);
+                // Defense-in-depth: refuse head===base even if the 'submittableBranch'
+                // API guard and the syncSubmitPr backstop were both somehow bypassed
+                // (e.g. a task queued before this check shipped). PermanentTaskError
+                // (not a plain Error) so this fails immediately instead of burning
+                // the retry budget on an identical doomed request -- retrying can
+                // never make the branch not be the base branch.
+                if (sanitizeBranchName(branch) === this.sanitizedBaseBranch) {
+                    throw new PermanentTaskError(`Refusing to push-and-create-or-update-pr for "${branch}": it is the base branch -- submitting the base branch is never valid`);
+                }
                 await this.pushBranchToGitHub(branch);
                 const result = await createOrUpdatePullRequest({
                     octokit: this.octokit,
                     owner: this.config.githubOwner,
                     repo: this.config.githubRepo,
                     head: branch,
-                    base: optionalString(payload, 'baseBranch', this.baseBranch),
+                    base,
                     title: optionalString(payload, 'title', `Submit ${branch}`),
                     body: optionalString(payload, 'body', ''),
                     // Content submits (api/github-sync.ts) set this; settings-branch
@@ -572,8 +657,21 @@ export class CmsWorker {
             };
             if (result.prUrl)
                 updates.pullRequestUrl = result.prUrl;
-            if (result.prNumber)
+            if (result.prNumber) {
                 updates.pullRequestNumber = result.prNumber;
+                // As soon as the PR exists the field should read 'open' -- without
+                // this it would stay absent until the next poll cycle observes it.
+                // Exception: the git-sync loop may have archived this branch (PR
+                // merged/closed) while this task was in flight; a late task
+                // completion must not downgrade that terminal state back to 'open'.
+                const current = await BranchMetadataFileManager.loadOnly(branchPath);
+                const terminal = current?.branch.status === 'archived' ||
+                    current?.branch.pullRequestState === 'merged' ||
+                    current?.branch.pullRequestState === 'closed';
+                if (!terminal) {
+                    updates.pullRequestState = 'open';
+                }
+            }
             await meta.save({ branch: updates });
         }
         catch (err) {
@@ -645,6 +743,7 @@ export class CmsWorker {
         if (!this.running)
             return;
         console.log('Syncing git...');
+        const cycleStartedAt = Date.now();
         const git = simpleGit({
             baseDir: this.remoteGitPath,
             // DEP-H1: a hung fetch/push would stall the sync loop forever
@@ -652,35 +751,384 @@ export class CmsWorker {
             // is inactivity-based, so a slow-but-flowing transfer is unaffected.
             timeout: { block: this.taskTimeoutMs },
         });
-        // Fetch all branches from GitHub using direct URL (no named remote)
-        // We use raw git commands since simple-git's fetch() with a URL
-        // doesn't support --prune directly
-        await git.raw(['fetch', this.buildGitHubUrl(), '--prune', '+refs/heads/*:refs/heads/*']);
-        console.log('Fetched from GitHub');
-        // Push settings branches to GitHub (belt-and-suspenders for task queue).
-        // Ensures settings reach GitHub even if a task queue entry is lost.
-        await this.pushSettingsBranches(git);
-        await this.rebaseActiveBranches();
-        // Periodically clean up old completed/failed tasks
-        await cleanupOldTasks(this.taskDir, undefined, this.log);
+        // PR-W1: the whole cycle is wrapped so both outcomes -- success and
+        // hard failure (e.g. the fetch throwing against a poisoned remote.git)
+        // -- record a worker-status.json snapshot. The status write itself is
+        // always best-effort (.catch below): it must never turn an otherwise
+        // successful cycle into a failure, and must never mask the real error
+        // on a failed one. On failure we rethrow so scheduleLoop's existing
+        // per-cycle catch stays the loud path.
+        try {
+            // Fetch all branches from GitHub using direct URL (no named remote)
+            // We use raw git commands since simple-git's fetch() with a URL
+            // doesn't support --prune directly
+            await git.raw(['fetch', this.buildGitHubUrl(), '--prune', '+refs/heads/*:refs/heads/*']);
+            console.log('Fetched from GitHub');
+            // Push settings branches to GitHub (belt-and-suspenders for task queue).
+            // Ensures settings reach GitHub even if a task queue entry is lost.
+            await this.pushSettingsBranches(git);
+            await this.refreshBaseBranchWorkspace();
+            const rebaseSummary = await this.rebaseActiveBranches();
+            // Periodically clean up old completed/failed tasks
+            await cleanupOldTasks(this.taskDir, undefined, this.log);
+            // [C1] Sweep branch directories the admin purge action trashed more
+            // than TRASH_RETENTION_MS ago. Worker-only by design: purge itself
+            // never deletes anything (reversible), and this cycle is the sole
+            // place actual removal happens.
+            const trashRemoved = await this.cleanupTrashedBranchDirs();
+            if (trashRemoved > 0) {
+                console.log(`Removed ${trashRemoved} expired trashed branch dir(s)`);
+            }
+            const report = this.ensureStatusReport();
+            report.lastGitSyncAt = new Date().toISOString();
+            delete report.lastGitSyncError;
+            report.lastGitSync = {
+                durationMs: Date.now() - cycleStartedAt,
+                rebased: rebaseSummary.rebased,
+                skippedDirty: rebaseSummary.skippedDirty,
+                failed: rebaseSummary.failed,
+            };
+            await writeWorkerStatus(this.taskDir, report).catch((writeErr) => console.error('Failed to write worker status:', getErrorMessage(writeErr)));
+        }
+        catch (err) {
+            const report = this.ensureStatusReport();
+            // [HIGH-1] Persisted to worker-status.json and served to the browser
+            // by the admin panel -- a fetch/push failure's message can embed the
+            // bot token via buildGitHubUrl().
+            report.lastGitSyncError = {
+                message: redactCredentials(getErrorMessage(err)),
+                at: new Date().toISOString(),
+            };
+            await writeWorkerStatus(this.taskDir, report).catch((writeErr) => console.error('Failed to write worker status:', getErrorMessage(writeErr)));
+            throw err;
+        }
+    }
+    /**
+     * [C1] Remove `.trash-*` branch directories (created by the admin purge
+     * action, api/admin-branch-health.ts) whose name-embedded stamp is older
+     * than {@link TRASH_RETENTION_MS}. Names that don't match the expected
+     * `.trash-{dirName}-{STAMP}` shape, or whose stamp fails to parse, are
+     * left alone (logged once per cycle, not per file, to avoid flooding logs
+     * if something odd accumulates) -- purge is the only writer of this
+     * naming scheme, so an unparseable name is unexpected and worth a human
+     * looking rather than a silent skip.
+     */
+    async cleanupTrashedBranchDirs() {
+        let entries;
+        try {
+            entries = await fs.readdir(this.contentBranchesPath);
+        }
+        catch (err) {
+            if (isNodeError(err) && err.code === 'ENOENT')
+                return 0;
+            throw err;
+        }
+        const now = Date.now();
+        let removed = 0;
+        let loggedUnparseable = false;
+        for (const name of entries) {
+            if (!name.startsWith('.trash-'))
+                continue;
+            const stampMatch = TRASH_DIR_STAMP_RE.exec(name);
+            const stampDate = stampMatch ? parseTrashStamp(stampMatch[1]) : null;
+            if (!stampDate) {
+                if (!loggedUnparseable) {
+                    console.log(`CanopyCMS: Skipping trash dir with unparseable stamp: ${name}`);
+                    loggedUnparseable = true;
+                }
+                continue;
+            }
+            if (now - stampDate.getTime() < TRASH_RETENTION_MS)
+                continue;
+            try {
+                await fs.rm(path.join(this.contentBranchesPath, name), {
+                    recursive: true,
+                    force: true,
+                    maxRetries: 3,
+                    retryDelay: 100,
+                });
+                removed++;
+            }
+            catch (err) {
+                console.error(`CanopyCMS: Failed to remove trashed branch dir ${name}:`, getErrorMessage(err));
+            }
+        }
+        return removed;
+    }
+    /**
+     * Fast-forward the base branch's own working-tree clone
+     * (content-branches/<baseBranch>) to match origin/<baseBranch>.
+     *
+     * Previously this clone was refreshed only incidentally, by the generic
+     * rebase loop below (rebaseActiveBranches): for a branch with status
+     * 'editing', rebasing onto origin/<baseBranch> degenerates to a
+     * fast-forward when the clone IS the base branch. But that loop's skip
+     * paths -- a dirty tree, a missing .git -- are silent, which is the
+     * suspected live failure mode: a wedged base clone with no diagnosable
+     * signal in the logs. This dedicated step makes the refresh explicit,
+     * ff-only, and loud, so a stuck base view (an editor forking a new branch
+     * "from base" that's actually a stale snapshot) is diagnosable from logs.
+     * This runs every sync cycle so the drift window is bounded by
+     * gitSyncInterval.
+     *
+     * ff-only on purpose: this clone must stay a linear mirror of
+     * origin/<baseBranch>, so a merge that isn't a fast-forward (diverged
+     * local history) is treated as a should-never-happen condition and left
+     * untouched rather than force-resolved.
+     */
+    async refreshBaseBranchWorkspace() {
+        // Sanitized name for the workspace directory (a base branch containing
+        // e.g. '/' would otherwise stat a wrong nested path here forever).
+        const basePath = path.join(this.contentBranchesPath, this.sanitizedBaseBranch);
+        const gitDir = path.join(basePath, '.git');
+        try {
+            let gitDirStat;
+            try {
+                gitDirStat = await fs.stat(gitDir);
+            }
+            catch {
+                gitDirStat = null;
+            }
+            if (!gitDirStat || !gitDirStat.isDirectory()) {
+                console.log(`Base branch workspace (${this.baseBranch}): not yet provisioned, skipping refresh`);
+                return;
+            }
+            const baseGit = simpleGit({
+                baseDir: basePath,
+                // Keep git non-interactive during the merge so it never blocks on an
+                // editor. simple-git >=3.32 blocks setting core.editor unless
+                // explicitly opted in; the value here is a hardcoded literal
+                // ("true", the shell no-op), not user input, so enabling
+                // allowUnsafeEditor carries no injection risk (same as the rebase
+                // loop's git config below).
+                config: ['core.editor=true'],
+                unsafe: { allowUnsafeEditor: true },
+                // DEP-H1: a hung fetch/merge against this EFS-backed clone would
+                // stall the sync loop forever (scheduleLoop only reschedules after
+                // completion). The block timeout is inactivity-based, so a
+                // slow-but-flowing transfer is unaffected -- same as syncGit()'s
+                // remote handle.
+                timeout: { block: this.taskTimeoutMs },
+            });
+            // Nothing makes this clone read-only. A direct edit here (or a stray
+            // process) wedges every editor's view of the base branch until an
+            // operator intervenes, so a dirty tree is loud, not a quiet skip.
+            // Only TRACKED changes block the refresh: a stray untracked file (e.g.
+            // runtime metadata missing from .git/info/exclude) must not wedge the
+            // fast-forward forever — if an untracked file would collide with
+            // incoming content, the --ff-only merge below refuses on its own and
+            // that failure is already logged loudly.
+            const status = await baseGit.status();
+            const trackedDirty = status.files.filter((f) => f.index !== '?' || f.working_dir !== '?');
+            if (trackedDirty.length > 0) {
+                console.error(`Base branch workspace (${this.baseBranch}) has uncommitted changes -- skipping refresh. Dirty files: ${trackedDirty.map((f) => f.path).join(', ')}`);
+                return;
+            }
+            // Raw (unsanitized) name from here on: these are git ref operations
+            // against origin/<baseBranch>, not filesystem paths, so they must use
+            // the same name GitHub knows the branch by.
+            await baseGit.fetch('origin', this.baseBranch);
+            // Use rev-list instead of status.behind for the same reason as the
+            // rebase loop below: status.behind only works with an upstream
+            // tracking branch configured, which isn't guaranteed here.
+            // Compare against the just-fetched tip rather than origin/<base>:
+            // workspaces are cloned --single-branch (git-manager.ts), so their
+            // fetch refspec only materializes a remote-tracking ref for the branch
+            // they were cloned from — for any other base branch, origin/<base>
+            // simply never exists and rev-list dies with "ambiguous argument".
+            // Pin FETCH_HEAD to a SHA immediately: FETCH_HEAD is one shared mutable
+            // file per repo, silently repointed by ANY other fetch in this clone.
+            const fetchedTip = (await baseGit.revparse(['FETCH_HEAD'])).trim();
+            const behindCount = parseInt((await baseGit.raw(['rev-list', '--count', `HEAD..${fetchedTip}`])).trim(), 10);
+            if (behindCount > 0) {
+                try {
+                    await baseGit.merge(['--ff-only', fetchedTip]);
+                }
+                catch (err) {
+                    console.error(`Base branch workspace (${this.baseBranch}) failed to fast-forward (diverged local history?): ${getErrorMessage(err)}`);
+                    return;
+                }
+                await invalidateBranchContentCaches(basePath);
+            }
+            // Hygiene: conflictStatus/conflictFiles are meaningless for the base
+            // branch's own metadata and may be left over from before the base was
+            // excluded from rebaseActiveBranches()'s conflict-resolution loop.
+            // Reuse the already-clean no-op-save guard from that loop: save()
+            // eager-regenerates the branch registry (O(branch count) EFS reads),
+            // so skip it when there's nothing to clear.
+            const currentMeta = await BranchMetadataFileManager.loadOnly(basePath);
+            const conflictStatus = currentMeta?.branch.conflictStatus;
+            const conflictFiles = currentMeta?.branch.conflictFiles;
+            const alreadyClean = (conflictStatus === undefined || conflictStatus === 'clean') &&
+                (conflictFiles === undefined || conflictFiles.length === 0);
+            if (!alreadyClean) {
+                const meta = getBranchMetadataFileManager(basePath, this.contentBranchesPath);
+                await meta.save({
+                    branch: { name: this.baseBranch, conflictStatus: 'clean', conflictFiles: [] },
+                });
+            }
+            // One concise per-cycle line -- the diagnostic for the next live deploy.
+            console.log(behindCount > 0
+                ? `Base branch workspace (${this.baseBranch}): fast-forwarded ${behindCount} commit(s)`
+                : `Base branch workspace (${this.baseBranch}): up to date`);
+        }
+        catch (err) {
+            console.error(`Base branch workspace (${this.baseBranch}) refresh failed: ${getErrorMessage(err)}`);
+        }
+    }
+    /**
+     * Poll GitHub for a submitted/approved branch's PR resolution.
+     *
+     * submitted/approved branches sit outside the rebase loop and get no
+     * other signal that their PR resolved on GitHub -- nothing pushes a
+     * merge/close webhook back into the branch workspace. merged ->
+     * auto-archive via buildMergedBranchUpdate (shared with the manual
+     * markAsMerged API so both paths produce identical archived-branch
+     * metadata). closed-without-merge -> record pullRequestState only; an
+     * admin decides the workflow transition from there. Best-effort: any
+     * failure here is logged and swallowed, retried next sync cycle.
+     */
+    async pollMergeState(branchDir, branchPath, metaFile) {
+        const prNumber = metaFile?.branch.pullRequestNumber;
+        if (!prNumber)
+            return;
+        const controller = new AbortController();
+        const timer = setTimeout(() => controller.abort(), this.taskTimeoutMs);
+        try {
+            const { data } = await this.octokit.pulls.get({
+                owner: this.config.githubOwner,
+                repo: this.config.githubRepo,
+                pull_number: prNumber,
+                request: { signal: controller.signal },
+            });
+            if (data.merged) {
+                const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
+                // Use GitHub's actual merge time when available; buildMergedBranchUpdate
+                // falls back to "now" (its default `now` param) when merged_at is absent.
+                await meta.save({
+                    branch: buildMergedBranchUpdate(branchDir, data.merged_at ? new Date(data.merged_at) : undefined),
+                });
+                console.log(`  PR #${prNumber} for ${branchDir} is merged -> archived`);
+                return;
+            }
+            const newState = data.state === 'closed' ? 'closed' : 'open';
+            // Re-load fresh (not the loop-top `metaFile` snapshot passed in) -- a
+            // concurrent Lambda write (e.g. an editor re-submitting) may have
+            // landed since that snapshot was taken.
+            const currentMeta = await BranchMetadataFileManager.loadOnly(branchPath);
+            if (currentMeta?.branch.pullRequestState === newState)
+                return;
+            const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
+            await meta.save({ branch: { name: branchDir, pullRequestState: newState } });
+            console.log(`  PR #${prNumber} for ${branchDir}: pullRequestState -> ${newState}`);
+        }
+        catch (err) {
+            // Non-fatal: transient GitHub/network errors are retried next cycle.
+            console.warn(`  Failed to poll PR #${prNumber} for ${branchDir}: ${getErrorMessage(err)}`);
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }
+    /**
+     * Persist a per-branch rebase failure to branch.json (PR-W2), bounded to
+     * roughly one save per failing branch per hour: a branch stuck failing
+     * every cycle must not turn into unbounded save-per-cycle x N-failing-
+     * branches write amplification -- save() eager-regenerates the branch
+     * registry (branch-metadata.ts's invalidateRegistry(), O(branch count) EFS
+     * reads), the same concern the `alreadyClean` no-op guard above exists
+     * for.
+     *
+     * Best-effort and non-fatal like every other metadata write in this
+     * loop's error paths: a corrupt branch.json, a lock-contention error, or
+     * any other save failure here must never abort the per-branch iteration.
+     * This matters doubly at the two call sites -- one is inside the outer
+     * per-branch catch, with no further catch of its own around this call --
+     * so the whole method is wrapped, not just the load.
+     */
+    async recordRebaseFailure(branchPath, branchDir, message) {
+        const RECORD_REFRESH_MS = 60 * 60 * 1000; // 1 hour
+        // [HIGH-1] Defense-in-depth redaction: rebaseFailure.message is
+        // persisted to branch.json and served to the browser via the
+        // branch-health admin endpoint. Both call sites in rebaseActiveBranches
+        // already redact before passing in (the failed.push sites below),
+        // redactCredentials is idempotent, so redacting again here is free and
+        // keeps this method safe on its own.
+        const redactedMessage = redactCredentials(message);
+        try {
+            const existing = await BranchMetadataFileManager.loadOnly(branchPath);
+            const prior = existing?.branch.rebaseFailure;
+            const sameMessage = prior?.message === redactedMessage;
+            const now = new Date();
+            if (sameMessage) {
+                const lastAtMs = Date.parse(prior.lastAt);
+                if (!Number.isNaN(lastAtMs) && now.getTime() - lastAtMs < RECORD_REFRESH_MS) {
+                    // Same failure, refreshed within the last hour -- skip the save.
+                    return;
+                }
+            }
+            const nowIso = now.toISOString();
+            const firstAt = sameMessage ? prior.firstAt : nowIso;
+            const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
+            await meta.save({
+                branch: {
+                    name: branchDir,
+                    rebaseFailure: { message: redactedMessage, firstAt, lastAt: nowIso },
+                },
+            });
+        }
+        catch (err) {
+            // Includes BranchMetadataCorruptError from the load above (a save()
+            // against the same corrupt file would just throw again) as well as
+            // any other load/save failure -- recording is best-effort
+            // observability, never allowed to abort the branch loop.
+            console.warn(`  Failed to record rebase failure for ${branchDir}: ${getErrorMessage(err)}`);
+        }
     }
     async rebaseActiveBranches() {
+        // PR-W1: collected across the loop below and returned as a summary
+        // (folded into worker-status.json by syncGit()). Purely additive
+        // bookkeeping -- doesn't change any control flow or existing logging.
+        const rebased = [];
+        const skippedDirty = [];
+        const failed = [];
         let branchDirs;
         try {
             branchDirs = await fs.readdir(this.contentBranchesPath);
         }
         catch {
-            return;
+            return { rebased, skippedDirty, failed };
         }
         for (const branchDir of branchDirs) {
+            // Known structural entries under content-branches/, not branch
+            // workspaces: branches.json (registry snapshot) and dot-prefixed
+            // entries (.canopy-meta/, transient lock dirs). Skip them silently so
+            // the no-.git skip logs below don't fire for them every cycle.
+            // Everything else still logs loudly on skip.
+            if (branchDir.startsWith('.') || branchDir === 'branches.json') {
+                continue;
+            }
             const branchPath = path.join(this.contentBranchesPath, branchDir);
             const gitDir = path.join(branchPath, '.git');
             try {
                 const stat = await fs.stat(gitDir);
-                if (!stat.isDirectory())
+                if (!stat.isDirectory()) {
+                    console.log(`  Skipping ${branchDir}: .git is not a directory`);
                     continue;
+                }
             }
             catch {
+                console.log(`  Skipping ${branchDir}: no .git directory (not a branch workspace)`);
+                continue;
+            }
+            // The base branch's own clone is refreshed ff-only by
+            // refreshBaseBranchWorkspace() earlier in syncGit(). Routing it
+            // through this conflict-resolution rebase loop could rewrite its
+            // history (the --theirs loop below) and stamp meaningless conflict
+            // metadata on it. Compare sanitized-vs-sanitized: branchDir is a
+            // filesystem name (already sanitized), this.baseBranch is raw.
+            if (branchDir === this.sanitizedBaseBranch) {
+                console.log(`  Skipping ${branchDir}: base branch (refreshed separately)`);
                 continue;
             }
             try {
@@ -688,11 +1136,17 @@ export class CmsWorker {
                 const metaFile = await BranchMetadataFileManager.loadOnly(branchPath);
                 const branchStatus = metaFile?.branch.status;
                 // Skip branches that shouldn't be mutated:
-                // - submitted/approved: in review, don't rewrite history under an active PR
-                // - archived: already merged, no reason to rebase
-                if (branchStatus === 'submitted' ||
-                    branchStatus === 'approved' ||
-                    branchStatus === 'archived') {
+                // - submitted/approved: in review, don't rewrite history under an
+                //   active PR -- but do poll GitHub for the PR's resolution, since
+                //   nothing else tells the worker a merge/close happened.
+                // - archived: already merged, no reason to rebase and no PR left to
+                //   poll (avoid the wasted API call).
+                if (branchStatus === 'submitted' || branchStatus === 'approved') {
+                    console.log(`  Skipping ${branchDir} (${branchStatus})`);
+                    await this.pollMergeState(branchDir, branchPath, metaFile);
+                    continue;
+                }
+                if (branchStatus === 'archived') {
                     console.log(`  Skipping ${branchDir} (${branchStatus})`);
                     continue;
                 }
@@ -712,13 +1166,20 @@ export class CmsWorker {
                 const dirtyCheck = await branchGit.status();
                 if (dirtyCheck.files.length > 0) {
                     console.log(`  Skipping ${branchDir}: has uncommitted changes`);
+                    skippedDirty.push(branchDir);
                     continue;
                 }
                 await branchGit.fetch('origin', this.baseBranch);
                 // Use rev-list instead of status.behind — status.behind only works when the
                 // branch has an upstream tracking branch configured, which isn't guaranteed
                 // (checkoutBranch fallback paths create branches without --track).
-                const behindCount = parseInt((await branchGit.raw(['rev-list', '--count', `HEAD..origin/${this.baseBranch}`])).trim(), 10);
+                // The just-fetched tip, not origin/<base>: branch clones are
+                // --single-branch, so no remote-tracking ref exists for a base branch
+                // other than the one they were cloned from (see the base-refresh
+                // comment above). Pinned to a SHA immediately — FETCH_HEAD is one
+                // shared mutable file per repo, repointed by any concurrent fetch.
+                const fetchedBaseTip = (await branchGit.revparse(['FETCH_HEAD'])).trim();
+                const behindCount = parseInt((await branchGit.raw(['rev-list', '--count', `HEAD..${fetchedBaseTip}`])).trim(), 10);
                 const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
                 if (behindCount === 0) {
                     // Already in sync. This is the overwhelmingly common outcome per
@@ -735,8 +1196,13 @@ export class CmsWorker {
                     const currentMeta = await BranchMetadataFileManager.loadOnly(branchPath);
                     const conflictStatus = currentMeta?.branch.conflictStatus;
                     const conflictFiles = currentMeta?.branch.conflictFiles;
-                    const alreadyClean = (conflictStatus === undefined || conflictStatus === 'clean') &&
+                    const conflictAlreadyClean = (conflictStatus === undefined || conflictStatus === 'clean') &&
                         (conflictFiles === undefined || conflictFiles.length === 0);
+                    // PR-W2: a lingering rebaseFailure must also be cleared once the
+                    // branch catches up clean -- otherwise it sticks as a stale
+                    // warning forever (nothing else touches this branch once it's
+                    // caught up, so no other save site would ever clear it).
+                    const alreadyClean = conflictAlreadyClean && currentMeta?.branch.rebaseFailure === undefined;
                     if (alreadyClean) {
                         continue;
                     }
@@ -745,6 +1211,7 @@ export class CmsWorker {
                             name: branchDir,
                             conflictStatus: 'clean',
                             conflictFiles: [],
+                            rebaseFailure: undefined,
                         },
                     });
                     continue;
@@ -755,11 +1222,16 @@ export class CmsWorker {
                 const conflictedFiles = [];
                 let nextAction = 'start';
                 let completed = false;
+                // PR-W1: captured only on the "unexpected error" exit below, for the
+                // failed-summary entry pushed at the `if (!completed)` check.
+                let failureReason;
                 const MAX_ROUNDS = 50; // safety limit against infinite loops
                 for (let round = 0; round < MAX_ROUNDS && !completed; round++) {
                     try {
                         if (nextAction === 'start') {
-                            await branchGit.rebase([`origin/${this.baseBranch}`]);
+                            // The pinned base tip fetched above (single-branch clones have
+                            // no origin/<base> remote-tracking ref for other branches).
+                            await branchGit.rebase([fetchedBaseTip]);
                         }
                         else if (nextAction === 'continue') {
                             await branchGit.rebase(['--continue']);
@@ -796,6 +1268,7 @@ export class CmsWorker {
                                 // state. Previous metadata (possibly stale) is preserved until the
                                 // next successful rebase cycle corrects it.
                                 console.warn(`  Unexpected rebase error in ${branchDir}: ${msg || 'Unknown error'}`);
+                                failureReason = msg || 'Unknown error';
                                 await branchGit.rebase(['--abort']).catch(() => { });
                                 break;
                             }
@@ -803,8 +1276,30 @@ export class CmsWorker {
                     }
                 }
                 if (!completed) {
-                    console.warn(`  Rebase of ${branchDir} did not complete within ${MAX_ROUNDS} rounds, aborting`);
+                    // PR-W2 (M1 rider): failureReason is only set on the "unexpected
+                    // error" break above -- MAX_ROUNDS exhaustion is a distinct exit
+                    // path with no error message of its own, so the warn text must
+                    // not conflate the two.
+                    console.warn(failureReason !== undefined
+                        ? `  Rebase of ${branchDir} aborted due to unexpected error: ${failureReason}`
+                        : `  Rebase of ${branchDir} did not complete within ${MAX_ROUNDS} rounds, aborting`);
                     await branchGit.rebase(['--abort']).catch(() => { });
+                    const rebaseFailureMessage = failureReason ?? `did not complete within ${MAX_ROUNDS} rounds`;
+                    // [HIGH-1] failed[] folds into worker-status.json's
+                    // lastGitSync.failed, served to the browser -- failureReason can
+                    // be an arbitrary git error message that embeds the bot token.
+                    const redactedRebaseFailureMessage = redactCredentials(rebaseFailureMessage);
+                    failed.push({
+                        branch: branchDir,
+                        error: redactedRebaseFailureMessage,
+                    });
+                    // PR-W2: record once here for the "!completed" exit -- the
+                    // unexpected-error break above is NOT disjoint from this block (it
+                    // always falls through here), so recording at the break itself
+                    // would double-record. The outer catch below is the only other
+                    // record site (a distinct, non-overlapping failure class: errors
+                    // outside this round loop, e.g. fetch/rev-list failures).
+                    await this.recordRebaseFailure(branchPath, branchDir, redactedRebaseFailureMessage);
                     continue;
                 }
                 // The rebase rewrote the branch clone's working tree — mark ContentStore
@@ -845,14 +1340,32 @@ export class CmsWorker {
                         name: branchDir,
                         conflictStatus: hadConflicts ? 'conflicts-detected' : 'clean',
                         conflictFiles: conflictIdsDeduped,
+                        // PR-W2: the cycle completed successfully -- clear any prior
+                        // failure record regardless of conflict outcome.
+                        rebaseFailure: undefined,
                     },
                 });
+                // PR-W1: the branch was behind and the rebase completed (with or
+                // without --theirs conflict resolution) -- it moved, so it belongs
+                // in the summary. Branches already up to date `continue`d above and
+                // are deliberately not listed here.
+                rebased.push(branchDir);
             }
             catch (err) {
                 const message = err instanceof Error ? err.message : 'Unknown error';
                 console.warn(`  Failed to sync ${branchDir}: ${message}`);
+                // [HIGH-1] Same rationale as the `if (!completed)` push site above --
+                // this catches fetch/rev-list/unexpected errors, whose message can
+                // embed the bot token.
+                const redactedMessage = redactCredentials(message);
+                failed.push({ branch: branchDir, error: redactedMessage });
+                // PR-W2: second (and only other) record site -- see the comment at
+                // the `if (!completed)` block above for why these two sites are
+                // disjoint.
+                await this.recordRebaseFailure(branchPath, branchDir, redactedMessage);
             }
         }
+        return { rebased, skippedDirty, failed };
     }
     async refreshAuthCache() {
         if (!this.running || !this.config.refreshAuthCache)