canopycms 0.0.65 → 0.0.66-int.82

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (60) hide show
  1. package/README.md +1 -3
  2. package/dist/ai/generate.d.ts +13 -0
  3. package/dist/ai/generate.js +9 -3
  4. package/dist/ai/to-plain-text.js +4 -0
  5. package/dist/ai/types.d.ts +20 -1
  6. package/dist/api/admin-branch-health.js +13 -12
  7. package/dist/api/branch.js +2 -2
  8. package/dist/branch-health.js +1 -1
  9. package/dist/branch-metadata-file.d.ts +58 -0
  10. package/dist/branch-metadata-file.js +65 -0
  11. package/dist/branch-metadata.d.ts +2 -26
  12. package/dist/branch-metadata.js +6 -33
  13. package/dist/branch-registry.js +4 -2
  14. package/dist/build/generate-ai-content.js +103 -0
  15. package/dist/cli/cli.js +432 -317
  16. package/dist/cli/generate-ai-content.js +264 -160
  17. package/dist/content-id-index.js +22 -0
  18. package/dist/content-listing.js +3 -0
  19. package/dist/content-reader.d.ts +25 -1
  20. package/dist/content-reader.js +15 -0
  21. package/dist/content-store.d.ts +50 -1
  22. package/dist/content-store.js +102 -4
  23. package/dist/context.d.ts +36 -4
  24. package/dist/context.js +15 -4
  25. package/dist/editor/admin/SystemHealthPanel.js +2 -2
  26. package/dist/entry-schema.d.ts +22 -0
  27. package/dist/git-manager.d.ts +9 -8
  28. package/dist/git-manager.js +35 -9
  29. package/dist/github-service.d.ts +1 -1
  30. package/dist/github-service.js +1 -1
  31. package/dist/paths/branch-name.d.ts +1 -1
  32. package/dist/paths/branch-name.js +1 -1
  33. package/dist/schema/index.d.ts +1 -1
  34. package/dist/schema/index.js +1 -1
  35. package/dist/schema/meta-loader.js +1 -1
  36. package/dist/services.d.ts +22 -0
  37. package/dist/types.d.ts +1 -1
  38. package/dist/url-exclusivity-fixtures.d.ts +76 -0
  39. package/dist/url-exclusivity-fixtures.js +119 -0
  40. package/dist/url-path-resolver.js +8 -4
  41. package/dist/utils/content-write-lock.d.ts +2 -1
  42. package/dist/utils/content-write-lock.js +2 -1
  43. package/dist/utils/error.d.ts +17 -0
  44. package/dist/utils/error.js +17 -0
  45. package/dist/utils/git.d.ts +1 -1
  46. package/dist/utils/git.js +1 -1
  47. package/dist/utils/occ-json-write.js +1 -1
  48. package/dist/worker/cms-worker.d.ts +22 -288
  49. package/dist/worker/cms-worker.js +118 -1995
  50. package/dist/worker/git-sync.d.ts +166 -0
  51. package/dist/worker/git-sync.js +554 -0
  52. package/dist/worker/history-rewrite.d.ts +129 -0
  53. package/dist/worker/history-rewrite.js +216 -0
  54. package/dist/worker/rebase.d.ts +171 -0
  55. package/dist/worker/rebase.js +859 -0
  56. package/dist/worker/task-runner.d.ts +93 -0
  57. package/dist/worker/task-runner.js +529 -0
  58. package/dist/worker/worker-context.d.ts +133 -0
  59. package/dist/worker/worker-context.js +1 -0
  60. package/package.json +1 -1
@@ -2,21 +2,21 @@ import fs from 'node:fs/promises';
2
2
  import path from 'node:path';
3
3
  import { simpleGit } from 'simple-git';
4
4
  import lockfile from 'proper-lockfile';
5
- import { enqueueTask, listTasks, dequeueTask, completeTask, failTask, retryTask, recoverOrphanedTasks, cleanupOldTasks, cmsTaskQueueLogger, } from './task-queue.js';
6
- import { createOrUpdatePullRequest, createCanopyOctokit } from '../github-service.js';
7
- import { getBranchMetadataFileManager, BranchMetadataFileManager, buildMergedBranchUpdate, } from '../branch-metadata.js';
8
- import { extractIdFromFilename } from '../content-id-index.js';
9
- import { invalidateBranchContentCaches } from '../content-index-generation.js';
10
- import { ROOT_COLLECTION_ID } from '../paths/types.js';
5
+ import { recoverOrphanedTasks, cmsTaskQueueLogger } from './task-queue.js';
6
+ import { createCanopyOctokit } from '../github-service.js';
11
7
  import { sanitizeBranchName, RESERVED_SETTINGS_BRANCH_PREFIX } from '../paths/branch-name.js';
12
- import { normalizeFilesystemPath } from '../paths/normalize.js';
13
- import { GITHUB_TRACKING_REF_PREFIX, gitChildEnv, gitNetworkChildEnv } from '../git-manager.js';
14
8
  import { resolveDeploymentName } from '../operating-mode/deployment-name.js';
15
- import { tryAcquireContentWriteLock } from '../utils/content-write-lock.js';
16
9
  import { getErrorMessage, isNodeError, redactCredentials } from '../utils/error.js';
17
- import { isNonFastForwardRejection, isRebaseInProgress, isStaleLeaseRejection } from '../utils/git.js';
18
10
  import { writeWorkerStatus } from './worker-status.js';
19
11
  import { workerLog, workerLogWarn, workerLogError } from './log.js';
12
+ import { executeTask, orphanRecoveryMaxAgeMs, processTaskQueue, pushBranchToGitHub, updateBranchMetadata, } from './task-runner.js';
13
+ import { pollMergeState, runRebaseCycle } from './rebase.js';
14
+ import { cleanupTrashedBranchDirs, pushSettingsBranches, refreshBaseBranchWorkspace, syncGit, } from './git-sync.js';
15
+ // Re-exported because this module is the package's advertised worker
16
+ // entrypoint (`canopycms/worker/cms-worker`) and both were exported from here
17
+ // before the task-queue cluster moved to ./task-runner. cms-worker.test.ts
18
+ // imports them from this path.
19
+ export { PermanentTaskError, isPermanentTaskFailure } from './task-runner.js';
20
20
  // Re-exported so the AWS entrypoint (packages/canopycms-cdk/worker/index.ts)
21
21
  // can prefix its own startup lines through the same helpers without adding a
22
22
  // new package entrypoint - `canopycms/worker/cms-worker` already exists. Every
@@ -26,128 +26,6 @@ export { workerLog, workerLogWarn, workerLogError, installWorkerLogger } from '.
26
26
  const DEFAULT_TASK_TIMEOUT = 60_000;
27
27
  const DEFAULT_MAX_RETRIES = 3;
28
28
  const DEFAULT_LOCK_STALE_MS = 60_000;
29
- /**
30
- * [C1] Retention window for `.trash-*` branch directories left behind by the
31
- * admin purge action (api/admin-branch-health.ts). Matches
32
- * cleanupOldTasks's default task retention for consistency.
33
- */
34
- const TRASH_RETENTION_MS = 30 * 24 * 60 * 60_000;
35
- /** Matches `.trash-{dirName}-{STAMP}` names, capturing the trailing stamp. */
36
- const TRASH_DIR_STAMP_RE = /-(\d{8}T\d{6}Z)$/;
37
- /**
38
- * Parse a purge-generated `YYYYMMDDTHHMMSSZ` stamp into a Date, or null if
39
- * malformed. Age comes ONLY from this name-embedded stamp, never the dir's
40
- * own mtime -- `fs.rename` preserves the original directory's mtime, so an
41
- * mtime-based retention check would delete a months-stale orphan's trash on
42
- * the very first cleanup pass after purge.
43
- */
44
- function parseTrashStamp(stamp) {
45
- const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(stamp);
46
- if (!match)
47
- return null;
48
- const [, year, month, day, hour, minute, second] = match;
49
- const date = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.000Z`);
50
- return Number.isNaN(date.getTime()) ? null : date;
51
- }
52
- /**
53
- * An error inherent to the task itself (malformed payload, unknown action):
54
- * retrying can never succeed, so the task should fail fast instead of
55
- * burning its retry budget.
56
- */
57
- export class PermanentTaskError extends Error {
58
- }
59
- /**
60
- * Classify a task failure as permanent (fail fast) or transient (retry).
61
- *
62
- * Transient — worth retrying with backoff:
63
- * - network errors / anything without an HTTP status (git failures included:
64
- * most push/fetch failures are connectivity or contention and the retry
65
- * budget bounds the pathological cases)
66
- * - HTTP 408 (request timeout) and 429 (rate limited)
67
- * - HTTP 403 that carries a rate-limit signal (see `isRateLimitSignal403`):
68
- * GitHub returns 403, not 429, for both primary and secondary/abuse rate
69
- * limits. The throttling plugin (see github-service.ts createCanopyOctokit)
70
- * proactively retries short waits, but this carve-out remains the safety
71
- * net for waits the plugin gives up on (`shouldRetryRateLimit`/
72
- * `shouldRetrySecondaryRateLimit`) and for errors it never sees — without
73
- * it a rate-limited push-and-create-or-update-pr task would fail
74
- * permanently and wedge the branch (`sync-failed`, no retry).
75
- * - HTTP 5xx (server-side, usually recovers)
76
- *
77
- * Permanent — retrying the identical request cannot succeed:
78
- * - PermanentTaskError (malformed payload, unknown action)
79
- * - other HTTP 4xx (e.g. 401/404/422): the request itself is bad
80
- * - plain HTTP 403 with no rate-limit signal: a real permission denial
81
- */
82
- export function isPermanentTaskFailure(err) {
83
- if (err instanceof PermanentTaskError)
84
- return true;
85
- const status = getHttpStatus(err);
86
- if (status === null)
87
- return false;
88
- if (status === 408 || status === 429)
89
- return false;
90
- if (status === 403 && isRateLimitSignal403(err))
91
- return false;
92
- return status >= 400 && status < 500;
93
- }
94
- /** Extract an HTTP status from an error, if present (Octokit RequestError shape). */
95
- function getHttpStatus(err) {
96
- if (err instanceof Error && 'status' in err) {
97
- const status = err.status;
98
- if (typeof status === 'number')
99
- return status;
100
- }
101
- return null;
102
- }
103
- /** Narrow an unknown value to a response-headers record, if present (Octokit lowercases header names). */
104
- function getResponseHeaders(err) {
105
- if (typeof err !== 'object' || err === null || !('response' in err))
106
- return null;
107
- const response = err.response;
108
- if (typeof response !== 'object' || response === null || !('headers' in response))
109
- return null;
110
- const headers = response.headers;
111
- if (typeof headers !== 'object' || headers === null)
112
- return null;
113
- return headers;
114
- }
115
- /**
116
- * Detect whether a 403 is a GitHub rate-limit response rather than a plain
117
- * permission denial. GitHub signals rate limiting on 403s three ways: the
118
- * primary limit zeroes out `x-ratelimit-remaining`, secondary/abuse limits
119
- * often include a `retry-after` header, and both cases produce a message
120
- * containing "rate limit" (e.g. "You have exceeded a secondary rate limit").
121
- */
122
- function isRateLimitSignal403(err) {
123
- const headers = getResponseHeaders(err);
124
- if (headers) {
125
- if (headers['x-ratelimit-remaining'] === '0')
126
- return true;
127
- if (typeof headers['retry-after'] === 'string' && headers['retry-after'].length > 0)
128
- return true;
129
- }
130
- if (err instanceof Error && /rate limit/i.test(err.message))
131
- return true;
132
- return false;
133
- }
134
- // Payload validation helpers — fail fast with clear errors instead of silent `as` casts
135
- function requireString(payload, key) {
136
- const val = payload[key];
137
- if (typeof val !== 'string')
138
- throw new PermanentTaskError(`Task payload missing required string field: ${key}`);
139
- return val;
140
- }
141
- function requireNumber(payload, key) {
142
- const val = payload[key];
143
- if (typeof val !== 'number')
144
- throw new PermanentTaskError(`Task payload missing required number field: ${key}`);
145
- return val;
146
- }
147
- function optionalString(payload, key, fallback) {
148
- const val = payload[key];
149
- return typeof val === 'string' ? val : fallback;
150
- }
151
29
  /**
152
30
  * CMS Worker daemon.
153
31
  * Handles operations that Lambda (with no internet) cannot perform:
@@ -231,6 +109,46 @@ export class CmsWorker {
231
109
  }
232
110
  return this.settingsBranchResolved;
233
111
  }
112
+ /**
113
+ * Build the {@link WorkerContext} handed to the extracted clusters
114
+ * (task-runner.ts, git-sync.ts, rebase.ts, history-rewrite.ts).
115
+ *
116
+ * Built FRESH on every call rather than once in the constructor, and the
117
+ * instance-backed members are functions rather than copied values. Both
118
+ * choices exist for the same reason: the test suite drives this class by
119
+ * replacing `octokit` and `buildGitHubUrl` ON THE INSTANCE, by setting
120
+ * `running` directly, and by subclassing to override the two rebase test
121
+ * hooks. A context that captured any of those at construction time would hand
122
+ * the extracted code the pre-test value -- which for `buildGitHubUrl` means a
123
+ * test's push going to github.com for real instead of its local fixture repo.
124
+ * See WorkerContext's doc comment for the full list.
125
+ */
126
+ ctx() {
127
+ return {
128
+ githubOwner: this.config.githubOwner,
129
+ githubRepo: this.config.githubRepo,
130
+ baseBranch: this.baseBranch,
131
+ sanitizedBaseBranch: this.sanitizedBaseBranch,
132
+ taskDir: this.taskDir,
133
+ remoteGitPath: this.remoteGitPath,
134
+ contentBranchesPath: this.contentBranchesPath,
135
+ contentRoot: this.contentRoot,
136
+ taskTimeoutMs: this.taskTimeoutMs,
137
+ maxTasksPerCycle: this.maxTasksPerCycle,
138
+ maxRetries: this.maxRetries,
139
+ log: this.log,
140
+ octokit: () => this.octokit,
141
+ buildGitHubUrl: () => this.buildGitHubUrl(),
142
+ branchWorkspacePath: (branchRefName) => this.branchWorkspacePath(branchRefName),
143
+ executeTask: (task, signal) => this.executeTask(task, signal),
144
+ pushBranchToGitHub: (branch) => this.pushBranchToGitHub(branch),
145
+ isRunning: () => this.running,
146
+ ensureStatusReport: () => this.ensureStatusReport(),
147
+ ensureSettingsBranch: () => this.ensureSettingsBranch(),
148
+ afterConflictDetectedForTesting: () => this.afterConflictDetectedForTesting(),
149
+ afterRebaseCompletedForTesting: () => this.afterRebaseCompletedForTesting(),
150
+ };
151
+ }
234
152
  async start() {
235
153
  this.running = true;
236
154
  workerLog('CMS Worker starting...');
@@ -265,7 +183,7 @@ export class CmsWorker {
265
183
  // doc comment for why a boot-only call is insufficient once the
266
184
  // worker's ASG rolls on every `cdk deploy` (CanopyCmsService's
267
185
  // UpdatePolicy).
268
- const recovered = await recoverOrphanedTasks(this.taskDir, this.orphanRecoveryMaxAgeMs(), this.log);
186
+ const recovered = await recoverOrphanedTasks(this.taskDir, orphanRecoveryMaxAgeMs(this.ctx()), this.log);
269
187
  if (recovered > 0) {
270
188
  workerLog(`Recovered ${recovered} orphaned task(s)`);
271
189
  }
@@ -283,7 +201,7 @@ export class CmsWorker {
283
201
  // a status-write failure must never block releasing the lock.
284
202
  const report = this.ensureStatusReport();
285
203
  report.lastFatalError = {
286
- // [HIGH-1] Persisted to worker-status.json and served to the
204
+ // [REDACT] Persisted to worker-status.json and served to the
287
205
  // browser by the admin panel -- must never carry the bot token
288
206
  // that a poisoned/failed git URL (buildGitHubUrl()) can embed.
289
207
  message: redactCredentials(getErrorMessage(err)),
@@ -594,319 +512,37 @@ export class CmsWorker {
594
512
  await fs.rename(stagingPath, this.remoteGitPath);
595
513
  workerLog('remote.git initialized');
596
514
  }
597
- /**
598
- * Staleness threshold for recoverOrphanedTasks, derived from the
599
- * configured task timeout rather than fixed: the safety argument for
600
- * running recovery on every poll cycle is "no legitimately in-flight task
601
- * can be this old, because executeTaskWithTimeout bounds every attempt by
602
- * taskTimeoutMs" -- which is only true if this threshold scales with
603
- * taskTimeoutMs. 2x leaves the same comfortable margin the defaults have
604
- * (60s timeout vs 5min threshold); the 5-minute floor preserves the
605
- * long-standing default for a replacement instance's boot window.
606
- */
607
- orphanRecoveryMaxAgeMs() {
608
- return Math.max(5 * 60_000, this.taskTimeoutMs * 2);
609
- }
610
- /**
611
- * Process queued tasks from Lambda.
612
- * Polls .tasks/pending/ directory and executes each task.
613
- * Processes up to maxTasksPerCycle tasks per invocation.
614
- * Retries transient failures with exponential backoff.
615
- */
515
+ // --- Task-queue cluster (worker/task-runner.ts) ------------------------
516
+ //
517
+ // READ THIS BEFORE STUBBING ANY DELEGATOR BELOW. They are not all the same,
518
+ // and the difference is invisible from here.
519
+ //
520
+ // `processTaskQueue` is the public loop entry `scheduleLoop` drives, so it IS
521
+ // on the production path. The other three exist ONLY so the existing test
522
+ // files can reach the implementations through the instance; production never
523
+ // dispatches through them, because the extracted modules call each other
524
+ // directly at module level.
525
+ //
526
+ // The consequence: replacing one of these on an instance only affects
527
+ // production behaviour if the context also routes it. Two do --
528
+ // `executeTask` and `pushBranchToGitHub` are on `WorkerContext` precisely
529
+ // because cms-worker.test.ts assigns over them and then asserts on the
530
+ // replacement. `updateBranchMetadata` is NOT: the test calls it directly and
531
+ // never stubs it, so a stub installed there today would be a silent no-op.
532
+ // If you need to stub a method that isn't on the context, add it to
533
+ // WorkerContext and route the internal caller through `ctx` -- do not assume
534
+ // the delegator's existence means the stub takes effect.
616
535
  async processTaskQueue() {
617
- if (!this.running)
618
- return;
619
- // Recover tasks orphaned in processing/ on EVERY cycle, not only at boot
620
- // (start()'s call above). Before this fix, recoverOrphanedTasks() ran
621
- // exactly once, at start() - fine when an instance was replaced rarely
622
- // (a spot interruption), but CanopyCmsService's worker ASG now rolls on
623
- // every `cdk deploy` (see its UpdatePolicy), making replacement routine.
624
- // A replacement instance's user-data (yum install git/unzip/nodejs/
625
- // efs-utils, mount EFS) takes roughly 2-4 minutes, well under
626
- // recoverOrphanedTasks()'s 5-minute default staleness threshold - so
627
- // THAT ONE boot-time call would see the just-orphaned file as "too
628
- // fresh" and skip it, and nothing rescanned afterward: the task (and its
629
- // branch's syncStatus) would be wedged forever. Re-checking every poll
630
- // means the file's age eventually crosses the threshold and it gets
631
- // recovered without operator intervention.
632
- //
633
- // Safe to run this often: executeTaskWithTimeout() guarantees every task
634
- // THIS process dequeues is completed, failed, or retried (all three
635
- // remove the processing/ file) within taskTimeoutMs - and the staleness
636
- // threshold is derived from taskTimeoutMs (orphanRecoveryMaxAgeMs below)
637
- // precisely so that guarantee holds for ANY configured timeout, not just
638
- // the 60s default: a fixed 5-minute threshold would have this call steal
639
- // the worker's own still-in-flight task back to pending whenever an
640
- // adopter configured taskTimeoutMs above ~5 minutes.
641
- const recovered = await recoverOrphanedTasks(this.taskDir, this.orphanRecoveryMaxAgeMs(), this.log);
642
- if (recovered > 0) {
643
- workerLog(`Recovered ${recovered} orphaned task(s)`);
644
- }
645
- let processed = 0;
646
- let task;
647
- while (processed < this.maxTasksPerCycle &&
648
- (task = await dequeueTask(this.taskDir, this.log)) !== null) {
649
- try {
650
- const result = await this.executeTaskWithTimeout(task);
651
- await completeTask(this.taskDir, task.id, result, this.log);
652
- await this.updateBranchMetadata(task, result);
653
- }
654
- catch (err) {
655
- const message = getErrorMessage(err);
656
- workerLogError(`Task ${task.id} (${task.action}) failed:`, message);
657
- // [HIGH-1] task.error is persisted (pending/failed task JSON) and
658
- // served to the browser by the admin panel's Tasks tab -- a push
659
- // failure's message can embed the bot token via buildGitHubUrl().
660
- // Console output above stays raw (journald/CloudWatch is trusted).
661
- const persistedMessage = redactCredentials(message);
662
- // DEP-L1: only transient failures (network, 429/5xx, timeouts) are
663
- // worth retrying; permanent ones (malformed payload, other 4xx) would
664
- // just burn the retry budget on an identical doomed request.
665
- const permanent = isPermanentTaskFailure(err);
666
- const retryCount = task.retryCount ?? 0;
667
- const maxRetries = task.maxRetries ?? this.maxRetries;
668
- if (!permanent && retryCount < maxRetries) {
669
- await retryTask(this.taskDir, task.id, persistedMessage, this.log);
670
- workerLog(` Will retry (attempt ${retryCount + 1}/${maxRetries})`);
671
- }
672
- else {
673
- await failTask(this.taskDir, task.id, persistedMessage, this.log);
674
- await this.updateBranchMetadataOnFailure(task, persistedMessage);
675
- workerLogError(permanent
676
- ? ' Permanently failed (non-retryable error)'
677
- : ` Permanently failed after ${maxRetries} retries`);
678
- }
679
- }
680
- processed++;
681
- }
682
- // PR-W1: only stamp/write when work actually happened this poll --
683
- // otherwise every idle 5s poll would hit worker-status.json, an EFS
684
- // write treadmill for no signal (liveness is already covered by the
685
- // lock heartbeat; see api/admin.ts's classifyWorkerLiveness).
686
- if (processed > 0) {
687
- const report = this.ensureStatusReport();
688
- report.lastTaskCycleAt = new Date().toISOString();
689
- await writeWorkerStatus(this.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
690
- }
691
- }
692
- /**
693
- * Execute a task bounded by taskTimeoutMs (DEP-H1). Two layers:
694
- * - An AbortSignal cancels Octokit HTTP calls promptly.
695
- * - A Promise.race rejects when the timeout fires, so work that cannot
696
- * observe the signal (git subprocesses via simple-git) still fails the
697
- * attempt and the worker moves on instead of stalling forever.
698
- * pushBranchToGitHub additionally kills stalled git processes via
699
- * simple-git's block timeout, so a hung push doesn't leak a process.
700
- */
701
- async executeTaskWithTimeout(task) {
702
- const controller = new AbortController();
703
- const timer = setTimeout(() => controller.abort(), this.taskTimeoutMs);
704
- try {
705
- const work = this.executeTask(task, controller.signal);
706
- // If the timeout wins the race, the losing promise must not surface an
707
- // unhandled rejection when it eventually settles.
708
- work.catch(() => { });
709
- const timedOut = new Promise((_, reject) => {
710
- controller.signal.addEventListener('abort', () => reject(new Error(`Task timed out after ${this.taskTimeoutMs}ms`)), { once: true });
711
- });
712
- return await Promise.race([work, timedOut]);
713
- }
714
- finally {
715
- clearTimeout(timer);
716
- }
536
+ return processTaskQueue(this.ctx());
717
537
  }
718
538
  async executeTask(task, signal) {
719
- const { action, payload } = task;
720
- switch (action) {
721
- case 'push-branch': {
722
- const branch = requireString(payload, 'branch');
723
- await this.pushBranchToGitHub(branch);
724
- return { pushed: true };
725
- }
726
- case 'push-and-create-pr': {
727
- const branch = requireString(payload, 'branch');
728
- await this.pushBranchToGitHub(branch);
729
- const pr = await this.octokit.pulls.create({
730
- owner: this.config.githubOwner,
731
- repo: this.config.githubRepo,
732
- head: branch,
733
- base: optionalString(payload, 'baseBranch', this.baseBranch),
734
- title: optionalString(payload, 'title', `Submit ${branch}`),
735
- body: optionalString(payload, 'body', ''),
736
- request: { signal },
737
- });
738
- workerLog(`Created PR #${pr.data.number} for ${branch}`);
739
- return { prUrl: pr.data.html_url, prNumber: pr.data.number };
740
- }
741
- case 'push-and-update-pr': {
742
- const branch = requireString(payload, 'branch');
743
- const prNumber = requireNumber(payload, 'pullRequestNumber');
744
- await this.pushBranchToGitHub(branch);
745
- await this.octokit.pulls.update({
746
- owner: this.config.githubOwner,
747
- repo: this.config.githubRepo,
748
- pull_number: prNumber,
749
- title: optionalString(payload, 'title', `Submit ${branch}`),
750
- body: optionalString(payload, 'body', ''),
751
- request: { signal },
752
- });
753
- workerLog(`Updated PR #${prNumber} for ${branch}`);
754
- return { prNumber };
755
- }
756
- case 'push-and-create-or-update-pr': {
757
- // GIT-H1: idempotent create-or-update. Used for both content-branch
758
- // submits and settings-branch syncs so a retry after a crash (task
759
- // completed on GitHub but branch metadata never recorded the PR
760
- // number) recovers the existing PR instead of hitting the 422 that
761
- // a blind `pulls.create` would throw on a duplicate head+base.
762
- // Delegates to the shared helper (also used by GitHubService's
763
- // direct-API path) so the list->tiebreak->update/create logic and
764
- // the draft->ready conversion live in one place.
765
- const branch = requireString(payload, 'branch');
766
- const base = optionalString(payload, 'baseBranch', this.baseBranch);
767
- // Defense-in-depth: refuse head===base even if the 'submittableBranch'
768
- // API guard and the syncSubmitPr backstop were both somehow bypassed
769
- // (e.g. a task queued before this check shipped). PermanentTaskError
770
- // (not a plain Error) so this fails immediately instead of burning
771
- // the retry budget on an identical doomed request -- retrying can
772
- // never make the branch not be the base branch.
773
- if (sanitizeBranchName(branch) === this.sanitizedBaseBranch) {
774
- throw new PermanentTaskError(`Refusing to push-and-create-or-update-pr for "${branch}": it is the base branch -- submitting the base branch is never valid`);
775
- }
776
- await this.pushBranchToGitHub(branch);
777
- const result = await createOrUpdatePullRequest({
778
- octokit: this.octokit,
779
- owner: this.config.githubOwner,
780
- repo: this.config.githubRepo,
781
- head: branch,
782
- base,
783
- title: optionalString(payload, 'title', `Submit ${branch}`),
784
- body: optionalString(payload, 'body', ''),
785
- // Content submits (api/github-sync.ts) set this; settings-branch
786
- // syncs (services.ts) deliberately don't.
787
- markReadyIfDraft: payload.markReadyIfDraft === true,
788
- signal,
789
- });
790
- workerLog(result.created
791
- ? `Created PR #${result.number} for ${branch}`
792
- : `Updated existing PR #${result.number} for ${branch}`);
793
- return { prUrl: result.url, prNumber: result.number };
794
- }
795
- case 'convert-to-draft': {
796
- const draftPrNumber = requireNumber(payload, 'pullRequestNumber');
797
- // GitHub REST API doesn't support converting to draft directly.
798
- // Use the GraphQL API via Octokit.
799
- const { data: pr } = await this.octokit.pulls.get({
800
- owner: this.config.githubOwner,
801
- repo: this.config.githubRepo,
802
- pull_number: draftPrNumber,
803
- request: { signal },
804
- });
805
- await this.octokit.graphql(`mutation($id: ID!) { convertPullRequestToDraft(input: { pullRequestId: $id }) { pullRequest { isDraft } } }`, { id: pr.node_id, request: { signal } });
806
- workerLog(`Converted PR #${draftPrNumber} to draft`);
807
- return { prNumber: draftPrNumber, draft: true };
808
- }
809
- case 'close-pr': {
810
- const closePrNumber = requireNumber(payload, 'pullRequestNumber');
811
- await this.octokit.pulls.update({
812
- owner: this.config.githubOwner,
813
- repo: this.config.githubRepo,
814
- pull_number: closePrNumber,
815
- state: 'closed',
816
- request: { signal },
817
- });
818
- return { closed: true };
819
- }
820
- case 'delete-remote-branch': {
821
- const branch = requireString(payload, 'branch');
822
- await this.octokit.git.deleteRef({
823
- owner: this.config.githubOwner,
824
- repo: this.config.githubRepo,
825
- ref: `heads/${branch}`,
826
- request: { signal },
827
- });
828
- return { deleted: true };
829
- }
830
- default:
831
- throw new PermanentTaskError(`Unknown task action: ${action}`);
832
- }
539
+ return executeTask(this.ctx(), task, signal);
833
540
  }
834
- /**
835
- * Update branch metadata after successful task completion.
836
- * Writes PR URL/number and sets syncStatus to 'synced'.
837
- */
838
541
  async updateBranchMetadata(task, result) {
839
- const branch = typeof task.payload.branch === 'string' ? task.payload.branch : null;
840
- if (!branch)
841
- return;
842
- const branchPath = this.branchWorkspacePath(branch);
843
- try {
844
- await fs.stat(branchPath);
845
- }
846
- catch {
847
- return; // Branch directory doesn't exist
848
- }
849
- try {
850
- const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
851
- const updates = {
852
- name: branch,
853
- syncStatus: 'synced',
854
- // save()'s merge only overwrites keys present in this update (see
855
- // BranchMetadataFileManager.save()'s spread order) -- a prior
856
- // syncFailureReason from an earlier failed attempt would otherwise
857
- // survive forever past this successful sync. Explicitly clear it,
858
- // same pattern as rebaseFailure's PR-W2 M2 clearing elsewhere.
859
- syncFailureReason: undefined,
860
- };
861
- if (result.prUrl)
862
- updates.pullRequestUrl = result.prUrl;
863
- if (result.prNumber) {
864
- updates.pullRequestNumber = result.prNumber;
865
- // As soon as the PR exists the field should read 'open' -- without
866
- // this it would stay absent until the next poll cycle observes it.
867
- // Exception: the git-sync loop may have archived this branch (PR
868
- // merged/closed) while this task was in flight; a late task
869
- // completion must not downgrade that terminal state back to 'open'.
870
- const current = await BranchMetadataFileManager.loadOnly(branchPath);
871
- const terminal = current?.branch.status === 'archived' ||
872
- current?.branch.pullRequestState === 'merged' ||
873
- current?.branch.pullRequestState === 'closed';
874
- if (!terminal) {
875
- updates.pullRequestState = 'open';
876
- }
877
- }
878
- await meta.save({ branch: updates });
879
- }
880
- catch (err) {
881
- workerLogError(`Failed to update metadata for ${branch}:`, err instanceof Error ? err.message : err);
882
- }
542
+ return updateBranchMetadata(this.ctx(), task, result);
883
543
  }
884
- /**
885
- * Update branch metadata after permanent task failure.
886
- * Sets syncStatus to 'sync-failed' and records `error` (already redacted
887
- * by the caller, processTaskQueue -- see [HIGH-1] there) as
888
- * syncFailureReason, so the editor can show WHY, not just that it failed.
889
- */
890
- async updateBranchMetadataOnFailure(task, error) {
891
- const branch = typeof task.payload.branch === 'string' ? task.payload.branch : null;
892
- if (!branch)
893
- return;
894
- const branchPath = this.branchWorkspacePath(branch);
895
- try {
896
- await fs.stat(branchPath);
897
- }
898
- catch {
899
- return;
900
- }
901
- try {
902
- const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
903
- await meta.save({
904
- branch: { name: branch, syncStatus: 'sync-failed', syncFailureReason: error },
905
- });
906
- }
907
- catch (err) {
908
- workerLogError(`Failed to update failure metadata for ${branch}:`, err instanceof Error ? err.message : err);
909
- }
544
+ async pushBranchToGitHub(branch) {
545
+ return pushBranchToGitHub(this.ctx(), branch);
910
546
  }
911
547
  buildGitHubUrl() {
912
548
  return `https://x-access-token:${this.config.githubToken}@github.com/${this.config.githubOwner}/${this.config.githubRepo}.git`;
@@ -930,941 +566,6 @@ export class CmsWorker {
930
566
  branchWorkspacePath(branchRefName) {
931
567
  return path.join(this.contentBranchesPath, sanitizeBranchName(branchRefName));
932
568
  }
933
- /**
934
- * What `remote.git` currently holds for `branchRef`, or null when this
935
- * branch was never published there (never submitted) or the ref is
936
- * unreadable.
937
- *
938
- * Uses the same explicit `--git-dir` shape as verifyBaseBranchExists()
939
- * above: reading a bare repo that way does not depend on
940
- * `safe.bareRepository` being permissive, which is why prod code takes
941
- * this route rather than the config override the test-only `openBareRepo`
942
- * helper uses.
943
- */
944
- async readPublishedSha(branchRef) {
945
- try {
946
- const out = await simpleGit().raw([
947
- '--git-dir',
948
- this.remoteGitPath,
949
- 'rev-parse',
950
- '--verify',
951
- `refs/heads/${branchRef}`,
952
- ]);
953
- const sha = out.trim();
954
- return sha.length > 0 ? sha : null;
955
- }
956
- catch {
957
- // Absent from remote.git: this branch was never submitted, so none of
958
- // its history was ever published. Not an error.
959
- return null;
960
- }
961
- }
962
- /**
963
- * Record that this worker rewrote `expectedSha` out of a branch's already
964
- * published history (see BranchMetadata.historyRewrittenFrom).
965
- *
966
- * Set-once: if a marker is already present it is LEFT ALONE. Across two
967
- * rebases before any GitHub push lands, GitHub still holds the commit the
968
- * FIRST rebase replaced, so advancing the marker would aim the lease at a
969
- * commit GitHub never had and permanently wedge the branch.
970
- *
971
- * Re-reads metadata rather than trusting the caller's loop-top snapshot:
972
- * the task loop runs concurrently with syncGit() (see scheduleLoop) and may
973
- * have cleared the marker while this branch was rebasing.
974
- */
975
- async markHistoryRewritten(branchPath, branchDir, expectedSha) {
976
- const current = await BranchMetadataFileManager.loadOnly(branchPath);
977
- if (typeof current?.branch.historyRewrittenFrom === 'string')
978
- return;
979
- const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
980
- await meta.save({ branch: { name: branchDir, historyRewrittenFrom: expectedSha } });
981
- }
982
- /**
983
- * Clear the marker once GitHub is confirmed to hold the rewritten history.
984
- *
985
- * Best-effort only in the sense that a failure here destroys nothing: the
986
- * lease still refuses anything unexpected, and the plain-push fallback only
987
- * ever fast-forwards. It is NOT harmless. A marker that outlives its
988
- * episode can wedge the NEXT one -- if the base advances before the stale
989
- * marker is revisited, the queued push leases a commit GitHub has already
990
- * moved off, falls back to a plain push of a rebased (non-ancestor)
991
- * history, and fails permanently with a "something else moved it on
992
- * GitHub" diagnosis that is false: we did.
993
- *
994
- * Tracked, with the concurrent-clear race that can drop a marker
995
- * mid-arming, in
996
- * .claude/future-tasks/worker-history-rewrite-marker-races.md.
997
- */
998
- async clearHistoryRewrittenMarker(branchPath, branchDir) {
999
- try {
1000
- const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
1001
- // Explicit undefined clears the key -- save()'s merge only overwrites
1002
- // keys present in the update (same pattern as rebaseFailure).
1003
- await meta.save({ branch: { name: branchDir, historyRewrittenFrom: undefined } });
1004
- }
1005
- catch (err) {
1006
- workerLogWarn(` Failed to clear history-rewrite marker for ${branchDir}: ${getErrorMessage(err)}`);
1007
- }
1008
- }
1009
- /**
1010
- * Publish a branch clone's rebased history into `remote.git`, replacing
1011
- * EXACTLY `expectedSha` and nothing else.
1012
- *
1013
- * The lease is the entire safety argument. `--force-with-lease=<ref>:<sha>`
1014
- * refuses unless `remote.git` still stands at `<sha>`, so this can only
1015
- * ever undo the commit our own rebase rewrote away. Callers must never pass
1016
- * "whatever remote.git currently holds" -- see the arming guard in
1017
- * rebaseActiveBranches() for the interleaving where that would silently
1018
- * delete a reviewer's direct push.
1019
- *
1020
- * Returns whether the push landed. A refused lease means a concurrent
1021
- * Lambda push moved the ref; that is logged and retried by the self-heal
1022
- * pass on a later cycle, never thrown.
1023
- */
1024
- async forcePublishToLocalRemote(branchPath, branchRef, expectedSha) {
1025
- // A dedicated instance rather than the caller's: `.env()` replaces the
1026
- // whole child environment, and the rebase loop's instance must keep its
1027
- // ambient one. gitChildEnv (not gitNetworkChildEnv) because this push
1028
- // targets the local bare repo -- and the locale pin is what keeps
1029
- // isStaleLeaseRejection below from silently becoming a no-op.
1030
- const pushGit = simpleGit({ baseDir: branchPath, timeout: { block: this.taskTimeoutMs } });
1031
- pushGit.env(gitChildEnv({}));
1032
- try {
1033
- await pushGit.raw([
1034
- 'push',
1035
- `--force-with-lease=${branchRef}:${expectedSha}`,
1036
- // Real flags must precede --end-of-options; everything after it is
1037
- // positional (see GitManager.push() for the same guard).
1038
- '--end-of-options',
1039
- 'origin',
1040
- `${branchRef}:${branchRef}`,
1041
- ]);
1042
- workerLog(` Published rebased ${branchRef} into remote.git`);
1043
- return true;
1044
- }
1045
- catch (err) {
1046
- const message = redactCredentials(getErrorMessage(err));
1047
- workerLogWarn(isStaleLeaseRejection(message)
1048
- ? ` Did not publish rebased ${branchRef} into remote.git: it moved since this cycle read it (concurrent submit?) -- retrying next cycle`
1049
- : ` Failed to publish rebased ${branchRef} into remote.git: ${message}`);
1050
- return false;
1051
- }
1052
- }
1053
- /**
1054
- * Queue the GitHub hop for a branch whose rewritten history now sits in
1055
- * `remote.git`, so an open PR's head follows the rebase within a cycle
1056
- * instead of waiting for the editor's next submit.
1057
- *
1058
- * Deliberately NOT skipped when a marker is already set: inferring "a task
1059
- * must already be queued" from the marker starves this hop whenever a task
1060
- * was lost, failed permanently, or was never written. Duplicate push tasks
1061
- * are bounded by base-branch advances and are idempotent (a repeat push is
1062
- * a no-op once GitHub holds the tip).
1063
- */
1064
- async enqueueGitHubPush(branchRef) {
1065
- try {
1066
- // Dedupe against tasks actually in flight rather than against the
1067
- // marker: a branch whose GitHub push keeps failing would otherwise
1068
- // gain one task per sync cycle forever.
1069
- for (const status of ['pending', 'processing']) {
1070
- const inFlight = await listTasks(this.taskDir, status);
1071
- if (inFlight.some((t) => t.action === 'push-branch' && t.payload.branch === branchRef)) {
1072
- return;
1073
- }
1074
- }
1075
- await enqueueTask(this.taskDir, { action: 'push-branch', payload: { branch: branchRef } });
1076
- workerLog(` Queued GitHub push for ${branchRef}`);
1077
- }
1078
- catch (err) {
1079
- workerLogWarn(` Failed to queue GitHub push for ${branchRef}: ${getErrorMessage(err)}`);
1080
- }
1081
- }
1082
- /**
1083
- * Complete a rewrite this worker started but did not finish: get the
1084
- * rebased history into `remote.git` and queued for GitHub.
1085
- *
1086
- * Runs from the rebase loop for any branch carrying a marker, whether or
1087
- * not it is behind base this cycle, so an interrupted publish converges
1088
- * without waiting for the next base-branch advance.
1089
- *
1090
- * Always leases on the MARKER, never on remote.git's current tip -- the
1091
- * marker is the one commit we know our own rebase replaced.
1092
- */
1093
- async reconcilePendingRewrite(options) {
1094
- const { branchPath, branchDir, branchRef, headSha, marker } = options;
1095
- const publishedSha = await this.readPublishedSha(branchRef);
1096
- if (publishedSha === null) {
1097
- workerLogWarn(` ${branchDir}: a rewritten history is recorded but ${branchRef} is gone from remote.git -- nothing to publish`);
1098
- return;
1099
- }
1100
- if (publishedSha === headSha) {
1101
- // remote.git already carries the rewrite; only the GitHub hop is left
1102
- // (a crash between the push and the queue, or a task that was lost).
1103
- await this.enqueueGitHubPush(branchRef);
1104
- return;
1105
- }
1106
- if (publishedSha === marker) {
1107
- // The publish into remote.git never landed -- a crash right after the
1108
- // marker was written, or a lease refused by a concurrent submit.
1109
- if (await this.forcePublishToLocalRemote(branchPath, branchRef, marker)) {
1110
- await this.enqueueGitHubPush(branchRef);
1111
- }
1112
- return;
1113
- }
1114
- // Neither this branch's rebased tip nor the commit its rewrite replaced:
1115
- // something else moved remote.git. Never force over that.
1116
- workerLogWarn(` ${branchDir}: remote.git is at ${publishedSha} for ${branchRef}, which is neither the ` +
1117
- `rebased tip nor the commit the rewrite replaced -- left untouched`);
1118
- }
1119
- async pushBranchToGitHub(branch) {
1120
- const git = simpleGit({
1121
- baseDir: this.remoteGitPath,
1122
- // DEP-H1: kill the git process if it produces no output for
1123
- // taskTimeoutMs (network stall, credential prompt) instead of letting
1124
- // it hang past the task timeout.
1125
- timeout: { block: this.taskTimeoutMs },
1126
- });
1127
- // Force stable (English) git output so isNonFastForwardRejection below
1128
- // can reliably match it -- git's rejection text is gettext-translated, so
1129
- // a non-English host would silently turn that classifier into a no-op.
1130
- // gitNetworkChildEnv (NOT gitChildEnv) because this call talks to GitHub:
1131
- // it keeps ambient HTTPS_PROXY/GIT_SSL_*/GIT_SSH_COMMAND, which
1132
- // gitChildEnv's local-ops allowlist deliberately drops.
1133
- git.env(gitNetworkChildEnv());
1134
- // [SYNC-H1] If the rebase loop rewrote this branch's already-published
1135
- // history, GitHub still holds the commit it replaced, so an ordinary
1136
- // push is non-fast-forward forever. Push under a lease keyed to exactly
1137
- // that commit: it moves GitHub off the commit we rewrote away, and
1138
- // refuses in every other case (including a commit someone else pushed).
1139
- const branchPath = this.branchWorkspacePath(branch);
1140
- const metaFile = await BranchMetadataFileManager.loadOnly(branchPath).catch(() => null);
1141
- const marker = metaFile?.branch.historyRewrittenFrom;
1142
- // What this push will actually send: remote.git's tip for the branch.
1143
- const outgoingSha = await this.readPublishedSha(branch);
1144
- try {
1145
- if (marker) {
1146
- await git.raw([
1147
- 'push',
1148
- `--force-with-lease=${branch}:${marker}`,
1149
- '--end-of-options',
1150
- this.buildGitHubUrl(),
1151
- `${branch}:${branch}`,
1152
- ]);
1153
- }
1154
- else {
1155
- // Pass URL directly to avoid persisting the token in remote.git/config
1156
- await git.push(this.buildGitHubUrl(), branch);
1157
- }
1158
- }
1159
- catch (err) {
1160
- const message = getErrorMessage(err);
1161
- // A refused lease means GitHub is not at the commit we rewrote, so the
1162
- // marker is stale -- which is routine, not exceptional: tasks are
1163
- // re-run after a crash (recoverOrphanedTasks), and the marker survives
1164
- // any failure to clear it. Two benign shapes reach here, and both are
1165
- // ordinary fast-forwards that a lease has no business blocking:
1166
- // GitHub already holds the rewritten history from an earlier attempt,
1167
- // or the branch has since moved past it.
1168
- //
1169
- // Retry PLAIN, and let git adjudicate. A non-forced push succeeds if
1170
- // and only if it fast-forwards, so it can never destroy anything --
1171
- // there is no ancestry check to get wrong here, and no extra network
1172
- // round trip to read GitHub's tip. Only if THAT is also rejected has
1173
- // the branch genuinely diverged.
1174
- //
1175
- // (Verified: git evaluates the lease only when it actually has an
1176
- // update to apply. An up-to-date ref with a stale lease prints
1177
- // "Everything up-to-date" and exits 0, so the already-landed case is
1178
- // usually absorbed above and never even reaches this branch.)
1179
- if (marker && isStaleLeaseRejection(message)) {
1180
- try {
1181
- await git.push(this.buildGitHubUrl(), branch);
1182
- }
1183
- catch (retryErr) {
1184
- const retryMessage = getErrorMessage(retryErr);
1185
- if (isNonFastForwardRejection(retryMessage)) {
1186
- throw new PermanentTaskError(`Push rejected for branch "${branch}": GitHub's tip is neither the commit this ` +
1187
- `deployment last published nor an ancestor of what it is pushing, so the branch ` +
1188
- `has genuinely diverged and nothing was overwritten. Something else moved it on ` +
1189
- `GitHub -- a direct push, or another CanopyCMS deployment sharing this repository.`);
1190
- }
1191
- throw retryErr;
1192
- }
1193
- // The lease was refused, so GitHub is provably not at the marker: it
1194
- // has moved past the rewritten commit and the marker is spent.
1195
- await this.clearHistoryRewrittenMarker(branchPath, branch);
1196
- workerLog(`Pushed ${branch} to GitHub (GitHub had already moved past the rewritten commit)`);
1197
- return;
1198
- }
1199
- // An ordinary non-fast-forward rejection: GitHub has commits this
1200
- // deployment never published. Retrying the identical push can never
1201
- // succeed (DEP-L1's git-failure-is-transient carve-out does NOT apply
1202
- // here), so fail fast instead of burning the task's retry budget.
1203
- // Deliberately does NOT advise renaming the branch: a branch reaching
1204
- // this point usually has an open PR, and renaming would orphan it.
1205
- if (isNonFastForwardRejection(message)) {
1206
- throw new PermanentTaskError(`Push rejected for branch "${branch}": GitHub's tip is not what this deployment last ` +
1207
- `published, so the branch has diverged and needs reconciling. Something else moved it ` +
1208
- `on GitHub -- a direct push, or another CanopyCMS deployment sharing this repository.`);
1209
- }
1210
- throw err;
1211
- }
1212
- // Clear the marker only once GitHub is confirmed to hold something other
1213
- // than the commit we rewrote. A push that sent nothing new (remote.git
1214
- // still at the marker because its own publish has not landed yet) must
1215
- // leave the marker set -- it is the sole trigger for the self-heal pass.
1216
- if (marker && outgoingSha && outgoingSha !== marker) {
1217
- await this.clearHistoryRewrittenMarker(branchPath, branch);
1218
- }
1219
- workerLog(`Pushed ${branch} to GitHub`);
1220
- }
1221
- /**
1222
- * Push THIS deployment's own settings branch (`ensureSettingsBranch()`) from
1223
- * remote.git to GitHub. Non-fatal: a no-op push for an up-to-date branch
1224
- * just succeeds quietly.
1225
- *
1226
- * Deliberately narrowed to one branch — this used to push EVERY local
1227
- * branch matching `canopycms-settings-*`. With the tracking-namespace fetch
1228
- * fix (GITHUB_TRACKING_REF_PREFIX), `reconcileTrackedBranches` creates local
1229
- * heads for branches that exist on GitHub, so ANOTHER deployment's settings
1230
- * branch (sharing this same GitHub repo) can legitimately show up as a
1231
- * local head here too. Pushing it would be this deployment shipping
1232
- * settings state it doesn't own.
1233
- */
1234
- async pushSettingsBranches(git, trackedNames) {
1235
- try {
1236
- // Resolved once here rather than at each use below: ensureSettingsBranch()
1237
- // is idempotent, but a local reads better and keeps the eight references
1238
- // in this method obviously talking about one value.
1239
- const settingsBranch = this.ensureSettingsBranch();
1240
- const branches = await git.branch();
1241
- const settingsBranches = branches.all.filter((b) => b.startsWith(RESERVED_SETTINGS_BRANCH_PREFIX));
1242
- const foreign = settingsBranches.filter((b) => b !== settingsBranch);
1243
- if (foreign.length > 0) {
1244
- // Signal, not an error: this is exactly the "two deployments, one repo"
1245
- // condition this workstream exists to make visible. Never push these.
1246
- workerLogWarn(`Found settings branch(es) not owned by this deployment (${settingsBranch}): ` +
1247
- `${foreign.join(', ')}. Another CanopyCMS deployment may share this GitHub repo. Not pushing them.`);
1248
- }
1249
- // Check the full branch list, not the `canopycms-settings-*` subset: an
1250
- // adopter-supplied `settingsBranch` override need not carry that prefix.
1251
- const ownBranchMissing = !branches.all.includes(settingsBranch);
1252
- // [SYNC-M3] A settings branch present in remote.git but absent from
1253
- // GitHub's tracking refs was pushed here LOCALLY -- only this
1254
- // deployment's own API can write to remote.git -- and has never
1255
- // reached GitHub. That is the discriminating signature: in the
1256
- // SUPPORTED two-deployments-one-repo case the foreign branch arrives
1257
- // through the GitHub fetch and therefore always has a tracking ref, so
1258
- // the warn above cannot tell the two apart on its own and a
1259
- // "owned-branch-absent" test alone would fire on every deployment that
1260
- // simply has not had a settings edit yet.
1261
- //
1262
- // With this deployment's own branch also missing, the API and this
1263
- // worker have resolved different deploymentNames, and every settings
1264
- // change the API commits is stranded in remote.git forever.
1265
- const strandedLocal = foreign.filter((b) => !trackedNames.has(b));
1266
- if (strandedLocal.length > 0) {
1267
- workerLogWarn(ownBranchMissing
1268
- ? `Settings branch mismatch: this worker owns "${settingsBranch}", which does not ` +
1269
- `exist in remote.git, while ${strandedLocal.join(', ')} exist(s) here and has never ` +
1270
- `been pushed to GitHub. The API and this worker disagree about deploymentName, so ` +
1271
- `settings changes are NOT reaching GitHub. Set CANOPYCMS_DEPLOYMENT_NAME (or ` +
1272
- `settingsBranch) on this worker to match what the API resolves.`
1273
- : `Settings branch(es) ${strandedLocal.join(', ')} exist in remote.git but not on ` +
1274
- `GitHub, and are not owned by this deployment (${settingsBranch}) -- nothing ` +
1275
- `will ever push them onward. Check that deploymentName matches across this ` +
1276
- `deployment's API and worker.`);
1277
- }
1278
- if (ownBranchMissing) {
1279
- // Not created locally yet — nothing to push.
1280
- return;
1281
- }
1282
- try {
1283
- await git.push(this.buildGitHubUrl(), settingsBranch);
1284
- workerLog(`Pushed settings branch ${settingsBranch} to GitHub`);
1285
- }
1286
- catch (err) {
1287
- // Non-fatal: branch may already be up-to-date. This call site has no
1288
- // task to throw a PermanentTaskError into (unlike pushBranchToGitHub)
1289
- // and is deliberately not restructured to add one -- but a
1290
- // non-fast-forward rejection here is the highest-signal instance of
1291
- // this whole failure class: it means another CanopyCMS deployment's
1292
- // worker already pushed ITS OWN state to its own settings branch on
1293
- // GitHub (an actual settings-branch name collision, not just the
1294
- // "foreign branch found locally" case warned about above), so make
1295
- // that explicit instead of a generic "push failed" line.
1296
- const message = getErrorMessage(err);
1297
- if (isNonFastForwardRejection(message)) {
1298
- workerLogWarn(`Settings push for ${settingsBranch} was rejected (non-fast-forward): another ` +
1299
- `CanopyCMS deployment appears to own this settings branch on GitHub. Settings from ` +
1300
- `that deployment will NOT be overwritten; this deployment's local settings changes ` +
1301
- `were not pushed. Rename this deployment's settings branch (config.settingsBranch or ` +
1302
- `deploymentName) to resolve the collision.`);
1303
- }
1304
- else {
1305
- workerLogWarn(`Settings push for ${settingsBranch}:`, message);
1306
- }
1307
- }
1308
- }
1309
- catch (err) {
1310
- workerLogWarn('Failed to list branches for settings push:', err instanceof Error ? err.message : err);
1311
- }
1312
- }
1313
- /**
1314
- * Bring `refs/heads/*` in `remote.git` toward what was just fetched into
1315
- * `GITHUB_TRACKING_REF_PREFIX` -- WITHOUT ever force-rewinding or deleting
1316
- * a local head. This is the non-destructive replacement for what the old
1317
- * `+refs/heads/*:refs/heads/*` fetch refspec used to do implicitly (and
1318
- * destructively) as part of the fetch itself; see
1319
- * `GITHUB_TRACKING_REF_PREFIX`'s doc comment for the two failure modes
1320
- * that refspec caused.
1321
- *
1322
- * Per tracked branch:
1323
- * - no local `refs/heads/<name>` yet -> create it at the tracked commit
1324
- * (a branch created on GitHub, or by another deployment sharing this
1325
- * GitHub repo, becomes visible locally).
1326
- * - local is a strict ancestor of tracked (behind) -> fast-forward it.
1327
- * - local === tracked -> nothing to do.
1328
- * - tracked is a strict ancestor of local (ahead) -> LEAVE IT ALONE. This
1329
- * is unpushed editor/settings work; the queued push task (or
1330
- * pushSettingsBranches) ships it. This is exactly the branch state the
1331
- * old refspec used to destroy.
1332
- * - neither is an ancestor of the other (diverged) -> LEAVE IT ALONE and
1333
- * count/log it. A real collision (e.g. another deployment moved the
1334
- * same branch name on GitHub); the next push attempt will be rejected
1335
- * non-fast-forward, which is the correct, visible outcome -- this
1336
- * method must never silently pick a winner. The one expected, benign
1337
- * form of this -- our own rebase loop having published a rewrite into
1338
- * remote.git with the GitHub push still queued -- is split out into the
1339
- * `rewritten` bucket so the collision warning stays meaningful.
1340
- *
1341
- * Never deletes a local head: a branch removed on GitHub simply stops
1342
- * being tracked here; the local ref persists until removed through its
1343
- * own explicit path (the sync loop must not be one of them).
1344
- *
1345
- * `remote.git` is bare, so there is no worktree to invalidate by moving
1346
- * these refs -- unlike a non-bare repo, updating the ref that happens to
1347
- * be "checked out" is a non-issue here.
1348
- *
1349
- * Concurrency: `remote.git` is bare and on EFS, and the Lambda pushes into
1350
- * it concurrently (`GitManager.push()`'s `target:target` refspec) while
1351
- * this runs. Every `update-ref` below passes the expected old value (the
1352
- * all-zeros OID for "must not exist yet" on creation, the previously-read
1353
- * SHA for the fast-forward case) so a concurrent Lambda write landing in
1354
- * the gap between the read and the write loses the ref update instead of
1355
- * being silently clobbered -- the branch is simply revisited next cycle.
1356
- */
1357
- async reconcileTrackedBranches(git) {
1358
- const GIT_ZERO_OID = '0000000000000000000000000000000000000000';
1359
- const created = [];
1360
- const fastForwarded = [];
1361
- const ahead = [];
1362
- const diverged = [];
1363
- const rewritten = [];
1364
- // One invocation enumerates both namespaces: refs/heads/<name> (what the
1365
- // Lambda pushes into and branch clones read from) and
1366
- // GITHUB_TRACKING_REF_PREFIX<name> (GitHub's tip, just fetched above).
1367
- const raw = await git.raw([
1368
- 'for-each-ref',
1369
- '--format=%(refname) %(objectname)',
1370
- 'refs/heads/',
1371
- GITHUB_TRACKING_REF_PREFIX,
1372
- ]);
1373
- const heads = new Map();
1374
- const tracked = new Map();
1375
- for (const line of raw.split('\n')) {
1376
- const trimmed = line.trim();
1377
- if (!trimmed)
1378
- continue;
1379
- const [refname, sha] = trimmed.split(' ');
1380
- if (refname.startsWith('refs/heads/')) {
1381
- heads.set(refname.slice('refs/heads/'.length), sha);
1382
- }
1383
- else if (refname.startsWith(GITHUB_TRACKING_REF_PREFIX)) {
1384
- tracked.set(refname.slice(GITHUB_TRACKING_REF_PREFIX.length), sha);
1385
- }
1386
- }
1387
- for (const [name, trackedSha] of tracked) {
1388
- const localSha = heads.get(name);
1389
- const localRef = `refs/heads/${name}`;
1390
- if (!localSha) {
1391
- try {
1392
- // Zero old-value asserts the ref does not already exist -- guards
1393
- // against a concurrent Lambda push creating this exact branch
1394
- // name between the for-each-ref read above and this update.
1395
- await git.raw(['update-ref', localRef, trackedSha, GIT_ZERO_OID]);
1396
- created.push(name);
1397
- }
1398
- catch (err) {
1399
- workerLogWarn(` Tracked-branch reconcile: failed to create local ref for ${name} (concurrent update?): ${getErrorMessage(err)}`);
1400
- }
1401
- continue;
1402
- }
1403
- if (localSha === trackedSha)
1404
- continue; // nothing to do
1405
- // [SYNC-M2] Everything below is per-branch best-effort. The rev-list
1406
- // used to sit between the two guarded update-ref calls with no guard
1407
- // of its own, so a single ref pointing at a missing or partially
1408
- // written object -- plausible on EFS with the Lambda writing
1409
- // concurrently -- threw out of this loop, out of syncGit()'s try, and
1410
- // skipped pushSettingsBranches(), refreshBaseBranchWorkspace() and
1411
- // rebaseActiveBranches() entirely. Nothing self-healed, so it recurred
1412
- // every cycle. One unreadable ref must cost its own branch, not the
1413
- // whole sync cycle.
1414
- try {
1415
- // Count commits unique to each side in one call: left = commits local
1416
- // has that tracked doesn't (ahead), right = commits tracked has that
1417
- // local doesn't (behind). Exits 0 regardless of ancestry direction,
1418
- // unlike `merge-base --is-ancestor` (which simple-git would throw on
1419
- // a non-zero exit for the common "not an ancestor" case).
1420
- const counts = (await git.raw(['rev-list', '--left-right', '--count', `${localSha}...${trackedSha}`])).trim();
1421
- const [leftStr, rightStr] = counts.split(/\s+/);
1422
- const localAheadCount = parseInt(leftStr, 10);
1423
- const localBehindCount = parseInt(rightStr, 10);
1424
- if (!Number.isInteger(localAheadCount) || !Number.isInteger(localBehindCount)) {
1425
- // Unparseable output means we could not read this branch, NOT that
1426
- // it diverged: falling through to the `diverged` bucket below would
1427
- // warn operators about a cross-deployment collision that never
1428
- // happened (parseInt yields NaN, and NaN fails both comparisons).
1429
- workerLogWarn(` Tracked-branch reconcile: skipping ${name}: unparseable rev-list output ${JSON.stringify(counts)}`);
1430
- continue;
1431
- }
1432
- if (localAheadCount === 0 && localBehindCount > 0) {
1433
- try {
1434
- await git.raw(['update-ref', localRef, trackedSha, localSha]);
1435
- fastForwarded.push(name);
1436
- }
1437
- catch (err) {
1438
- // Concurrent Lambda push moved the ref since the read above --
1439
- // the guard did its job; this branch is simply revisited next cycle.
1440
- workerLogWarn(` Tracked-branch reconcile: failed to fast-forward ${name} (concurrent update?): ${getErrorMessage(err)}`);
1441
- }
1442
- }
1443
- else if (localBehindCount === 0 && localAheadCount > 0) {
1444
- // Unpushed local work -- exactly what the old destructive refspec
1445
- // used to force-rewind or delete. Leave it.
1446
- ahead.push(name);
1447
- }
1448
- else if (await this.hasPendingHistoryRewrite(name)) {
1449
- // [SYNC-H1] Our own rebase published a rewrite into remote.git and
1450
- // the GitHub push has not landed yet. Ref-level this is identical
1451
- // to a collision, but it is expected and self-resolving, so it must
1452
- // not fire the collision warning below.
1453
- rewritten.push(name);
1454
- }
1455
- else {
1456
- // Neither side is an ancestor of the other. Leave both alone.
1457
- diverged.push(name);
1458
- }
1459
- }
1460
- catch (err) {
1461
- workerLogWarn(` Tracked-branch reconcile: skipping ${name} (unreadable ref or object?): ${getErrorMessage(err)}`);
1462
- continue;
1463
- }
1464
- }
1465
- if (diverged.length > 0) {
1466
- workerLogWarn(` Tracked-branch reconcile: ${diverged.length} branch(es) diverged from GitHub and were left untouched: ${diverged.join(', ')}`);
1467
- }
1468
- if (rewritten.length > 0) {
1469
- workerLog(` Tracked-branch reconcile: ${rewritten.length} branch(es) rebased locally with the GitHub push still pending: ${rewritten.join(', ')}`);
1470
- }
1471
- // trackedNames is returned alongside the summary (rather than folded into
1472
- // it) because it is a working set for pushSettingsBranches' stranded-
1473
- // branch check, not part of the worker-status snapshot -- it lists every
1474
- // branch on GitHub and would bloat worker-status.json for no reader.
1475
- return {
1476
- summary: { created, fastForwarded, ahead, diverged, rewritten },
1477
- trackedNames: new Set(tracked.keys()),
1478
- };
1479
- }
1480
- /**
1481
- * Whether this branch carries a pending history rewrite -- the rebase loop
1482
- * rewrote already-published history and the GitHub push has not landed yet
1483
- * (see BranchMetadata.historyRewrittenFrom).
1484
- *
1485
- * `branchName` is a git ref name; branch workspaces are directories named
1486
- * with the sanitized form, hence the conversion. Best-effort: a settings
1487
- * branch (no workspace at all), a missing directory or an unreadable
1488
- * branch.json all mean "no known rewrite", which is the conservative
1489
- * answer -- it keeps the branch in the louder `diverged` bucket.
1490
- */
1491
- async hasPendingHistoryRewrite(branchName) {
1492
- try {
1493
- const branchPath = path.join(this.contentBranchesPath, sanitizeBranchName(branchName));
1494
- const metaFile = await BranchMetadataFileManager.loadOnly(branchPath);
1495
- return typeof metaFile?.branch.historyRewrittenFrom === 'string';
1496
- }
1497
- catch {
1498
- return false;
1499
- }
1500
- }
1501
- async syncGit() {
1502
- if (!this.running)
1503
- return;
1504
- workerLog('Syncing git...');
1505
- const cycleStartedAt = Date.now();
1506
- const git = simpleGit({
1507
- baseDir: this.remoteGitPath,
1508
- // DEP-H1: a hung fetch/push would stall the sync loop forever
1509
- // (scheduleLoop only reschedules after completion). The block timeout
1510
- // is inactivity-based, so a slow-but-flowing transfer is unaffected.
1511
- timeout: { block: this.taskTimeoutMs },
1512
- });
1513
- // This instance also drives pushSettingsBranches(git) below, whose
1514
- // classification of a rejected settings-branch push (see that method)
1515
- // needs the same stable-English guarantee as pushBranchToGitHub. Network
1516
- // env for the same reason: the fetch and push here reach GitHub.
1517
- git.env(gitNetworkChildEnv());
1518
- // PR-W1: the whole cycle is wrapped so both outcomes -- success and
1519
- // hard failure (e.g. the fetch throwing against a poisoned remote.git)
1520
- // -- record a worker-status.json snapshot. The status write itself is
1521
- // always best-effort (.catch below): it must never turn an otherwise
1522
- // successful cycle into a failure, and must never mask the real error
1523
- // on a failed one. On failure we rethrow so scheduleLoop's existing
1524
- // per-cycle catch stays the loud path.
1525
- try {
1526
- // Fetch all branches from GitHub using direct URL (no named remote),
1527
- // into the GITHUB_TRACKING_REF_PREFIX remote-tracking namespace rather
1528
- // than refs/heads/* directly -- see that constant's doc comment for
1529
- // the destructive-fetch bug this avoids. We use raw git commands since
1530
- // simple-git's fetch() with a URL doesn't support --prune directly.
1531
- await git.raw([
1532
- 'fetch',
1533
- this.buildGitHubUrl(),
1534
- '--prune',
1535
- `+refs/heads/*:${GITHUB_TRACKING_REF_PREFIX}*`,
1536
- ]);
1537
- workerLog('Fetched from GitHub');
1538
- // Bring refs/heads/* toward what was just fetched, WITHOUT ever
1539
- // force-rewinding or deleting a local head -- see
1540
- // reconcileTrackedBranches()'s doc comment.
1541
- const { summary: trackedSummary, trackedNames } = await this.reconcileTrackedBranches(git);
1542
- // Push settings branches to GitHub (belt-and-suspenders for task queue).
1543
- // Ensures settings reach GitHub even if a task queue entry is lost.
1544
- // Ordering relative to the fetch/reconcile above is no longer a
1545
- // correctness dependency now that the fetch can't clobber refs/heads/*
1546
- // -- this could run before or after them just as safely.
1547
- await this.pushSettingsBranches(git, trackedNames);
1548
- await this.refreshBaseBranchWorkspace();
1549
- const rebaseSummary = await this.rebaseActiveBranches();
1550
- // Periodically clean up old completed/failed tasks
1551
- await cleanupOldTasks(this.taskDir, undefined, this.log);
1552
- // [C1] Sweep branch directories the admin purge action trashed more
1553
- // than TRASH_RETENTION_MS ago. Worker-only by design: purge itself
1554
- // never deletes anything (reversible), and this cycle is the sole
1555
- // place actual removal happens.
1556
- const trashRemoved = await this.cleanupTrashedBranchDirs();
1557
- if (trashRemoved > 0) {
1558
- workerLog(`Removed ${trashRemoved} expired trashed branch dir(s)`);
1559
- }
1560
- const report = this.ensureStatusReport();
1561
- report.lastGitSyncAt = new Date().toISOString();
1562
- delete report.lastGitSyncError;
1563
- report.lastGitSync = {
1564
- durationMs: Date.now() - cycleStartedAt,
1565
- rebased: rebaseSummary.rebased,
1566
- skippedDirty: rebaseSummary.skippedDirty,
1567
- skippedLocked: rebaseSummary.skippedLocked,
1568
- failed: rebaseSummary.failed,
1569
- tracked: trackedSummary,
1570
- };
1571
- await writeWorkerStatus(this.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
1572
- }
1573
- catch (err) {
1574
- const report = this.ensureStatusReport();
1575
- // [HIGH-1] Persisted to worker-status.json and served to the browser
1576
- // by the admin panel -- a fetch/push failure's message can embed the
1577
- // bot token via buildGitHubUrl().
1578
- report.lastGitSyncError = {
1579
- message: redactCredentials(getErrorMessage(err)),
1580
- at: new Date().toISOString(),
1581
- };
1582
- await writeWorkerStatus(this.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
1583
- throw err;
1584
- }
1585
- }
1586
- /**
1587
- * [C1] Remove `.trash-*` branch directories (created by the admin purge
1588
- * action, api/admin-branch-health.ts) whose name-embedded stamp is older
1589
- * than {@link TRASH_RETENTION_MS}. Names that don't match the expected
1590
- * `.trash-{dirName}-{STAMP}` shape, or whose stamp fails to parse, are
1591
- * left alone (logged once per cycle, not per file, to avoid flooding logs
1592
- * if something odd accumulates) -- purge is the only writer of this
1593
- * naming scheme, so an unparseable name is unexpected and worth a human
1594
- * looking rather than a silent skip.
1595
- */
1596
- async cleanupTrashedBranchDirs() {
1597
- let entries;
1598
- try {
1599
- entries = await fs.readdir(this.contentBranchesPath);
1600
- }
1601
- catch (err) {
1602
- if (isNodeError(err) && err.code === 'ENOENT')
1603
- return 0;
1604
- throw err;
1605
- }
1606
- const now = Date.now();
1607
- let removed = 0;
1608
- let loggedUnparseable = false;
1609
- for (const name of entries) {
1610
- if (!name.startsWith('.trash-'))
1611
- continue;
1612
- const stampMatch = TRASH_DIR_STAMP_RE.exec(name);
1613
- const stampDate = stampMatch ? parseTrashStamp(stampMatch[1]) : null;
1614
- if (!stampDate) {
1615
- if (!loggedUnparseable) {
1616
- workerLog(`CanopyCMS: Skipping trash dir with unparseable stamp: ${name}`);
1617
- loggedUnparseable = true;
1618
- }
1619
- continue;
1620
- }
1621
- if (now - stampDate.getTime() < TRASH_RETENTION_MS)
1622
- continue;
1623
- try {
1624
- await fs.rm(path.join(this.contentBranchesPath, name), {
1625
- recursive: true,
1626
- force: true,
1627
- maxRetries: 3,
1628
- retryDelay: 100,
1629
- });
1630
- removed++;
1631
- }
1632
- catch (err) {
1633
- workerLogError(`CanopyCMS: Failed to remove trashed branch dir ${name}:`, getErrorMessage(err));
1634
- }
1635
- }
1636
- return removed;
1637
- }
1638
- /**
1639
- * Fast-forward the base branch's own working-tree clone
1640
- * (content-branches/<baseBranch>) to match origin/<baseBranch>.
1641
- *
1642
- * Previously this clone was refreshed only incidentally, by the generic
1643
- * rebase loop below (rebaseActiveBranches): for a branch with status
1644
- * 'editing', rebasing onto origin/<baseBranch> degenerates to a
1645
- * fast-forward when the clone IS the base branch. But that loop's skip
1646
- * paths -- a dirty tree, a missing .git -- are silent, which is the
1647
- * suspected live failure mode: a wedged base clone with no diagnosable
1648
- * signal in the logs. This dedicated step makes the refresh explicit,
1649
- * ff-only, and loud, so a stuck base view (an editor forking a new branch
1650
- * "from base" that's actually a stale snapshot) is diagnosable from logs.
1651
- * This runs every sync cycle so the drift window is bounded by
1652
- * gitSyncInterval.
1653
- *
1654
- * ff-only on purpose: this clone must stay a linear mirror of
1655
- * origin/<baseBranch>, so a merge that isn't a fast-forward (diverged
1656
- * local history) is treated as a should-never-happen condition and left
1657
- * untouched rather than force-resolved.
1658
- */
1659
- async refreshBaseBranchWorkspace() {
1660
- // Sanitized name for the workspace directory (a base branch containing
1661
- // e.g. '/' would otherwise stat a wrong nested path here forever).
1662
- const basePath = path.join(this.contentBranchesPath, this.sanitizedBaseBranch);
1663
- const gitDir = path.join(basePath, '.git');
1664
- try {
1665
- let gitDirStat;
1666
- try {
1667
- gitDirStat = await fs.stat(gitDir);
1668
- }
1669
- catch {
1670
- gitDirStat = null;
1671
- }
1672
- if (!gitDirStat || !gitDirStat.isDirectory()) {
1673
- workerLog(`Base branch workspace (${this.baseBranch}): not yet provisioned, skipping refresh`);
1674
- return;
1675
- }
1676
- const baseGit = simpleGit({
1677
- baseDir: basePath,
1678
- // Keep git non-interactive during the merge so it never blocks on an
1679
- // editor. simple-git >=3.32 blocks setting core.editor unless
1680
- // explicitly opted in; the value here is a hardcoded literal
1681
- // ("true", the shell no-op), not user input, so enabling
1682
- // allowUnsafeEditor carries no injection risk (same as the rebase
1683
- // loop's git config below).
1684
- config: ['core.editor=true'],
1685
- unsafe: { allowUnsafeEditor: true },
1686
- // DEP-H1: a hung fetch/merge against this EFS-backed clone would
1687
- // stall the sync loop forever (scheduleLoop only reschedules after
1688
- // completion). The block timeout is inactivity-based, so a
1689
- // slow-but-flowing transfer is unaffected -- same as syncGit()'s
1690
- // remote handle.
1691
- timeout: { block: this.taskTimeoutMs },
1692
- });
1693
- // Nothing makes this clone read-only. A direct edit here (or a stray
1694
- // process) wedges every editor's view of the base branch until an
1695
- // operator intervenes, so a dirty tree is loud, not a quiet skip.
1696
- // Only TRACKED changes block the refresh: a stray untracked file (e.g.
1697
- // runtime metadata missing from .git/info/exclude) must not wedge the
1698
- // fast-forward forever — if an untracked file would collide with
1699
- // incoming content, the --ff-only merge below refuses on its own and
1700
- // that failure is already logged loudly.
1701
- const status = await baseGit.status();
1702
- const trackedDirty = status.files.filter((f) => f.index !== '?' || f.working_dir !== '?');
1703
- if (trackedDirty.length > 0) {
1704
- workerLogError(`Base branch workspace (${this.baseBranch}) has uncommitted changes -- skipping refresh. Dirty files: ${trackedDirty.map((f) => f.path).join(', ')}`);
1705
- return;
1706
- }
1707
- // Raw (unsanitized) name from here on: these are git ref operations
1708
- // against origin/<baseBranch>, not filesystem paths, so they must use
1709
- // the same name GitHub knows the branch by.
1710
- await baseGit.fetch('origin', this.baseBranch);
1711
- // Use rev-list instead of status.behind for the same reason as the
1712
- // rebase loop below: status.behind only works with an upstream
1713
- // tracking branch configured, which isn't guaranteed here.
1714
- // Compare against the just-fetched tip rather than origin/<base>:
1715
- // workspaces are cloned --single-branch (git-manager.ts), so their
1716
- // fetch refspec only materializes a remote-tracking ref for the branch
1717
- // they were cloned from — for any other base branch, origin/<base>
1718
- // simply never exists and rev-list dies with "ambiguous argument".
1719
- // Pin FETCH_HEAD to a SHA immediately: FETCH_HEAD is one shared mutable
1720
- // file per repo, silently repointed by ANY other fetch in this clone.
1721
- const fetchedTip = (await baseGit.revparse(['FETCH_HEAD'])).trim();
1722
- const behindCount = parseInt((await baseGit.raw(['rev-list', '--count', `HEAD..${fetchedTip}`])).trim(), 10);
1723
- if (behindCount > 0) {
1724
- try {
1725
- await baseGit.merge(['--ff-only', fetchedTip]);
1726
- }
1727
- catch (err) {
1728
- workerLogError(`Base branch workspace (${this.baseBranch}) failed to fast-forward (diverged local history?): ${getErrorMessage(err)}`);
1729
- return;
1730
- }
1731
- await invalidateBranchContentCaches(basePath);
1732
- }
1733
- // Hygiene: conflictStatus/conflictFiles are meaningless for the base
1734
- // branch's own metadata and may be left over from before the base was
1735
- // excluded from rebaseActiveBranches()'s conflict-resolution loop.
1736
- // Reuse the already-clean no-op-save guard from that loop: save()
1737
- // eager-regenerates the branch registry (O(branch count) EFS reads),
1738
- // so skip it when there's nothing to clear.
1739
- const currentMeta = await BranchMetadataFileManager.loadOnly(basePath);
1740
- const conflictStatus = currentMeta?.branch.conflictStatus;
1741
- const conflictFiles = currentMeta?.branch.conflictFiles;
1742
- const alreadyClean = (conflictStatus === undefined || conflictStatus === 'clean') &&
1743
- (conflictFiles === undefined || conflictFiles.length === 0);
1744
- if (!alreadyClean) {
1745
- const meta = getBranchMetadataFileManager(basePath, this.contentBranchesPath);
1746
- await meta.save({
1747
- branch: { name: this.baseBranch, conflictStatus: 'clean', conflictFiles: [] },
1748
- });
1749
- }
1750
- // One concise per-cycle line -- the diagnostic for the next live deploy.
1751
- workerLog(behindCount > 0
1752
- ? `Base branch workspace (${this.baseBranch}): fast-forwarded ${behindCount} commit(s)`
1753
- : `Base branch workspace (${this.baseBranch}): up to date`);
1754
- }
1755
- catch (err) {
1756
- workerLogError(`Base branch workspace (${this.baseBranch}) refresh failed: ${getErrorMessage(err)}`);
1757
- }
1758
- }
1759
- /**
1760
- * Poll GitHub for a submitted/approved branch's PR resolution.
1761
- *
1762
- * submitted/approved branches sit outside the rebase loop and get no
1763
- * other signal that their PR resolved on GitHub -- nothing pushes a
1764
- * merge/close webhook back into the branch workspace. merged ->
1765
- * auto-archive via buildMergedBranchUpdate (shared with the manual
1766
- * markAsMerged API so both paths produce identical archived-branch
1767
- * metadata). closed-without-merge -> record pullRequestState only; an
1768
- * admin decides the workflow transition from there. Best-effort: any
1769
- * failure here is logged and swallowed, retried next sync cycle.
1770
- */
1771
- async pollMergeState(branchDir, branchPath, metaFile) {
1772
- const prNumber = metaFile?.branch.pullRequestNumber;
1773
- if (!prNumber)
1774
- return;
1775
- const controller = new AbortController();
1776
- const timer = setTimeout(() => controller.abort(), this.taskTimeoutMs);
1777
- try {
1778
- const { data } = await this.octokit.pulls.get({
1779
- owner: this.config.githubOwner,
1780
- repo: this.config.githubRepo,
1781
- pull_number: prNumber,
1782
- request: { signal: controller.signal },
1783
- });
1784
- if (data.merged) {
1785
- const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
1786
- // Use GitHub's actual merge time when available; buildMergedBranchUpdate
1787
- // falls back to "now" (its default `now` param) when merged_at is absent.
1788
- await meta.save({
1789
- branch: buildMergedBranchUpdate(branchDir, data.merged_at ? new Date(data.merged_at) : undefined),
1790
- });
1791
- workerLog(` PR #${prNumber} for ${branchDir} is merged -> archived`);
1792
- return;
1793
- }
1794
- const newState = data.state === 'closed' ? 'closed' : 'open';
1795
- // Re-load fresh (not the loop-top `metaFile` snapshot passed in) -- a
1796
- // concurrent Lambda write (e.g. an editor re-submitting) may have
1797
- // landed since that snapshot was taken.
1798
- const currentMeta = await BranchMetadataFileManager.loadOnly(branchPath);
1799
- if (currentMeta?.branch.pullRequestState === newState)
1800
- return;
1801
- const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
1802
- await meta.save({ branch: { name: branchDir, pullRequestState: newState } });
1803
- workerLog(` PR #${prNumber} for ${branchDir}: pullRequestState -> ${newState}`);
1804
- }
1805
- catch (err) {
1806
- // Non-fatal: transient GitHub/network errors are retried next cycle.
1807
- workerLogWarn(` Failed to poll PR #${prNumber} for ${branchDir}: ${getErrorMessage(err)}`);
1808
- }
1809
- finally {
1810
- clearTimeout(timer);
1811
- }
1812
- }
1813
- /**
1814
- * Persist a per-branch rebase failure to branch.json (PR-W2), bounded to
1815
- * roughly one save per failing branch per hour: a branch stuck failing
1816
- * every cycle must not turn into unbounded save-per-cycle x N-failing-
1817
- * branches write amplification -- save() eager-regenerates the branch
1818
- * registry (branch-metadata.ts's invalidateRegistry(), O(branch count) EFS
1819
- * reads), the same concern the `alreadyClean` no-op guard above exists
1820
- * for.
1821
- *
1822
- * Best-effort and non-fatal like every other metadata write in this
1823
- * loop's error paths: a corrupt branch.json, a lock-contention error, or
1824
- * any other save failure here must never abort the per-branch iteration.
1825
- * This matters doubly at the two call sites -- one is inside the outer
1826
- * per-branch catch, with no further catch of its own around this call --
1827
- * so the whole method is wrapped, not just the load.
1828
- */
1829
- async recordRebaseFailure(branchPath, branchDir, message) {
1830
- const RECORD_REFRESH_MS = 60 * 60 * 1000; // 1 hour
1831
- // [HIGH-1] Defense-in-depth redaction: rebaseFailure.message is
1832
- // persisted to branch.json and served to the browser via the
1833
- // branch-health admin endpoint. Both call sites in rebaseActiveBranches
1834
- // already redact before passing in (the failed.push sites below),
1835
- // redactCredentials is idempotent, so redacting again here is free and
1836
- // keeps this method safe on its own.
1837
- const redactedMessage = redactCredentials(message);
1838
- try {
1839
- const existing = await BranchMetadataFileManager.loadOnly(branchPath);
1840
- const prior = existing?.branch.rebaseFailure;
1841
- const sameMessage = prior?.message === redactedMessage;
1842
- const now = new Date();
1843
- if (sameMessage) {
1844
- const lastAtMs = Date.parse(prior.lastAt);
1845
- if (!Number.isNaN(lastAtMs) && now.getTime() - lastAtMs < RECORD_REFRESH_MS) {
1846
- // Same failure, refreshed within the last hour -- skip the save.
1847
- return;
1848
- }
1849
- }
1850
- const nowIso = now.toISOString();
1851
- const firstAt = sameMessage ? prior.firstAt : nowIso;
1852
- const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
1853
- await meta.save({
1854
- branch: {
1855
- name: branchDir,
1856
- rebaseFailure: { message: redactedMessage, firstAt, lastAt: nowIso },
1857
- },
1858
- });
1859
- }
1860
- catch (err) {
1861
- // Includes BranchMetadataCorruptError from the load above (a save()
1862
- // against the same corrupt file would just throw again) as well as
1863
- // any other load/save failure -- recording is best-effort
1864
- // observability, never allowed to abort the branch loop.
1865
- workerLogWarn(` Failed to record rebase failure for ${branchDir}: ${getErrorMessage(err)}`);
1866
- }
1867
- }
1868
569
  /**
1869
570
  * Test-only seam: awaited inside `rebaseActiveBranches()`'s conflict round,
1870
571
  * after `git rebase` reported conflicted files and BEFORE the
@@ -1886,626 +587,48 @@ export class CmsWorker {
1886
587
  * would strand a rewritten history.
1887
588
  */
1888
589
  async afterRebaseCompletedForTesting() { }
590
+ // --- Rebase loop (worker/rebase.ts) ------------------------------------
591
+ //
592
+ // Both are TEST-ONLY entry points -- see the task-queue block above. Nothing
593
+ // in production dispatches through either; `syncGit` calls `runRebaseCycle`
594
+ // directly and that calls `pollMergeState` directly. Four test files call
595
+ // `rebaseActiveBranches` (it is the single entry point the whole rebase suite
596
+ // drives), and cms-worker-merge-poll.test.ts calls `pollMergeState` with a
597
+ // mocked `octokit`. Neither is on WorkerContext, so a stub installed on
598
+ // either is a no-op as far as a production code path is concerned.
599
+ //
600
+ // `rebaseActiveBranches` also has a consumer outside the test files:
601
+ // apps/test-app/app/api/e2e-test/rebase/route.ts drives it from an e2e
602
+ // fixture route. Do not delete it as unused.
1889
603
  async rebaseActiveBranches() {
1890
- // PR-W1: collected across the loop below and returned as a summary
1891
- // (folded into worker-status.json by syncGit()). Purely additive
1892
- // bookkeeping -- doesn't change any control flow or existing logging.
1893
- const rebased = [];
1894
- const skippedDirty = [];
1895
- const skippedLocked = [];
1896
- const failed = [];
1897
- let branchDirs;
1898
- try {
1899
- branchDirs = await fs.readdir(this.contentBranchesPath);
1900
- }
1901
- catch {
1902
- return { rebased, skippedDirty, skippedLocked, failed };
1903
- }
1904
- for (const branchDir of branchDirs) {
1905
- // Known structural entries under content-branches/, not branch
1906
- // workspaces: branches.json (registry snapshot) and dot-prefixed
1907
- // entries (.canopy-meta/, transient lock dirs). Skip them silently so
1908
- // the no-.git skip logs below don't fire for them every cycle.
1909
- // Everything else still logs loudly on skip.
1910
- if (branchDir.startsWith('.') || branchDir === 'branches.json') {
1911
- continue;
1912
- }
1913
- const branchPath = path.join(this.contentBranchesPath, branchDir);
1914
- const gitDir = path.join(branchPath, '.git');
1915
- try {
1916
- const stat = await fs.stat(gitDir);
1917
- if (!stat.isDirectory()) {
1918
- workerLog(` Skipping ${branchDir}: .git is not a directory`);
1919
- continue;
1920
- }
1921
- }
1922
- catch {
1923
- workerLog(` Skipping ${branchDir}: no .git directory (not a branch workspace)`);
1924
- continue;
1925
- }
1926
- // The base branch's own clone is refreshed ff-only by
1927
- // refreshBaseBranchWorkspace() earlier in syncGit(). Routing it
1928
- // through this conflict-resolution rebase loop could rewrite its
1929
- // history (the --theirs loop below) and stamp meaningless conflict
1930
- // metadata on it. Compare sanitized-vs-sanitized: branchDir is a
1931
- // filesystem name (already sanitized), this.baseBranch is raw.
1932
- if (branchDir === this.sanitizedBaseBranch) {
1933
- workerLog(` Skipping ${branchDir}: base branch (refreshed separately)`);
1934
- continue;
1935
- }
1936
- try {
1937
- // Load metadata before any git ops to check branch status
1938
- const metaFile = await BranchMetadataFileManager.loadOnly(branchPath);
1939
- const branchStatus = metaFile?.branch.status;
1940
- // Skip branches that shouldn't be mutated:
1941
- // - submitted/approved: in review, don't rewrite history under an
1942
- // active PR -- but do poll GitHub for the PR's resolution, since
1943
- // nothing else tells the worker a merge/close happened.
1944
- // - archived: already merged, no reason to rebase and no PR left to
1945
- // poll (avoid the wasted API call).
1946
- if (branchStatus === 'submitted' || branchStatus === 'approved') {
1947
- workerLog(` Skipping ${branchDir} (${branchStatus})`);
1948
- await this.pollMergeState(branchDir, branchPath, metaFile);
1949
- continue;
1950
- }
1951
- if (branchStatus === 'archived') {
1952
- workerLog(` Skipping ${branchDir} (${branchStatus})`);
1953
- continue;
1954
- }
1955
- const branchGit = simpleGit({
1956
- baseDir: branchPath,
1957
- // Keep git non-interactive during rebase/merge so it never blocks on an editor.
1958
- // simple-git >=3.32 blocks setting core.editor unless explicitly opted in; the
1959
- // value here is a hardcoded literal ("true", the shell no-op), not user input,
1960
- // so enabling allowUnsafeEditor carries no injection risk.
1961
- config: ['core.editor=true'],
1962
- unsafe: { allowUnsafeEditor: true },
1963
- });
1964
- // [SYNC-C1] Take the branch's cross-host content-write lock BEFORE the
1965
- // dirty check, and hold it for the whole rebase.
1966
- //
1967
- // The dirty check alone is check-then-act. The old comment here claimed
1968
- // the residual window was safe ("the rebase will fail and the catch
1969
- // block will abort safely"), which only holds for a save landing before
1970
- // `git rebase` STARTS. After that -- a window spanning fetch, replay and
1971
- // N conflict rounds of awaited git subprocesses on EFS -- a save is
1972
- // destroyed two ways: `checkout --theirs` below overwrites the
1973
- // just-saved file with the branch's committed version and the rebase
1974
- // then SUCCEEDS (nothing logs a failure at all), and `rebase --abort`
1975
- // hard-resets the tree. The editor already got its 200 either way.
1976
- //
1977
- // Zero-retry acquisition, deliberately: on contention this branch is
1978
- // skipped and retried on the next sync cycle (~5 min), which is the
1979
- // same principle as the skip-dirty-branches behavior below. Writers get
1980
- // the patient side of the asymmetry -- see utils/content-write-lock.ts.
1981
- //
1982
- // The heartbeat that keeps this lock fresh (proper-lockfile refreshes
1983
- // every `stale`/2 = 15s) is a timer on this event loop; every git step
1984
- // below is an awaited subprocess, never a synchronous block, so the
1985
- // refresh keeps firing for the whole hold.
1986
- let releaseContentLock;
1987
- // [SYNC-C1] If the lock is lost mid-hold, a writer may now be live
1988
- // against this same tree. Every git step below is destructive, so
1989
- // record it and bail before the next one rather than replaying over
1990
- // an editor's concurrent save.
1991
- let contentLockCompromised = false;
1992
- try {
1993
- releaseContentLock = await tryAcquireContentWriteLock(branchPath, (lockErr) => {
1994
- contentLockCompromised = true;
1995
- workerLogWarn(` Content-write lock compromised mid-rebase for ${branchDir}: ${getErrorMessage(lockErr)}`);
1996
- });
1997
- }
1998
- catch (lockErr) {
1999
- if (isNodeError(lockErr) && lockErr.code === 'ELOCKED') {
2000
- workerLog(` Skipping ${branchDir}: content write in progress (retrying next cycle)`);
2001
- skippedLocked.push(branchDir);
2002
- continue;
2003
- }
2004
- // Anything else (ENOENT on a branch dir deleted mid-cycle, EACCES,
2005
- // ...) is a real failure: let the outer catch record it.
2006
- throw lockErr;
2007
- }
2008
- try {
2009
- // Recover an INTERRUPTED rebase before anything else looks at this
2010
- // tree. A clone left with .git/rebase-merge (or rebase-apply) reports
2011
- // uncommitted changes, so without this the dirty check below would
2012
- // classify it `skippedDirty` on every cycle FOREVER while
2013
- // branch-health scanned it as healthy -- and editors would meanwhile
2014
- // read, and be able to save over, conflict-marker content.
2015
- //
2016
- // An in-progress rebase is always this worker's own abandoned work:
2017
- // it is the only thing that ever rebases these clones, and it got
2018
- // here via a crash, an OOM, a spot interruption, or the ASG rolling
2019
- // the instance (which happens on EVERY `cdk deploy`, while `stop()`
2020
- // drains for at most taskTimeoutMs).
2021
- //
2022
- // NOT LOSSLESS, and it is important not to claim otherwise. `git
2023
- // rebase --abort` hard-resets tracked files to the pre-rebase head.
2024
- // While the worker was DOWN nothing held the [SYNC-C1] content-write
2025
- // lock, so an editor could have saved into this wedged clone and
2026
- // received a 200; that save is a working-tree modification, and the
2027
- // abort reverts it. (New, untracked entry files survive; edits to
2028
- // existing ones do not.) Taking the lock here stops any FURTHER save
2029
- // racing the abort, but cannot recover one that already landed.
2030
- //
2031
- // Aborting anyway is still the right call: the alternative is a
2032
- // branch wedged forever whose tree serves conflict-marker content to
2033
- // editors. What must not happen is doing it SILENTLY -- so anything
2034
- // modified beyond the rebase's own conflict state is logged by path
2035
- // first, which is the only record an operator would have.
2036
- if (await isRebaseInProgress(branchPath)) {
2037
- const preAbort = await branchGit.status().catch(() => null);
2038
- // Keyed on the WORKING-TREE column only. The two porcelain columns
2039
- // mean different things here, and conflating them produces a false
2040
- // data-loss report on essentially every conflict-wedged recovery
2041
- // (verified against real git, mid-rebase):
2042
- //
2043
- // `M ` index=M, wd=' ' -- the interrupted replay's own cleanly
2044
- // merged files, already STAGED. These
2045
- // are committed history and survive the
2046
- // abort untouched. Not collateral.
2047
- // ` M` index=' ', wd=M -- a working-tree modification nothing
2048
- // staged: an editor's save landing while
2049
- // the worker was down. The abort
2050
- // discards exactly these.
2051
- // `??` -- untracked; the abort leaves them.
2052
- //
2053
- // KNOWN GAP, stated rather than hidden: a save onto one of the
2054
- // rebase's own conflicted paths (the "saved over conflict-marker
2055
- // content" case) is excluded below, because the file reads `UU`
2056
- // whether or not an editor touched it -- status alone cannot tell
2057
- // the two apart. Those discards go unlogged.
2058
- const collateral = (preAbort?.files ?? [])
2059
- .filter((f) => !preAbort?.conflicted.includes(f.path))
2060
- .filter((f) => f.working_dir !== ' ' && f.working_dir !== '?')
2061
- .map((f) => f.path);
2062
- if (collateral.length > 0) {
2063
- workerLogWarn(` ${branchDir}: aborting the interrupted rebase will DISCARD working-tree changes to ` +
2064
- `${collateral.length} file(s) saved while the worker was down: ${collateral.join(', ')}`);
2065
- }
2066
- workerLogWarn(` ${branchDir}: found an interrupted rebase (this worker's own abandoned work) -- aborting it to recover the branch`);
2067
- try {
2068
- await branchGit.rebase(['--abort']);
2069
- }
2070
- catch (abortErr) {
2071
- // Leave it for the next cycle rather than pressing on: every step
2072
- // below assumes a clean tree.
2073
- const reason = redactCredentials(`could not abort interrupted rebase: ${getErrorMessage(abortErr)}`);
2074
- workerLogWarn(` Skipping ${branchDir}: ${reason}`);
2075
- failed.push({ branch: branchDir, error: reason });
2076
- // Record on branch metadata too, like the other two failure exits
2077
- // (`!completed` and the outer catch). Without this a persistently
2078
- // un-abortable wedge appeared in worker-status.json but never set
2079
- // a `syncFailureReason`, so the admin branch panel showed nothing
2080
- // -- and this is precisely the state that needs an operator,
2081
- // since it is the one the next cycle cannot fix by itself.
2082
- await this.recordRebaseFailure(branchPath, branchDir, reason);
2083
- continue;
2084
- }
2085
- }
2086
- // Skip dirty branches — editor has unsaved changes that can't be rebased.
2087
- // Now inside the lock, so no write can land between this check and the
2088
- // rebase below.
2089
- const dirtyCheck = await branchGit.status();
2090
- if (dirtyCheck.files.length > 0) {
2091
- workerLog(` Skipping ${branchDir}: has uncommitted changes`);
2092
- skippedDirty.push(branchDir);
2093
- continue;
2094
- }
2095
- // The clone's own ref name: branchDir is the sanitized DIRECTORY
2096
- // name and need not match it. A literal 'HEAD' means a detached
2097
- // clone (e.g. a crashed rebase left one behind) -- nothing below can
2098
- // safely name a ref then, so every publish path stays disarmed,
2099
- // which is the safe direction.
2100
- const branchRef = (await branchGit.revparse(['--abbrev-ref', 'HEAD'])).trim();
2101
- const canPublish = branchRef.length > 0 && branchRef !== 'HEAD';
2102
- // [SYNC-H1] Self-heal: finish an interrupted publish even when this
2103
- // branch is not behind base. Every crash window in the arming
2104
- // sequence below leaves the marker set with the work unfinished, and
2105
- // this is what completes it -- without it, one lost lease race would
2106
- // strand the branch until the base branch happened to advance again.
2107
- // Gated on the loop-top snapshot, so unmarked branches (nearly all
2108
- // of them, every cycle) cost nothing extra.
2109
- if (canPublish && metaFile?.branch.historyRewrittenFrom) {
2110
- await this.reconcilePendingRewrite({
2111
- branchPath,
2112
- branchDir,
2113
- branchRef,
2114
- headSha: (await branchGit.revparse(['HEAD'])).trim(),
2115
- marker: metaFile.branch.historyRewrittenFrom,
2116
- });
2117
- }
2118
- await branchGit.fetch('origin', this.baseBranch);
2119
- // Use rev-list instead of status.behind — status.behind only works when the
2120
- // branch has an upstream tracking branch configured, which isn't guaranteed
2121
- // (checkoutBranch fallback paths create branches without --track).
2122
- // The just-fetched tip, not origin/<base>: branch clones are
2123
- // --single-branch, so no remote-tracking ref exists for a base branch
2124
- // other than the one they were cloned from (see the base-refresh
2125
- // comment above). Pinned to a SHA immediately — FETCH_HEAD is one
2126
- // shared mutable file per repo, repointed by any concurrent fetch.
2127
- const fetchedBaseTip = (await branchGit.revparse(['FETCH_HEAD'])).trim();
2128
- const behindCount = parseInt((await branchGit.raw(['rev-list', '--count', `HEAD..${fetchedBaseTip}`])).trim(), 10);
2129
- const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
2130
- if (behindCount === 0) {
2131
- // Already in sync. This is the overwhelmingly common outcome per
2132
- // branch per cycle (most branches are caught up most of the
2133
- // time), so skip the save entirely when metadata already reflects
2134
- // a clean state -- every save() now eager-regenerates the branch
2135
- // registry (branch-metadata.ts's invalidateRegistry(), O(branch
2136
- // count) fs reads on EFS), so an unconditional save here turns
2137
- // every rebase cycle into O(N^2) registry work across N branches
2138
- // for what is otherwise a true no-op. Re-load fresh (not the
2139
- // `metaFile` snapshot from before the fetch/rev-list above) so a
2140
- // concurrent editor-driven metadata change during that window
2141
- // isn't clobbered by a stale skip decision.
2142
- const currentMeta = await BranchMetadataFileManager.loadOnly(branchPath);
2143
- const conflictStatus = currentMeta?.branch.conflictStatus;
2144
- const conflictFiles = currentMeta?.branch.conflictFiles;
2145
- const conflictAlreadyClean = (conflictStatus === undefined || conflictStatus === 'clean') &&
2146
- (conflictFiles === undefined || conflictFiles.length === 0);
2147
- // PR-W2: a lingering rebaseFailure must also be cleared once the
2148
- // branch catches up clean -- otherwise it sticks as a stale
2149
- // warning forever (nothing else touches this branch once it's
2150
- // caught up, so no other save site would ever clear it).
2151
- const alreadyClean = conflictAlreadyClean && currentMeta?.branch.rebaseFailure === undefined;
2152
- if (alreadyClean) {
2153
- continue;
2154
- }
2155
- await meta.save({
2156
- branch: {
2157
- name: branchDir,
2158
- conflictStatus: 'clean',
2159
- conflictFiles: [],
2160
- rebaseFailure: undefined,
2161
- },
2162
- });
2163
- continue;
2164
- }
2165
- workerLog(`Rebasing ${branchDir} (${behindCount} commits behind)...`);
2166
- // Read BOTH sides before rewriting anything: the arming guard after
2167
- // the rebase compares what remote.git published against what this
2168
- // clone is about to rebase away. Reading them afterwards would be
2169
- // useless -- the clone's pre-rebase tip is exactly what disappears.
2170
- const preRebaseHead = (await branchGit.revparse(['HEAD'])).trim();
2171
- const publishedSha = canPublish ? await this.readPublishedSha(branchRef) : null;
2172
- // Resolve-and-continue loop: keep branch version for conflicting files, then continue
2173
- // Non-conflicting files get main's changes; conflicting files keep branch version.
2174
- const conflictedFiles = [];
2175
- let nextAction = 'start';
2176
- let completed = false;
2177
- // PR-W1: captured only on the "unexpected error" exit below, for the
2178
- // failed-summary entry pushed at the `if (!completed)` check.
2179
- let failureReason;
2180
- const MAX_ROUNDS = 50; // safety limit against infinite loops
2181
- for (let round = 0; round < MAX_ROUNDS && !completed; round++) {
2182
- // Re-checked every round: the compromise can land between rounds,
2183
- // and `--continue`/`--skip` are as destructive as the initial
2184
- // rebase. Handled after the loop so it cannot be mistaken for the
2185
- // `!completed` rebase-FAILURE path below.
2186
- if (contentLockCompromised)
2187
- break;
2188
- try {
2189
- if (nextAction === 'start') {
2190
- // The pinned base tip fetched above (single-branch clones have
2191
- // no origin/<base> remote-tracking ref for other branches).
2192
- await branchGit.rebase([fetchedBaseTip]);
2193
- }
2194
- else if (nextAction === 'continue') {
2195
- await branchGit.rebase(['--continue']);
2196
- }
2197
- else {
2198
- await branchGit.rebase(['--skip']);
2199
- }
2200
- completed = true;
2201
- }
2202
- catch (rebaseErr) {
2203
- nextAction = 'continue';
2204
- const st = await branchGit.status();
2205
- if (st.conflicted.length > 0) {
2206
- await this.afterConflictDetectedForTesting();
2207
- // During rebase, --theirs = the branch being replayed (editor's work).
2208
- // (git rebase reverses ours/theirs: "ours" is the rebase target, "theirs" is the branch.)
2209
- //
2210
- // MODIFY/DELETE conflicts have no "their version" to check out
2211
- // and must be resolved by staging a delete or an add instead.
2212
- // `git checkout --theirs` on one exits non-zero ("path ... does
2213
- // not have their version") and simple-git throws -- and because
2214
- // this loop body IS the round loop's catch, that throw escapes
2215
- // the round loop entirely, skipping BOTH `rebase --abort` sites
2216
- // below and leaving the clone wedged mid-rebase forever. The
2217
- // index/working-tree code pair identifies which side deleted
2218
- // (verified against real git, not inferred):
2219
- //
2220
- // U/D "deleted by them" -- the BRANCH deleted it, base
2221
- // modified it. Git leaves base's version in the tree.
2222
- // Keep-branch-version means honouring the delete: git rm.
2223
- // D/U "deleted by us" -- base deleted it, the BRANCH
2224
- // modified it. Git leaves the branch's version in the
2225
- // tree. Keep-branch-version means keeping it: git add.
2226
- //
2227
- // Any per-file resolution that STILL fails routes into the
2228
- // `!completed` path below (which aborts and records) instead of
2229
- // escaping -- deliberately NOT a rethrow, since a throw from
2230
- // here is exactly the bug being fixed.
2231
- const conflictKind = new Map(st.files.map((f) => [f.path, `${f.index}${f.working_dir}`]));
2232
- let resolutionFailure;
2233
- for (const file of st.conflicted) {
2234
- const kind = conflictKind.get(file);
2235
- try {
2236
- if (kind === 'UD') {
2237
- await branchGit.raw(['rm', '-f', '--', file]);
2238
- }
2239
- else if (kind === 'DU') {
2240
- await branchGit.add(file);
2241
- }
2242
- else {
2243
- await branchGit.raw(['checkout', '--theirs', file]);
2244
- await branchGit.add(file);
2245
- }
2246
- }
2247
- catch (resolveErr) {
2248
- resolutionFailure =
2249
- `failed to resolve conflicted file '${file}' (status ${kind ?? '??'}): ` +
2250
- getErrorMessage(resolveErr);
2251
- break;
2252
- }
2253
- conflictedFiles.push(file);
2254
- }
2255
- if (resolutionFailure !== undefined) {
2256
- // Same exit shape as the "unexpected error" branch below: set
2257
- // failureReason and break, letting the `!completed` block do
2258
- // the single `rebase --abort` and record the failure once.
2259
- failureReason = resolutionFailure;
2260
- break;
2261
- }
2262
- // nextAction stays 'continue'
2263
- }
2264
- else {
2265
- const msg = rebaseErr instanceof Error ? rebaseErr.message : '';
2266
- if (msg.toLowerCase().includes('nothing to commit') ||
2267
- msg.toLowerCase().includes('apply --skip')) {
2268
- // Empty commit after --theirs resolution — skip it
2269
- nextAction = 'skip';
2270
- }
2271
- else {
2272
- // Unexpected error — abort and leave branch behind.
2273
- // We intentionally don't update conflictStatus/conflictFiles here:
2274
- // the rebase didn't complete so we can't determine the true conflict
2275
- // state. Previous metadata (possibly stale) is preserved until the
2276
- // next successful rebase cycle corrects it.
2277
- workerLogWarn(` Unexpected rebase error in ${branchDir}: ${msg || 'Unknown error'}`);
2278
- failureReason = msg || 'Unknown error';
2279
- await branchGit.rebase(['--abort']).catch(() => { });
2280
- break;
2281
- }
2282
- }
2283
- }
2284
- }
2285
- // Outside the round loop's try/catch, so a throwing test hook can
2286
- // never be misread as a rebase error.
2287
- if (completed)
2288
- await this.afterRebaseCompletedForTesting();
2289
- // [SYNC-C1] A lost lock is a RETRY, not a rebase failure: nothing is
2290
- // wrong with the branch, we simply can no longer prove we were the
2291
- // only writer. Handled ahead of the `!completed` block so it never
2292
- // records a user-visible rebaseFailure or lands in `failed[]`.
2293
- //
2294
- // ONLY when the rebase did not complete. Bailing out of a COMPLETED
2295
- // rebase would be worse than useless: the history is already
2296
- // rewritten (so `--abort` is a no-op), and skipping the completion
2297
- // path below strands three things the next cycle will never redo,
2298
- // because a caught-up branch short-circuits at the `behindCount === 0`
2299
- // check above -- the [SYNC-H1] `markHistoryRewritten` marker (without
2300
- // which the editor's next submit is rejected non-fast-forward and
2301
- // mis-diagnosed as another deployment), the content-cache
2302
- // invalidation for a tree that DID change, and the conflictStatus
2303
- // save. That converts a transient lock compromise into a permanently
2304
- // wedged published branch.
2305
- //
2306
- // Nor does bailing protect a racing save: a rebase replays COMMITTED
2307
- // history, while a concurrent save is uncommitted working-tree state,
2308
- // and that writer is already told to retry via
2309
- // ContentWriteLockBusyError. So when the rebase completed, log the
2310
- // lost exclusivity loudly and finish the job.
2311
- if (contentLockCompromised && !completed) {
2312
- workerLogWarn(` Skipping ${branchDir}: content-write lock was compromised mid-rebase (retrying next cycle)`);
2313
- // No-op when no rebase is in progress; failure is expected there.
2314
- await branchGit.rebase(['--abort']).catch(() => { });
2315
- skippedLocked.push(branchDir);
2316
- continue;
2317
- }
2318
- if (contentLockCompromised) {
2319
- workerLogWarn(` Content-write lock for ${branchDir} was compromised, but its rebase had already completed -- finishing the sync (history is rewritten; skipping now would strand the history-rewrite marker and wedge the branch)`);
2320
- }
2321
- if (!completed) {
2322
- // PR-W2 (M1 rider): failureReason is only set on the "unexpected
2323
- // error" break above -- MAX_ROUNDS exhaustion is a distinct exit
2324
- // path with no error message of its own, so the warn text must
2325
- // not conflate the two.
2326
- workerLogWarn(failureReason !== undefined
2327
- ? ` Rebase of ${branchDir} aborted due to unexpected error: ${failureReason}`
2328
- : ` Rebase of ${branchDir} did not complete within ${MAX_ROUNDS} rounds, aborting`);
2329
- await branchGit.rebase(['--abort']).catch(() => { });
2330
- const rebaseFailureMessage = failureReason ?? `did not complete within ${MAX_ROUNDS} rounds`;
2331
- // [HIGH-1] failed[] folds into worker-status.json's
2332
- // lastGitSync.failed, served to the browser -- failureReason can
2333
- // be an arbitrary git error message that embeds the bot token.
2334
- const redactedRebaseFailureMessage = redactCredentials(rebaseFailureMessage);
2335
- failed.push({
2336
- branch: branchDir,
2337
- error: redactedRebaseFailureMessage,
2338
- });
2339
- // PR-W2: record once here for the "!completed" exit -- the
2340
- // unexpected-error break above is NOT disjoint from this block (it
2341
- // always falls through here), so recording at the break itself
2342
- // would double-record. The outer catch below is the only other
2343
- // record site (a distinct, non-overlapping failure class: errors
2344
- // outside this round loop, e.g. fetch/rev-list failures).
2345
- await this.recordRebaseFailure(branchPath, branchDir, redactedRebaseFailureMessage);
2346
- continue;
2347
- }
2348
- // The rebase rewrote the branch clone's working tree — mark ContentStore
2349
- // ID indexes rooted here stale so lookups rebuild from disk, in this
2350
- // process and (via the on-disk generation marker) in the Lambda
2351
- // containers sharing this filesystem.
2352
- await invalidateBranchContentCaches(branchPath);
2353
- // Convert file paths to ContentIds — immutable, survives slug renames.
2354
- // Entry files have IDs in their filename (e.g., "post.slug.a1b2c3d4e5f6.mdx").
2355
- // .collection.json files have no ID themselves (extractIdFromFilename returns null
2356
- // for dot-prefixed files), so we extract the ID from the parent directory instead.
2357
- // The root content directory (e.g., "content/", or "cms/content/" for a
2358
- // multi-segment contentRoot) has no embedded ID, so we use ROOT_COLLECTION_ID
2359
- // as a sentinel — but only for the configured contentRoot.
2360
- //
2361
- // Two different notions of "parent" are needed below, and conflating them
2362
- // reproduces the exact bug this comparison guards against (see
2363
- // schema-store.ts's contentRootName doc comment for the same shape elsewhere):
2364
- // - `parentDir` (a basename) recovers a SUB-collection's own embedded ID
2365
- // (e.g. "posts.cNbR5xFm2Kpd" -> "cNbR5xFm2Kpd") — correct as a basename,
2366
- // since a collection directory carries its ID in its own name, one path
2367
- // segment.
2368
- // - `parentPath` (the full relative parent path, normalized) is what must be
2369
- // compared against `this.contentRoot`, because `contentRoot` is documented
2370
- // (config/helpers.ts) as allowed to span multiple segments (e.g.
2371
- // "cms/content"). Comparing a basename ("content") against that full value
2372
- // is always false, which silently drops the root collection's conflict.
2373
- // Git reports POSIX-style paths and the configured value may be authored
2374
- // with either separator, so both sides go through normalizeFilesystemPath
2375
- // before comparing.
2376
- const normalizedContentRoot = normalizeFilesystemPath(this.contentRoot);
2377
- const conflictIds = [...new Set(conflictedFiles)]
2378
- .map((f) => {
2379
- const fileId = extractIdFromFilename(path.basename(f));
2380
- if (fileId)
2381
- return fileId;
2382
- const parentDir = path.basename(path.dirname(f));
2383
- const dirId = extractIdFromFilename(parentDir);
2384
- if (dirId)
2385
- return dirId;
2386
- // Only assign ROOT_COLLECTION_ID when the file's parent directory IS the
2387
- // configured content root. Other unrecognized paths are filtered out.
2388
- const parentPath = normalizeFilesystemPath(path.dirname(f));
2389
- if (path.basename(f) === '.collection.json' && parentPath === normalizedContentRoot) {
2390
- return ROOT_COLLECTION_ID;
2391
- }
2392
- return null;
2393
- })
2394
- .filter((id) => id !== null);
2395
- const conflictIdsDeduped = [...new Set(conflictIds)];
2396
- const hadConflicts = conflictIdsDeduped.length > 0;
2397
- workerLog(hadConflicts
2398
- ? ` Rebased ${branchDir} (kept branch version for ${conflictIdsDeduped.length} conflicting file(s))`
2399
- : ` Rebased ${branchDir} successfully`);
2400
- await meta.save({
2401
- branch: {
2402
- name: branchDir,
2403
- conflictStatus: hadConflicts ? 'conflicts-detected' : 'clean',
2404
- conflictFiles: conflictIdsDeduped,
2405
- // PR-W2: the cycle completed successfully -- clear any prior
2406
- // failure record regardless of conflict outcome.
2407
- rebaseFailure: undefined,
2408
- },
2409
- });
2410
- // PR-W1: the branch was behind and the rebase completed (with or
2411
- // without --theirs conflict resolution) -- it moved, so it belongs
2412
- // in the summary. Branches already up to date `continue`d above and
2413
- // are deliberately not listed here.
2414
- rebased.push(branchDir);
2415
- // [SYNC-H1] The rebase just rewrote this clone's history. If that
2416
- // history was already published, nothing else will ever reconcile
2417
- // remote.git (and GitHub) with it -- the editor's next submit would
2418
- // simply be rejected non-fast-forward. Carry the rewrite forward.
2419
- if (publishedSha !== null) {
2420
- if (publishedSha === preRebaseHead) {
2421
- // ARMING GUARD. remote.git holds EXACTLY what this clone just
2422
- // rebased away and nothing more, so a lease keyed to it can only
2423
- // undo our own rewrite.
2424
- //
2425
- // The inequality case below is not defensive padding: branch
2426
- // clones never fetch their own branch (GitManager's clone is
2427
- // --single-branch and checkoutBranch only checks out an existing
2428
- // local branch), while reconcileTrackedBranches fast-forwards
2429
- // remote.git to GitHub's tip. So after a reviewer pushes a fixup
2430
- // straight to the PR branch, remote.git legitimately holds a
2431
- // commit this clone has never seen. Leasing on "whatever
2432
- // remote.git currently holds" would be SATISFIED there and would
2433
- // delete that fixup from remote.git and then from GitHub,
2434
- // silently. Keying the lease to the pre-rebase tip turns that
2435
- // case into the visible divergence it should be.
2436
- //
2437
- // Order matters: mark, then push, then queue. A crash after any
2438
- // step leaves the marker set with the work unfinished, which
2439
- // reconcilePendingRewrite() completes on a later cycle. Pushing
2440
- // first would leave remote.git rewritten, GitHub stale and
2441
- // nothing recorded -- unrecoverable, and landing on exactly the
2442
- // false "another deployment" diagnosis this change removes.
2443
- await this.markHistoryRewritten(branchPath, branchDir, publishedSha);
2444
- if (await this.forcePublishToLocalRemote(branchPath, branchRef, publishedSha)) {
2445
- await this.enqueueGitHubPush(branchRef);
2446
- }
2447
- }
2448
- else {
2449
- const divergence = `rebased locally, but remote.git holds ${publishedSha} for ${branchRef}, which this ` +
2450
- `clone never had (a direct push to the branch?). Left untouched -- reconcile it ` +
2451
- `before submitting again.`;
2452
- workerLogWarn(` ${branchDir}: ${divergence}`);
2453
- await this.recordRebaseFailure(branchPath, branchDir, divergence);
2454
- }
2455
- }
2456
- }
2457
- finally {
2458
- // Last-resort guarantee that NO exit path leaves this clone
2459
- // mid-rebase -- including an unexpected throw from any git step
2460
- // above, which lands in the outer catch and previously only logged.
2461
- //
2462
- // It must live HERE rather than in that outer catch: the catch runs
2463
- // AFTER this finally has released the content-write lock, so aborting
2464
- // there would hard-reset a working tree an editor's save could
2465
- // already be racing -- precisely the [SYNC-C1] hazard the lock
2466
- // exists to prevent. Inside the finally the lock is still held --
2467
- // EXCEPT on the narrow path where it was compromised mid-hold, in
2468
- // which case a newly-admitted writer may already be live and this
2469
- // abort carries the same exposure as the compromise path's own abort
2470
- // above. Not special-cased: leaving a clone wedged mid-rebase is the
2471
- // worse outcome, and the writer in that window is already being told
2472
- // to retry.
2473
- //
2474
- // Guarded on actual rebase state so the happy path and the `continue`
2475
- // exits cost one stat and do nothing.
2476
- try {
2477
- if (await isRebaseInProgress(branchPath)) {
2478
- workerLogWarn(` ${branchDir}: rebase still in progress on exit -- aborting so the clone is not left wedged`);
2479
- await branchGit.rebase(['--abort']);
2480
- }
2481
- }
2482
- catch (abortErr) {
2483
- // Best effort: the next cycle's recovery check retries this.
2484
- workerLogWarn(` Failed to abort in-progress rebase for ${branchDir}: ${getErrorMessage(abortErr)}`);
2485
- }
2486
- // [SYNC-C1] Released on EVERY exit -- the `continue`s above, a throw
2487
- // into the outer catch, and the happy path alike. A stranded lock
2488
- // would wedge every write to this branch until it went stale.
2489
- await releaseContentLock?.().catch((releaseErr) => {
2490
- workerLogWarn(` Failed to release content-write lock for ${branchDir}: ${getErrorMessage(releaseErr)}`);
2491
- });
2492
- }
2493
- }
2494
- catch (err) {
2495
- const message = err instanceof Error ? err.message : 'Unknown error';
2496
- workerLogWarn(` Failed to sync ${branchDir}: ${message}`);
2497
- // [HIGH-1] Same rationale as the `if (!completed)` push site above --
2498
- // this catches fetch/rev-list/unexpected errors, whose message can
2499
- // embed the bot token.
2500
- const redactedMessage = redactCredentials(message);
2501
- failed.push({ branch: branchDir, error: redactedMessage });
2502
- // PR-W2: second (and only other) record site -- see the comment at
2503
- // the `if (!completed)` block above for why these two sites are
2504
- // disjoint.
2505
- await this.recordRebaseFailure(branchPath, branchDir, redactedMessage);
2506
- }
2507
- }
2508
- return { rebased, skippedDirty, skippedLocked, failed };
604
+ return runRebaseCycle(this.ctx());
605
+ }
606
+ // --- Git-sync cluster (worker/git-sync.ts) -----------------------------
607
+ //
608
+ // `syncGit` is the public loop entry `scheduleLoop` drives. The three private
609
+ // ones below it are TEST-ONLY entry points -- see the task-queue block above
610
+ // -- each called directly by one test file (cms-worker-sync-reconcile for the
611
+ // settings push, cms-worker-base-refresh for the base workspace,
612
+ // cms-worker.test.ts for the trash sweep). `syncGit` reaches all three as
613
+ // module-level calls, so a stub installed on one of these is a no-op.
614
+ //
615
+ // `reconcileTrackedBranches` deliberately has NO delegator: no test reaches
616
+ // it through the instance. Comments elsewhere that point at it as living in
617
+ // this file are stale by definition -- it is in git-sync.ts.
618
+ async syncGit() {
619
+ return syncGit(this.ctx());
620
+ }
621
+ async pushSettingsBranches(git, trackedNames) {
622
+ return pushSettingsBranches(this.ctx(), git, trackedNames);
623
+ }
624
+ async refreshBaseBranchWorkspace() {
625
+ return refreshBaseBranchWorkspace(this.ctx());
626
+ }
627
+ async cleanupTrashedBranchDirs() {
628
+ return cleanupTrashedBranchDirs(this.ctx());
629
+ }
630
+ async pollMergeState(branchDir, branchPath, metaFile) {
631
+ return pollMergeState(this.ctx(), branchDir, branchPath, metaFile);
2509
632
  }
2510
633
  async refreshAuthCache() {
2511
634
  if (!this.running || !this.config.refreshAuthCache)