canopycms 0.0.65 → 0.0.66-int.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/ai/generate.d.ts +13 -0
- package/dist/ai/generate.js +9 -3
- package/dist/ai/to-plain-text.js +4 -0
- package/dist/ai/types.d.ts +20 -1
- package/dist/api/admin-branch-health.js +13 -12
- package/dist/api/branch.js +2 -2
- package/dist/branch-health.js +1 -1
- package/dist/branch-metadata-file.d.ts +58 -0
- package/dist/branch-metadata-file.js +65 -0
- package/dist/branch-metadata.d.ts +2 -26
- package/dist/branch-metadata.js +6 -33
- package/dist/branch-registry.js +4 -2
- package/dist/build/generate-ai-content.js +103 -0
- package/dist/cli/cli.js +432 -317
- package/dist/cli/generate-ai-content.js +264 -160
- package/dist/content-id-index.js +22 -0
- package/dist/content-listing.js +3 -0
- package/dist/content-reader.d.ts +25 -1
- package/dist/content-reader.js +15 -0
- package/dist/content-store.d.ts +50 -1
- package/dist/content-store.js +102 -4
- package/dist/context.d.ts +36 -4
- package/dist/context.js +15 -4
- package/dist/editor/admin/SystemHealthPanel.js +2 -2
- package/dist/entry-schema.d.ts +22 -0
- package/dist/git-manager.d.ts +9 -8
- package/dist/git-manager.js +35 -9
- package/dist/github-service.d.ts +1 -1
- package/dist/github-service.js +1 -1
- package/dist/paths/branch-name.d.ts +1 -1
- package/dist/paths/branch-name.js +1 -1
- package/dist/schema/index.d.ts +1 -1
- package/dist/schema/index.js +1 -1
- package/dist/schema/meta-loader.js +1 -1
- package/dist/services.d.ts +22 -0
- package/dist/types.d.ts +1 -1
- package/dist/url-exclusivity-fixtures.d.ts +76 -0
- package/dist/url-exclusivity-fixtures.js +119 -0
- package/dist/url-path-resolver.js +8 -4
- package/dist/utils/content-write-lock.d.ts +2 -1
- package/dist/utils/content-write-lock.js +2 -1
- package/dist/utils/error.d.ts +17 -0
- package/dist/utils/error.js +17 -0
- package/dist/utils/git.d.ts +1 -1
- package/dist/utils/git.js +1 -1
- package/dist/utils/occ-json-write.js +1 -1
- package/dist/worker/cms-worker.d.ts +22 -288
- package/dist/worker/cms-worker.js +118 -1995
- package/dist/worker/git-sync.d.ts +166 -0
- package/dist/worker/git-sync.js +554 -0
- package/dist/worker/history-rewrite.d.ts +129 -0
- package/dist/worker/history-rewrite.js +216 -0
- package/dist/worker/rebase.d.ts +171 -0
- package/dist/worker/rebase.js +859 -0
- package/dist/worker/task-runner.d.ts +93 -0
- package/dist/worker/task-runner.js +529 -0
- package/dist/worker/worker-context.d.ts +133 -0
- package/dist/worker/worker-context.js +1 -0
- package/package.json +1 -1
|
@@ -2,21 +2,21 @@ import fs from 'node:fs/promises';
|
|
|
2
2
|
import path from 'node:path';
|
|
3
3
|
import { simpleGit } from 'simple-git';
|
|
4
4
|
import lockfile from 'proper-lockfile';
|
|
5
|
-
import {
|
|
6
|
-
import {
|
|
7
|
-
import { getBranchMetadataFileManager, BranchMetadataFileManager, buildMergedBranchUpdate, } from '../branch-metadata.js';
|
|
8
|
-
import { extractIdFromFilename } from '../content-id-index.js';
|
|
9
|
-
import { invalidateBranchContentCaches } from '../content-index-generation.js';
|
|
10
|
-
import { ROOT_COLLECTION_ID } from '../paths/types.js';
|
|
5
|
+
import { recoverOrphanedTasks, cmsTaskQueueLogger } from './task-queue.js';
|
|
6
|
+
import { createCanopyOctokit } from '../github-service.js';
|
|
11
7
|
import { sanitizeBranchName, RESERVED_SETTINGS_BRANCH_PREFIX } from '../paths/branch-name.js';
|
|
12
|
-
import { normalizeFilesystemPath } from '../paths/normalize.js';
|
|
13
|
-
import { GITHUB_TRACKING_REF_PREFIX, gitChildEnv, gitNetworkChildEnv } from '../git-manager.js';
|
|
14
8
|
import { resolveDeploymentName } from '../operating-mode/deployment-name.js';
|
|
15
|
-
import { tryAcquireContentWriteLock } from '../utils/content-write-lock.js';
|
|
16
9
|
import { getErrorMessage, isNodeError, redactCredentials } from '../utils/error.js';
|
|
17
|
-
import { isNonFastForwardRejection, isRebaseInProgress, isStaleLeaseRejection } from '../utils/git.js';
|
|
18
10
|
import { writeWorkerStatus } from './worker-status.js';
|
|
19
11
|
import { workerLog, workerLogWarn, workerLogError } from './log.js';
|
|
12
|
+
import { executeTask, orphanRecoveryMaxAgeMs, processTaskQueue, pushBranchToGitHub, updateBranchMetadata, } from './task-runner.js';
|
|
13
|
+
import { pollMergeState, runRebaseCycle } from './rebase.js';
|
|
14
|
+
import { cleanupTrashedBranchDirs, pushSettingsBranches, refreshBaseBranchWorkspace, syncGit, } from './git-sync.js';
|
|
15
|
+
// Re-exported because this module is the package's advertised worker
|
|
16
|
+
// entrypoint (`canopycms/worker/cms-worker`) and both were exported from here
|
|
17
|
+
// before the task-queue cluster moved to ./task-runner. cms-worker.test.ts
|
|
18
|
+
// imports them from this path.
|
|
19
|
+
export { PermanentTaskError, isPermanentTaskFailure } from './task-runner.js';
|
|
20
20
|
// Re-exported so the AWS entrypoint (packages/canopycms-cdk/worker/index.ts)
|
|
21
21
|
// can prefix its own startup lines through the same helpers without adding a
|
|
22
22
|
// new package entrypoint - `canopycms/worker/cms-worker` already exists. Every
|
|
@@ -26,128 +26,6 @@ export { workerLog, workerLogWarn, workerLogError, installWorkerLogger } from '.
|
|
|
26
26
|
const DEFAULT_TASK_TIMEOUT = 60_000;
|
|
27
27
|
const DEFAULT_MAX_RETRIES = 3;
|
|
28
28
|
const DEFAULT_LOCK_STALE_MS = 60_000;
|
|
29
|
-
/**
|
|
30
|
-
* [C1] Retention window for `.trash-*` branch directories left behind by the
|
|
31
|
-
* admin purge action (api/admin-branch-health.ts). Matches
|
|
32
|
-
* cleanupOldTasks's default task retention for consistency.
|
|
33
|
-
*/
|
|
34
|
-
const TRASH_RETENTION_MS = 30 * 24 * 60 * 60_000;
|
|
35
|
-
/** Matches `.trash-{dirName}-{STAMP}` names, capturing the trailing stamp. */
|
|
36
|
-
const TRASH_DIR_STAMP_RE = /-(\d{8}T\d{6}Z)$/;
|
|
37
|
-
/**
|
|
38
|
-
* Parse a purge-generated `YYYYMMDDTHHMMSSZ` stamp into a Date, or null if
|
|
39
|
-
* malformed. Age comes ONLY from this name-embedded stamp, never the dir's
|
|
40
|
-
* own mtime -- `fs.rename` preserves the original directory's mtime, so an
|
|
41
|
-
* mtime-based retention check would delete a months-stale orphan's trash on
|
|
42
|
-
* the very first cleanup pass after purge.
|
|
43
|
-
*/
|
|
44
|
-
function parseTrashStamp(stamp) {
|
|
45
|
-
const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(stamp);
|
|
46
|
-
if (!match)
|
|
47
|
-
return null;
|
|
48
|
-
const [, year, month, day, hour, minute, second] = match;
|
|
49
|
-
const date = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.000Z`);
|
|
50
|
-
return Number.isNaN(date.getTime()) ? null : date;
|
|
51
|
-
}
|
|
52
|
-
/**
|
|
53
|
-
* An error inherent to the task itself (malformed payload, unknown action):
|
|
54
|
-
* retrying can never succeed, so the task should fail fast instead of
|
|
55
|
-
* burning its retry budget.
|
|
56
|
-
*/
|
|
57
|
-
export class PermanentTaskError extends Error {
|
|
58
|
-
}
|
|
59
|
-
/**
|
|
60
|
-
* Classify a task failure as permanent (fail fast) or transient (retry).
|
|
61
|
-
*
|
|
62
|
-
* Transient — worth retrying with backoff:
|
|
63
|
-
* - network errors / anything without an HTTP status (git failures included:
|
|
64
|
-
* most push/fetch failures are connectivity or contention and the retry
|
|
65
|
-
* budget bounds the pathological cases)
|
|
66
|
-
* - HTTP 408 (request timeout) and 429 (rate limited)
|
|
67
|
-
* - HTTP 403 that carries a rate-limit signal (see `isRateLimitSignal403`):
|
|
68
|
-
* GitHub returns 403, not 429, for both primary and secondary/abuse rate
|
|
69
|
-
* limits. The throttling plugin (see github-service.ts createCanopyOctokit)
|
|
70
|
-
* proactively retries short waits, but this carve-out remains the safety
|
|
71
|
-
* net for waits the plugin gives up on (`shouldRetryRateLimit`/
|
|
72
|
-
* `shouldRetrySecondaryRateLimit`) and for errors it never sees — without
|
|
73
|
-
* it a rate-limited push-and-create-or-update-pr task would fail
|
|
74
|
-
* permanently and wedge the branch (`sync-failed`, no retry).
|
|
75
|
-
* - HTTP 5xx (server-side, usually recovers)
|
|
76
|
-
*
|
|
77
|
-
* Permanent — retrying the identical request cannot succeed:
|
|
78
|
-
* - PermanentTaskError (malformed payload, unknown action)
|
|
79
|
-
* - other HTTP 4xx (e.g. 401/404/422): the request itself is bad
|
|
80
|
-
* - plain HTTP 403 with no rate-limit signal: a real permission denial
|
|
81
|
-
*/
|
|
82
|
-
export function isPermanentTaskFailure(err) {
|
|
83
|
-
if (err instanceof PermanentTaskError)
|
|
84
|
-
return true;
|
|
85
|
-
const status = getHttpStatus(err);
|
|
86
|
-
if (status === null)
|
|
87
|
-
return false;
|
|
88
|
-
if (status === 408 || status === 429)
|
|
89
|
-
return false;
|
|
90
|
-
if (status === 403 && isRateLimitSignal403(err))
|
|
91
|
-
return false;
|
|
92
|
-
return status >= 400 && status < 500;
|
|
93
|
-
}
|
|
94
|
-
/** Extract an HTTP status from an error, if present (Octokit RequestError shape). */
|
|
95
|
-
function getHttpStatus(err) {
|
|
96
|
-
if (err instanceof Error && 'status' in err) {
|
|
97
|
-
const status = err.status;
|
|
98
|
-
if (typeof status === 'number')
|
|
99
|
-
return status;
|
|
100
|
-
}
|
|
101
|
-
return null;
|
|
102
|
-
}
|
|
103
|
-
/** Narrow an unknown value to a response-headers record, if present (Octokit lowercases header names). */
|
|
104
|
-
function getResponseHeaders(err) {
|
|
105
|
-
if (typeof err !== 'object' || err === null || !('response' in err))
|
|
106
|
-
return null;
|
|
107
|
-
const response = err.response;
|
|
108
|
-
if (typeof response !== 'object' || response === null || !('headers' in response))
|
|
109
|
-
return null;
|
|
110
|
-
const headers = response.headers;
|
|
111
|
-
if (typeof headers !== 'object' || headers === null)
|
|
112
|
-
return null;
|
|
113
|
-
return headers;
|
|
114
|
-
}
|
|
115
|
-
/**
|
|
116
|
-
* Detect whether a 403 is a GitHub rate-limit response rather than a plain
|
|
117
|
-
* permission denial. GitHub signals rate limiting on 403s three ways: the
|
|
118
|
-
* primary limit zeroes out `x-ratelimit-remaining`, secondary/abuse limits
|
|
119
|
-
* often include a `retry-after` header, and both cases produce a message
|
|
120
|
-
* containing "rate limit" (e.g. "You have exceeded a secondary rate limit").
|
|
121
|
-
*/
|
|
122
|
-
function isRateLimitSignal403(err) {
|
|
123
|
-
const headers = getResponseHeaders(err);
|
|
124
|
-
if (headers) {
|
|
125
|
-
if (headers['x-ratelimit-remaining'] === '0')
|
|
126
|
-
return true;
|
|
127
|
-
if (typeof headers['retry-after'] === 'string' && headers['retry-after'].length > 0)
|
|
128
|
-
return true;
|
|
129
|
-
}
|
|
130
|
-
if (err instanceof Error && /rate limit/i.test(err.message))
|
|
131
|
-
return true;
|
|
132
|
-
return false;
|
|
133
|
-
}
|
|
134
|
-
// Payload validation helpers — fail fast with clear errors instead of silent `as` casts
|
|
135
|
-
function requireString(payload, key) {
|
|
136
|
-
const val = payload[key];
|
|
137
|
-
if (typeof val !== 'string')
|
|
138
|
-
throw new PermanentTaskError(`Task payload missing required string field: ${key}`);
|
|
139
|
-
return val;
|
|
140
|
-
}
|
|
141
|
-
function requireNumber(payload, key) {
|
|
142
|
-
const val = payload[key];
|
|
143
|
-
if (typeof val !== 'number')
|
|
144
|
-
throw new PermanentTaskError(`Task payload missing required number field: ${key}`);
|
|
145
|
-
return val;
|
|
146
|
-
}
|
|
147
|
-
function optionalString(payload, key, fallback) {
|
|
148
|
-
const val = payload[key];
|
|
149
|
-
return typeof val === 'string' ? val : fallback;
|
|
150
|
-
}
|
|
151
29
|
/**
|
|
152
30
|
* CMS Worker daemon.
|
|
153
31
|
* Handles operations that Lambda (with no internet) cannot perform:
|
|
@@ -231,6 +109,46 @@ export class CmsWorker {
|
|
|
231
109
|
}
|
|
232
110
|
return this.settingsBranchResolved;
|
|
233
111
|
}
|
|
112
|
+
/**
|
|
113
|
+
* Build the {@link WorkerContext} handed to the extracted clusters
|
|
114
|
+
* (task-runner.ts, git-sync.ts, rebase.ts, history-rewrite.ts).
|
|
115
|
+
*
|
|
116
|
+
* Built FRESH on every call rather than once in the constructor, and the
|
|
117
|
+
* instance-backed members are functions rather than copied values. Both
|
|
118
|
+
* choices exist for the same reason: the test suite drives this class by
|
|
119
|
+
* replacing `octokit` and `buildGitHubUrl` ON THE INSTANCE, by setting
|
|
120
|
+
* `running` directly, and by subclassing to override the two rebase test
|
|
121
|
+
* hooks. A context that captured any of those at construction time would hand
|
|
122
|
+
* the extracted code the pre-test value -- which for `buildGitHubUrl` means a
|
|
123
|
+
* test's push going to github.com for real instead of its local fixture repo.
|
|
124
|
+
* See WorkerContext's doc comment for the full list.
|
|
125
|
+
*/
|
|
126
|
+
ctx() {
|
|
127
|
+
return {
|
|
128
|
+
githubOwner: this.config.githubOwner,
|
|
129
|
+
githubRepo: this.config.githubRepo,
|
|
130
|
+
baseBranch: this.baseBranch,
|
|
131
|
+
sanitizedBaseBranch: this.sanitizedBaseBranch,
|
|
132
|
+
taskDir: this.taskDir,
|
|
133
|
+
remoteGitPath: this.remoteGitPath,
|
|
134
|
+
contentBranchesPath: this.contentBranchesPath,
|
|
135
|
+
contentRoot: this.contentRoot,
|
|
136
|
+
taskTimeoutMs: this.taskTimeoutMs,
|
|
137
|
+
maxTasksPerCycle: this.maxTasksPerCycle,
|
|
138
|
+
maxRetries: this.maxRetries,
|
|
139
|
+
log: this.log,
|
|
140
|
+
octokit: () => this.octokit,
|
|
141
|
+
buildGitHubUrl: () => this.buildGitHubUrl(),
|
|
142
|
+
branchWorkspacePath: (branchRefName) => this.branchWorkspacePath(branchRefName),
|
|
143
|
+
executeTask: (task, signal) => this.executeTask(task, signal),
|
|
144
|
+
pushBranchToGitHub: (branch) => this.pushBranchToGitHub(branch),
|
|
145
|
+
isRunning: () => this.running,
|
|
146
|
+
ensureStatusReport: () => this.ensureStatusReport(),
|
|
147
|
+
ensureSettingsBranch: () => this.ensureSettingsBranch(),
|
|
148
|
+
afterConflictDetectedForTesting: () => this.afterConflictDetectedForTesting(),
|
|
149
|
+
afterRebaseCompletedForTesting: () => this.afterRebaseCompletedForTesting(),
|
|
150
|
+
};
|
|
151
|
+
}
|
|
234
152
|
async start() {
|
|
235
153
|
this.running = true;
|
|
236
154
|
workerLog('CMS Worker starting...');
|
|
@@ -265,7 +183,7 @@ export class CmsWorker {
|
|
|
265
183
|
// doc comment for why a boot-only call is insufficient once the
|
|
266
184
|
// worker's ASG rolls on every `cdk deploy` (CanopyCmsService's
|
|
267
185
|
// UpdatePolicy).
|
|
268
|
-
const recovered = await recoverOrphanedTasks(this.taskDir, this.
|
|
186
|
+
const recovered = await recoverOrphanedTasks(this.taskDir, orphanRecoveryMaxAgeMs(this.ctx()), this.log);
|
|
269
187
|
if (recovered > 0) {
|
|
270
188
|
workerLog(`Recovered ${recovered} orphaned task(s)`);
|
|
271
189
|
}
|
|
@@ -283,7 +201,7 @@ export class CmsWorker {
|
|
|
283
201
|
// a status-write failure must never block releasing the lock.
|
|
284
202
|
const report = this.ensureStatusReport();
|
|
285
203
|
report.lastFatalError = {
|
|
286
|
-
// [
|
|
204
|
+
// [REDACT] Persisted to worker-status.json and served to the
|
|
287
205
|
// browser by the admin panel -- must never carry the bot token
|
|
288
206
|
// that a poisoned/failed git URL (buildGitHubUrl()) can embed.
|
|
289
207
|
message: redactCredentials(getErrorMessage(err)),
|
|
@@ -594,319 +512,37 @@ export class CmsWorker {
|
|
|
594
512
|
await fs.rename(stagingPath, this.remoteGitPath);
|
|
595
513
|
workerLog('remote.git initialized');
|
|
596
514
|
}
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
515
|
+
// --- Task-queue cluster (worker/task-runner.ts) ------------------------
|
|
516
|
+
//
|
|
517
|
+
// READ THIS BEFORE STUBBING ANY DELEGATOR BELOW. They are not all the same,
|
|
518
|
+
// and the difference is invisible from here.
|
|
519
|
+
//
|
|
520
|
+
// `processTaskQueue` is the public loop entry `scheduleLoop` drives, so it IS
|
|
521
|
+
// on the production path. The other three exist ONLY so the existing test
|
|
522
|
+
// files can reach the implementations through the instance; production never
|
|
523
|
+
// dispatches through them, because the extracted modules call each other
|
|
524
|
+
// directly at module level.
|
|
525
|
+
//
|
|
526
|
+
// The consequence: replacing one of these on an instance only affects
|
|
527
|
+
// production behaviour if the context also routes it. Two do --
|
|
528
|
+
// `executeTask` and `pushBranchToGitHub` are on `WorkerContext` precisely
|
|
529
|
+
// because cms-worker.test.ts assigns over them and then asserts on the
|
|
530
|
+
// replacement. `updateBranchMetadata` is NOT: the test calls it directly and
|
|
531
|
+
// never stubs it, so a stub installed there today would be a silent no-op.
|
|
532
|
+
// If you need to stub a method that isn't on the context, add it to
|
|
533
|
+
// WorkerContext and route the internal caller through `ctx` -- do not assume
|
|
534
|
+
// the delegator's existence means the stub takes effect.
|
|
616
535
|
async processTaskQueue() {
|
|
617
|
-
|
|
618
|
-
return;
|
|
619
|
-
// Recover tasks orphaned in processing/ on EVERY cycle, not only at boot
|
|
620
|
-
// (start()'s call above). Before this fix, recoverOrphanedTasks() ran
|
|
621
|
-
// exactly once, at start() - fine when an instance was replaced rarely
|
|
622
|
-
// (a spot interruption), but CanopyCmsService's worker ASG now rolls on
|
|
623
|
-
// every `cdk deploy` (see its UpdatePolicy), making replacement routine.
|
|
624
|
-
// A replacement instance's user-data (yum install git/unzip/nodejs/
|
|
625
|
-
// efs-utils, mount EFS) takes roughly 2-4 minutes, well under
|
|
626
|
-
// recoverOrphanedTasks()'s 5-minute default staleness threshold - so
|
|
627
|
-
// THAT ONE boot-time call would see the just-orphaned file as "too
|
|
628
|
-
// fresh" and skip it, and nothing rescanned afterward: the task (and its
|
|
629
|
-
// branch's syncStatus) would be wedged forever. Re-checking every poll
|
|
630
|
-
// means the file's age eventually crosses the threshold and it gets
|
|
631
|
-
// recovered without operator intervention.
|
|
632
|
-
//
|
|
633
|
-
// Safe to run this often: executeTaskWithTimeout() guarantees every task
|
|
634
|
-
// THIS process dequeues is completed, failed, or retried (all three
|
|
635
|
-
// remove the processing/ file) within taskTimeoutMs - and the staleness
|
|
636
|
-
// threshold is derived from taskTimeoutMs (orphanRecoveryMaxAgeMs below)
|
|
637
|
-
// precisely so that guarantee holds for ANY configured timeout, not just
|
|
638
|
-
// the 60s default: a fixed 5-minute threshold would have this call steal
|
|
639
|
-
// the worker's own still-in-flight task back to pending whenever an
|
|
640
|
-
// adopter configured taskTimeoutMs above ~5 minutes.
|
|
641
|
-
const recovered = await recoverOrphanedTasks(this.taskDir, this.orphanRecoveryMaxAgeMs(), this.log);
|
|
642
|
-
if (recovered > 0) {
|
|
643
|
-
workerLog(`Recovered ${recovered} orphaned task(s)`);
|
|
644
|
-
}
|
|
645
|
-
let processed = 0;
|
|
646
|
-
let task;
|
|
647
|
-
while (processed < this.maxTasksPerCycle &&
|
|
648
|
-
(task = await dequeueTask(this.taskDir, this.log)) !== null) {
|
|
649
|
-
try {
|
|
650
|
-
const result = await this.executeTaskWithTimeout(task);
|
|
651
|
-
await completeTask(this.taskDir, task.id, result, this.log);
|
|
652
|
-
await this.updateBranchMetadata(task, result);
|
|
653
|
-
}
|
|
654
|
-
catch (err) {
|
|
655
|
-
const message = getErrorMessage(err);
|
|
656
|
-
workerLogError(`Task ${task.id} (${task.action}) failed:`, message);
|
|
657
|
-
// [HIGH-1] task.error is persisted (pending/failed task JSON) and
|
|
658
|
-
// served to the browser by the admin panel's Tasks tab -- a push
|
|
659
|
-
// failure's message can embed the bot token via buildGitHubUrl().
|
|
660
|
-
// Console output above stays raw (journald/CloudWatch is trusted).
|
|
661
|
-
const persistedMessage = redactCredentials(message);
|
|
662
|
-
// DEP-L1: only transient failures (network, 429/5xx, timeouts) are
|
|
663
|
-
// worth retrying; permanent ones (malformed payload, other 4xx) would
|
|
664
|
-
// just burn the retry budget on an identical doomed request.
|
|
665
|
-
const permanent = isPermanentTaskFailure(err);
|
|
666
|
-
const retryCount = task.retryCount ?? 0;
|
|
667
|
-
const maxRetries = task.maxRetries ?? this.maxRetries;
|
|
668
|
-
if (!permanent && retryCount < maxRetries) {
|
|
669
|
-
await retryTask(this.taskDir, task.id, persistedMessage, this.log);
|
|
670
|
-
workerLog(` Will retry (attempt ${retryCount + 1}/${maxRetries})`);
|
|
671
|
-
}
|
|
672
|
-
else {
|
|
673
|
-
await failTask(this.taskDir, task.id, persistedMessage, this.log);
|
|
674
|
-
await this.updateBranchMetadataOnFailure(task, persistedMessage);
|
|
675
|
-
workerLogError(permanent
|
|
676
|
-
? ' Permanently failed (non-retryable error)'
|
|
677
|
-
: ` Permanently failed after ${maxRetries} retries`);
|
|
678
|
-
}
|
|
679
|
-
}
|
|
680
|
-
processed++;
|
|
681
|
-
}
|
|
682
|
-
// PR-W1: only stamp/write when work actually happened this poll --
|
|
683
|
-
// otherwise every idle 5s poll would hit worker-status.json, an EFS
|
|
684
|
-
// write treadmill for no signal (liveness is already covered by the
|
|
685
|
-
// lock heartbeat; see api/admin.ts's classifyWorkerLiveness).
|
|
686
|
-
if (processed > 0) {
|
|
687
|
-
const report = this.ensureStatusReport();
|
|
688
|
-
report.lastTaskCycleAt = new Date().toISOString();
|
|
689
|
-
await writeWorkerStatus(this.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
|
|
690
|
-
}
|
|
691
|
-
}
|
|
692
|
-
/**
|
|
693
|
-
* Execute a task bounded by taskTimeoutMs (DEP-H1). Two layers:
|
|
694
|
-
* - An AbortSignal cancels Octokit HTTP calls promptly.
|
|
695
|
-
* - A Promise.race rejects when the timeout fires, so work that cannot
|
|
696
|
-
* observe the signal (git subprocesses via simple-git) still fails the
|
|
697
|
-
* attempt and the worker moves on instead of stalling forever.
|
|
698
|
-
* pushBranchToGitHub additionally kills stalled git processes via
|
|
699
|
-
* simple-git's block timeout, so a hung push doesn't leak a process.
|
|
700
|
-
*/
|
|
701
|
-
async executeTaskWithTimeout(task) {
|
|
702
|
-
const controller = new AbortController();
|
|
703
|
-
const timer = setTimeout(() => controller.abort(), this.taskTimeoutMs);
|
|
704
|
-
try {
|
|
705
|
-
const work = this.executeTask(task, controller.signal);
|
|
706
|
-
// If the timeout wins the race, the losing promise must not surface an
|
|
707
|
-
// unhandled rejection when it eventually settles.
|
|
708
|
-
work.catch(() => { });
|
|
709
|
-
const timedOut = new Promise((_, reject) => {
|
|
710
|
-
controller.signal.addEventListener('abort', () => reject(new Error(`Task timed out after ${this.taskTimeoutMs}ms`)), { once: true });
|
|
711
|
-
});
|
|
712
|
-
return await Promise.race([work, timedOut]);
|
|
713
|
-
}
|
|
714
|
-
finally {
|
|
715
|
-
clearTimeout(timer);
|
|
716
|
-
}
|
|
536
|
+
return processTaskQueue(this.ctx());
|
|
717
537
|
}
|
|
718
538
|
async executeTask(task, signal) {
|
|
719
|
-
|
|
720
|
-
switch (action) {
|
|
721
|
-
case 'push-branch': {
|
|
722
|
-
const branch = requireString(payload, 'branch');
|
|
723
|
-
await this.pushBranchToGitHub(branch);
|
|
724
|
-
return { pushed: true };
|
|
725
|
-
}
|
|
726
|
-
case 'push-and-create-pr': {
|
|
727
|
-
const branch = requireString(payload, 'branch');
|
|
728
|
-
await this.pushBranchToGitHub(branch);
|
|
729
|
-
const pr = await this.octokit.pulls.create({
|
|
730
|
-
owner: this.config.githubOwner,
|
|
731
|
-
repo: this.config.githubRepo,
|
|
732
|
-
head: branch,
|
|
733
|
-
base: optionalString(payload, 'baseBranch', this.baseBranch),
|
|
734
|
-
title: optionalString(payload, 'title', `Submit ${branch}`),
|
|
735
|
-
body: optionalString(payload, 'body', ''),
|
|
736
|
-
request: { signal },
|
|
737
|
-
});
|
|
738
|
-
workerLog(`Created PR #${pr.data.number} for ${branch}`);
|
|
739
|
-
return { prUrl: pr.data.html_url, prNumber: pr.data.number };
|
|
740
|
-
}
|
|
741
|
-
case 'push-and-update-pr': {
|
|
742
|
-
const branch = requireString(payload, 'branch');
|
|
743
|
-
const prNumber = requireNumber(payload, 'pullRequestNumber');
|
|
744
|
-
await this.pushBranchToGitHub(branch);
|
|
745
|
-
await this.octokit.pulls.update({
|
|
746
|
-
owner: this.config.githubOwner,
|
|
747
|
-
repo: this.config.githubRepo,
|
|
748
|
-
pull_number: prNumber,
|
|
749
|
-
title: optionalString(payload, 'title', `Submit ${branch}`),
|
|
750
|
-
body: optionalString(payload, 'body', ''),
|
|
751
|
-
request: { signal },
|
|
752
|
-
});
|
|
753
|
-
workerLog(`Updated PR #${prNumber} for ${branch}`);
|
|
754
|
-
return { prNumber };
|
|
755
|
-
}
|
|
756
|
-
case 'push-and-create-or-update-pr': {
|
|
757
|
-
// GIT-H1: idempotent create-or-update. Used for both content-branch
|
|
758
|
-
// submits and settings-branch syncs so a retry after a crash (task
|
|
759
|
-
// completed on GitHub but branch metadata never recorded the PR
|
|
760
|
-
// number) recovers the existing PR instead of hitting the 422 that
|
|
761
|
-
// a blind `pulls.create` would throw on a duplicate head+base.
|
|
762
|
-
// Delegates to the shared helper (also used by GitHubService's
|
|
763
|
-
// direct-API path) so the list->tiebreak->update/create logic and
|
|
764
|
-
// the draft->ready conversion live in one place.
|
|
765
|
-
const branch = requireString(payload, 'branch');
|
|
766
|
-
const base = optionalString(payload, 'baseBranch', this.baseBranch);
|
|
767
|
-
// Defense-in-depth: refuse head===base even if the 'submittableBranch'
|
|
768
|
-
// API guard and the syncSubmitPr backstop were both somehow bypassed
|
|
769
|
-
// (e.g. a task queued before this check shipped). PermanentTaskError
|
|
770
|
-
// (not a plain Error) so this fails immediately instead of burning
|
|
771
|
-
// the retry budget on an identical doomed request -- retrying can
|
|
772
|
-
// never make the branch not be the base branch.
|
|
773
|
-
if (sanitizeBranchName(branch) === this.sanitizedBaseBranch) {
|
|
774
|
-
throw new PermanentTaskError(`Refusing to push-and-create-or-update-pr for "${branch}": it is the base branch -- submitting the base branch is never valid`);
|
|
775
|
-
}
|
|
776
|
-
await this.pushBranchToGitHub(branch);
|
|
777
|
-
const result = await createOrUpdatePullRequest({
|
|
778
|
-
octokit: this.octokit,
|
|
779
|
-
owner: this.config.githubOwner,
|
|
780
|
-
repo: this.config.githubRepo,
|
|
781
|
-
head: branch,
|
|
782
|
-
base,
|
|
783
|
-
title: optionalString(payload, 'title', `Submit ${branch}`),
|
|
784
|
-
body: optionalString(payload, 'body', ''),
|
|
785
|
-
// Content submits (api/github-sync.ts) set this; settings-branch
|
|
786
|
-
// syncs (services.ts) deliberately don't.
|
|
787
|
-
markReadyIfDraft: payload.markReadyIfDraft === true,
|
|
788
|
-
signal,
|
|
789
|
-
});
|
|
790
|
-
workerLog(result.created
|
|
791
|
-
? `Created PR #${result.number} for ${branch}`
|
|
792
|
-
: `Updated existing PR #${result.number} for ${branch}`);
|
|
793
|
-
return { prUrl: result.url, prNumber: result.number };
|
|
794
|
-
}
|
|
795
|
-
case 'convert-to-draft': {
|
|
796
|
-
const draftPrNumber = requireNumber(payload, 'pullRequestNumber');
|
|
797
|
-
// GitHub REST API doesn't support converting to draft directly.
|
|
798
|
-
// Use the GraphQL API via Octokit.
|
|
799
|
-
const { data: pr } = await this.octokit.pulls.get({
|
|
800
|
-
owner: this.config.githubOwner,
|
|
801
|
-
repo: this.config.githubRepo,
|
|
802
|
-
pull_number: draftPrNumber,
|
|
803
|
-
request: { signal },
|
|
804
|
-
});
|
|
805
|
-
await this.octokit.graphql(`mutation($id: ID!) { convertPullRequestToDraft(input: { pullRequestId: $id }) { pullRequest { isDraft } } }`, { id: pr.node_id, request: { signal } });
|
|
806
|
-
workerLog(`Converted PR #${draftPrNumber} to draft`);
|
|
807
|
-
return { prNumber: draftPrNumber, draft: true };
|
|
808
|
-
}
|
|
809
|
-
case 'close-pr': {
|
|
810
|
-
const closePrNumber = requireNumber(payload, 'pullRequestNumber');
|
|
811
|
-
await this.octokit.pulls.update({
|
|
812
|
-
owner: this.config.githubOwner,
|
|
813
|
-
repo: this.config.githubRepo,
|
|
814
|
-
pull_number: closePrNumber,
|
|
815
|
-
state: 'closed',
|
|
816
|
-
request: { signal },
|
|
817
|
-
});
|
|
818
|
-
return { closed: true };
|
|
819
|
-
}
|
|
820
|
-
case 'delete-remote-branch': {
|
|
821
|
-
const branch = requireString(payload, 'branch');
|
|
822
|
-
await this.octokit.git.deleteRef({
|
|
823
|
-
owner: this.config.githubOwner,
|
|
824
|
-
repo: this.config.githubRepo,
|
|
825
|
-
ref: `heads/${branch}`,
|
|
826
|
-
request: { signal },
|
|
827
|
-
});
|
|
828
|
-
return { deleted: true };
|
|
829
|
-
}
|
|
830
|
-
default:
|
|
831
|
-
throw new PermanentTaskError(`Unknown task action: ${action}`);
|
|
832
|
-
}
|
|
539
|
+
return executeTask(this.ctx(), task, signal);
|
|
833
540
|
}
|
|
834
|
-
/**
|
|
835
|
-
* Update branch metadata after successful task completion.
|
|
836
|
-
* Writes PR URL/number and sets syncStatus to 'synced'.
|
|
837
|
-
*/
|
|
838
541
|
async updateBranchMetadata(task, result) {
|
|
839
|
-
|
|
840
|
-
if (!branch)
|
|
841
|
-
return;
|
|
842
|
-
const branchPath = this.branchWorkspacePath(branch);
|
|
843
|
-
try {
|
|
844
|
-
await fs.stat(branchPath);
|
|
845
|
-
}
|
|
846
|
-
catch {
|
|
847
|
-
return; // Branch directory doesn't exist
|
|
848
|
-
}
|
|
849
|
-
try {
|
|
850
|
-
const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
|
|
851
|
-
const updates = {
|
|
852
|
-
name: branch,
|
|
853
|
-
syncStatus: 'synced',
|
|
854
|
-
// save()'s merge only overwrites keys present in this update (see
|
|
855
|
-
// BranchMetadataFileManager.save()'s spread order) -- a prior
|
|
856
|
-
// syncFailureReason from an earlier failed attempt would otherwise
|
|
857
|
-
// survive forever past this successful sync. Explicitly clear it,
|
|
858
|
-
// same pattern as rebaseFailure's PR-W2 M2 clearing elsewhere.
|
|
859
|
-
syncFailureReason: undefined,
|
|
860
|
-
};
|
|
861
|
-
if (result.prUrl)
|
|
862
|
-
updates.pullRequestUrl = result.prUrl;
|
|
863
|
-
if (result.prNumber) {
|
|
864
|
-
updates.pullRequestNumber = result.prNumber;
|
|
865
|
-
// As soon as the PR exists the field should read 'open' -- without
|
|
866
|
-
// this it would stay absent until the next poll cycle observes it.
|
|
867
|
-
// Exception: the git-sync loop may have archived this branch (PR
|
|
868
|
-
// merged/closed) while this task was in flight; a late task
|
|
869
|
-
// completion must not downgrade that terminal state back to 'open'.
|
|
870
|
-
const current = await BranchMetadataFileManager.loadOnly(branchPath);
|
|
871
|
-
const terminal = current?.branch.status === 'archived' ||
|
|
872
|
-
current?.branch.pullRequestState === 'merged' ||
|
|
873
|
-
current?.branch.pullRequestState === 'closed';
|
|
874
|
-
if (!terminal) {
|
|
875
|
-
updates.pullRequestState = 'open';
|
|
876
|
-
}
|
|
877
|
-
}
|
|
878
|
-
await meta.save({ branch: updates });
|
|
879
|
-
}
|
|
880
|
-
catch (err) {
|
|
881
|
-
workerLogError(`Failed to update metadata for ${branch}:`, err instanceof Error ? err.message : err);
|
|
882
|
-
}
|
|
542
|
+
return updateBranchMetadata(this.ctx(), task, result);
|
|
883
543
|
}
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
* Sets syncStatus to 'sync-failed' and records `error` (already redacted
|
|
887
|
-
* by the caller, processTaskQueue -- see [HIGH-1] there) as
|
|
888
|
-
* syncFailureReason, so the editor can show WHY, not just that it failed.
|
|
889
|
-
*/
|
|
890
|
-
async updateBranchMetadataOnFailure(task, error) {
|
|
891
|
-
const branch = typeof task.payload.branch === 'string' ? task.payload.branch : null;
|
|
892
|
-
if (!branch)
|
|
893
|
-
return;
|
|
894
|
-
const branchPath = this.branchWorkspacePath(branch);
|
|
895
|
-
try {
|
|
896
|
-
await fs.stat(branchPath);
|
|
897
|
-
}
|
|
898
|
-
catch {
|
|
899
|
-
return;
|
|
900
|
-
}
|
|
901
|
-
try {
|
|
902
|
-
const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
|
|
903
|
-
await meta.save({
|
|
904
|
-
branch: { name: branch, syncStatus: 'sync-failed', syncFailureReason: error },
|
|
905
|
-
});
|
|
906
|
-
}
|
|
907
|
-
catch (err) {
|
|
908
|
-
workerLogError(`Failed to update failure metadata for ${branch}:`, err instanceof Error ? err.message : err);
|
|
909
|
-
}
|
|
544
|
+
async pushBranchToGitHub(branch) {
|
|
545
|
+
return pushBranchToGitHub(this.ctx(), branch);
|
|
910
546
|
}
|
|
911
547
|
buildGitHubUrl() {
|
|
912
548
|
return `https://x-access-token:${this.config.githubToken}@github.com/${this.config.githubOwner}/${this.config.githubRepo}.git`;
|
|
@@ -930,941 +566,6 @@ export class CmsWorker {
|
|
|
930
566
|
branchWorkspacePath(branchRefName) {
|
|
931
567
|
return path.join(this.contentBranchesPath, sanitizeBranchName(branchRefName));
|
|
932
568
|
}
|
|
933
|
-
/**
|
|
934
|
-
* What `remote.git` currently holds for `branchRef`, or null when this
|
|
935
|
-
* branch was never published there (never submitted) or the ref is
|
|
936
|
-
* unreadable.
|
|
937
|
-
*
|
|
938
|
-
* Uses the same explicit `--git-dir` shape as verifyBaseBranchExists()
|
|
939
|
-
* above: reading a bare repo that way does not depend on
|
|
940
|
-
* `safe.bareRepository` being permissive, which is why prod code takes
|
|
941
|
-
* this route rather than the config override the test-only `openBareRepo`
|
|
942
|
-
* helper uses.
|
|
943
|
-
*/
|
|
944
|
-
async readPublishedSha(branchRef) {
|
|
945
|
-
try {
|
|
946
|
-
const out = await simpleGit().raw([
|
|
947
|
-
'--git-dir',
|
|
948
|
-
this.remoteGitPath,
|
|
949
|
-
'rev-parse',
|
|
950
|
-
'--verify',
|
|
951
|
-
`refs/heads/${branchRef}`,
|
|
952
|
-
]);
|
|
953
|
-
const sha = out.trim();
|
|
954
|
-
return sha.length > 0 ? sha : null;
|
|
955
|
-
}
|
|
956
|
-
catch {
|
|
957
|
-
// Absent from remote.git: this branch was never submitted, so none of
|
|
958
|
-
// its history was ever published. Not an error.
|
|
959
|
-
return null;
|
|
960
|
-
}
|
|
961
|
-
}
|
|
962
|
-
/**
|
|
963
|
-
* Record that this worker rewrote `expectedSha` out of a branch's already
|
|
964
|
-
* published history (see BranchMetadata.historyRewrittenFrom).
|
|
965
|
-
*
|
|
966
|
-
* Set-once: if a marker is already present it is LEFT ALONE. Across two
|
|
967
|
-
* rebases before any GitHub push lands, GitHub still holds the commit the
|
|
968
|
-
* FIRST rebase replaced, so advancing the marker would aim the lease at a
|
|
969
|
-
* commit GitHub never had and permanently wedge the branch.
|
|
970
|
-
*
|
|
971
|
-
* Re-reads metadata rather than trusting the caller's loop-top snapshot:
|
|
972
|
-
* the task loop runs concurrently with syncGit() (see scheduleLoop) and may
|
|
973
|
-
* have cleared the marker while this branch was rebasing.
|
|
974
|
-
*/
|
|
975
|
-
async markHistoryRewritten(branchPath, branchDir, expectedSha) {
|
|
976
|
-
const current = await BranchMetadataFileManager.loadOnly(branchPath);
|
|
977
|
-
if (typeof current?.branch.historyRewrittenFrom === 'string')
|
|
978
|
-
return;
|
|
979
|
-
const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
|
|
980
|
-
await meta.save({ branch: { name: branchDir, historyRewrittenFrom: expectedSha } });
|
|
981
|
-
}
|
|
982
|
-
/**
|
|
983
|
-
* Clear the marker once GitHub is confirmed to hold the rewritten history.
|
|
984
|
-
*
|
|
985
|
-
* Best-effort only in the sense that a failure here destroys nothing: the
|
|
986
|
-
* lease still refuses anything unexpected, and the plain-push fallback only
|
|
987
|
-
* ever fast-forwards. It is NOT harmless. A marker that outlives its
|
|
988
|
-
* episode can wedge the NEXT one -- if the base advances before the stale
|
|
989
|
-
* marker is revisited, the queued push leases a commit GitHub has already
|
|
990
|
-
* moved off, falls back to a plain push of a rebased (non-ancestor)
|
|
991
|
-
* history, and fails permanently with a "something else moved it on
|
|
992
|
-
* GitHub" diagnosis that is false: we did.
|
|
993
|
-
*
|
|
994
|
-
* Tracked, with the concurrent-clear race that can drop a marker
|
|
995
|
-
* mid-arming, in
|
|
996
|
-
* .claude/future-tasks/worker-history-rewrite-marker-races.md.
|
|
997
|
-
*/
|
|
998
|
-
async clearHistoryRewrittenMarker(branchPath, branchDir) {
|
|
999
|
-
try {
|
|
1000
|
-
const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
|
|
1001
|
-
// Explicit undefined clears the key -- save()'s merge only overwrites
|
|
1002
|
-
// keys present in the update (same pattern as rebaseFailure).
|
|
1003
|
-
await meta.save({ branch: { name: branchDir, historyRewrittenFrom: undefined } });
|
|
1004
|
-
}
|
|
1005
|
-
catch (err) {
|
|
1006
|
-
workerLogWarn(` Failed to clear history-rewrite marker for ${branchDir}: ${getErrorMessage(err)}`);
|
|
1007
|
-
}
|
|
1008
|
-
}
|
|
1009
|
-
/**
|
|
1010
|
-
* Publish a branch clone's rebased history into `remote.git`, replacing
|
|
1011
|
-
* EXACTLY `expectedSha` and nothing else.
|
|
1012
|
-
*
|
|
1013
|
-
* The lease is the entire safety argument. `--force-with-lease=<ref>:<sha>`
|
|
1014
|
-
* refuses unless `remote.git` still stands at `<sha>`, so this can only
|
|
1015
|
-
* ever undo the commit our own rebase rewrote away. Callers must never pass
|
|
1016
|
-
* "whatever remote.git currently holds" -- see the arming guard in
|
|
1017
|
-
* rebaseActiveBranches() for the interleaving where that would silently
|
|
1018
|
-
* delete a reviewer's direct push.
|
|
1019
|
-
*
|
|
1020
|
-
* Returns whether the push landed. A refused lease means a concurrent
|
|
1021
|
-
* Lambda push moved the ref; that is logged and retried by the self-heal
|
|
1022
|
-
* pass on a later cycle, never thrown.
|
|
1023
|
-
*/
|
|
1024
|
-
async forcePublishToLocalRemote(branchPath, branchRef, expectedSha) {
|
|
1025
|
-
// A dedicated instance rather than the caller's: `.env()` replaces the
|
|
1026
|
-
// whole child environment, and the rebase loop's instance must keep its
|
|
1027
|
-
// ambient one. gitChildEnv (not gitNetworkChildEnv) because this push
|
|
1028
|
-
// targets the local bare repo -- and the locale pin is what keeps
|
|
1029
|
-
// isStaleLeaseRejection below from silently becoming a no-op.
|
|
1030
|
-
const pushGit = simpleGit({ baseDir: branchPath, timeout: { block: this.taskTimeoutMs } });
|
|
1031
|
-
pushGit.env(gitChildEnv({}));
|
|
1032
|
-
try {
|
|
1033
|
-
await pushGit.raw([
|
|
1034
|
-
'push',
|
|
1035
|
-
`--force-with-lease=${branchRef}:${expectedSha}`,
|
|
1036
|
-
// Real flags must precede --end-of-options; everything after it is
|
|
1037
|
-
// positional (see GitManager.push() for the same guard).
|
|
1038
|
-
'--end-of-options',
|
|
1039
|
-
'origin',
|
|
1040
|
-
`${branchRef}:${branchRef}`,
|
|
1041
|
-
]);
|
|
1042
|
-
workerLog(` Published rebased ${branchRef} into remote.git`);
|
|
1043
|
-
return true;
|
|
1044
|
-
}
|
|
1045
|
-
catch (err) {
|
|
1046
|
-
const message = redactCredentials(getErrorMessage(err));
|
|
1047
|
-
workerLogWarn(isStaleLeaseRejection(message)
|
|
1048
|
-
? ` Did not publish rebased ${branchRef} into remote.git: it moved since this cycle read it (concurrent submit?) -- retrying next cycle`
|
|
1049
|
-
: ` Failed to publish rebased ${branchRef} into remote.git: ${message}`);
|
|
1050
|
-
return false;
|
|
1051
|
-
}
|
|
1052
|
-
}
|
|
1053
|
-
/**
|
|
1054
|
-
* Queue the GitHub hop for a branch whose rewritten history now sits in
|
|
1055
|
-
* `remote.git`, so an open PR's head follows the rebase within a cycle
|
|
1056
|
-
* instead of waiting for the editor's next submit.
|
|
1057
|
-
*
|
|
1058
|
-
* Deliberately NOT skipped when a marker is already set: inferring "a task
|
|
1059
|
-
* must already be queued" from the marker starves this hop whenever a task
|
|
1060
|
-
* was lost, failed permanently, or was never written. Duplicate push tasks
|
|
1061
|
-
* are bounded by base-branch advances and are idempotent (a repeat push is
|
|
1062
|
-
* a no-op once GitHub holds the tip).
|
|
1063
|
-
*/
|
|
1064
|
-
async enqueueGitHubPush(branchRef) {
|
|
1065
|
-
try {
|
|
1066
|
-
// Dedupe against tasks actually in flight rather than against the
|
|
1067
|
-
// marker: a branch whose GitHub push keeps failing would otherwise
|
|
1068
|
-
// gain one task per sync cycle forever.
|
|
1069
|
-
for (const status of ['pending', 'processing']) {
|
|
1070
|
-
const inFlight = await listTasks(this.taskDir, status);
|
|
1071
|
-
if (inFlight.some((t) => t.action === 'push-branch' && t.payload.branch === branchRef)) {
|
|
1072
|
-
return;
|
|
1073
|
-
}
|
|
1074
|
-
}
|
|
1075
|
-
await enqueueTask(this.taskDir, { action: 'push-branch', payload: { branch: branchRef } });
|
|
1076
|
-
workerLog(` Queued GitHub push for ${branchRef}`);
|
|
1077
|
-
}
|
|
1078
|
-
catch (err) {
|
|
1079
|
-
workerLogWarn(` Failed to queue GitHub push for ${branchRef}: ${getErrorMessage(err)}`);
|
|
1080
|
-
}
|
|
1081
|
-
}
|
|
1082
|
-
/**
|
|
1083
|
-
* Complete a rewrite this worker started but did not finish: get the
|
|
1084
|
-
* rebased history into `remote.git` and queued for GitHub.
|
|
1085
|
-
*
|
|
1086
|
-
* Runs from the rebase loop for any branch carrying a marker, whether or
|
|
1087
|
-
* not it is behind base this cycle, so an interrupted publish converges
|
|
1088
|
-
* without waiting for the next base-branch advance.
|
|
1089
|
-
*
|
|
1090
|
-
* Always leases on the MARKER, never on remote.git's current tip -- the
|
|
1091
|
-
* marker is the one commit we know our own rebase replaced.
|
|
1092
|
-
*/
|
|
1093
|
-
async reconcilePendingRewrite(options) {
|
|
1094
|
-
const { branchPath, branchDir, branchRef, headSha, marker } = options;
|
|
1095
|
-
const publishedSha = await this.readPublishedSha(branchRef);
|
|
1096
|
-
if (publishedSha === null) {
|
|
1097
|
-
workerLogWarn(` ${branchDir}: a rewritten history is recorded but ${branchRef} is gone from remote.git -- nothing to publish`);
|
|
1098
|
-
return;
|
|
1099
|
-
}
|
|
1100
|
-
if (publishedSha === headSha) {
|
|
1101
|
-
// remote.git already carries the rewrite; only the GitHub hop is left
|
|
1102
|
-
// (a crash between the push and the queue, or a task that was lost).
|
|
1103
|
-
await this.enqueueGitHubPush(branchRef);
|
|
1104
|
-
return;
|
|
1105
|
-
}
|
|
1106
|
-
if (publishedSha === marker) {
|
|
1107
|
-
// The publish into remote.git never landed -- a crash right after the
|
|
1108
|
-
// marker was written, or a lease refused by a concurrent submit.
|
|
1109
|
-
if (await this.forcePublishToLocalRemote(branchPath, branchRef, marker)) {
|
|
1110
|
-
await this.enqueueGitHubPush(branchRef);
|
|
1111
|
-
}
|
|
1112
|
-
return;
|
|
1113
|
-
}
|
|
1114
|
-
// Neither this branch's rebased tip nor the commit its rewrite replaced:
|
|
1115
|
-
// something else moved remote.git. Never force over that.
|
|
1116
|
-
workerLogWarn(` ${branchDir}: remote.git is at ${publishedSha} for ${branchRef}, which is neither the ` +
|
|
1117
|
-
`rebased tip nor the commit the rewrite replaced -- left untouched`);
|
|
1118
|
-
}
|
|
1119
|
-
async pushBranchToGitHub(branch) {
|
|
1120
|
-
const git = simpleGit({
|
|
1121
|
-
baseDir: this.remoteGitPath,
|
|
1122
|
-
// DEP-H1: kill the git process if it produces no output for
|
|
1123
|
-
// taskTimeoutMs (network stall, credential prompt) instead of letting
|
|
1124
|
-
// it hang past the task timeout.
|
|
1125
|
-
timeout: { block: this.taskTimeoutMs },
|
|
1126
|
-
});
|
|
1127
|
-
// Force stable (English) git output so isNonFastForwardRejection below
|
|
1128
|
-
// can reliably match it -- git's rejection text is gettext-translated, so
|
|
1129
|
-
// a non-English host would silently turn that classifier into a no-op.
|
|
1130
|
-
// gitNetworkChildEnv (NOT gitChildEnv) because this call talks to GitHub:
|
|
1131
|
-
// it keeps ambient HTTPS_PROXY/GIT_SSL_*/GIT_SSH_COMMAND, which
|
|
1132
|
-
// gitChildEnv's local-ops allowlist deliberately drops.
|
|
1133
|
-
git.env(gitNetworkChildEnv());
|
|
1134
|
-
// [SYNC-H1] If the rebase loop rewrote this branch's already-published
|
|
1135
|
-
// history, GitHub still holds the commit it replaced, so an ordinary
|
|
1136
|
-
// push is non-fast-forward forever. Push under a lease keyed to exactly
|
|
1137
|
-
// that commit: it moves GitHub off the commit we rewrote away, and
|
|
1138
|
-
// refuses in every other case (including a commit someone else pushed).
|
|
1139
|
-
const branchPath = this.branchWorkspacePath(branch);
|
|
1140
|
-
const metaFile = await BranchMetadataFileManager.loadOnly(branchPath).catch(() => null);
|
|
1141
|
-
const marker = metaFile?.branch.historyRewrittenFrom;
|
|
1142
|
-
// What this push will actually send: remote.git's tip for the branch.
|
|
1143
|
-
const outgoingSha = await this.readPublishedSha(branch);
|
|
1144
|
-
try {
|
|
1145
|
-
if (marker) {
|
|
1146
|
-
await git.raw([
|
|
1147
|
-
'push',
|
|
1148
|
-
`--force-with-lease=${branch}:${marker}`,
|
|
1149
|
-
'--end-of-options',
|
|
1150
|
-
this.buildGitHubUrl(),
|
|
1151
|
-
`${branch}:${branch}`,
|
|
1152
|
-
]);
|
|
1153
|
-
}
|
|
1154
|
-
else {
|
|
1155
|
-
// Pass URL directly to avoid persisting the token in remote.git/config
|
|
1156
|
-
await git.push(this.buildGitHubUrl(), branch);
|
|
1157
|
-
}
|
|
1158
|
-
}
|
|
1159
|
-
catch (err) {
|
|
1160
|
-
const message = getErrorMessage(err);
|
|
1161
|
-
// A refused lease means GitHub is not at the commit we rewrote, so the
|
|
1162
|
-
// marker is stale -- which is routine, not exceptional: tasks are
|
|
1163
|
-
// re-run after a crash (recoverOrphanedTasks), and the marker survives
|
|
1164
|
-
// any failure to clear it. Two benign shapes reach here, and both are
|
|
1165
|
-
// ordinary fast-forwards that a lease has no business blocking:
|
|
1166
|
-
// GitHub already holds the rewritten history from an earlier attempt,
|
|
1167
|
-
// or the branch has since moved past it.
|
|
1168
|
-
//
|
|
1169
|
-
// Retry PLAIN, and let git adjudicate. A non-forced push succeeds if
|
|
1170
|
-
// and only if it fast-forwards, so it can never destroy anything --
|
|
1171
|
-
// there is no ancestry check to get wrong here, and no extra network
|
|
1172
|
-
// round trip to read GitHub's tip. Only if THAT is also rejected has
|
|
1173
|
-
// the branch genuinely diverged.
|
|
1174
|
-
//
|
|
1175
|
-
// (Verified: git evaluates the lease only when it actually has an
|
|
1176
|
-
// update to apply. An up-to-date ref with a stale lease prints
|
|
1177
|
-
// "Everything up-to-date" and exits 0, so the already-landed case is
|
|
1178
|
-
// usually absorbed above and never even reaches this branch.)
|
|
1179
|
-
if (marker && isStaleLeaseRejection(message)) {
|
|
1180
|
-
try {
|
|
1181
|
-
await git.push(this.buildGitHubUrl(), branch);
|
|
1182
|
-
}
|
|
1183
|
-
catch (retryErr) {
|
|
1184
|
-
const retryMessage = getErrorMessage(retryErr);
|
|
1185
|
-
if (isNonFastForwardRejection(retryMessage)) {
|
|
1186
|
-
throw new PermanentTaskError(`Push rejected for branch "${branch}": GitHub's tip is neither the commit this ` +
|
|
1187
|
-
`deployment last published nor an ancestor of what it is pushing, so the branch ` +
|
|
1188
|
-
`has genuinely diverged and nothing was overwritten. Something else moved it on ` +
|
|
1189
|
-
`GitHub -- a direct push, or another CanopyCMS deployment sharing this repository.`);
|
|
1190
|
-
}
|
|
1191
|
-
throw retryErr;
|
|
1192
|
-
}
|
|
1193
|
-
// The lease was refused, so GitHub is provably not at the marker: it
|
|
1194
|
-
// has moved past the rewritten commit and the marker is spent.
|
|
1195
|
-
await this.clearHistoryRewrittenMarker(branchPath, branch);
|
|
1196
|
-
workerLog(`Pushed ${branch} to GitHub (GitHub had already moved past the rewritten commit)`);
|
|
1197
|
-
return;
|
|
1198
|
-
}
|
|
1199
|
-
// An ordinary non-fast-forward rejection: GitHub has commits this
|
|
1200
|
-
// deployment never published. Retrying the identical push can never
|
|
1201
|
-
// succeed (DEP-L1's git-failure-is-transient carve-out does NOT apply
|
|
1202
|
-
// here), so fail fast instead of burning the task's retry budget.
|
|
1203
|
-
// Deliberately does NOT advise renaming the branch: a branch reaching
|
|
1204
|
-
// this point usually has an open PR, and renaming would orphan it.
|
|
1205
|
-
if (isNonFastForwardRejection(message)) {
|
|
1206
|
-
throw new PermanentTaskError(`Push rejected for branch "${branch}": GitHub's tip is not what this deployment last ` +
|
|
1207
|
-
`published, so the branch has diverged and needs reconciling. Something else moved it ` +
|
|
1208
|
-
`on GitHub -- a direct push, or another CanopyCMS deployment sharing this repository.`);
|
|
1209
|
-
}
|
|
1210
|
-
throw err;
|
|
1211
|
-
}
|
|
1212
|
-
// Clear the marker only once GitHub is confirmed to hold something other
|
|
1213
|
-
// than the commit we rewrote. A push that sent nothing new (remote.git
|
|
1214
|
-
// still at the marker because its own publish has not landed yet) must
|
|
1215
|
-
// leave the marker set -- it is the sole trigger for the self-heal pass.
|
|
1216
|
-
if (marker && outgoingSha && outgoingSha !== marker) {
|
|
1217
|
-
await this.clearHistoryRewrittenMarker(branchPath, branch);
|
|
1218
|
-
}
|
|
1219
|
-
workerLog(`Pushed ${branch} to GitHub`);
|
|
1220
|
-
}
|
|
1221
|
-
/**
|
|
1222
|
-
* Push THIS deployment's own settings branch (`ensureSettingsBranch()`) from
|
|
1223
|
-
* remote.git to GitHub. Non-fatal: a no-op push for an up-to-date branch
|
|
1224
|
-
* just succeeds quietly.
|
|
1225
|
-
*
|
|
1226
|
-
* Deliberately narrowed to one branch — this used to push EVERY local
|
|
1227
|
-
* branch matching `canopycms-settings-*`. With the tracking-namespace fetch
|
|
1228
|
-
* fix (GITHUB_TRACKING_REF_PREFIX), `reconcileTrackedBranches` creates local
|
|
1229
|
-
* heads for branches that exist on GitHub, so ANOTHER deployment's settings
|
|
1230
|
-
* branch (sharing this same GitHub repo) can legitimately show up as a
|
|
1231
|
-
* local head here too. Pushing it would be this deployment shipping
|
|
1232
|
-
* settings state it doesn't own.
|
|
1233
|
-
*/
|
|
1234
|
-
async pushSettingsBranches(git, trackedNames) {
|
|
1235
|
-
try {
|
|
1236
|
-
// Resolved once here rather than at each use below: ensureSettingsBranch()
|
|
1237
|
-
// is idempotent, but a local reads better and keeps the eight references
|
|
1238
|
-
// in this method obviously talking about one value.
|
|
1239
|
-
const settingsBranch = this.ensureSettingsBranch();
|
|
1240
|
-
const branches = await git.branch();
|
|
1241
|
-
const settingsBranches = branches.all.filter((b) => b.startsWith(RESERVED_SETTINGS_BRANCH_PREFIX));
|
|
1242
|
-
const foreign = settingsBranches.filter((b) => b !== settingsBranch);
|
|
1243
|
-
if (foreign.length > 0) {
|
|
1244
|
-
// Signal, not an error: this is exactly the "two deployments, one repo"
|
|
1245
|
-
// condition this workstream exists to make visible. Never push these.
|
|
1246
|
-
workerLogWarn(`Found settings branch(es) not owned by this deployment (${settingsBranch}): ` +
|
|
1247
|
-
`${foreign.join(', ')}. Another CanopyCMS deployment may share this GitHub repo. Not pushing them.`);
|
|
1248
|
-
}
|
|
1249
|
-
// Check the full branch list, not the `canopycms-settings-*` subset: an
|
|
1250
|
-
// adopter-supplied `settingsBranch` override need not carry that prefix.
|
|
1251
|
-
const ownBranchMissing = !branches.all.includes(settingsBranch);
|
|
1252
|
-
// [SYNC-M3] A settings branch present in remote.git but absent from
|
|
1253
|
-
// GitHub's tracking refs was pushed here LOCALLY -- only this
|
|
1254
|
-
// deployment's own API can write to remote.git -- and has never
|
|
1255
|
-
// reached GitHub. That is the discriminating signature: in the
|
|
1256
|
-
// SUPPORTED two-deployments-one-repo case the foreign branch arrives
|
|
1257
|
-
// through the GitHub fetch and therefore always has a tracking ref, so
|
|
1258
|
-
// the warn above cannot tell the two apart on its own and a
|
|
1259
|
-
// "owned-branch-absent" test alone would fire on every deployment that
|
|
1260
|
-
// simply has not had a settings edit yet.
|
|
1261
|
-
//
|
|
1262
|
-
// With this deployment's own branch also missing, the API and this
|
|
1263
|
-
// worker have resolved different deploymentNames, and every settings
|
|
1264
|
-
// change the API commits is stranded in remote.git forever.
|
|
1265
|
-
const strandedLocal = foreign.filter((b) => !trackedNames.has(b));
|
|
1266
|
-
if (strandedLocal.length > 0) {
|
|
1267
|
-
workerLogWarn(ownBranchMissing
|
|
1268
|
-
? `Settings branch mismatch: this worker owns "${settingsBranch}", which does not ` +
|
|
1269
|
-
`exist in remote.git, while ${strandedLocal.join(', ')} exist(s) here and has never ` +
|
|
1270
|
-
`been pushed to GitHub. The API and this worker disagree about deploymentName, so ` +
|
|
1271
|
-
`settings changes are NOT reaching GitHub. Set CANOPYCMS_DEPLOYMENT_NAME (or ` +
|
|
1272
|
-
`settingsBranch) on this worker to match what the API resolves.`
|
|
1273
|
-
: `Settings branch(es) ${strandedLocal.join(', ')} exist in remote.git but not on ` +
|
|
1274
|
-
`GitHub, and are not owned by this deployment (${settingsBranch}) -- nothing ` +
|
|
1275
|
-
`will ever push them onward. Check that deploymentName matches across this ` +
|
|
1276
|
-
`deployment's API and worker.`);
|
|
1277
|
-
}
|
|
1278
|
-
if (ownBranchMissing) {
|
|
1279
|
-
// Not created locally yet — nothing to push.
|
|
1280
|
-
return;
|
|
1281
|
-
}
|
|
1282
|
-
try {
|
|
1283
|
-
await git.push(this.buildGitHubUrl(), settingsBranch);
|
|
1284
|
-
workerLog(`Pushed settings branch ${settingsBranch} to GitHub`);
|
|
1285
|
-
}
|
|
1286
|
-
catch (err) {
|
|
1287
|
-
// Non-fatal: branch may already be up-to-date. This call site has no
|
|
1288
|
-
// task to throw a PermanentTaskError into (unlike pushBranchToGitHub)
|
|
1289
|
-
// and is deliberately not restructured to add one -- but a
|
|
1290
|
-
// non-fast-forward rejection here is the highest-signal instance of
|
|
1291
|
-
// this whole failure class: it means another CanopyCMS deployment's
|
|
1292
|
-
// worker already pushed ITS OWN state to its own settings branch on
|
|
1293
|
-
// GitHub (an actual settings-branch name collision, not just the
|
|
1294
|
-
// "foreign branch found locally" case warned about above), so make
|
|
1295
|
-
// that explicit instead of a generic "push failed" line.
|
|
1296
|
-
const message = getErrorMessage(err);
|
|
1297
|
-
if (isNonFastForwardRejection(message)) {
|
|
1298
|
-
workerLogWarn(`Settings push for ${settingsBranch} was rejected (non-fast-forward): another ` +
|
|
1299
|
-
`CanopyCMS deployment appears to own this settings branch on GitHub. Settings from ` +
|
|
1300
|
-
`that deployment will NOT be overwritten; this deployment's local settings changes ` +
|
|
1301
|
-
`were not pushed. Rename this deployment's settings branch (config.settingsBranch or ` +
|
|
1302
|
-
`deploymentName) to resolve the collision.`);
|
|
1303
|
-
}
|
|
1304
|
-
else {
|
|
1305
|
-
workerLogWarn(`Settings push for ${settingsBranch}:`, message);
|
|
1306
|
-
}
|
|
1307
|
-
}
|
|
1308
|
-
}
|
|
1309
|
-
catch (err) {
|
|
1310
|
-
workerLogWarn('Failed to list branches for settings push:', err instanceof Error ? err.message : err);
|
|
1311
|
-
}
|
|
1312
|
-
}
|
|
1313
|
-
/**
|
|
1314
|
-
* Bring `refs/heads/*` in `remote.git` toward what was just fetched into
|
|
1315
|
-
* `GITHUB_TRACKING_REF_PREFIX` -- WITHOUT ever force-rewinding or deleting
|
|
1316
|
-
* a local head. This is the non-destructive replacement for what the old
|
|
1317
|
-
* `+refs/heads/*:refs/heads/*` fetch refspec used to do implicitly (and
|
|
1318
|
-
* destructively) as part of the fetch itself; see
|
|
1319
|
-
* `GITHUB_TRACKING_REF_PREFIX`'s doc comment for the two failure modes
|
|
1320
|
-
* that refspec caused.
|
|
1321
|
-
*
|
|
1322
|
-
* Per tracked branch:
|
|
1323
|
-
* - no local `refs/heads/<name>` yet -> create it at the tracked commit
|
|
1324
|
-
* (a branch created on GitHub, or by another deployment sharing this
|
|
1325
|
-
* GitHub repo, becomes visible locally).
|
|
1326
|
-
* - local is a strict ancestor of tracked (behind) -> fast-forward it.
|
|
1327
|
-
* - local === tracked -> nothing to do.
|
|
1328
|
-
* - tracked is a strict ancestor of local (ahead) -> LEAVE IT ALONE. This
|
|
1329
|
-
* is unpushed editor/settings work; the queued push task (or
|
|
1330
|
-
* pushSettingsBranches) ships it. This is exactly the branch state the
|
|
1331
|
-
* old refspec used to destroy.
|
|
1332
|
-
* - neither is an ancestor of the other (diverged) -> LEAVE IT ALONE and
|
|
1333
|
-
* count/log it. A real collision (e.g. another deployment moved the
|
|
1334
|
-
* same branch name on GitHub); the next push attempt will be rejected
|
|
1335
|
-
* non-fast-forward, which is the correct, visible outcome -- this
|
|
1336
|
-
* method must never silently pick a winner. The one expected, benign
|
|
1337
|
-
* form of this -- our own rebase loop having published a rewrite into
|
|
1338
|
-
* remote.git with the GitHub push still queued -- is split out into the
|
|
1339
|
-
* `rewritten` bucket so the collision warning stays meaningful.
|
|
1340
|
-
*
|
|
1341
|
-
* Never deletes a local head: a branch removed on GitHub simply stops
|
|
1342
|
-
* being tracked here; the local ref persists until removed through its
|
|
1343
|
-
* own explicit path (the sync loop must not be one of them).
|
|
1344
|
-
*
|
|
1345
|
-
* `remote.git` is bare, so there is no worktree to invalidate by moving
|
|
1346
|
-
* these refs -- unlike a non-bare repo, updating the ref that happens to
|
|
1347
|
-
* be "checked out" is a non-issue here.
|
|
1348
|
-
*
|
|
1349
|
-
* Concurrency: `remote.git` is bare and on EFS, and the Lambda pushes into
|
|
1350
|
-
* it concurrently (`GitManager.push()`'s `target:target` refspec) while
|
|
1351
|
-
* this runs. Every `update-ref` below passes the expected old value (the
|
|
1352
|
-
* all-zeros OID for "must not exist yet" on creation, the previously-read
|
|
1353
|
-
* SHA for the fast-forward case) so a concurrent Lambda write landing in
|
|
1354
|
-
* the gap between the read and the write loses the ref update instead of
|
|
1355
|
-
* being silently clobbered -- the branch is simply revisited next cycle.
|
|
1356
|
-
*/
|
|
1357
|
-
async reconcileTrackedBranches(git) {
|
|
1358
|
-
const GIT_ZERO_OID = '0000000000000000000000000000000000000000';
|
|
1359
|
-
const created = [];
|
|
1360
|
-
const fastForwarded = [];
|
|
1361
|
-
const ahead = [];
|
|
1362
|
-
const diverged = [];
|
|
1363
|
-
const rewritten = [];
|
|
1364
|
-
// One invocation enumerates both namespaces: refs/heads/<name> (what the
|
|
1365
|
-
// Lambda pushes into and branch clones read from) and
|
|
1366
|
-
// GITHUB_TRACKING_REF_PREFIX<name> (GitHub's tip, just fetched above).
|
|
1367
|
-
const raw = await git.raw([
|
|
1368
|
-
'for-each-ref',
|
|
1369
|
-
'--format=%(refname) %(objectname)',
|
|
1370
|
-
'refs/heads/',
|
|
1371
|
-
GITHUB_TRACKING_REF_PREFIX,
|
|
1372
|
-
]);
|
|
1373
|
-
const heads = new Map();
|
|
1374
|
-
const tracked = new Map();
|
|
1375
|
-
for (const line of raw.split('\n')) {
|
|
1376
|
-
const trimmed = line.trim();
|
|
1377
|
-
if (!trimmed)
|
|
1378
|
-
continue;
|
|
1379
|
-
const [refname, sha] = trimmed.split(' ');
|
|
1380
|
-
if (refname.startsWith('refs/heads/')) {
|
|
1381
|
-
heads.set(refname.slice('refs/heads/'.length), sha);
|
|
1382
|
-
}
|
|
1383
|
-
else if (refname.startsWith(GITHUB_TRACKING_REF_PREFIX)) {
|
|
1384
|
-
tracked.set(refname.slice(GITHUB_TRACKING_REF_PREFIX.length), sha);
|
|
1385
|
-
}
|
|
1386
|
-
}
|
|
1387
|
-
for (const [name, trackedSha] of tracked) {
|
|
1388
|
-
const localSha = heads.get(name);
|
|
1389
|
-
const localRef = `refs/heads/${name}`;
|
|
1390
|
-
if (!localSha) {
|
|
1391
|
-
try {
|
|
1392
|
-
// Zero old-value asserts the ref does not already exist -- guards
|
|
1393
|
-
// against a concurrent Lambda push creating this exact branch
|
|
1394
|
-
// name between the for-each-ref read above and this update.
|
|
1395
|
-
await git.raw(['update-ref', localRef, trackedSha, GIT_ZERO_OID]);
|
|
1396
|
-
created.push(name);
|
|
1397
|
-
}
|
|
1398
|
-
catch (err) {
|
|
1399
|
-
workerLogWarn(` Tracked-branch reconcile: failed to create local ref for ${name} (concurrent update?): ${getErrorMessage(err)}`);
|
|
1400
|
-
}
|
|
1401
|
-
continue;
|
|
1402
|
-
}
|
|
1403
|
-
if (localSha === trackedSha)
|
|
1404
|
-
continue; // nothing to do
|
|
1405
|
-
// [SYNC-M2] Everything below is per-branch best-effort. The rev-list
|
|
1406
|
-
// used to sit between the two guarded update-ref calls with no guard
|
|
1407
|
-
// of its own, so a single ref pointing at a missing or partially
|
|
1408
|
-
// written object -- plausible on EFS with the Lambda writing
|
|
1409
|
-
// concurrently -- threw out of this loop, out of syncGit()'s try, and
|
|
1410
|
-
// skipped pushSettingsBranches(), refreshBaseBranchWorkspace() and
|
|
1411
|
-
// rebaseActiveBranches() entirely. Nothing self-healed, so it recurred
|
|
1412
|
-
// every cycle. One unreadable ref must cost its own branch, not the
|
|
1413
|
-
// whole sync cycle.
|
|
1414
|
-
try {
|
|
1415
|
-
// Count commits unique to each side in one call: left = commits local
|
|
1416
|
-
// has that tracked doesn't (ahead), right = commits tracked has that
|
|
1417
|
-
// local doesn't (behind). Exits 0 regardless of ancestry direction,
|
|
1418
|
-
// unlike `merge-base --is-ancestor` (which simple-git would throw on
|
|
1419
|
-
// a non-zero exit for the common "not an ancestor" case).
|
|
1420
|
-
const counts = (await git.raw(['rev-list', '--left-right', '--count', `${localSha}...${trackedSha}`])).trim();
|
|
1421
|
-
const [leftStr, rightStr] = counts.split(/\s+/);
|
|
1422
|
-
const localAheadCount = parseInt(leftStr, 10);
|
|
1423
|
-
const localBehindCount = parseInt(rightStr, 10);
|
|
1424
|
-
if (!Number.isInteger(localAheadCount) || !Number.isInteger(localBehindCount)) {
|
|
1425
|
-
// Unparseable output means we could not read this branch, NOT that
|
|
1426
|
-
// it diverged: falling through to the `diverged` bucket below would
|
|
1427
|
-
// warn operators about a cross-deployment collision that never
|
|
1428
|
-
// happened (parseInt yields NaN, and NaN fails both comparisons).
|
|
1429
|
-
workerLogWarn(` Tracked-branch reconcile: skipping ${name}: unparseable rev-list output ${JSON.stringify(counts)}`);
|
|
1430
|
-
continue;
|
|
1431
|
-
}
|
|
1432
|
-
if (localAheadCount === 0 && localBehindCount > 0) {
|
|
1433
|
-
try {
|
|
1434
|
-
await git.raw(['update-ref', localRef, trackedSha, localSha]);
|
|
1435
|
-
fastForwarded.push(name);
|
|
1436
|
-
}
|
|
1437
|
-
catch (err) {
|
|
1438
|
-
// Concurrent Lambda push moved the ref since the read above --
|
|
1439
|
-
// the guard did its job; this branch is simply revisited next cycle.
|
|
1440
|
-
workerLogWarn(` Tracked-branch reconcile: failed to fast-forward ${name} (concurrent update?): ${getErrorMessage(err)}`);
|
|
1441
|
-
}
|
|
1442
|
-
}
|
|
1443
|
-
else if (localBehindCount === 0 && localAheadCount > 0) {
|
|
1444
|
-
// Unpushed local work -- exactly what the old destructive refspec
|
|
1445
|
-
// used to force-rewind or delete. Leave it.
|
|
1446
|
-
ahead.push(name);
|
|
1447
|
-
}
|
|
1448
|
-
else if (await this.hasPendingHistoryRewrite(name)) {
|
|
1449
|
-
// [SYNC-H1] Our own rebase published a rewrite into remote.git and
|
|
1450
|
-
// the GitHub push has not landed yet. Ref-level this is identical
|
|
1451
|
-
// to a collision, but it is expected and self-resolving, so it must
|
|
1452
|
-
// not fire the collision warning below.
|
|
1453
|
-
rewritten.push(name);
|
|
1454
|
-
}
|
|
1455
|
-
else {
|
|
1456
|
-
// Neither side is an ancestor of the other. Leave both alone.
|
|
1457
|
-
diverged.push(name);
|
|
1458
|
-
}
|
|
1459
|
-
}
|
|
1460
|
-
catch (err) {
|
|
1461
|
-
workerLogWarn(` Tracked-branch reconcile: skipping ${name} (unreadable ref or object?): ${getErrorMessage(err)}`);
|
|
1462
|
-
continue;
|
|
1463
|
-
}
|
|
1464
|
-
}
|
|
1465
|
-
if (diverged.length > 0) {
|
|
1466
|
-
workerLogWarn(` Tracked-branch reconcile: ${diverged.length} branch(es) diverged from GitHub and were left untouched: ${diverged.join(', ')}`);
|
|
1467
|
-
}
|
|
1468
|
-
if (rewritten.length > 0) {
|
|
1469
|
-
workerLog(` Tracked-branch reconcile: ${rewritten.length} branch(es) rebased locally with the GitHub push still pending: ${rewritten.join(', ')}`);
|
|
1470
|
-
}
|
|
1471
|
-
// trackedNames is returned alongside the summary (rather than folded into
|
|
1472
|
-
// it) because it is a working set for pushSettingsBranches' stranded-
|
|
1473
|
-
// branch check, not part of the worker-status snapshot -- it lists every
|
|
1474
|
-
// branch on GitHub and would bloat worker-status.json for no reader.
|
|
1475
|
-
return {
|
|
1476
|
-
summary: { created, fastForwarded, ahead, diverged, rewritten },
|
|
1477
|
-
trackedNames: new Set(tracked.keys()),
|
|
1478
|
-
};
|
|
1479
|
-
}
|
|
1480
|
-
/**
|
|
1481
|
-
* Whether this branch carries a pending history rewrite -- the rebase loop
|
|
1482
|
-
* rewrote already-published history and the GitHub push has not landed yet
|
|
1483
|
-
* (see BranchMetadata.historyRewrittenFrom).
|
|
1484
|
-
*
|
|
1485
|
-
* `branchName` is a git ref name; branch workspaces are directories named
|
|
1486
|
-
* with the sanitized form, hence the conversion. Best-effort: a settings
|
|
1487
|
-
* branch (no workspace at all), a missing directory or an unreadable
|
|
1488
|
-
* branch.json all mean "no known rewrite", which is the conservative
|
|
1489
|
-
* answer -- it keeps the branch in the louder `diverged` bucket.
|
|
1490
|
-
*/
|
|
1491
|
-
async hasPendingHistoryRewrite(branchName) {
|
|
1492
|
-
try {
|
|
1493
|
-
const branchPath = path.join(this.contentBranchesPath, sanitizeBranchName(branchName));
|
|
1494
|
-
const metaFile = await BranchMetadataFileManager.loadOnly(branchPath);
|
|
1495
|
-
return typeof metaFile?.branch.historyRewrittenFrom === 'string';
|
|
1496
|
-
}
|
|
1497
|
-
catch {
|
|
1498
|
-
return false;
|
|
1499
|
-
}
|
|
1500
|
-
}
|
|
1501
|
-
async syncGit() {
|
|
1502
|
-
if (!this.running)
|
|
1503
|
-
return;
|
|
1504
|
-
workerLog('Syncing git...');
|
|
1505
|
-
const cycleStartedAt = Date.now();
|
|
1506
|
-
const git = simpleGit({
|
|
1507
|
-
baseDir: this.remoteGitPath,
|
|
1508
|
-
// DEP-H1: a hung fetch/push would stall the sync loop forever
|
|
1509
|
-
// (scheduleLoop only reschedules after completion). The block timeout
|
|
1510
|
-
// is inactivity-based, so a slow-but-flowing transfer is unaffected.
|
|
1511
|
-
timeout: { block: this.taskTimeoutMs },
|
|
1512
|
-
});
|
|
1513
|
-
// This instance also drives pushSettingsBranches(git) below, whose
|
|
1514
|
-
// classification of a rejected settings-branch push (see that method)
|
|
1515
|
-
// needs the same stable-English guarantee as pushBranchToGitHub. Network
|
|
1516
|
-
// env for the same reason: the fetch and push here reach GitHub.
|
|
1517
|
-
git.env(gitNetworkChildEnv());
|
|
1518
|
-
// PR-W1: the whole cycle is wrapped so both outcomes -- success and
|
|
1519
|
-
// hard failure (e.g. the fetch throwing against a poisoned remote.git)
|
|
1520
|
-
// -- record a worker-status.json snapshot. The status write itself is
|
|
1521
|
-
// always best-effort (.catch below): it must never turn an otherwise
|
|
1522
|
-
// successful cycle into a failure, and must never mask the real error
|
|
1523
|
-
// on a failed one. On failure we rethrow so scheduleLoop's existing
|
|
1524
|
-
// per-cycle catch stays the loud path.
|
|
1525
|
-
try {
|
|
1526
|
-
// Fetch all branches from GitHub using direct URL (no named remote),
|
|
1527
|
-
// into the GITHUB_TRACKING_REF_PREFIX remote-tracking namespace rather
|
|
1528
|
-
// than refs/heads/* directly -- see that constant's doc comment for
|
|
1529
|
-
// the destructive-fetch bug this avoids. We use raw git commands since
|
|
1530
|
-
// simple-git's fetch() with a URL doesn't support --prune directly.
|
|
1531
|
-
await git.raw([
|
|
1532
|
-
'fetch',
|
|
1533
|
-
this.buildGitHubUrl(),
|
|
1534
|
-
'--prune',
|
|
1535
|
-
`+refs/heads/*:${GITHUB_TRACKING_REF_PREFIX}*`,
|
|
1536
|
-
]);
|
|
1537
|
-
workerLog('Fetched from GitHub');
|
|
1538
|
-
// Bring refs/heads/* toward what was just fetched, WITHOUT ever
|
|
1539
|
-
// force-rewinding or deleting a local head -- see
|
|
1540
|
-
// reconcileTrackedBranches()'s doc comment.
|
|
1541
|
-
const { summary: trackedSummary, trackedNames } = await this.reconcileTrackedBranches(git);
|
|
1542
|
-
// Push settings branches to GitHub (belt-and-suspenders for task queue).
|
|
1543
|
-
// Ensures settings reach GitHub even if a task queue entry is lost.
|
|
1544
|
-
// Ordering relative to the fetch/reconcile above is no longer a
|
|
1545
|
-
// correctness dependency now that the fetch can't clobber refs/heads/*
|
|
1546
|
-
// -- this could run before or after them just as safely.
|
|
1547
|
-
await this.pushSettingsBranches(git, trackedNames);
|
|
1548
|
-
await this.refreshBaseBranchWorkspace();
|
|
1549
|
-
const rebaseSummary = await this.rebaseActiveBranches();
|
|
1550
|
-
// Periodically clean up old completed/failed tasks
|
|
1551
|
-
await cleanupOldTasks(this.taskDir, undefined, this.log);
|
|
1552
|
-
// [C1] Sweep branch directories the admin purge action trashed more
|
|
1553
|
-
// than TRASH_RETENTION_MS ago. Worker-only by design: purge itself
|
|
1554
|
-
// never deletes anything (reversible), and this cycle is the sole
|
|
1555
|
-
// place actual removal happens.
|
|
1556
|
-
const trashRemoved = await this.cleanupTrashedBranchDirs();
|
|
1557
|
-
if (trashRemoved > 0) {
|
|
1558
|
-
workerLog(`Removed ${trashRemoved} expired trashed branch dir(s)`);
|
|
1559
|
-
}
|
|
1560
|
-
const report = this.ensureStatusReport();
|
|
1561
|
-
report.lastGitSyncAt = new Date().toISOString();
|
|
1562
|
-
delete report.lastGitSyncError;
|
|
1563
|
-
report.lastGitSync = {
|
|
1564
|
-
durationMs: Date.now() - cycleStartedAt,
|
|
1565
|
-
rebased: rebaseSummary.rebased,
|
|
1566
|
-
skippedDirty: rebaseSummary.skippedDirty,
|
|
1567
|
-
skippedLocked: rebaseSummary.skippedLocked,
|
|
1568
|
-
failed: rebaseSummary.failed,
|
|
1569
|
-
tracked: trackedSummary,
|
|
1570
|
-
};
|
|
1571
|
-
await writeWorkerStatus(this.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
|
|
1572
|
-
}
|
|
1573
|
-
catch (err) {
|
|
1574
|
-
const report = this.ensureStatusReport();
|
|
1575
|
-
// [HIGH-1] Persisted to worker-status.json and served to the browser
|
|
1576
|
-
// by the admin panel -- a fetch/push failure's message can embed the
|
|
1577
|
-
// bot token via buildGitHubUrl().
|
|
1578
|
-
report.lastGitSyncError = {
|
|
1579
|
-
message: redactCredentials(getErrorMessage(err)),
|
|
1580
|
-
at: new Date().toISOString(),
|
|
1581
|
-
};
|
|
1582
|
-
await writeWorkerStatus(this.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
|
|
1583
|
-
throw err;
|
|
1584
|
-
}
|
|
1585
|
-
}
|
|
1586
|
-
/**
|
|
1587
|
-
* [C1] Remove `.trash-*` branch directories (created by the admin purge
|
|
1588
|
-
* action, api/admin-branch-health.ts) whose name-embedded stamp is older
|
|
1589
|
-
* than {@link TRASH_RETENTION_MS}. Names that don't match the expected
|
|
1590
|
-
* `.trash-{dirName}-{STAMP}` shape, or whose stamp fails to parse, are
|
|
1591
|
-
* left alone (logged once per cycle, not per file, to avoid flooding logs
|
|
1592
|
-
* if something odd accumulates) -- purge is the only writer of this
|
|
1593
|
-
* naming scheme, so an unparseable name is unexpected and worth a human
|
|
1594
|
-
* looking rather than a silent skip.
|
|
1595
|
-
*/
|
|
1596
|
-
async cleanupTrashedBranchDirs() {
|
|
1597
|
-
let entries;
|
|
1598
|
-
try {
|
|
1599
|
-
entries = await fs.readdir(this.contentBranchesPath);
|
|
1600
|
-
}
|
|
1601
|
-
catch (err) {
|
|
1602
|
-
if (isNodeError(err) && err.code === 'ENOENT')
|
|
1603
|
-
return 0;
|
|
1604
|
-
throw err;
|
|
1605
|
-
}
|
|
1606
|
-
const now = Date.now();
|
|
1607
|
-
let removed = 0;
|
|
1608
|
-
let loggedUnparseable = false;
|
|
1609
|
-
for (const name of entries) {
|
|
1610
|
-
if (!name.startsWith('.trash-'))
|
|
1611
|
-
continue;
|
|
1612
|
-
const stampMatch = TRASH_DIR_STAMP_RE.exec(name);
|
|
1613
|
-
const stampDate = stampMatch ? parseTrashStamp(stampMatch[1]) : null;
|
|
1614
|
-
if (!stampDate) {
|
|
1615
|
-
if (!loggedUnparseable) {
|
|
1616
|
-
workerLog(`CanopyCMS: Skipping trash dir with unparseable stamp: ${name}`);
|
|
1617
|
-
loggedUnparseable = true;
|
|
1618
|
-
}
|
|
1619
|
-
continue;
|
|
1620
|
-
}
|
|
1621
|
-
if (now - stampDate.getTime() < TRASH_RETENTION_MS)
|
|
1622
|
-
continue;
|
|
1623
|
-
try {
|
|
1624
|
-
await fs.rm(path.join(this.contentBranchesPath, name), {
|
|
1625
|
-
recursive: true,
|
|
1626
|
-
force: true,
|
|
1627
|
-
maxRetries: 3,
|
|
1628
|
-
retryDelay: 100,
|
|
1629
|
-
});
|
|
1630
|
-
removed++;
|
|
1631
|
-
}
|
|
1632
|
-
catch (err) {
|
|
1633
|
-
workerLogError(`CanopyCMS: Failed to remove trashed branch dir ${name}:`, getErrorMessage(err));
|
|
1634
|
-
}
|
|
1635
|
-
}
|
|
1636
|
-
return removed;
|
|
1637
|
-
}
|
|
1638
|
-
/**
|
|
1639
|
-
* Fast-forward the base branch's own working-tree clone
|
|
1640
|
-
* (content-branches/<baseBranch>) to match origin/<baseBranch>.
|
|
1641
|
-
*
|
|
1642
|
-
* Previously this clone was refreshed only incidentally, by the generic
|
|
1643
|
-
* rebase loop below (rebaseActiveBranches): for a branch with status
|
|
1644
|
-
* 'editing', rebasing onto origin/<baseBranch> degenerates to a
|
|
1645
|
-
* fast-forward when the clone IS the base branch. But that loop's skip
|
|
1646
|
-
* paths -- a dirty tree, a missing .git -- are silent, which is the
|
|
1647
|
-
* suspected live failure mode: a wedged base clone with no diagnosable
|
|
1648
|
-
* signal in the logs. This dedicated step makes the refresh explicit,
|
|
1649
|
-
* ff-only, and loud, so a stuck base view (an editor forking a new branch
|
|
1650
|
-
* "from base" that's actually a stale snapshot) is diagnosable from logs.
|
|
1651
|
-
* This runs every sync cycle so the drift window is bounded by
|
|
1652
|
-
* gitSyncInterval.
|
|
1653
|
-
*
|
|
1654
|
-
* ff-only on purpose: this clone must stay a linear mirror of
|
|
1655
|
-
* origin/<baseBranch>, so a merge that isn't a fast-forward (diverged
|
|
1656
|
-
* local history) is treated as a should-never-happen condition and left
|
|
1657
|
-
* untouched rather than force-resolved.
|
|
1658
|
-
*/
|
|
1659
|
-
async refreshBaseBranchWorkspace() {
|
|
1660
|
-
// Sanitized name for the workspace directory (a base branch containing
|
|
1661
|
-
// e.g. '/' would otherwise stat a wrong nested path here forever).
|
|
1662
|
-
const basePath = path.join(this.contentBranchesPath, this.sanitizedBaseBranch);
|
|
1663
|
-
const gitDir = path.join(basePath, '.git');
|
|
1664
|
-
try {
|
|
1665
|
-
let gitDirStat;
|
|
1666
|
-
try {
|
|
1667
|
-
gitDirStat = await fs.stat(gitDir);
|
|
1668
|
-
}
|
|
1669
|
-
catch {
|
|
1670
|
-
gitDirStat = null;
|
|
1671
|
-
}
|
|
1672
|
-
if (!gitDirStat || !gitDirStat.isDirectory()) {
|
|
1673
|
-
workerLog(`Base branch workspace (${this.baseBranch}): not yet provisioned, skipping refresh`);
|
|
1674
|
-
return;
|
|
1675
|
-
}
|
|
1676
|
-
const baseGit = simpleGit({
|
|
1677
|
-
baseDir: basePath,
|
|
1678
|
-
// Keep git non-interactive during the merge so it never blocks on an
|
|
1679
|
-
// editor. simple-git >=3.32 blocks setting core.editor unless
|
|
1680
|
-
// explicitly opted in; the value here is a hardcoded literal
|
|
1681
|
-
// ("true", the shell no-op), not user input, so enabling
|
|
1682
|
-
// allowUnsafeEditor carries no injection risk (same as the rebase
|
|
1683
|
-
// loop's git config below).
|
|
1684
|
-
config: ['core.editor=true'],
|
|
1685
|
-
unsafe: { allowUnsafeEditor: true },
|
|
1686
|
-
// DEP-H1: a hung fetch/merge against this EFS-backed clone would
|
|
1687
|
-
// stall the sync loop forever (scheduleLoop only reschedules after
|
|
1688
|
-
// completion). The block timeout is inactivity-based, so a
|
|
1689
|
-
// slow-but-flowing transfer is unaffected -- same as syncGit()'s
|
|
1690
|
-
// remote handle.
|
|
1691
|
-
timeout: { block: this.taskTimeoutMs },
|
|
1692
|
-
});
|
|
1693
|
-
// Nothing makes this clone read-only. A direct edit here (or a stray
|
|
1694
|
-
// process) wedges every editor's view of the base branch until an
|
|
1695
|
-
// operator intervenes, so a dirty tree is loud, not a quiet skip.
|
|
1696
|
-
// Only TRACKED changes block the refresh: a stray untracked file (e.g.
|
|
1697
|
-
// runtime metadata missing from .git/info/exclude) must not wedge the
|
|
1698
|
-
// fast-forward forever — if an untracked file would collide with
|
|
1699
|
-
// incoming content, the --ff-only merge below refuses on its own and
|
|
1700
|
-
// that failure is already logged loudly.
|
|
1701
|
-
const status = await baseGit.status();
|
|
1702
|
-
const trackedDirty = status.files.filter((f) => f.index !== '?' || f.working_dir !== '?');
|
|
1703
|
-
if (trackedDirty.length > 0) {
|
|
1704
|
-
workerLogError(`Base branch workspace (${this.baseBranch}) has uncommitted changes -- skipping refresh. Dirty files: ${trackedDirty.map((f) => f.path).join(', ')}`);
|
|
1705
|
-
return;
|
|
1706
|
-
}
|
|
1707
|
-
// Raw (unsanitized) name from here on: these are git ref operations
|
|
1708
|
-
// against origin/<baseBranch>, not filesystem paths, so they must use
|
|
1709
|
-
// the same name GitHub knows the branch by.
|
|
1710
|
-
await baseGit.fetch('origin', this.baseBranch);
|
|
1711
|
-
// Use rev-list instead of status.behind for the same reason as the
|
|
1712
|
-
// rebase loop below: status.behind only works with an upstream
|
|
1713
|
-
// tracking branch configured, which isn't guaranteed here.
|
|
1714
|
-
// Compare against the just-fetched tip rather than origin/<base>:
|
|
1715
|
-
// workspaces are cloned --single-branch (git-manager.ts), so their
|
|
1716
|
-
// fetch refspec only materializes a remote-tracking ref for the branch
|
|
1717
|
-
// they were cloned from — for any other base branch, origin/<base>
|
|
1718
|
-
// simply never exists and rev-list dies with "ambiguous argument".
|
|
1719
|
-
// Pin FETCH_HEAD to a SHA immediately: FETCH_HEAD is one shared mutable
|
|
1720
|
-
// file per repo, silently repointed by ANY other fetch in this clone.
|
|
1721
|
-
const fetchedTip = (await baseGit.revparse(['FETCH_HEAD'])).trim();
|
|
1722
|
-
const behindCount = parseInt((await baseGit.raw(['rev-list', '--count', `HEAD..${fetchedTip}`])).trim(), 10);
|
|
1723
|
-
if (behindCount > 0) {
|
|
1724
|
-
try {
|
|
1725
|
-
await baseGit.merge(['--ff-only', fetchedTip]);
|
|
1726
|
-
}
|
|
1727
|
-
catch (err) {
|
|
1728
|
-
workerLogError(`Base branch workspace (${this.baseBranch}) failed to fast-forward (diverged local history?): ${getErrorMessage(err)}`);
|
|
1729
|
-
return;
|
|
1730
|
-
}
|
|
1731
|
-
await invalidateBranchContentCaches(basePath);
|
|
1732
|
-
}
|
|
1733
|
-
// Hygiene: conflictStatus/conflictFiles are meaningless for the base
|
|
1734
|
-
// branch's own metadata and may be left over from before the base was
|
|
1735
|
-
// excluded from rebaseActiveBranches()'s conflict-resolution loop.
|
|
1736
|
-
// Reuse the already-clean no-op-save guard from that loop: save()
|
|
1737
|
-
// eager-regenerates the branch registry (O(branch count) EFS reads),
|
|
1738
|
-
// so skip it when there's nothing to clear.
|
|
1739
|
-
const currentMeta = await BranchMetadataFileManager.loadOnly(basePath);
|
|
1740
|
-
const conflictStatus = currentMeta?.branch.conflictStatus;
|
|
1741
|
-
const conflictFiles = currentMeta?.branch.conflictFiles;
|
|
1742
|
-
const alreadyClean = (conflictStatus === undefined || conflictStatus === 'clean') &&
|
|
1743
|
-
(conflictFiles === undefined || conflictFiles.length === 0);
|
|
1744
|
-
if (!alreadyClean) {
|
|
1745
|
-
const meta = getBranchMetadataFileManager(basePath, this.contentBranchesPath);
|
|
1746
|
-
await meta.save({
|
|
1747
|
-
branch: { name: this.baseBranch, conflictStatus: 'clean', conflictFiles: [] },
|
|
1748
|
-
});
|
|
1749
|
-
}
|
|
1750
|
-
// One concise per-cycle line -- the diagnostic for the next live deploy.
|
|
1751
|
-
workerLog(behindCount > 0
|
|
1752
|
-
? `Base branch workspace (${this.baseBranch}): fast-forwarded ${behindCount} commit(s)`
|
|
1753
|
-
: `Base branch workspace (${this.baseBranch}): up to date`);
|
|
1754
|
-
}
|
|
1755
|
-
catch (err) {
|
|
1756
|
-
workerLogError(`Base branch workspace (${this.baseBranch}) refresh failed: ${getErrorMessage(err)}`);
|
|
1757
|
-
}
|
|
1758
|
-
}
|
|
1759
|
-
/**
|
|
1760
|
-
* Poll GitHub for a submitted/approved branch's PR resolution.
|
|
1761
|
-
*
|
|
1762
|
-
* submitted/approved branches sit outside the rebase loop and get no
|
|
1763
|
-
* other signal that their PR resolved on GitHub -- nothing pushes a
|
|
1764
|
-
* merge/close webhook back into the branch workspace. merged ->
|
|
1765
|
-
* auto-archive via buildMergedBranchUpdate (shared with the manual
|
|
1766
|
-
* markAsMerged API so both paths produce identical archived-branch
|
|
1767
|
-
* metadata). closed-without-merge -> record pullRequestState only; an
|
|
1768
|
-
* admin decides the workflow transition from there. Best-effort: any
|
|
1769
|
-
* failure here is logged and swallowed, retried next sync cycle.
|
|
1770
|
-
*/
|
|
1771
|
-
async pollMergeState(branchDir, branchPath, metaFile) {
|
|
1772
|
-
const prNumber = metaFile?.branch.pullRequestNumber;
|
|
1773
|
-
if (!prNumber)
|
|
1774
|
-
return;
|
|
1775
|
-
const controller = new AbortController();
|
|
1776
|
-
const timer = setTimeout(() => controller.abort(), this.taskTimeoutMs);
|
|
1777
|
-
try {
|
|
1778
|
-
const { data } = await this.octokit.pulls.get({
|
|
1779
|
-
owner: this.config.githubOwner,
|
|
1780
|
-
repo: this.config.githubRepo,
|
|
1781
|
-
pull_number: prNumber,
|
|
1782
|
-
request: { signal: controller.signal },
|
|
1783
|
-
});
|
|
1784
|
-
if (data.merged) {
|
|
1785
|
-
const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
|
|
1786
|
-
// Use GitHub's actual merge time when available; buildMergedBranchUpdate
|
|
1787
|
-
// falls back to "now" (its default `now` param) when merged_at is absent.
|
|
1788
|
-
await meta.save({
|
|
1789
|
-
branch: buildMergedBranchUpdate(branchDir, data.merged_at ? new Date(data.merged_at) : undefined),
|
|
1790
|
-
});
|
|
1791
|
-
workerLog(` PR #${prNumber} for ${branchDir} is merged -> archived`);
|
|
1792
|
-
return;
|
|
1793
|
-
}
|
|
1794
|
-
const newState = data.state === 'closed' ? 'closed' : 'open';
|
|
1795
|
-
// Re-load fresh (not the loop-top `metaFile` snapshot passed in) -- a
|
|
1796
|
-
// concurrent Lambda write (e.g. an editor re-submitting) may have
|
|
1797
|
-
// landed since that snapshot was taken.
|
|
1798
|
-
const currentMeta = await BranchMetadataFileManager.loadOnly(branchPath);
|
|
1799
|
-
if (currentMeta?.branch.pullRequestState === newState)
|
|
1800
|
-
return;
|
|
1801
|
-
const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
|
|
1802
|
-
await meta.save({ branch: { name: branchDir, pullRequestState: newState } });
|
|
1803
|
-
workerLog(` PR #${prNumber} for ${branchDir}: pullRequestState -> ${newState}`);
|
|
1804
|
-
}
|
|
1805
|
-
catch (err) {
|
|
1806
|
-
// Non-fatal: transient GitHub/network errors are retried next cycle.
|
|
1807
|
-
workerLogWarn(` Failed to poll PR #${prNumber} for ${branchDir}: ${getErrorMessage(err)}`);
|
|
1808
|
-
}
|
|
1809
|
-
finally {
|
|
1810
|
-
clearTimeout(timer);
|
|
1811
|
-
}
|
|
1812
|
-
}
|
|
1813
|
-
/**
|
|
1814
|
-
* Persist a per-branch rebase failure to branch.json (PR-W2), bounded to
|
|
1815
|
-
* roughly one save per failing branch per hour: a branch stuck failing
|
|
1816
|
-
* every cycle must not turn into unbounded save-per-cycle x N-failing-
|
|
1817
|
-
* branches write amplification -- save() eager-regenerates the branch
|
|
1818
|
-
* registry (branch-metadata.ts's invalidateRegistry(), O(branch count) EFS
|
|
1819
|
-
* reads), the same concern the `alreadyClean` no-op guard above exists
|
|
1820
|
-
* for.
|
|
1821
|
-
*
|
|
1822
|
-
* Best-effort and non-fatal like every other metadata write in this
|
|
1823
|
-
* loop's error paths: a corrupt branch.json, a lock-contention error, or
|
|
1824
|
-
* any other save failure here must never abort the per-branch iteration.
|
|
1825
|
-
* This matters doubly at the two call sites -- one is inside the outer
|
|
1826
|
-
* per-branch catch, with no further catch of its own around this call --
|
|
1827
|
-
* so the whole method is wrapped, not just the load.
|
|
1828
|
-
*/
|
|
1829
|
-
async recordRebaseFailure(branchPath, branchDir, message) {
|
|
1830
|
-
const RECORD_REFRESH_MS = 60 * 60 * 1000; // 1 hour
|
|
1831
|
-
// [HIGH-1] Defense-in-depth redaction: rebaseFailure.message is
|
|
1832
|
-
// persisted to branch.json and served to the browser via the
|
|
1833
|
-
// branch-health admin endpoint. Both call sites in rebaseActiveBranches
|
|
1834
|
-
// already redact before passing in (the failed.push sites below),
|
|
1835
|
-
// redactCredentials is idempotent, so redacting again here is free and
|
|
1836
|
-
// keeps this method safe on its own.
|
|
1837
|
-
const redactedMessage = redactCredentials(message);
|
|
1838
|
-
try {
|
|
1839
|
-
const existing = await BranchMetadataFileManager.loadOnly(branchPath);
|
|
1840
|
-
const prior = existing?.branch.rebaseFailure;
|
|
1841
|
-
const sameMessage = prior?.message === redactedMessage;
|
|
1842
|
-
const now = new Date();
|
|
1843
|
-
if (sameMessage) {
|
|
1844
|
-
const lastAtMs = Date.parse(prior.lastAt);
|
|
1845
|
-
if (!Number.isNaN(lastAtMs) && now.getTime() - lastAtMs < RECORD_REFRESH_MS) {
|
|
1846
|
-
// Same failure, refreshed within the last hour -- skip the save.
|
|
1847
|
-
return;
|
|
1848
|
-
}
|
|
1849
|
-
}
|
|
1850
|
-
const nowIso = now.toISOString();
|
|
1851
|
-
const firstAt = sameMessage ? prior.firstAt : nowIso;
|
|
1852
|
-
const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
|
|
1853
|
-
await meta.save({
|
|
1854
|
-
branch: {
|
|
1855
|
-
name: branchDir,
|
|
1856
|
-
rebaseFailure: { message: redactedMessage, firstAt, lastAt: nowIso },
|
|
1857
|
-
},
|
|
1858
|
-
});
|
|
1859
|
-
}
|
|
1860
|
-
catch (err) {
|
|
1861
|
-
// Includes BranchMetadataCorruptError from the load above (a save()
|
|
1862
|
-
// against the same corrupt file would just throw again) as well as
|
|
1863
|
-
// any other load/save failure -- recording is best-effort
|
|
1864
|
-
// observability, never allowed to abort the branch loop.
|
|
1865
|
-
workerLogWarn(` Failed to record rebase failure for ${branchDir}: ${getErrorMessage(err)}`);
|
|
1866
|
-
}
|
|
1867
|
-
}
|
|
1868
569
|
/**
|
|
1869
570
|
* Test-only seam: awaited inside `rebaseActiveBranches()`'s conflict round,
|
|
1870
571
|
* after `git rebase` reported conflicted files and BEFORE the
|
|
@@ -1886,626 +587,48 @@ export class CmsWorker {
|
|
|
1886
587
|
* would strand a rewritten history.
|
|
1887
588
|
*/
|
|
1888
589
|
async afterRebaseCompletedForTesting() { }
|
|
590
|
+
// --- Rebase loop (worker/rebase.ts) ------------------------------------
|
|
591
|
+
//
|
|
592
|
+
// Both are TEST-ONLY entry points -- see the task-queue block above. Nothing
|
|
593
|
+
// in production dispatches through either; `syncGit` calls `runRebaseCycle`
|
|
594
|
+
// directly and that calls `pollMergeState` directly. Four test files call
|
|
595
|
+
// `rebaseActiveBranches` (it is the single entry point the whole rebase suite
|
|
596
|
+
// drives), and cms-worker-merge-poll.test.ts calls `pollMergeState` with a
|
|
597
|
+
// mocked `octokit`. Neither is on WorkerContext, so a stub installed on
|
|
598
|
+
// either is a no-op as far as a production code path is concerned.
|
|
599
|
+
//
|
|
600
|
+
// `rebaseActiveBranches` also has a consumer outside the test files:
|
|
601
|
+
// apps/test-app/app/api/e2e-test/rebase/route.ts drives it from an e2e
|
|
602
|
+
// fixture route. Do not delete it as unused.
|
|
1889
603
|
async rebaseActiveBranches() {
|
|
1890
|
-
|
|
1891
|
-
|
|
1892
|
-
|
|
1893
|
-
|
|
1894
|
-
|
|
1895
|
-
|
|
1896
|
-
|
|
1897
|
-
|
|
1898
|
-
|
|
1899
|
-
|
|
1900
|
-
|
|
1901
|
-
|
|
1902
|
-
|
|
1903
|
-
|
|
1904
|
-
|
|
1905
|
-
|
|
1906
|
-
|
|
1907
|
-
|
|
1908
|
-
|
|
1909
|
-
|
|
1910
|
-
|
|
1911
|
-
|
|
1912
|
-
|
|
1913
|
-
|
|
1914
|
-
|
|
1915
|
-
|
|
1916
|
-
|
|
1917
|
-
|
|
1918
|
-
workerLog(` Skipping ${branchDir}: .git is not a directory`);
|
|
1919
|
-
continue;
|
|
1920
|
-
}
|
|
1921
|
-
}
|
|
1922
|
-
catch {
|
|
1923
|
-
workerLog(` Skipping ${branchDir}: no .git directory (not a branch workspace)`);
|
|
1924
|
-
continue;
|
|
1925
|
-
}
|
|
1926
|
-
// The base branch's own clone is refreshed ff-only by
|
|
1927
|
-
// refreshBaseBranchWorkspace() earlier in syncGit(). Routing it
|
|
1928
|
-
// through this conflict-resolution rebase loop could rewrite its
|
|
1929
|
-
// history (the --theirs loop below) and stamp meaningless conflict
|
|
1930
|
-
// metadata on it. Compare sanitized-vs-sanitized: branchDir is a
|
|
1931
|
-
// filesystem name (already sanitized), this.baseBranch is raw.
|
|
1932
|
-
if (branchDir === this.sanitizedBaseBranch) {
|
|
1933
|
-
workerLog(` Skipping ${branchDir}: base branch (refreshed separately)`);
|
|
1934
|
-
continue;
|
|
1935
|
-
}
|
|
1936
|
-
try {
|
|
1937
|
-
// Load metadata before any git ops to check branch status
|
|
1938
|
-
const metaFile = await BranchMetadataFileManager.loadOnly(branchPath);
|
|
1939
|
-
const branchStatus = metaFile?.branch.status;
|
|
1940
|
-
// Skip branches that shouldn't be mutated:
|
|
1941
|
-
// - submitted/approved: in review, don't rewrite history under an
|
|
1942
|
-
// active PR -- but do poll GitHub for the PR's resolution, since
|
|
1943
|
-
// nothing else tells the worker a merge/close happened.
|
|
1944
|
-
// - archived: already merged, no reason to rebase and no PR left to
|
|
1945
|
-
// poll (avoid the wasted API call).
|
|
1946
|
-
if (branchStatus === 'submitted' || branchStatus === 'approved') {
|
|
1947
|
-
workerLog(` Skipping ${branchDir} (${branchStatus})`);
|
|
1948
|
-
await this.pollMergeState(branchDir, branchPath, metaFile);
|
|
1949
|
-
continue;
|
|
1950
|
-
}
|
|
1951
|
-
if (branchStatus === 'archived') {
|
|
1952
|
-
workerLog(` Skipping ${branchDir} (${branchStatus})`);
|
|
1953
|
-
continue;
|
|
1954
|
-
}
|
|
1955
|
-
const branchGit = simpleGit({
|
|
1956
|
-
baseDir: branchPath,
|
|
1957
|
-
// Keep git non-interactive during rebase/merge so it never blocks on an editor.
|
|
1958
|
-
// simple-git >=3.32 blocks setting core.editor unless explicitly opted in; the
|
|
1959
|
-
// value here is a hardcoded literal ("true", the shell no-op), not user input,
|
|
1960
|
-
// so enabling allowUnsafeEditor carries no injection risk.
|
|
1961
|
-
config: ['core.editor=true'],
|
|
1962
|
-
unsafe: { allowUnsafeEditor: true },
|
|
1963
|
-
});
|
|
1964
|
-
// [SYNC-C1] Take the branch's cross-host content-write lock BEFORE the
|
|
1965
|
-
// dirty check, and hold it for the whole rebase.
|
|
1966
|
-
//
|
|
1967
|
-
// The dirty check alone is check-then-act. The old comment here claimed
|
|
1968
|
-
// the residual window was safe ("the rebase will fail and the catch
|
|
1969
|
-
// block will abort safely"), which only holds for a save landing before
|
|
1970
|
-
// `git rebase` STARTS. After that -- a window spanning fetch, replay and
|
|
1971
|
-
// N conflict rounds of awaited git subprocesses on EFS -- a save is
|
|
1972
|
-
// destroyed two ways: `checkout --theirs` below overwrites the
|
|
1973
|
-
// just-saved file with the branch's committed version and the rebase
|
|
1974
|
-
// then SUCCEEDS (nothing logs a failure at all), and `rebase --abort`
|
|
1975
|
-
// hard-resets the tree. The editor already got its 200 either way.
|
|
1976
|
-
//
|
|
1977
|
-
// Zero-retry acquisition, deliberately: on contention this branch is
|
|
1978
|
-
// skipped and retried on the next sync cycle (~5 min), which is the
|
|
1979
|
-
// same principle as the skip-dirty-branches behavior below. Writers get
|
|
1980
|
-
// the patient side of the asymmetry -- see utils/content-write-lock.ts.
|
|
1981
|
-
//
|
|
1982
|
-
// The heartbeat that keeps this lock fresh (proper-lockfile refreshes
|
|
1983
|
-
// every `stale`/2 = 15s) is a timer on this event loop; every git step
|
|
1984
|
-
// below is an awaited subprocess, never a synchronous block, so the
|
|
1985
|
-
// refresh keeps firing for the whole hold.
|
|
1986
|
-
let releaseContentLock;
|
|
1987
|
-
// [SYNC-C1] If the lock is lost mid-hold, a writer may now be live
|
|
1988
|
-
// against this same tree. Every git step below is destructive, so
|
|
1989
|
-
// record it and bail before the next one rather than replaying over
|
|
1990
|
-
// an editor's concurrent save.
|
|
1991
|
-
let contentLockCompromised = false;
|
|
1992
|
-
try {
|
|
1993
|
-
releaseContentLock = await tryAcquireContentWriteLock(branchPath, (lockErr) => {
|
|
1994
|
-
contentLockCompromised = true;
|
|
1995
|
-
workerLogWarn(` Content-write lock compromised mid-rebase for ${branchDir}: ${getErrorMessage(lockErr)}`);
|
|
1996
|
-
});
|
|
1997
|
-
}
|
|
1998
|
-
catch (lockErr) {
|
|
1999
|
-
if (isNodeError(lockErr) && lockErr.code === 'ELOCKED') {
|
|
2000
|
-
workerLog(` Skipping ${branchDir}: content write in progress (retrying next cycle)`);
|
|
2001
|
-
skippedLocked.push(branchDir);
|
|
2002
|
-
continue;
|
|
2003
|
-
}
|
|
2004
|
-
// Anything else (ENOENT on a branch dir deleted mid-cycle, EACCES,
|
|
2005
|
-
// ...) is a real failure: let the outer catch record it.
|
|
2006
|
-
throw lockErr;
|
|
2007
|
-
}
|
|
2008
|
-
try {
|
|
2009
|
-
// Recover an INTERRUPTED rebase before anything else looks at this
|
|
2010
|
-
// tree. A clone left with .git/rebase-merge (or rebase-apply) reports
|
|
2011
|
-
// uncommitted changes, so without this the dirty check below would
|
|
2012
|
-
// classify it `skippedDirty` on every cycle FOREVER while
|
|
2013
|
-
// branch-health scanned it as healthy -- and editors would meanwhile
|
|
2014
|
-
// read, and be able to save over, conflict-marker content.
|
|
2015
|
-
//
|
|
2016
|
-
// An in-progress rebase is always this worker's own abandoned work:
|
|
2017
|
-
// it is the only thing that ever rebases these clones, and it got
|
|
2018
|
-
// here via a crash, an OOM, a spot interruption, or the ASG rolling
|
|
2019
|
-
// the instance (which happens on EVERY `cdk deploy`, while `stop()`
|
|
2020
|
-
// drains for at most taskTimeoutMs).
|
|
2021
|
-
//
|
|
2022
|
-
// NOT LOSSLESS, and it is important not to claim otherwise. `git
|
|
2023
|
-
// rebase --abort` hard-resets tracked files to the pre-rebase head.
|
|
2024
|
-
// While the worker was DOWN nothing held the [SYNC-C1] content-write
|
|
2025
|
-
// lock, so an editor could have saved into this wedged clone and
|
|
2026
|
-
// received a 200; that save is a working-tree modification, and the
|
|
2027
|
-
// abort reverts it. (New, untracked entry files survive; edits to
|
|
2028
|
-
// existing ones do not.) Taking the lock here stops any FURTHER save
|
|
2029
|
-
// racing the abort, but cannot recover one that already landed.
|
|
2030
|
-
//
|
|
2031
|
-
// Aborting anyway is still the right call: the alternative is a
|
|
2032
|
-
// branch wedged forever whose tree serves conflict-marker content to
|
|
2033
|
-
// editors. What must not happen is doing it SILENTLY -- so anything
|
|
2034
|
-
// modified beyond the rebase's own conflict state is logged by path
|
|
2035
|
-
// first, which is the only record an operator would have.
|
|
2036
|
-
if (await isRebaseInProgress(branchPath)) {
|
|
2037
|
-
const preAbort = await branchGit.status().catch(() => null);
|
|
2038
|
-
// Keyed on the WORKING-TREE column only. The two porcelain columns
|
|
2039
|
-
// mean different things here, and conflating them produces a false
|
|
2040
|
-
// data-loss report on essentially every conflict-wedged recovery
|
|
2041
|
-
// (verified against real git, mid-rebase):
|
|
2042
|
-
//
|
|
2043
|
-
// `M ` index=M, wd=' ' -- the interrupted replay's own cleanly
|
|
2044
|
-
// merged files, already STAGED. These
|
|
2045
|
-
// are committed history and survive the
|
|
2046
|
-
// abort untouched. Not collateral.
|
|
2047
|
-
// ` M` index=' ', wd=M -- a working-tree modification nothing
|
|
2048
|
-
// staged: an editor's save landing while
|
|
2049
|
-
// the worker was down. The abort
|
|
2050
|
-
// discards exactly these.
|
|
2051
|
-
// `??` -- untracked; the abort leaves them.
|
|
2052
|
-
//
|
|
2053
|
-
// KNOWN GAP, stated rather than hidden: a save onto one of the
|
|
2054
|
-
// rebase's own conflicted paths (the "saved over conflict-marker
|
|
2055
|
-
// content" case) is excluded below, because the file reads `UU`
|
|
2056
|
-
// whether or not an editor touched it -- status alone cannot tell
|
|
2057
|
-
// the two apart. Those discards go unlogged.
|
|
2058
|
-
const collateral = (preAbort?.files ?? [])
|
|
2059
|
-
.filter((f) => !preAbort?.conflicted.includes(f.path))
|
|
2060
|
-
.filter((f) => f.working_dir !== ' ' && f.working_dir !== '?')
|
|
2061
|
-
.map((f) => f.path);
|
|
2062
|
-
if (collateral.length > 0) {
|
|
2063
|
-
workerLogWarn(` ${branchDir}: aborting the interrupted rebase will DISCARD working-tree changes to ` +
|
|
2064
|
-
`${collateral.length} file(s) saved while the worker was down: ${collateral.join(', ')}`);
|
|
2065
|
-
}
|
|
2066
|
-
workerLogWarn(` ${branchDir}: found an interrupted rebase (this worker's own abandoned work) -- aborting it to recover the branch`);
|
|
2067
|
-
try {
|
|
2068
|
-
await branchGit.rebase(['--abort']);
|
|
2069
|
-
}
|
|
2070
|
-
catch (abortErr) {
|
|
2071
|
-
// Leave it for the next cycle rather than pressing on: every step
|
|
2072
|
-
// below assumes a clean tree.
|
|
2073
|
-
const reason = redactCredentials(`could not abort interrupted rebase: ${getErrorMessage(abortErr)}`);
|
|
2074
|
-
workerLogWarn(` Skipping ${branchDir}: ${reason}`);
|
|
2075
|
-
failed.push({ branch: branchDir, error: reason });
|
|
2076
|
-
// Record on branch metadata too, like the other two failure exits
|
|
2077
|
-
// (`!completed` and the outer catch). Without this a persistently
|
|
2078
|
-
// un-abortable wedge appeared in worker-status.json but never set
|
|
2079
|
-
// a `syncFailureReason`, so the admin branch panel showed nothing
|
|
2080
|
-
// -- and this is precisely the state that needs an operator,
|
|
2081
|
-
// since it is the one the next cycle cannot fix by itself.
|
|
2082
|
-
await this.recordRebaseFailure(branchPath, branchDir, reason);
|
|
2083
|
-
continue;
|
|
2084
|
-
}
|
|
2085
|
-
}
|
|
2086
|
-
// Skip dirty branches — editor has unsaved changes that can't be rebased.
|
|
2087
|
-
// Now inside the lock, so no write can land between this check and the
|
|
2088
|
-
// rebase below.
|
|
2089
|
-
const dirtyCheck = await branchGit.status();
|
|
2090
|
-
if (dirtyCheck.files.length > 0) {
|
|
2091
|
-
workerLog(` Skipping ${branchDir}: has uncommitted changes`);
|
|
2092
|
-
skippedDirty.push(branchDir);
|
|
2093
|
-
continue;
|
|
2094
|
-
}
|
|
2095
|
-
// The clone's own ref name: branchDir is the sanitized DIRECTORY
|
|
2096
|
-
// name and need not match it. A literal 'HEAD' means a detached
|
|
2097
|
-
// clone (e.g. a crashed rebase left one behind) -- nothing below can
|
|
2098
|
-
// safely name a ref then, so every publish path stays disarmed,
|
|
2099
|
-
// which is the safe direction.
|
|
2100
|
-
const branchRef = (await branchGit.revparse(['--abbrev-ref', 'HEAD'])).trim();
|
|
2101
|
-
const canPublish = branchRef.length > 0 && branchRef !== 'HEAD';
|
|
2102
|
-
// [SYNC-H1] Self-heal: finish an interrupted publish even when this
|
|
2103
|
-
// branch is not behind base. Every crash window in the arming
|
|
2104
|
-
// sequence below leaves the marker set with the work unfinished, and
|
|
2105
|
-
// this is what completes it -- without it, one lost lease race would
|
|
2106
|
-
// strand the branch until the base branch happened to advance again.
|
|
2107
|
-
// Gated on the loop-top snapshot, so unmarked branches (nearly all
|
|
2108
|
-
// of them, every cycle) cost nothing extra.
|
|
2109
|
-
if (canPublish && metaFile?.branch.historyRewrittenFrom) {
|
|
2110
|
-
await this.reconcilePendingRewrite({
|
|
2111
|
-
branchPath,
|
|
2112
|
-
branchDir,
|
|
2113
|
-
branchRef,
|
|
2114
|
-
headSha: (await branchGit.revparse(['HEAD'])).trim(),
|
|
2115
|
-
marker: metaFile.branch.historyRewrittenFrom,
|
|
2116
|
-
});
|
|
2117
|
-
}
|
|
2118
|
-
await branchGit.fetch('origin', this.baseBranch);
|
|
2119
|
-
// Use rev-list instead of status.behind — status.behind only works when the
|
|
2120
|
-
// branch has an upstream tracking branch configured, which isn't guaranteed
|
|
2121
|
-
// (checkoutBranch fallback paths create branches without --track).
|
|
2122
|
-
// The just-fetched tip, not origin/<base>: branch clones are
|
|
2123
|
-
// --single-branch, so no remote-tracking ref exists for a base branch
|
|
2124
|
-
// other than the one they were cloned from (see the base-refresh
|
|
2125
|
-
// comment above). Pinned to a SHA immediately — FETCH_HEAD is one
|
|
2126
|
-
// shared mutable file per repo, repointed by any concurrent fetch.
|
|
2127
|
-
const fetchedBaseTip = (await branchGit.revparse(['FETCH_HEAD'])).trim();
|
|
2128
|
-
const behindCount = parseInt((await branchGit.raw(['rev-list', '--count', `HEAD..${fetchedBaseTip}`])).trim(), 10);
|
|
2129
|
-
const meta = getBranchMetadataFileManager(branchPath, this.contentBranchesPath);
|
|
2130
|
-
if (behindCount === 0) {
|
|
2131
|
-
// Already in sync. This is the overwhelmingly common outcome per
|
|
2132
|
-
// branch per cycle (most branches are caught up most of the
|
|
2133
|
-
// time), so skip the save entirely when metadata already reflects
|
|
2134
|
-
// a clean state -- every save() now eager-regenerates the branch
|
|
2135
|
-
// registry (branch-metadata.ts's invalidateRegistry(), O(branch
|
|
2136
|
-
// count) fs reads on EFS), so an unconditional save here turns
|
|
2137
|
-
// every rebase cycle into O(N^2) registry work across N branches
|
|
2138
|
-
// for what is otherwise a true no-op. Re-load fresh (not the
|
|
2139
|
-
// `metaFile` snapshot from before the fetch/rev-list above) so a
|
|
2140
|
-
// concurrent editor-driven metadata change during that window
|
|
2141
|
-
// isn't clobbered by a stale skip decision.
|
|
2142
|
-
const currentMeta = await BranchMetadataFileManager.loadOnly(branchPath);
|
|
2143
|
-
const conflictStatus = currentMeta?.branch.conflictStatus;
|
|
2144
|
-
const conflictFiles = currentMeta?.branch.conflictFiles;
|
|
2145
|
-
const conflictAlreadyClean = (conflictStatus === undefined || conflictStatus === 'clean') &&
|
|
2146
|
-
(conflictFiles === undefined || conflictFiles.length === 0);
|
|
2147
|
-
// PR-W2: a lingering rebaseFailure must also be cleared once the
|
|
2148
|
-
// branch catches up clean -- otherwise it sticks as a stale
|
|
2149
|
-
// warning forever (nothing else touches this branch once it's
|
|
2150
|
-
// caught up, so no other save site would ever clear it).
|
|
2151
|
-
const alreadyClean = conflictAlreadyClean && currentMeta?.branch.rebaseFailure === undefined;
|
|
2152
|
-
if (alreadyClean) {
|
|
2153
|
-
continue;
|
|
2154
|
-
}
|
|
2155
|
-
await meta.save({
|
|
2156
|
-
branch: {
|
|
2157
|
-
name: branchDir,
|
|
2158
|
-
conflictStatus: 'clean',
|
|
2159
|
-
conflictFiles: [],
|
|
2160
|
-
rebaseFailure: undefined,
|
|
2161
|
-
},
|
|
2162
|
-
});
|
|
2163
|
-
continue;
|
|
2164
|
-
}
|
|
2165
|
-
workerLog(`Rebasing ${branchDir} (${behindCount} commits behind)...`);
|
|
2166
|
-
// Read BOTH sides before rewriting anything: the arming guard after
|
|
2167
|
-
// the rebase compares what remote.git published against what this
|
|
2168
|
-
// clone is about to rebase away. Reading them afterwards would be
|
|
2169
|
-
// useless -- the clone's pre-rebase tip is exactly what disappears.
|
|
2170
|
-
const preRebaseHead = (await branchGit.revparse(['HEAD'])).trim();
|
|
2171
|
-
const publishedSha = canPublish ? await this.readPublishedSha(branchRef) : null;
|
|
2172
|
-
// Resolve-and-continue loop: keep branch version for conflicting files, then continue
|
|
2173
|
-
// Non-conflicting files get main's changes; conflicting files keep branch version.
|
|
2174
|
-
const conflictedFiles = [];
|
|
2175
|
-
let nextAction = 'start';
|
|
2176
|
-
let completed = false;
|
|
2177
|
-
// PR-W1: captured only on the "unexpected error" exit below, for the
|
|
2178
|
-
// failed-summary entry pushed at the `if (!completed)` check.
|
|
2179
|
-
let failureReason;
|
|
2180
|
-
const MAX_ROUNDS = 50; // safety limit against infinite loops
|
|
2181
|
-
for (let round = 0; round < MAX_ROUNDS && !completed; round++) {
|
|
2182
|
-
// Re-checked every round: the compromise can land between rounds,
|
|
2183
|
-
// and `--continue`/`--skip` are as destructive as the initial
|
|
2184
|
-
// rebase. Handled after the loop so it cannot be mistaken for the
|
|
2185
|
-
// `!completed` rebase-FAILURE path below.
|
|
2186
|
-
if (contentLockCompromised)
|
|
2187
|
-
break;
|
|
2188
|
-
try {
|
|
2189
|
-
if (nextAction === 'start') {
|
|
2190
|
-
// The pinned base tip fetched above (single-branch clones have
|
|
2191
|
-
// no origin/<base> remote-tracking ref for other branches).
|
|
2192
|
-
await branchGit.rebase([fetchedBaseTip]);
|
|
2193
|
-
}
|
|
2194
|
-
else if (nextAction === 'continue') {
|
|
2195
|
-
await branchGit.rebase(['--continue']);
|
|
2196
|
-
}
|
|
2197
|
-
else {
|
|
2198
|
-
await branchGit.rebase(['--skip']);
|
|
2199
|
-
}
|
|
2200
|
-
completed = true;
|
|
2201
|
-
}
|
|
2202
|
-
catch (rebaseErr) {
|
|
2203
|
-
nextAction = 'continue';
|
|
2204
|
-
const st = await branchGit.status();
|
|
2205
|
-
if (st.conflicted.length > 0) {
|
|
2206
|
-
await this.afterConflictDetectedForTesting();
|
|
2207
|
-
// During rebase, --theirs = the branch being replayed (editor's work).
|
|
2208
|
-
// (git rebase reverses ours/theirs: "ours" is the rebase target, "theirs" is the branch.)
|
|
2209
|
-
//
|
|
2210
|
-
// MODIFY/DELETE conflicts have no "their version" to check out
|
|
2211
|
-
// and must be resolved by staging a delete or an add instead.
|
|
2212
|
-
// `git checkout --theirs` on one exits non-zero ("path ... does
|
|
2213
|
-
// not have their version") and simple-git throws -- and because
|
|
2214
|
-
// this loop body IS the round loop's catch, that throw escapes
|
|
2215
|
-
// the round loop entirely, skipping BOTH `rebase --abort` sites
|
|
2216
|
-
// below and leaving the clone wedged mid-rebase forever. The
|
|
2217
|
-
// index/working-tree code pair identifies which side deleted
|
|
2218
|
-
// (verified against real git, not inferred):
|
|
2219
|
-
//
|
|
2220
|
-
// U/D "deleted by them" -- the BRANCH deleted it, base
|
|
2221
|
-
// modified it. Git leaves base's version in the tree.
|
|
2222
|
-
// Keep-branch-version means honouring the delete: git rm.
|
|
2223
|
-
// D/U "deleted by us" -- base deleted it, the BRANCH
|
|
2224
|
-
// modified it. Git leaves the branch's version in the
|
|
2225
|
-
// tree. Keep-branch-version means keeping it: git add.
|
|
2226
|
-
//
|
|
2227
|
-
// Any per-file resolution that STILL fails routes into the
|
|
2228
|
-
// `!completed` path below (which aborts and records) instead of
|
|
2229
|
-
// escaping -- deliberately NOT a rethrow, since a throw from
|
|
2230
|
-
// here is exactly the bug being fixed.
|
|
2231
|
-
const conflictKind = new Map(st.files.map((f) => [f.path, `${f.index}${f.working_dir}`]));
|
|
2232
|
-
let resolutionFailure;
|
|
2233
|
-
for (const file of st.conflicted) {
|
|
2234
|
-
const kind = conflictKind.get(file);
|
|
2235
|
-
try {
|
|
2236
|
-
if (kind === 'UD') {
|
|
2237
|
-
await branchGit.raw(['rm', '-f', '--', file]);
|
|
2238
|
-
}
|
|
2239
|
-
else if (kind === 'DU') {
|
|
2240
|
-
await branchGit.add(file);
|
|
2241
|
-
}
|
|
2242
|
-
else {
|
|
2243
|
-
await branchGit.raw(['checkout', '--theirs', file]);
|
|
2244
|
-
await branchGit.add(file);
|
|
2245
|
-
}
|
|
2246
|
-
}
|
|
2247
|
-
catch (resolveErr) {
|
|
2248
|
-
resolutionFailure =
|
|
2249
|
-
`failed to resolve conflicted file '${file}' (status ${kind ?? '??'}): ` +
|
|
2250
|
-
getErrorMessage(resolveErr);
|
|
2251
|
-
break;
|
|
2252
|
-
}
|
|
2253
|
-
conflictedFiles.push(file);
|
|
2254
|
-
}
|
|
2255
|
-
if (resolutionFailure !== undefined) {
|
|
2256
|
-
// Same exit shape as the "unexpected error" branch below: set
|
|
2257
|
-
// failureReason and break, letting the `!completed` block do
|
|
2258
|
-
// the single `rebase --abort` and record the failure once.
|
|
2259
|
-
failureReason = resolutionFailure;
|
|
2260
|
-
break;
|
|
2261
|
-
}
|
|
2262
|
-
// nextAction stays 'continue'
|
|
2263
|
-
}
|
|
2264
|
-
else {
|
|
2265
|
-
const msg = rebaseErr instanceof Error ? rebaseErr.message : '';
|
|
2266
|
-
if (msg.toLowerCase().includes('nothing to commit') ||
|
|
2267
|
-
msg.toLowerCase().includes('apply --skip')) {
|
|
2268
|
-
// Empty commit after --theirs resolution — skip it
|
|
2269
|
-
nextAction = 'skip';
|
|
2270
|
-
}
|
|
2271
|
-
else {
|
|
2272
|
-
// Unexpected error — abort and leave branch behind.
|
|
2273
|
-
// We intentionally don't update conflictStatus/conflictFiles here:
|
|
2274
|
-
// the rebase didn't complete so we can't determine the true conflict
|
|
2275
|
-
// state. Previous metadata (possibly stale) is preserved until the
|
|
2276
|
-
// next successful rebase cycle corrects it.
|
|
2277
|
-
workerLogWarn(` Unexpected rebase error in ${branchDir}: ${msg || 'Unknown error'}`);
|
|
2278
|
-
failureReason = msg || 'Unknown error';
|
|
2279
|
-
await branchGit.rebase(['--abort']).catch(() => { });
|
|
2280
|
-
break;
|
|
2281
|
-
}
|
|
2282
|
-
}
|
|
2283
|
-
}
|
|
2284
|
-
}
|
|
2285
|
-
// Outside the round loop's try/catch, so a throwing test hook can
|
|
2286
|
-
// never be misread as a rebase error.
|
|
2287
|
-
if (completed)
|
|
2288
|
-
await this.afterRebaseCompletedForTesting();
|
|
2289
|
-
// [SYNC-C1] A lost lock is a RETRY, not a rebase failure: nothing is
|
|
2290
|
-
// wrong with the branch, we simply can no longer prove we were the
|
|
2291
|
-
// only writer. Handled ahead of the `!completed` block so it never
|
|
2292
|
-
// records a user-visible rebaseFailure or lands in `failed[]`.
|
|
2293
|
-
//
|
|
2294
|
-
// ONLY when the rebase did not complete. Bailing out of a COMPLETED
|
|
2295
|
-
// rebase would be worse than useless: the history is already
|
|
2296
|
-
// rewritten (so `--abort` is a no-op), and skipping the completion
|
|
2297
|
-
// path below strands three things the next cycle will never redo,
|
|
2298
|
-
// because a caught-up branch short-circuits at the `behindCount === 0`
|
|
2299
|
-
// check above -- the [SYNC-H1] `markHistoryRewritten` marker (without
|
|
2300
|
-
// which the editor's next submit is rejected non-fast-forward and
|
|
2301
|
-
// mis-diagnosed as another deployment), the content-cache
|
|
2302
|
-
// invalidation for a tree that DID change, and the conflictStatus
|
|
2303
|
-
// save. That converts a transient lock compromise into a permanently
|
|
2304
|
-
// wedged published branch.
|
|
2305
|
-
//
|
|
2306
|
-
// Nor does bailing protect a racing save: a rebase replays COMMITTED
|
|
2307
|
-
// history, while a concurrent save is uncommitted working-tree state,
|
|
2308
|
-
// and that writer is already told to retry via
|
|
2309
|
-
// ContentWriteLockBusyError. So when the rebase completed, log the
|
|
2310
|
-
// lost exclusivity loudly and finish the job.
|
|
2311
|
-
if (contentLockCompromised && !completed) {
|
|
2312
|
-
workerLogWarn(` Skipping ${branchDir}: content-write lock was compromised mid-rebase (retrying next cycle)`);
|
|
2313
|
-
// No-op when no rebase is in progress; failure is expected there.
|
|
2314
|
-
await branchGit.rebase(['--abort']).catch(() => { });
|
|
2315
|
-
skippedLocked.push(branchDir);
|
|
2316
|
-
continue;
|
|
2317
|
-
}
|
|
2318
|
-
if (contentLockCompromised) {
|
|
2319
|
-
workerLogWarn(` Content-write lock for ${branchDir} was compromised, but its rebase had already completed -- finishing the sync (history is rewritten; skipping now would strand the history-rewrite marker and wedge the branch)`);
|
|
2320
|
-
}
|
|
2321
|
-
if (!completed) {
|
|
2322
|
-
// PR-W2 (M1 rider): failureReason is only set on the "unexpected
|
|
2323
|
-
// error" break above -- MAX_ROUNDS exhaustion is a distinct exit
|
|
2324
|
-
// path with no error message of its own, so the warn text must
|
|
2325
|
-
// not conflate the two.
|
|
2326
|
-
workerLogWarn(failureReason !== undefined
|
|
2327
|
-
? ` Rebase of ${branchDir} aborted due to unexpected error: ${failureReason}`
|
|
2328
|
-
: ` Rebase of ${branchDir} did not complete within ${MAX_ROUNDS} rounds, aborting`);
|
|
2329
|
-
await branchGit.rebase(['--abort']).catch(() => { });
|
|
2330
|
-
const rebaseFailureMessage = failureReason ?? `did not complete within ${MAX_ROUNDS} rounds`;
|
|
2331
|
-
// [HIGH-1] failed[] folds into worker-status.json's
|
|
2332
|
-
// lastGitSync.failed, served to the browser -- failureReason can
|
|
2333
|
-
// be an arbitrary git error message that embeds the bot token.
|
|
2334
|
-
const redactedRebaseFailureMessage = redactCredentials(rebaseFailureMessage);
|
|
2335
|
-
failed.push({
|
|
2336
|
-
branch: branchDir,
|
|
2337
|
-
error: redactedRebaseFailureMessage,
|
|
2338
|
-
});
|
|
2339
|
-
// PR-W2: record once here for the "!completed" exit -- the
|
|
2340
|
-
// unexpected-error break above is NOT disjoint from this block (it
|
|
2341
|
-
// always falls through here), so recording at the break itself
|
|
2342
|
-
// would double-record. The outer catch below is the only other
|
|
2343
|
-
// record site (a distinct, non-overlapping failure class: errors
|
|
2344
|
-
// outside this round loop, e.g. fetch/rev-list failures).
|
|
2345
|
-
await this.recordRebaseFailure(branchPath, branchDir, redactedRebaseFailureMessage);
|
|
2346
|
-
continue;
|
|
2347
|
-
}
|
|
2348
|
-
// The rebase rewrote the branch clone's working tree — mark ContentStore
|
|
2349
|
-
// ID indexes rooted here stale so lookups rebuild from disk, in this
|
|
2350
|
-
// process and (via the on-disk generation marker) in the Lambda
|
|
2351
|
-
// containers sharing this filesystem.
|
|
2352
|
-
await invalidateBranchContentCaches(branchPath);
|
|
2353
|
-
// Convert file paths to ContentIds — immutable, survives slug renames.
|
|
2354
|
-
// Entry files have IDs in their filename (e.g., "post.slug.a1b2c3d4e5f6.mdx").
|
|
2355
|
-
// .collection.json files have no ID themselves (extractIdFromFilename returns null
|
|
2356
|
-
// for dot-prefixed files), so we extract the ID from the parent directory instead.
|
|
2357
|
-
// The root content directory (e.g., "content/", or "cms/content/" for a
|
|
2358
|
-
// multi-segment contentRoot) has no embedded ID, so we use ROOT_COLLECTION_ID
|
|
2359
|
-
// as a sentinel — but only for the configured contentRoot.
|
|
2360
|
-
//
|
|
2361
|
-
// Two different notions of "parent" are needed below, and conflating them
|
|
2362
|
-
// reproduces the exact bug this comparison guards against (see
|
|
2363
|
-
// schema-store.ts's contentRootName doc comment for the same shape elsewhere):
|
|
2364
|
-
// - `parentDir` (a basename) recovers a SUB-collection's own embedded ID
|
|
2365
|
-
// (e.g. "posts.cNbR5xFm2Kpd" -> "cNbR5xFm2Kpd") — correct as a basename,
|
|
2366
|
-
// since a collection directory carries its ID in its own name, one path
|
|
2367
|
-
// segment.
|
|
2368
|
-
// - `parentPath` (the full relative parent path, normalized) is what must be
|
|
2369
|
-
// compared against `this.contentRoot`, because `contentRoot` is documented
|
|
2370
|
-
// (config/helpers.ts) as allowed to span multiple segments (e.g.
|
|
2371
|
-
// "cms/content"). Comparing a basename ("content") against that full value
|
|
2372
|
-
// is always false, which silently drops the root collection's conflict.
|
|
2373
|
-
// Git reports POSIX-style paths and the configured value may be authored
|
|
2374
|
-
// with either separator, so both sides go through normalizeFilesystemPath
|
|
2375
|
-
// before comparing.
|
|
2376
|
-
const normalizedContentRoot = normalizeFilesystemPath(this.contentRoot);
|
|
2377
|
-
const conflictIds = [...new Set(conflictedFiles)]
|
|
2378
|
-
.map((f) => {
|
|
2379
|
-
const fileId = extractIdFromFilename(path.basename(f));
|
|
2380
|
-
if (fileId)
|
|
2381
|
-
return fileId;
|
|
2382
|
-
const parentDir = path.basename(path.dirname(f));
|
|
2383
|
-
const dirId = extractIdFromFilename(parentDir);
|
|
2384
|
-
if (dirId)
|
|
2385
|
-
return dirId;
|
|
2386
|
-
// Only assign ROOT_COLLECTION_ID when the file's parent directory IS the
|
|
2387
|
-
// configured content root. Other unrecognized paths are filtered out.
|
|
2388
|
-
const parentPath = normalizeFilesystemPath(path.dirname(f));
|
|
2389
|
-
if (path.basename(f) === '.collection.json' && parentPath === normalizedContentRoot) {
|
|
2390
|
-
return ROOT_COLLECTION_ID;
|
|
2391
|
-
}
|
|
2392
|
-
return null;
|
|
2393
|
-
})
|
|
2394
|
-
.filter((id) => id !== null);
|
|
2395
|
-
const conflictIdsDeduped = [...new Set(conflictIds)];
|
|
2396
|
-
const hadConflicts = conflictIdsDeduped.length > 0;
|
|
2397
|
-
workerLog(hadConflicts
|
|
2398
|
-
? ` Rebased ${branchDir} (kept branch version for ${conflictIdsDeduped.length} conflicting file(s))`
|
|
2399
|
-
: ` Rebased ${branchDir} successfully`);
|
|
2400
|
-
await meta.save({
|
|
2401
|
-
branch: {
|
|
2402
|
-
name: branchDir,
|
|
2403
|
-
conflictStatus: hadConflicts ? 'conflicts-detected' : 'clean',
|
|
2404
|
-
conflictFiles: conflictIdsDeduped,
|
|
2405
|
-
// PR-W2: the cycle completed successfully -- clear any prior
|
|
2406
|
-
// failure record regardless of conflict outcome.
|
|
2407
|
-
rebaseFailure: undefined,
|
|
2408
|
-
},
|
|
2409
|
-
});
|
|
2410
|
-
// PR-W1: the branch was behind and the rebase completed (with or
|
|
2411
|
-
// without --theirs conflict resolution) -- it moved, so it belongs
|
|
2412
|
-
// in the summary. Branches already up to date `continue`d above and
|
|
2413
|
-
// are deliberately not listed here.
|
|
2414
|
-
rebased.push(branchDir);
|
|
2415
|
-
// [SYNC-H1] The rebase just rewrote this clone's history. If that
|
|
2416
|
-
// history was already published, nothing else will ever reconcile
|
|
2417
|
-
// remote.git (and GitHub) with it -- the editor's next submit would
|
|
2418
|
-
// simply be rejected non-fast-forward. Carry the rewrite forward.
|
|
2419
|
-
if (publishedSha !== null) {
|
|
2420
|
-
if (publishedSha === preRebaseHead) {
|
|
2421
|
-
// ARMING GUARD. remote.git holds EXACTLY what this clone just
|
|
2422
|
-
// rebased away and nothing more, so a lease keyed to it can only
|
|
2423
|
-
// undo our own rewrite.
|
|
2424
|
-
//
|
|
2425
|
-
// The inequality case below is not defensive padding: branch
|
|
2426
|
-
// clones never fetch their own branch (GitManager's clone is
|
|
2427
|
-
// --single-branch and checkoutBranch only checks out an existing
|
|
2428
|
-
// local branch), while reconcileTrackedBranches fast-forwards
|
|
2429
|
-
// remote.git to GitHub's tip. So after a reviewer pushes a fixup
|
|
2430
|
-
// straight to the PR branch, remote.git legitimately holds a
|
|
2431
|
-
// commit this clone has never seen. Leasing on "whatever
|
|
2432
|
-
// remote.git currently holds" would be SATISFIED there and would
|
|
2433
|
-
// delete that fixup from remote.git and then from GitHub,
|
|
2434
|
-
// silently. Keying the lease to the pre-rebase tip turns that
|
|
2435
|
-
// case into the visible divergence it should be.
|
|
2436
|
-
//
|
|
2437
|
-
// Order matters: mark, then push, then queue. A crash after any
|
|
2438
|
-
// step leaves the marker set with the work unfinished, which
|
|
2439
|
-
// reconcilePendingRewrite() completes on a later cycle. Pushing
|
|
2440
|
-
// first would leave remote.git rewritten, GitHub stale and
|
|
2441
|
-
// nothing recorded -- unrecoverable, and landing on exactly the
|
|
2442
|
-
// false "another deployment" diagnosis this change removes.
|
|
2443
|
-
await this.markHistoryRewritten(branchPath, branchDir, publishedSha);
|
|
2444
|
-
if (await this.forcePublishToLocalRemote(branchPath, branchRef, publishedSha)) {
|
|
2445
|
-
await this.enqueueGitHubPush(branchRef);
|
|
2446
|
-
}
|
|
2447
|
-
}
|
|
2448
|
-
else {
|
|
2449
|
-
const divergence = `rebased locally, but remote.git holds ${publishedSha} for ${branchRef}, which this ` +
|
|
2450
|
-
`clone never had (a direct push to the branch?). Left untouched -- reconcile it ` +
|
|
2451
|
-
`before submitting again.`;
|
|
2452
|
-
workerLogWarn(` ${branchDir}: ${divergence}`);
|
|
2453
|
-
await this.recordRebaseFailure(branchPath, branchDir, divergence);
|
|
2454
|
-
}
|
|
2455
|
-
}
|
|
2456
|
-
}
|
|
2457
|
-
finally {
|
|
2458
|
-
// Last-resort guarantee that NO exit path leaves this clone
|
|
2459
|
-
// mid-rebase -- including an unexpected throw from any git step
|
|
2460
|
-
// above, which lands in the outer catch and previously only logged.
|
|
2461
|
-
//
|
|
2462
|
-
// It must live HERE rather than in that outer catch: the catch runs
|
|
2463
|
-
// AFTER this finally has released the content-write lock, so aborting
|
|
2464
|
-
// there would hard-reset a working tree an editor's save could
|
|
2465
|
-
// already be racing -- precisely the [SYNC-C1] hazard the lock
|
|
2466
|
-
// exists to prevent. Inside the finally the lock is still held --
|
|
2467
|
-
// EXCEPT on the narrow path where it was compromised mid-hold, in
|
|
2468
|
-
// which case a newly-admitted writer may already be live and this
|
|
2469
|
-
// abort carries the same exposure as the compromise path's own abort
|
|
2470
|
-
// above. Not special-cased: leaving a clone wedged mid-rebase is the
|
|
2471
|
-
// worse outcome, and the writer in that window is already being told
|
|
2472
|
-
// to retry.
|
|
2473
|
-
//
|
|
2474
|
-
// Guarded on actual rebase state so the happy path and the `continue`
|
|
2475
|
-
// exits cost one stat and do nothing.
|
|
2476
|
-
try {
|
|
2477
|
-
if (await isRebaseInProgress(branchPath)) {
|
|
2478
|
-
workerLogWarn(` ${branchDir}: rebase still in progress on exit -- aborting so the clone is not left wedged`);
|
|
2479
|
-
await branchGit.rebase(['--abort']);
|
|
2480
|
-
}
|
|
2481
|
-
}
|
|
2482
|
-
catch (abortErr) {
|
|
2483
|
-
// Best effort: the next cycle's recovery check retries this.
|
|
2484
|
-
workerLogWarn(` Failed to abort in-progress rebase for ${branchDir}: ${getErrorMessage(abortErr)}`);
|
|
2485
|
-
}
|
|
2486
|
-
// [SYNC-C1] Released on EVERY exit -- the `continue`s above, a throw
|
|
2487
|
-
// into the outer catch, and the happy path alike. A stranded lock
|
|
2488
|
-
// would wedge every write to this branch until it went stale.
|
|
2489
|
-
await releaseContentLock?.().catch((releaseErr) => {
|
|
2490
|
-
workerLogWarn(` Failed to release content-write lock for ${branchDir}: ${getErrorMessage(releaseErr)}`);
|
|
2491
|
-
});
|
|
2492
|
-
}
|
|
2493
|
-
}
|
|
2494
|
-
catch (err) {
|
|
2495
|
-
const message = err instanceof Error ? err.message : 'Unknown error';
|
|
2496
|
-
workerLogWarn(` Failed to sync ${branchDir}: ${message}`);
|
|
2497
|
-
// [HIGH-1] Same rationale as the `if (!completed)` push site above --
|
|
2498
|
-
// this catches fetch/rev-list/unexpected errors, whose message can
|
|
2499
|
-
// embed the bot token.
|
|
2500
|
-
const redactedMessage = redactCredentials(message);
|
|
2501
|
-
failed.push({ branch: branchDir, error: redactedMessage });
|
|
2502
|
-
// PR-W2: second (and only other) record site -- see the comment at
|
|
2503
|
-
// the `if (!completed)` block above for why these two sites are
|
|
2504
|
-
// disjoint.
|
|
2505
|
-
await this.recordRebaseFailure(branchPath, branchDir, redactedMessage);
|
|
2506
|
-
}
|
|
2507
|
-
}
|
|
2508
|
-
return { rebased, skippedDirty, skippedLocked, failed };
|
|
604
|
+
return runRebaseCycle(this.ctx());
|
|
605
|
+
}
|
|
606
|
+
// --- Git-sync cluster (worker/git-sync.ts) -----------------------------
|
|
607
|
+
//
|
|
608
|
+
// `syncGit` is the public loop entry `scheduleLoop` drives. The three private
|
|
609
|
+
// ones below it are TEST-ONLY entry points -- see the task-queue block above
|
|
610
|
+
// -- each called directly by one test file (cms-worker-sync-reconcile for the
|
|
611
|
+
// settings push, cms-worker-base-refresh for the base workspace,
|
|
612
|
+
// cms-worker.test.ts for the trash sweep). `syncGit` reaches all three as
|
|
613
|
+
// module-level calls, so a stub installed on one of these is a no-op.
|
|
614
|
+
//
|
|
615
|
+
// `reconcileTrackedBranches` deliberately has NO delegator: no test reaches
|
|
616
|
+
// it through the instance. Comments elsewhere that point at it as living in
|
|
617
|
+
// this file are stale by definition -- it is in git-sync.ts.
|
|
618
|
+
async syncGit() {
|
|
619
|
+
return syncGit(this.ctx());
|
|
620
|
+
}
|
|
621
|
+
async pushSettingsBranches(git, trackedNames) {
|
|
622
|
+
return pushSettingsBranches(this.ctx(), git, trackedNames);
|
|
623
|
+
}
|
|
624
|
+
async refreshBaseBranchWorkspace() {
|
|
625
|
+
return refreshBaseBranchWorkspace(this.ctx());
|
|
626
|
+
}
|
|
627
|
+
async cleanupTrashedBranchDirs() {
|
|
628
|
+
return cleanupTrashedBranchDirs(this.ctx());
|
|
629
|
+
}
|
|
630
|
+
async pollMergeState(branchDir, branchPath, metaFile) {
|
|
631
|
+
return pollMergeState(this.ctx(), branchDir, branchPath, metaFile);
|
|
2509
632
|
}
|
|
2510
633
|
async refreshAuthCache() {
|
|
2511
634
|
if (!this.running || !this.config.refreshAuthCache)
|