canopycms 0.0.65 → 0.0.66-int.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -3
- package/dist/ai/generate.d.ts +13 -0
- package/dist/ai/generate.js +9 -3
- package/dist/ai/to-plain-text.js +4 -0
- package/dist/ai/types.d.ts +20 -1
- package/dist/api/admin-branch-health.js +13 -12
- package/dist/api/branch.js +2 -2
- package/dist/branch-health.js +1 -1
- package/dist/branch-metadata-file.d.ts +58 -0
- package/dist/branch-metadata-file.js +65 -0
- package/dist/branch-metadata.d.ts +2 -26
- package/dist/branch-metadata.js +6 -33
- package/dist/branch-registry.js +4 -2
- package/dist/build/generate-ai-content.js +103 -0
- package/dist/cli/cli.js +432 -317
- package/dist/cli/generate-ai-content.js +264 -160
- package/dist/content-id-index.js +22 -0
- package/dist/content-listing.js +3 -0
- package/dist/content-reader.d.ts +25 -1
- package/dist/content-reader.js +15 -0
- package/dist/content-store.d.ts +50 -1
- package/dist/content-store.js +102 -4
- package/dist/context.d.ts +36 -4
- package/dist/context.js +15 -4
- package/dist/editor/admin/SystemHealthPanel.js +2 -2
- package/dist/entry-schema.d.ts +22 -0
- package/dist/git-manager.d.ts +9 -8
- package/dist/git-manager.js +35 -9
- package/dist/github-service.d.ts +1 -1
- package/dist/github-service.js +1 -1
- package/dist/paths/branch-name.d.ts +1 -1
- package/dist/paths/branch-name.js +1 -1
- package/dist/schema/index.d.ts +1 -1
- package/dist/schema/index.js +1 -1
- package/dist/schema/meta-loader.js +1 -1
- package/dist/services.d.ts +22 -0
- package/dist/types.d.ts +1 -1
- package/dist/url-exclusivity-fixtures.d.ts +76 -0
- package/dist/url-exclusivity-fixtures.js +119 -0
- package/dist/url-path-resolver.js +8 -4
- package/dist/utils/content-write-lock.d.ts +2 -1
- package/dist/utils/content-write-lock.js +2 -1
- package/dist/utils/error.d.ts +17 -0
- package/dist/utils/error.js +17 -0
- package/dist/utils/git.d.ts +1 -1
- package/dist/utils/git.js +1 -1
- package/dist/utils/occ-json-write.js +1 -1
- package/dist/worker/cms-worker.d.ts +22 -288
- package/dist/worker/cms-worker.js +118 -1995
- package/dist/worker/git-sync.d.ts +166 -0
- package/dist/worker/git-sync.js +554 -0
- package/dist/worker/history-rewrite.d.ts +129 -0
- package/dist/worker/history-rewrite.js +216 -0
- package/dist/worker/rebase.d.ts +171 -0
- package/dist/worker/rebase.js +859 -0
- package/dist/worker/task-runner.d.ts +93 -0
- package/dist/worker/task-runner.js +529 -0
- package/dist/worker/worker-context.d.ts +133 -0
- package/dist/worker/worker-context.js +1 -0
- package/package.json +1 -1
|
@@ -0,0 +1,554 @@
|
|
|
1
|
+
import fs from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { simpleGit } from 'simple-git';
|
|
4
|
+
import { BranchMetadataFileManager, getBranchMetadataFileManager } from '../branch-metadata.js';
|
|
5
|
+
import { invalidateBranchContentCaches } from '../content-index-generation.js';
|
|
6
|
+
import { GITHUB_TRACKING_REF_PREFIX, gitNetworkChildEnv } from '../git-manager.js';
|
|
7
|
+
import { RESERVED_SETTINGS_BRANCH_PREFIX } from '../paths/branch-name.js';
|
|
8
|
+
import { getErrorMessage, isNodeError, redactCredentials } from '../utils/error.js';
|
|
9
|
+
import { isNonFastForwardRejection } from '../utils/git.js';
|
|
10
|
+
import { hasPendingHistoryRewrite } from './history-rewrite.js';
|
|
11
|
+
import { runRebaseCycle } from './rebase.js';
|
|
12
|
+
import { cleanupOldTasks } from './task-queue.js';
|
|
13
|
+
import { writeWorkerStatus } from './worker-status.js';
|
|
14
|
+
import { workerLog, workerLogError, workerLogWarn } from './log.js';
|
|
15
|
+
/**
|
|
16
|
+
* [C1] Retention window for `.trash-*` branch directories left behind by the
|
|
17
|
+
* admin purge action (api/admin-branch-health.ts). Matches
|
|
18
|
+
* cleanupOldTasks's default task retention for consistency.
|
|
19
|
+
*/
|
|
20
|
+
const TRASH_RETENTION_MS = 30 * 24 * 60 * 60_000;
|
|
21
|
+
/** Matches `.trash-{dirName}-{STAMP}` names, capturing the trailing stamp. */
|
|
22
|
+
const TRASH_DIR_STAMP_RE = /-(\d{8}T\d{6}Z)$/;
|
|
23
|
+
/**
|
|
24
|
+
* Parse a purge-generated `YYYYMMDDTHHMMSSZ` stamp into a Date, or null if
|
|
25
|
+
* malformed. Age comes ONLY from this name-embedded stamp, never the dir's
|
|
26
|
+
* own mtime -- `fs.rename` preserves the original directory's mtime, so an
|
|
27
|
+
* mtime-based retention check would delete a months-stale orphan's trash on
|
|
28
|
+
* the very first cleanup pass after purge.
|
|
29
|
+
*/
|
|
30
|
+
function parseTrashStamp(stamp) {
|
|
31
|
+
const match = /^(\d{4})(\d{2})(\d{2})T(\d{2})(\d{2})(\d{2})Z$/.exec(stamp);
|
|
32
|
+
if (!match)
|
|
33
|
+
return null;
|
|
34
|
+
const [, year, month, day, hour, minute, second] = match;
|
|
35
|
+
const date = new Date(`${year}-${month}-${day}T${hour}:${minute}:${second}.000Z`);
|
|
36
|
+
return Number.isNaN(date.getTime()) ? null : date;
|
|
37
|
+
}
|
|
38
|
+
/**
|
|
39
|
+
* Push THIS deployment's own settings branch (`ensureSettingsBranch()`) from
|
|
40
|
+
* remote.git to GitHub. Non-fatal: a no-op push for an up-to-date branch
|
|
41
|
+
* just succeeds quietly.
|
|
42
|
+
*
|
|
43
|
+
* Deliberately narrowed to one branch — this used to push EVERY local
|
|
44
|
+
* branch matching `canopycms-settings-*`. With the tracking-namespace fetch
|
|
45
|
+
* fix (GITHUB_TRACKING_REF_PREFIX), `reconcileTrackedBranches` creates local
|
|
46
|
+
* heads for branches that exist on GitHub, so ANOTHER deployment's settings
|
|
47
|
+
* branch (sharing this same GitHub repo) can legitimately show up as a
|
|
48
|
+
* local head here too. Pushing it would be this deployment shipping
|
|
49
|
+
* settings state it doesn't own.
|
|
50
|
+
*/
|
|
51
|
+
export async function pushSettingsBranches(ctx, git, trackedNames) {
|
|
52
|
+
try {
|
|
53
|
+
// Resolved once here rather than at each use below: ensureSettingsBranch()
|
|
54
|
+
// is idempotent, but a local reads better and keeps the eight references
|
|
55
|
+
// in this method obviously talking about one value.
|
|
56
|
+
const settingsBranch = ctx.ensureSettingsBranch();
|
|
57
|
+
const branches = await git.branch();
|
|
58
|
+
const settingsBranches = branches.all.filter((b) => b.startsWith(RESERVED_SETTINGS_BRANCH_PREFIX));
|
|
59
|
+
const foreign = settingsBranches.filter((b) => b !== settingsBranch);
|
|
60
|
+
if (foreign.length > 0) {
|
|
61
|
+
// Signal, not an error: this is exactly the "two deployments, one repo"
|
|
62
|
+
// condition this workstream exists to make visible. Never push these.
|
|
63
|
+
workerLogWarn(`Found settings branch(es) not owned by this deployment (${settingsBranch}): ` +
|
|
64
|
+
`${foreign.join(', ')}. Another CanopyCMS deployment may share this GitHub repo. Not pushing them.`);
|
|
65
|
+
}
|
|
66
|
+
// Check the full branch list, not the `canopycms-settings-*` subset: an
|
|
67
|
+
// adopter-supplied `settingsBranch` override need not carry that prefix.
|
|
68
|
+
const ownBranchMissing = !branches.all.includes(settingsBranch);
|
|
69
|
+
// [SYNC-M3] A settings branch present in remote.git but absent from
|
|
70
|
+
// GitHub's tracking refs was pushed here LOCALLY -- only this
|
|
71
|
+
// deployment's own API can write to remote.git -- and has never
|
|
72
|
+
// reached GitHub. That is the discriminating signature: in the
|
|
73
|
+
// SUPPORTED two-deployments-one-repo case the foreign branch arrives
|
|
74
|
+
// through the GitHub fetch and therefore always has a tracking ref, so
|
|
75
|
+
// the warn above cannot tell the two apart on its own and a
|
|
76
|
+
// "owned-branch-absent" test alone would fire on every deployment that
|
|
77
|
+
// simply has not had a settings edit yet.
|
|
78
|
+
//
|
|
79
|
+
// With this deployment's own branch also missing, the API and this
|
|
80
|
+
// worker have resolved different deploymentNames, and every settings
|
|
81
|
+
// change the API commits is stranded in remote.git forever.
|
|
82
|
+
const strandedLocal = foreign.filter((b) => !trackedNames.has(b));
|
|
83
|
+
if (strandedLocal.length > 0) {
|
|
84
|
+
workerLogWarn(ownBranchMissing
|
|
85
|
+
? `Settings branch mismatch: this worker owns "${settingsBranch}", which does not ` +
|
|
86
|
+
`exist in remote.git, while ${strandedLocal.join(', ')} exist(s) here and has never ` +
|
|
87
|
+
`been pushed to GitHub. The API and this worker disagree about deploymentName, so ` +
|
|
88
|
+
`settings changes are NOT reaching GitHub. Set CANOPYCMS_DEPLOYMENT_NAME (or ` +
|
|
89
|
+
`settingsBranch) on this worker to match what the API resolves.`
|
|
90
|
+
: `Settings branch(es) ${strandedLocal.join(', ')} exist in remote.git but not on ` +
|
|
91
|
+
`GitHub, and are not owned by this deployment (${settingsBranch}) -- nothing ` +
|
|
92
|
+
`will ever push them onward. Check that deploymentName matches across this ` +
|
|
93
|
+
`deployment's API and worker.`);
|
|
94
|
+
}
|
|
95
|
+
if (ownBranchMissing) {
|
|
96
|
+
// Not created locally yet — nothing to push.
|
|
97
|
+
return;
|
|
98
|
+
}
|
|
99
|
+
try {
|
|
100
|
+
await git.push(ctx.buildGitHubUrl(), settingsBranch);
|
|
101
|
+
workerLog(`Pushed settings branch ${settingsBranch} to GitHub`);
|
|
102
|
+
}
|
|
103
|
+
catch (err) {
|
|
104
|
+
// Non-fatal: branch may already be up-to-date. This call site has no
|
|
105
|
+
// task to throw a PermanentTaskError into (unlike pushBranchToGitHub)
|
|
106
|
+
// and is deliberately not restructured to add one -- but a
|
|
107
|
+
// non-fast-forward rejection here is the highest-signal instance of
|
|
108
|
+
// this whole failure class: it means another CanopyCMS deployment's
|
|
109
|
+
// worker already pushed ITS OWN state to its own settings branch on
|
|
110
|
+
// GitHub (an actual settings-branch name collision, not just the
|
|
111
|
+
// "foreign branch found locally" case warned about above), so make
|
|
112
|
+
// that explicit instead of a generic "push failed" line.
|
|
113
|
+
const message = getErrorMessage(err);
|
|
114
|
+
if (isNonFastForwardRejection(message)) {
|
|
115
|
+
workerLogWarn(`Settings push for ${settingsBranch} was rejected (non-fast-forward): another ` +
|
|
116
|
+
`CanopyCMS deployment appears to own this settings branch on GitHub. Settings from ` +
|
|
117
|
+
`that deployment will NOT be overwritten; this deployment's local settings changes ` +
|
|
118
|
+
`were not pushed. Rename this deployment's settings branch (config.settingsBranch or ` +
|
|
119
|
+
`deploymentName) to resolve the collision.`);
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
workerLogWarn(`Settings push for ${settingsBranch}:`, message);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
catch (err) {
|
|
127
|
+
workerLogWarn('Failed to list branches for settings push:', err instanceof Error ? err.message : err);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Bring `refs/heads/*` in `remote.git` toward what was just fetched into
|
|
132
|
+
* `GITHUB_TRACKING_REF_PREFIX` -- WITHOUT ever force-rewinding or deleting
|
|
133
|
+
* a local head. This is the non-destructive replacement for what the old
|
|
134
|
+
* `+refs/heads/*:refs/heads/*` fetch refspec used to do implicitly (and
|
|
135
|
+
* destructively) as part of the fetch itself; see
|
|
136
|
+
* `GITHUB_TRACKING_REF_PREFIX`'s doc comment for the two failure modes
|
|
137
|
+
* that refspec caused.
|
|
138
|
+
*
|
|
139
|
+
* Per tracked branch:
|
|
140
|
+
* - no local `refs/heads/<name>` yet -> create it at the tracked commit
|
|
141
|
+
* (a branch created on GitHub, or by another deployment sharing this
|
|
142
|
+
* GitHub repo, becomes visible locally).
|
|
143
|
+
* - local is a strict ancestor of tracked (behind) -> fast-forward it.
|
|
144
|
+
* - local === tracked -> nothing to do.
|
|
145
|
+
* - tracked is a strict ancestor of local (ahead) -> LEAVE IT ALONE. This
|
|
146
|
+
* is unpushed editor/settings work; the queued push task (or
|
|
147
|
+
* pushSettingsBranches) ships it. This is exactly the branch state the
|
|
148
|
+
* old refspec used to destroy.
|
|
149
|
+
* - neither is an ancestor of the other (diverged) -> LEAVE IT ALONE and
|
|
150
|
+
* count/log it. A real collision (e.g. another deployment moved the
|
|
151
|
+
* same branch name on GitHub); the next push attempt will be rejected
|
|
152
|
+
* non-fast-forward, which is the correct, visible outcome -- this
|
|
153
|
+
* method must never silently pick a winner. The one expected, benign
|
|
154
|
+
* form of this -- our own rebase loop having published a rewrite into
|
|
155
|
+
* remote.git with the GitHub push still queued -- is split out into the
|
|
156
|
+
* `rewritten` bucket so the collision warning stays meaningful.
|
|
157
|
+
*
|
|
158
|
+
* Never deletes a local head: a branch removed on GitHub simply stops
|
|
159
|
+
* being tracked here; the local ref persists until removed through its
|
|
160
|
+
* own explicit path (the sync loop must not be one of them).
|
|
161
|
+
*
|
|
162
|
+
* `remote.git` is bare, so there is no worktree to invalidate by moving
|
|
163
|
+
* these refs -- unlike a non-bare repo, updating the ref that happens to
|
|
164
|
+
* be "checked out" is a non-issue here.
|
|
165
|
+
*
|
|
166
|
+
* Concurrency: `remote.git` is bare and on EFS, and the Lambda pushes into
|
|
167
|
+
* it concurrently (`GitManager.push()`'s `target:target` refspec) while
|
|
168
|
+
* this runs. Every `update-ref` below passes the expected old value (the
|
|
169
|
+
* all-zeros OID for "must not exist yet" on creation, the previously-read
|
|
170
|
+
* SHA for the fast-forward case) so a concurrent Lambda write landing in
|
|
171
|
+
* the gap between the read and the write loses the ref update instead of
|
|
172
|
+
* being silently clobbered -- the branch is simply revisited next cycle.
|
|
173
|
+
*/
|
|
174
|
+
export async function reconcileTrackedBranches(ctx, git) {
|
|
175
|
+
const GIT_ZERO_OID = '0000000000000000000000000000000000000000';
|
|
176
|
+
const created = [];
|
|
177
|
+
const fastForwarded = [];
|
|
178
|
+
const ahead = [];
|
|
179
|
+
const diverged = [];
|
|
180
|
+
const rewritten = [];
|
|
181
|
+
// One invocation enumerates both namespaces: refs/heads/<name> (what the
|
|
182
|
+
// Lambda pushes into and branch clones read from) and
|
|
183
|
+
// GITHUB_TRACKING_REF_PREFIX<name> (GitHub's tip, just fetched above).
|
|
184
|
+
const raw = await git.raw([
|
|
185
|
+
'for-each-ref',
|
|
186
|
+
'--format=%(refname) %(objectname)',
|
|
187
|
+
'refs/heads/',
|
|
188
|
+
GITHUB_TRACKING_REF_PREFIX,
|
|
189
|
+
]);
|
|
190
|
+
const heads = new Map();
|
|
191
|
+
const tracked = new Map();
|
|
192
|
+
for (const line of raw.split('\n')) {
|
|
193
|
+
const trimmed = line.trim();
|
|
194
|
+
if (!trimmed)
|
|
195
|
+
continue;
|
|
196
|
+
const [refname, sha] = trimmed.split(' ');
|
|
197
|
+
if (refname.startsWith('refs/heads/')) {
|
|
198
|
+
heads.set(refname.slice('refs/heads/'.length), sha);
|
|
199
|
+
}
|
|
200
|
+
else if (refname.startsWith(GITHUB_TRACKING_REF_PREFIX)) {
|
|
201
|
+
tracked.set(refname.slice(GITHUB_TRACKING_REF_PREFIX.length), sha);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
for (const [name, trackedSha] of tracked) {
|
|
205
|
+
const localSha = heads.get(name);
|
|
206
|
+
const localRef = `refs/heads/${name}`;
|
|
207
|
+
if (!localSha) {
|
|
208
|
+
try {
|
|
209
|
+
// Zero old-value asserts the ref does not already exist -- guards
|
|
210
|
+
// against a concurrent Lambda push creating this exact branch
|
|
211
|
+
// name between the for-each-ref read above and this update.
|
|
212
|
+
await git.raw(['update-ref', localRef, trackedSha, GIT_ZERO_OID]);
|
|
213
|
+
created.push(name);
|
|
214
|
+
}
|
|
215
|
+
catch (err) {
|
|
216
|
+
workerLogWarn(` Tracked-branch reconcile: failed to create local ref for ${name} (concurrent update?): ${getErrorMessage(err)}`);
|
|
217
|
+
}
|
|
218
|
+
continue;
|
|
219
|
+
}
|
|
220
|
+
if (localSha === trackedSha)
|
|
221
|
+
continue; // nothing to do
|
|
222
|
+
// [SYNC-M2] Everything below is per-branch best-effort. The rev-list
|
|
223
|
+
// used to sit between the two guarded update-ref calls with no guard
|
|
224
|
+
// of its own, so a single ref pointing at a missing or partially
|
|
225
|
+
// written object -- plausible on EFS with the Lambda writing
|
|
226
|
+
// concurrently -- threw out of this loop, out of syncGit()'s try, and
|
|
227
|
+
// skipped pushSettingsBranches(), refreshBaseBranchWorkspace() and
|
|
228
|
+
// rebaseActiveBranches() entirely. Nothing self-healed, so it recurred
|
|
229
|
+
// every cycle. One unreadable ref must cost its own branch, not the
|
|
230
|
+
// whole sync cycle.
|
|
231
|
+
try {
|
|
232
|
+
// Count commits unique to each side in one call: left = commits local
|
|
233
|
+
// has that tracked doesn't (ahead), right = commits tracked has that
|
|
234
|
+
// local doesn't (behind). Exits 0 regardless of ancestry direction,
|
|
235
|
+
// unlike `merge-base --is-ancestor` (which simple-git would throw on
|
|
236
|
+
// a non-zero exit for the common "not an ancestor" case).
|
|
237
|
+
const counts = (await git.raw(['rev-list', '--left-right', '--count', `${localSha}...${trackedSha}`])).trim();
|
|
238
|
+
const [leftStr, rightStr] = counts.split(/\s+/);
|
|
239
|
+
const localAheadCount = parseInt(leftStr, 10);
|
|
240
|
+
const localBehindCount = parseInt(rightStr, 10);
|
|
241
|
+
if (!Number.isInteger(localAheadCount) || !Number.isInteger(localBehindCount)) {
|
|
242
|
+
// Unparseable output means we could not read this branch, NOT that
|
|
243
|
+
// it diverged: falling through to the `diverged` bucket below would
|
|
244
|
+
// warn operators about a cross-deployment collision that never
|
|
245
|
+
// happened (parseInt yields NaN, and NaN fails both comparisons).
|
|
246
|
+
workerLogWarn(` Tracked-branch reconcile: skipping ${name}: unparseable rev-list output ${JSON.stringify(counts)}`);
|
|
247
|
+
continue;
|
|
248
|
+
}
|
|
249
|
+
if (localAheadCount === 0 && localBehindCount > 0) {
|
|
250
|
+
try {
|
|
251
|
+
await git.raw(['update-ref', localRef, trackedSha, localSha]);
|
|
252
|
+
fastForwarded.push(name);
|
|
253
|
+
}
|
|
254
|
+
catch (err) {
|
|
255
|
+
// Concurrent Lambda push moved the ref since the read above --
|
|
256
|
+
// the guard did its job; this branch is simply revisited next cycle.
|
|
257
|
+
workerLogWarn(` Tracked-branch reconcile: failed to fast-forward ${name} (concurrent update?): ${getErrorMessage(err)}`);
|
|
258
|
+
}
|
|
259
|
+
}
|
|
260
|
+
else if (localBehindCount === 0 && localAheadCount > 0) {
|
|
261
|
+
// Unpushed local work -- exactly what the old destructive refspec
|
|
262
|
+
// used to force-rewind or delete. Leave it.
|
|
263
|
+
ahead.push(name);
|
|
264
|
+
}
|
|
265
|
+
else if (await hasPendingHistoryRewrite(ctx, name)) {
|
|
266
|
+
// [SYNC-H1] Our own rebase published a rewrite into remote.git and
|
|
267
|
+
// the GitHub push has not landed yet. Ref-level this is identical
|
|
268
|
+
// to a collision, but it is expected and self-resolving, so it must
|
|
269
|
+
// not fire the collision warning below.
|
|
270
|
+
rewritten.push(name);
|
|
271
|
+
}
|
|
272
|
+
else {
|
|
273
|
+
// Neither side is an ancestor of the other. Leave both alone.
|
|
274
|
+
diverged.push(name);
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
catch (err) {
|
|
278
|
+
workerLogWarn(` Tracked-branch reconcile: skipping ${name} (unreadable ref or object?): ${getErrorMessage(err)}`);
|
|
279
|
+
continue;
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
if (diverged.length > 0) {
|
|
283
|
+
workerLogWarn(` Tracked-branch reconcile: ${diverged.length} branch(es) diverged from GitHub and were left untouched: ${diverged.join(', ')}`);
|
|
284
|
+
}
|
|
285
|
+
if (rewritten.length > 0) {
|
|
286
|
+
workerLog(` Tracked-branch reconcile: ${rewritten.length} branch(es) rebased locally with the GitHub push still pending: ${rewritten.join(', ')}`);
|
|
287
|
+
}
|
|
288
|
+
// trackedNames is returned alongside the summary (rather than folded into
|
|
289
|
+
// it) because it is a working set for pushSettingsBranches' stranded-
|
|
290
|
+
// branch check, not part of the worker-status snapshot -- it lists every
|
|
291
|
+
// branch on GitHub and would bloat worker-status.json for no reader.
|
|
292
|
+
return {
|
|
293
|
+
summary: { created, fastForwarded, ahead, diverged, rewritten },
|
|
294
|
+
trackedNames: new Set(tracked.keys()),
|
|
295
|
+
};
|
|
296
|
+
}
|
|
297
|
+
export async function syncGit(ctx) {
|
|
298
|
+
if (!ctx.isRunning())
|
|
299
|
+
return;
|
|
300
|
+
workerLog('Syncing git...');
|
|
301
|
+
const cycleStartedAt = Date.now();
|
|
302
|
+
const git = simpleGit({
|
|
303
|
+
baseDir: ctx.remoteGitPath,
|
|
304
|
+
// DEP-H1: a hung fetch/push would stall the sync loop forever
|
|
305
|
+
// (scheduleLoop only reschedules after completion). The block timeout
|
|
306
|
+
// is inactivity-based, so a slow-but-flowing transfer is unaffected.
|
|
307
|
+
timeout: { block: ctx.taskTimeoutMs },
|
|
308
|
+
});
|
|
309
|
+
// This instance also drives pushSettingsBranches(git) below, whose
|
|
310
|
+
// classification of a rejected settings-branch push (see that method)
|
|
311
|
+
// needs the same stable-English guarantee as pushBranchToGitHub. Network
|
|
312
|
+
// env for the same reason: the fetch and push here reach GitHub.
|
|
313
|
+
git.env(gitNetworkChildEnv());
|
|
314
|
+
// PR-W1: the whole cycle is wrapped so both outcomes -- success and
|
|
315
|
+
// hard failure (e.g. the fetch throwing against a poisoned remote.git)
|
|
316
|
+
// -- record a worker-status.json snapshot. The status write itself is
|
|
317
|
+
// always best-effort (.catch below): it must never turn an otherwise
|
|
318
|
+
// successful cycle into a failure, and must never mask the real error
|
|
319
|
+
// on a failed one. On failure we rethrow so scheduleLoop's existing
|
|
320
|
+
// per-cycle catch stays the loud path.
|
|
321
|
+
try {
|
|
322
|
+
// Fetch all branches from GitHub using direct URL (no named remote),
|
|
323
|
+
// into the GITHUB_TRACKING_REF_PREFIX remote-tracking namespace rather
|
|
324
|
+
// than refs/heads/* directly -- see that constant's doc comment for
|
|
325
|
+
// the destructive-fetch bug this avoids. We use raw git commands since
|
|
326
|
+
// simple-git's fetch() with a URL doesn't support --prune directly.
|
|
327
|
+
await git.raw([
|
|
328
|
+
'fetch',
|
|
329
|
+
ctx.buildGitHubUrl(),
|
|
330
|
+
'--prune',
|
|
331
|
+
`+refs/heads/*:${GITHUB_TRACKING_REF_PREFIX}*`,
|
|
332
|
+
]);
|
|
333
|
+
workerLog('Fetched from GitHub');
|
|
334
|
+
// Bring refs/heads/* toward what was just fetched, WITHOUT ever
|
|
335
|
+
// force-rewinding or deleting a local head -- see
|
|
336
|
+
// reconcileTrackedBranches()'s doc comment.
|
|
337
|
+
const { summary: trackedSummary, trackedNames } = await reconcileTrackedBranches(ctx, git);
|
|
338
|
+
// Push settings branches to GitHub (belt-and-suspenders for task queue).
|
|
339
|
+
// Ensures settings reach GitHub even if a task queue entry is lost.
|
|
340
|
+
// Ordering relative to the fetch/reconcile above is no longer a
|
|
341
|
+
// correctness dependency now that the fetch can't clobber refs/heads/*
|
|
342
|
+
// -- this could run before or after them just as safely.
|
|
343
|
+
await pushSettingsBranches(ctx, git, trackedNames);
|
|
344
|
+
await refreshBaseBranchWorkspace(ctx);
|
|
345
|
+
const rebaseSummary = await runRebaseCycle(ctx);
|
|
346
|
+
// Periodically clean up old completed/failed tasks
|
|
347
|
+
await cleanupOldTasks(ctx.taskDir, undefined, ctx.log);
|
|
348
|
+
// [C1] Sweep branch directories the admin purge action trashed more
|
|
349
|
+
// than TRASH_RETENTION_MS ago. Worker-only by design: purge itself
|
|
350
|
+
// never deletes anything (reversible), and this cycle is the sole
|
|
351
|
+
// place actual removal happens.
|
|
352
|
+
const trashRemoved = await cleanupTrashedBranchDirs(ctx);
|
|
353
|
+
if (trashRemoved > 0) {
|
|
354
|
+
workerLog(`Removed ${trashRemoved} expired trashed branch dir(s)`);
|
|
355
|
+
}
|
|
356
|
+
const report = ctx.ensureStatusReport();
|
|
357
|
+
report.lastGitSyncAt = new Date().toISOString();
|
|
358
|
+
delete report.lastGitSyncError;
|
|
359
|
+
report.lastGitSync = {
|
|
360
|
+
durationMs: Date.now() - cycleStartedAt,
|
|
361
|
+
rebased: rebaseSummary.rebased,
|
|
362
|
+
skippedDirty: rebaseSummary.skippedDirty,
|
|
363
|
+
skippedLocked: rebaseSummary.skippedLocked,
|
|
364
|
+
failed: rebaseSummary.failed,
|
|
365
|
+
tracked: trackedSummary,
|
|
366
|
+
};
|
|
367
|
+
await writeWorkerStatus(ctx.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
|
|
368
|
+
}
|
|
369
|
+
catch (err) {
|
|
370
|
+
const report = ctx.ensureStatusReport();
|
|
371
|
+
// [REDACT] Persisted to worker-status.json and served to the browser
|
|
372
|
+
// by the admin panel -- a fetch/push failure's message can embed the
|
|
373
|
+
// bot token via buildGitHubUrl().
|
|
374
|
+
report.lastGitSyncError = {
|
|
375
|
+
message: redactCredentials(getErrorMessage(err)),
|
|
376
|
+
at: new Date().toISOString(),
|
|
377
|
+
};
|
|
378
|
+
await writeWorkerStatus(ctx.taskDir, report).catch((writeErr) => workerLogError('Failed to write worker status:', getErrorMessage(writeErr)));
|
|
379
|
+
throw err;
|
|
380
|
+
}
|
|
381
|
+
}
|
|
382
|
+
/**
|
|
383
|
+
* [C1] Remove `.trash-*` branch directories (created by the admin purge
|
|
384
|
+
* action, api/admin-branch-health.ts) whose name-embedded stamp is older
|
|
385
|
+
* than {@link TRASH_RETENTION_MS}. Names that don't match the expected
|
|
386
|
+
* `.trash-{dirName}-{STAMP}` shape, or whose stamp fails to parse, are
|
|
387
|
+
* left alone (logged once per cycle, not per file, to avoid flooding logs
|
|
388
|
+
* if something odd accumulates) -- purge is the only writer of this
|
|
389
|
+
* naming scheme, so an unparseable name is unexpected and worth a human
|
|
390
|
+
* looking rather than a silent skip.
|
|
391
|
+
*/
|
|
392
|
+
export async function cleanupTrashedBranchDirs(ctx) {
|
|
393
|
+
let entries;
|
|
394
|
+
try {
|
|
395
|
+
entries = await fs.readdir(ctx.contentBranchesPath);
|
|
396
|
+
}
|
|
397
|
+
catch (err) {
|
|
398
|
+
if (isNodeError(err) && err.code === 'ENOENT')
|
|
399
|
+
return 0;
|
|
400
|
+
throw err;
|
|
401
|
+
}
|
|
402
|
+
const now = Date.now();
|
|
403
|
+
let removed = 0;
|
|
404
|
+
let loggedUnparseable = false;
|
|
405
|
+
for (const name of entries) {
|
|
406
|
+
if (!name.startsWith('.trash-'))
|
|
407
|
+
continue;
|
|
408
|
+
const stampMatch = TRASH_DIR_STAMP_RE.exec(name);
|
|
409
|
+
const stampDate = stampMatch ? parseTrashStamp(stampMatch[1]) : null;
|
|
410
|
+
if (!stampDate) {
|
|
411
|
+
if (!loggedUnparseable) {
|
|
412
|
+
workerLog(`CanopyCMS: Skipping trash dir with unparseable stamp: ${name}`);
|
|
413
|
+
loggedUnparseable = true;
|
|
414
|
+
}
|
|
415
|
+
continue;
|
|
416
|
+
}
|
|
417
|
+
if (now - stampDate.getTime() < TRASH_RETENTION_MS)
|
|
418
|
+
continue;
|
|
419
|
+
try {
|
|
420
|
+
await fs.rm(path.join(ctx.contentBranchesPath, name), {
|
|
421
|
+
recursive: true,
|
|
422
|
+
force: true,
|
|
423
|
+
maxRetries: 3,
|
|
424
|
+
retryDelay: 100,
|
|
425
|
+
});
|
|
426
|
+
removed++;
|
|
427
|
+
}
|
|
428
|
+
catch (err) {
|
|
429
|
+
workerLogError(`CanopyCMS: Failed to remove trashed branch dir ${name}:`, getErrorMessage(err));
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
return removed;
|
|
433
|
+
}
|
|
434
|
+
/**
|
|
435
|
+
* Fast-forward the base branch's own working-tree clone
|
|
436
|
+
* (content-branches/<baseBranch>) to match origin/<baseBranch>.
|
|
437
|
+
*
|
|
438
|
+
* Previously this clone was refreshed only incidentally, by the generic
|
|
439
|
+
* rebase loop below (rebaseActiveBranches): for a branch with status
|
|
440
|
+
* 'editing', rebasing onto origin/<baseBranch> degenerates to a
|
|
441
|
+
* fast-forward when the clone IS the base branch. But that loop's skip
|
|
442
|
+
* paths -- a dirty tree, a missing .git -- are silent, which is the
|
|
443
|
+
* suspected live failure mode: a wedged base clone with no diagnosable
|
|
444
|
+
* signal in the logs. This dedicated step makes the refresh explicit,
|
|
445
|
+
* ff-only, and loud, so a stuck base view (an editor forking a new branch
|
|
446
|
+
* "from base" that's actually a stale snapshot) is diagnosable from logs.
|
|
447
|
+
* This runs every sync cycle so the drift window is bounded by
|
|
448
|
+
* gitSyncInterval.
|
|
449
|
+
*
|
|
450
|
+
* ff-only on purpose: this clone must stay a linear mirror of
|
|
451
|
+
* origin/<baseBranch>, so a merge that isn't a fast-forward (diverged
|
|
452
|
+
* local history) is treated as a should-never-happen condition and left
|
|
453
|
+
* untouched rather than force-resolved.
|
|
454
|
+
*/
|
|
455
|
+
export async function refreshBaseBranchWorkspace(ctx) {
|
|
456
|
+
// Sanitized name for the workspace directory (a base branch containing
|
|
457
|
+
// e.g. '/' would otherwise stat a wrong nested path here forever).
|
|
458
|
+
const basePath = path.join(ctx.contentBranchesPath, ctx.sanitizedBaseBranch);
|
|
459
|
+
const gitDir = path.join(basePath, '.git');
|
|
460
|
+
try {
|
|
461
|
+
let gitDirStat;
|
|
462
|
+
try {
|
|
463
|
+
gitDirStat = await fs.stat(gitDir);
|
|
464
|
+
}
|
|
465
|
+
catch {
|
|
466
|
+
gitDirStat = null;
|
|
467
|
+
}
|
|
468
|
+
if (!gitDirStat || !gitDirStat.isDirectory()) {
|
|
469
|
+
workerLog(`Base branch workspace (${ctx.baseBranch}): not yet provisioned, skipping refresh`);
|
|
470
|
+
return;
|
|
471
|
+
}
|
|
472
|
+
const baseGit = simpleGit({
|
|
473
|
+
baseDir: basePath,
|
|
474
|
+
// Keep git non-interactive during the merge so it never blocks on an
|
|
475
|
+
// editor. simple-git >=3.32 blocks setting core.editor unless
|
|
476
|
+
// explicitly opted in; the value here is a hardcoded literal
|
|
477
|
+
// ("true", the shell no-op), not user input, so enabling
|
|
478
|
+
// allowUnsafeEditor carries no injection risk (same as the rebase
|
|
479
|
+
// loop's git config below).
|
|
480
|
+
config: ['core.editor=true'],
|
|
481
|
+
unsafe: { allowUnsafeEditor: true },
|
|
482
|
+
// DEP-H1: a hung fetch/merge against this EFS-backed clone would
|
|
483
|
+
// stall the sync loop forever (scheduleLoop only reschedules after
|
|
484
|
+
// completion). The block timeout is inactivity-based, so a
|
|
485
|
+
// slow-but-flowing transfer is unaffected -- same as syncGit()'s
|
|
486
|
+
// remote handle.
|
|
487
|
+
timeout: { block: ctx.taskTimeoutMs },
|
|
488
|
+
});
|
|
489
|
+
// Nothing makes this clone read-only. A direct edit here (or a stray
|
|
490
|
+
// process) wedges every editor's view of the base branch until an
|
|
491
|
+
// operator intervenes, so a dirty tree is loud, not a quiet skip.
|
|
492
|
+
// Only TRACKED changes block the refresh: a stray untracked file (e.g.
|
|
493
|
+
// runtime metadata missing from .git/info/exclude) must not wedge the
|
|
494
|
+
// fast-forward forever — if an untracked file would collide with
|
|
495
|
+
// incoming content, the --ff-only merge below refuses on its own and
|
|
496
|
+
// that failure is already logged loudly.
|
|
497
|
+
const status = await baseGit.status();
|
|
498
|
+
const trackedDirty = status.files.filter((f) => f.index !== '?' || f.working_dir !== '?');
|
|
499
|
+
if (trackedDirty.length > 0) {
|
|
500
|
+
workerLogError(`Base branch workspace (${ctx.baseBranch}) has uncommitted changes -- skipping refresh. Dirty files: ${trackedDirty.map((f) => f.path).join(', ')}`);
|
|
501
|
+
return;
|
|
502
|
+
}
|
|
503
|
+
// Raw (unsanitized) name from here on: these are git ref operations
|
|
504
|
+
// against origin/<baseBranch>, not filesystem paths, so they must use
|
|
505
|
+
// the same name GitHub knows the branch by.
|
|
506
|
+
await baseGit.fetch('origin', ctx.baseBranch);
|
|
507
|
+
// Use rev-list instead of status.behind for the same reason as the
|
|
508
|
+
// rebase loop below: status.behind only works with an upstream
|
|
509
|
+
// tracking branch configured, which isn't guaranteed here.
|
|
510
|
+
// Compare against the just-fetched tip rather than origin/<base>:
|
|
511
|
+
// workspaces are cloned --single-branch (git-manager.ts), so their
|
|
512
|
+
// fetch refspec only materializes a remote-tracking ref for the branch
|
|
513
|
+
// they were cloned from — for any other base branch, origin/<base>
|
|
514
|
+
// simply never exists and rev-list dies with "ambiguous argument".
|
|
515
|
+
// Pin FETCH_HEAD to a SHA immediately: FETCH_HEAD is one shared mutable
|
|
516
|
+
// file per repo, silently repointed by ANY other fetch in this clone.
|
|
517
|
+
const fetchedTip = (await baseGit.revparse(['FETCH_HEAD'])).trim();
|
|
518
|
+
const behindCount = parseInt((await baseGit.raw(['rev-list', '--count', `HEAD..${fetchedTip}`])).trim(), 10);
|
|
519
|
+
if (behindCount > 0) {
|
|
520
|
+
try {
|
|
521
|
+
await baseGit.merge(['--ff-only', fetchedTip]);
|
|
522
|
+
}
|
|
523
|
+
catch (err) {
|
|
524
|
+
workerLogError(`Base branch workspace (${ctx.baseBranch}) failed to fast-forward (diverged local history?): ${getErrorMessage(err)}`);
|
|
525
|
+
return;
|
|
526
|
+
}
|
|
527
|
+
await invalidateBranchContentCaches(basePath);
|
|
528
|
+
}
|
|
529
|
+
// Hygiene: conflictStatus/conflictFiles are meaningless for the base
|
|
530
|
+
// branch's own metadata and may be left over from before the base was
|
|
531
|
+
// excluded from rebaseActiveBranches()'s conflict-resolution loop.
|
|
532
|
+
// Reuse the already-clean no-op-save guard from that loop: save()
|
|
533
|
+
// eager-regenerates the branch registry (O(branch count) EFS reads),
|
|
534
|
+
// so skip it when there's nothing to clear.
|
|
535
|
+
const currentMeta = await BranchMetadataFileManager.loadOnly(basePath);
|
|
536
|
+
const conflictStatus = currentMeta?.branch.conflictStatus;
|
|
537
|
+
const conflictFiles = currentMeta?.branch.conflictFiles;
|
|
538
|
+
const alreadyClean = (conflictStatus === undefined || conflictStatus === 'clean') &&
|
|
539
|
+
(conflictFiles === undefined || conflictFiles.length === 0);
|
|
540
|
+
if (!alreadyClean) {
|
|
541
|
+
const meta = getBranchMetadataFileManager(basePath, ctx.contentBranchesPath);
|
|
542
|
+
await meta.save({
|
|
543
|
+
branch: { name: ctx.baseBranch, conflictStatus: 'clean', conflictFiles: [] },
|
|
544
|
+
});
|
|
545
|
+
}
|
|
546
|
+
// One concise per-cycle line -- the diagnostic for the next live deploy.
|
|
547
|
+
workerLog(behindCount > 0
|
|
548
|
+
? `Base branch workspace (${ctx.baseBranch}): fast-forwarded ${behindCount} commit(s)`
|
|
549
|
+
: `Base branch workspace (${ctx.baseBranch}): up to date`);
|
|
550
|
+
}
|
|
551
|
+
catch (err) {
|
|
552
|
+
workerLogError(`Base branch workspace (${ctx.baseBranch}) refresh failed: ${getErrorMessage(err)}`);
|
|
553
|
+
}
|
|
554
|
+
}
|