@north-light/crouter 0.3.255 → 0.3.256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.d.ts +13 -1
- package/dist/api/client.js +17 -0
- package/dist/api/dto/worktree.d.ts +17 -0
- package/dist/api/routes.d.ts +1 -0
- package/dist/api/routes.js +1 -0
- package/dist/clients/attach/viewer.js +936 -797
- package/dist/commands/api-client.js +4 -0
- package/dist/commands/sys/worktrees.d.ts +1 -0
- package/dist/commands/sys/worktrees.js +53 -0
- package/dist/commands/sys.js +2 -1
- package/dist/core/__tests__/integration/worktree-land.test.js +71 -1
- package/dist/core/__tests__/integration/worktree-reap.test.js +101 -0
- package/dist/core/__tests__/worktree-landing.test.d.ts +1 -0
- package/dist/core/__tests__/worktree-landing.test.js +11 -0
- package/dist/core/canvas/canvas.js +24 -8
- package/dist/core/canvas/pid.d.ts +6 -0
- package/dist/core/canvas/pid.js +48 -3
- package/dist/core/canvas/types.d.ts +41 -1
- package/dist/core/exclusive-lock.d.ts +6 -0
- package/dist/core/exclusive-lock.js +12 -0
- package/dist/core/git.d.ts +1 -1
- package/dist/core/git.js +5 -1
- package/dist/core/runtime/fleet.d.ts +6 -0
- package/dist/core/worktree-landing.d.ts +44 -0
- package/dist/core/worktree-landing.js +56 -0
- package/dist/core/worktree-quarantine.d.ts +19 -0
- package/dist/core/worktree-quarantine.js +54 -0
- package/dist/core/worktree-sweep.d.ts +60 -0
- package/dist/core/worktree-sweep.js +493 -0
- package/dist/core/worktree.d.ts +37 -0
- package/dist/core/worktree.js +149 -23
- package/dist/daemon/__tests__/startup-block-marker.test.d.ts +1 -0
- package/dist/daemon/__tests__/startup-block-marker.test.js +86 -0
- package/dist/daemon/api/handlers/worktree.js +5 -0
- package/dist/daemon/crtrd-cli.js +12 -7
- package/dist/daemon/crtrd.js +6 -1
- package/dist/daemon/fleet.d.ts +1 -0
- package/dist/daemon/fleet.js +3 -0
- package/dist/daemon/manage.js +22 -1
- package/dist/daemon/reconcilers/managed-worktree-sweep.d.ts +24 -0
- package/dist/daemon/reconcilers/managed-worktree-sweep.js +190 -0
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +11 -2
- package/dist/daemon/reconcilers/storage-maintenance.js +7 -2
- package/dist/daemon/startup-block-marker.d.ts +25 -0
- package/dist/daemon/startup-block-marker.js +99 -0
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
package/dist/core/worktree.js
CHANGED
|
@@ -7,6 +7,7 @@ import { isPidAlive } from './canvas/pid.js';
|
|
|
7
7
|
import { getNode, updateNode } from './canvas/index.js';
|
|
8
8
|
import { CONFIG_FILE, CRTR_DIR_NAME, STATE_FILE } from '../types.js';
|
|
9
9
|
import { withExclusiveDirectoryLock } from './exclusive-lock.js';
|
|
10
|
+
import { parseStatusPorcelain, planLandIntoCheckout, refDeleteTransactionStdin } from './worktree-landing.js';
|
|
10
11
|
import { shellQuote } from '../shared/shell-quote.js';
|
|
11
12
|
export class WorktreeError extends Error {
|
|
12
13
|
code;
|
|
@@ -74,7 +75,7 @@ function checkoutPathForBranch(repoRoot, branch) {
|
|
|
74
75
|
* callers that must not throw on a read failure (an absent-path proof that
|
|
75
76
|
* needs to fail closed rather than error) call `gitSync` themselves and pass
|
|
76
77
|
* the output here instead of going through `checkoutPathForBranch`. */
|
|
77
|
-
function checkoutPathForBranchInList(porcelain, branch) {
|
|
78
|
+
export function checkoutPathForBranchInList(porcelain, branch) {
|
|
78
79
|
let current = null;
|
|
79
80
|
for (const line of porcelain.split('\n')) {
|
|
80
81
|
if (line.startsWith('worktree '))
|
|
@@ -84,7 +85,36 @@ function checkoutPathForBranchInList(porcelain, branch) {
|
|
|
84
85
|
}
|
|
85
86
|
return null;
|
|
86
87
|
}
|
|
87
|
-
|
|
88
|
+
/** Are these two spellings the same directory?
|
|
89
|
+
*
|
|
90
|
+
* Git canonicalizes every path it prints in `worktree list --porcelain`, while
|
|
91
|
+
* a recorded managed-worktree path preserves the spelling of `CRTR_HOME`. When
|
|
92
|
+
* the canvas home is reached through a symlink (macOS's `/var` → `/private/var`
|
|
93
|
+
* is the everyday case) the two forms name one directory but differ as
|
|
94
|
+
* strings, so a raw `===` reads a checkout crtr created as unowned. Resolution
|
|
95
|
+
* failures fall back to `resolve`, which is the pre-canonical comparison — no
|
|
96
|
+
* worse than the string compare it replaces. */
|
|
97
|
+
export function samePath(a, b) {
|
|
98
|
+
return a === b || canonicalPath(a) === canonicalPath(b);
|
|
99
|
+
}
|
|
100
|
+
function canonicalPath(path) {
|
|
101
|
+
try {
|
|
102
|
+
return realpathSync.native(path);
|
|
103
|
+
}
|
|
104
|
+
catch {
|
|
105
|
+
return resolve(path);
|
|
106
|
+
}
|
|
107
|
+
}
|
|
108
|
+
export const WORKTREE_LOCK_WAIT_MS = 30_000;
|
|
109
|
+
/** The lock directory naming one repository's common Git dir. Every process
|
|
110
|
+
* that mutates a managed worktree — the synchronous commands here and the
|
|
111
|
+
* daemon's asynchronous sweep — derives its lock path through this function,
|
|
112
|
+
* so the two locking styles contend for the SAME name. */
|
|
113
|
+
export function worktreeLockPathForCommonDir(commonDir) {
|
|
114
|
+
const lockRoot = `${crtrHome()}/worktree-locks`;
|
|
115
|
+
mkdirSync(lockRoot, { recursive: true });
|
|
116
|
+
return `${lockRoot}/${createHash('sha256').update(commonDir).digest('hex')}`;
|
|
117
|
+
}
|
|
88
118
|
function commonGitDir(repoRoot) {
|
|
89
119
|
const commonDir = runGit(repoRoot, ['rev-parse', '--git-common-dir'], 'git_common_dir_failed', 'Inspect the repository git metadata, then retry.');
|
|
90
120
|
return realpathSync(resolve(repoRoot, commonDir));
|
|
@@ -94,10 +124,7 @@ function commonGitDir(repoRoot) {
|
|
|
94
124
|
* add instead of waiting, so callers take this transaction lock first. */
|
|
95
125
|
function withRepositoryWorktreeLock(repoRoot, action) {
|
|
96
126
|
const commonDir = commonGitDir(repoRoot);
|
|
97
|
-
|
|
98
|
-
mkdirSync(lockRoot, { recursive: true });
|
|
99
|
-
const lockPath = `${lockRoot}/${createHash('sha256').update(commonDir).digest('hex')}`;
|
|
100
|
-
return withExclusiveDirectoryLock(lockPath, action, {
|
|
127
|
+
return withExclusiveDirectoryLock(worktreeLockPathForCommonDir(commonDir), action, {
|
|
101
128
|
timeoutMs: WORKTREE_LOCK_WAIT_MS,
|
|
102
129
|
timeoutError: () => new WorktreeError('worktree_lock_timeout', `timed out waiting to create a managed worktree in ${commonDir}`, 'Wait for the other managed-worktree operation to finish, then retry.'),
|
|
103
130
|
});
|
|
@@ -110,7 +137,7 @@ function registeredWorktree(repoRoot, path, branch) {
|
|
|
110
137
|
for (const line of list.stdout.split('\n')) {
|
|
111
138
|
if (line.startsWith('worktree '))
|
|
112
139
|
current = line.slice('worktree '.length).trim();
|
|
113
|
-
else if (current
|
|
140
|
+
else if (current !== null && samePath(current, path) && line === `branch refs/heads/${branch}`)
|
|
114
141
|
return true;
|
|
115
142
|
}
|
|
116
143
|
return false;
|
|
@@ -245,9 +272,10 @@ function gitPath(cwd, logicalPath) {
|
|
|
245
272
|
export function isRebaseInProgress(cwd) {
|
|
246
273
|
return existsSync(gitPath(cwd, 'rebase-merge')) || existsSync(gitPath(cwd, 'rebase-apply'));
|
|
247
274
|
}
|
|
248
|
-
/** The ONE authoritative closed-worktree state transition, shared by
|
|
249
|
-
* closing
|
|
250
|
-
* `autoDropCleanManagedWorktreeLocked` push-final drop
|
|
275
|
+
/** The ONE authoritative closed-worktree state transition, shared by every
|
|
276
|
+
* closing path (explicit `closeManagedWorktreeLocked` land, the clean
|
|
277
|
+
* `autoDropCleanManagedWorktreeLocked` push-final drop, and the daemon sweep's
|
|
278
|
+
* `reconcileManagedWorktree`): persist
|
|
251
279
|
* `managed_worktree.state: 'closed'`, name its cleanup state, and repoint the
|
|
252
280
|
* node's launch `cwd` to the durable base checkout. The caller broker stays
|
|
253
281
|
* alive and is Pi's sole transcript writer, so its session header cannot be
|
|
@@ -255,15 +283,100 @@ export function isRebaseInProgress(cwd) {
|
|
|
255
283
|
* revive launch fresh instead. Both closing paths MUST go through this helper
|
|
256
284
|
* instead of hand-rolling the `updateNode` call, so neither can drift out of
|
|
257
285
|
* sync. */
|
|
258
|
-
function persistClosedWorktreeState(nodeId, wt, cleanup = 'pending') {
|
|
286
|
+
export function persistClosedWorktreeState(nodeId, wt, cleanup = 'pending') {
|
|
259
287
|
const node = getNode(nodeId);
|
|
260
288
|
const sessionInvalidated = node?.pi_session_id != null || node?.pi_session_file != null;
|
|
289
|
+
// Every caller reaches this only after CONCLUDING the land — it succeeded, or
|
|
290
|
+
// the record was proven to have nothing to land. So a recorded land intent is
|
|
291
|
+
// discharged here, and a recorded refusal describes a state that no longer
|
|
292
|
+
// holds; dropping both leaves the sweep re-examining this record fresh.
|
|
293
|
+
const { land_intent: _landIntent, sweep: _sweep, ...retained } = wt;
|
|
261
294
|
updateNode(nodeId, {
|
|
262
295
|
cwd: wt.repo_root,
|
|
263
296
|
...(sessionInvalidated ? { pi_session_id: null, pi_session_file: null } : {}),
|
|
264
|
-
managed_worktree: { ...
|
|
297
|
+
managed_worktree: { ...retained, state: 'closed', cleanup, closed: new Date().toISOString() },
|
|
265
298
|
});
|
|
266
299
|
}
|
|
300
|
+
/** Record that this node ASKED to land and was blocked, so the daemon sweep may
|
|
301
|
+
* later retry exactly that land once the base checkout stops blocking it. The
|
|
302
|
+
* sweep never lands commits for a node that did not get this far: a node that
|
|
303
|
+
* crashed mid-work and never called close has its worktree preserved for a
|
|
304
|
+
* human instead. Recording a fresh intent also clears any prior refusal state,
|
|
305
|
+
* so the sweep re-examines promptly rather than honoring an old backoff. */
|
|
306
|
+
function recordBlockedLandIntent(nodeId, blocked) {
|
|
307
|
+
const current = getNode(nodeId)?.managed_worktree;
|
|
308
|
+
if (current == null || current.state !== 'open')
|
|
309
|
+
return;
|
|
310
|
+
const { sweep: _sweep, ...retained } = current;
|
|
311
|
+
updateNode(nodeId, { managed_worktree: { ...retained, land_intent: { at: new Date().toISOString(), blocked } } });
|
|
312
|
+
}
|
|
313
|
+
/** Fast-forward a base branch that is CHECKED OUT somewhere, landing underneath
|
|
314
|
+
* whatever local changes that checkout holds.
|
|
315
|
+
*
|
|
316
|
+
* The base checkout is shared by every other node using this repository, and
|
|
317
|
+
* its local changes usually have nothing to do with the child that is closing.
|
|
318
|
+
* Disjoint changes ride along untouched; overlapping ones are stashed, the
|
|
319
|
+
* fast-forward runs underneath them, and they are re-applied. The re-apply is
|
|
320
|
+
* `stash apply`, never `stash pop`, so a conflict cannot consume the stash and
|
|
321
|
+
* lose the work: on conflict the checkout is reset to the landed commit and the
|
|
322
|
+
* stash is deliberately KEPT for a person to apply. Either way this never
|
|
323
|
+
* leaves conflict markers behind in a checkout other nodes are using. */
|
|
324
|
+
function landIntoBaseCheckout(baseCheckout, branch, baseSha, landedSha) {
|
|
325
|
+
const status = gitSync(['status', '--porcelain'], baseCheckout);
|
|
326
|
+
if (status.status !== 0)
|
|
327
|
+
return { ok: false, detail: (status.stderr.trim() || status.stdout.trim()).trim() };
|
|
328
|
+
const diff = gitSync(['diff', '--name-only', baseSha, landedSha], baseCheckout);
|
|
329
|
+
if (diff.status !== 0)
|
|
330
|
+
return { ok: false, detail: (diff.stderr.trim() || diff.stdout.trim()).trim() };
|
|
331
|
+
const incoming = diff.stdout.split('\n').map((l) => l.trim()).filter((l) => l !== '');
|
|
332
|
+
const plan = planLandIntoCheckout(parseStatusPorcelain(status.stdout), incoming);
|
|
333
|
+
if (plan.kind === 'refuse') {
|
|
334
|
+
return { ok: false, detail: `untracked files in ${baseCheckout} would be overwritten by the incoming commits: ${plan.collisions.join(', ')}` };
|
|
335
|
+
}
|
|
336
|
+
const merge = () => {
|
|
337
|
+
const res = gitSync(['merge', '--ff-only', landedSha], baseCheckout);
|
|
338
|
+
return res.status === 0 ? { ok: true } : { ok: false, detail: (res.stderr.trim() || res.stdout.trim()).trim() };
|
|
339
|
+
};
|
|
340
|
+
if (plan.kind === 'direct')
|
|
341
|
+
return merge();
|
|
342
|
+
// Identify the stash by the exact commit this push created. `refs/stash` can
|
|
343
|
+
// already name an UNRELATED stash the user made earlier, and a push that
|
|
344
|
+
// saved nothing leaves it untouched — applying or dropping that one would
|
|
345
|
+
// destroy work this code never owned.
|
|
346
|
+
const before = gitSync(['rev-parse', '--verify', '--quiet', 'refs/stash'], baseCheckout);
|
|
347
|
+
const beforeSha = before.status === 0 ? before.stdout.trim() : null;
|
|
348
|
+
const push = gitSync(['stash', 'push', '--message', `crtr landing ${branch}`], baseCheckout);
|
|
349
|
+
if (push.status !== 0)
|
|
350
|
+
return { ok: false, detail: (push.stderr.trim() || push.stdout.trim()).trim() };
|
|
351
|
+
const after = gitSync(['rev-parse', '--verify', '--quiet', 'refs/stash'], baseCheckout);
|
|
352
|
+
const stashSha = after.status === 0 ? after.stdout.trim() : null;
|
|
353
|
+
if (stashSha === null || stashSha === beforeSha)
|
|
354
|
+
return merge(); // nothing was saved
|
|
355
|
+
const landed = merge();
|
|
356
|
+
if (!landed.ok) {
|
|
357
|
+
// The failed merge left the tree exactly as the stash found it, so this
|
|
358
|
+
// re-apply is clean; restore before surfacing the refusal.
|
|
359
|
+
if (gitSync(['stash', 'apply', stashSha], baseCheckout).status === 0)
|
|
360
|
+
dropStashIfStillTip(baseCheckout, stashSha);
|
|
361
|
+
return landed;
|
|
362
|
+
}
|
|
363
|
+
if (gitSync(['stash', 'apply', stashSha], baseCheckout).status === 0) {
|
|
364
|
+
dropStashIfStillTip(baseCheckout, stashSha);
|
|
365
|
+
return { ok: true };
|
|
366
|
+
}
|
|
367
|
+
gitSync(['reset', '--hard', landedSha], baseCheckout);
|
|
368
|
+
return {
|
|
369
|
+
ok: true,
|
|
370
|
+
stash: `local changes in ${baseCheckout} conflicted with the landed commits and were PRESERVED as stash ${stashSha}; nothing was lost. Re-apply them with \`git -C ${shellQuote(baseCheckout)} stash apply ${stashSha}\` and resolve the conflict.`,
|
|
371
|
+
};
|
|
372
|
+
}
|
|
373
|
+
/** Drop a stash entry only while it is still the tip `stash@{0}` names, so a
|
|
374
|
+
* stash someone else pushed in between is never dropped instead. */
|
|
375
|
+
function dropStashIfStillTip(cwd, stashSha) {
|
|
376
|
+
const tip = gitSync(['rev-parse', '--verify', '--quiet', 'refs/stash'], cwd);
|
|
377
|
+
if (tip.status === 0 && tip.stdout.trim() === stashSha)
|
|
378
|
+
gitSync(['stash', 'drop'], cwd);
|
|
379
|
+
}
|
|
267
380
|
function deferredCleanupResult(wt) {
|
|
268
381
|
const remove = `git -C ${shellQuote(wt.repo_root)} worktree remove ${shellQuote(wt.path)}`;
|
|
269
382
|
return {
|
|
@@ -349,9 +462,16 @@ function closeManagedWorktreeLocked(nodeId) {
|
|
|
349
462
|
// captured baseSha — landing is purely local, nothing is ever pushed to
|
|
350
463
|
// origin. Both landing paths require the base to still be exactly at that
|
|
351
464
|
// captured tip, so neither can overwrite a concurrent base change.
|
|
465
|
+
// Every refusal below leaves a rebased, landable branch that this node ASKED
|
|
466
|
+
// to land. Recording that blocked intent is what later authorizes the daemon
|
|
467
|
+
// sweep to finish exactly this land once the blocker clears.
|
|
468
|
+
const blocked = (message, next, detail) => {
|
|
469
|
+
recordBlockedLandIntent(nodeId, message);
|
|
470
|
+
return new WorktreeError('land_base_failed', message, next, detail);
|
|
471
|
+
};
|
|
352
472
|
const fastForward = gitSync(['merge-base', '--is-ancestor', baseSha, landedSha], wt.path);
|
|
353
473
|
if (fastForward.status !== 0) {
|
|
354
|
-
throw
|
|
474
|
+
throw blocked(`rebased worktree is not a fast-forward of local base branch '${wt.base_ref}'`, 'Ask the user with `crtr human send` how to land this managed worktree.', (fastForward.stderr.trim() || fastForward.stdout.trim()).trim());
|
|
355
475
|
}
|
|
356
476
|
const baseCheckout = checkoutPathForBranch(wt.path, wt.base_ref);
|
|
357
477
|
if (baseCheckout !== null) {
|
|
@@ -361,17 +481,22 @@ function closeManagedWorktreeLocked(nodeId) {
|
|
|
361
481
|
const detail = checkoutHead.status !== 0
|
|
362
482
|
? (checkoutHead.stderr.trim() || actualHead).trim()
|
|
363
483
|
: `expected ${baseSha}, found ${actualHead}`;
|
|
364
|
-
throw
|
|
484
|
+
throw blocked(`local base branch '${wt.base_ref}' changed while this worktree was closing`, `Reconcile the base checkout at ${baseCheckout}, then rerun \`crtr node worktree close\`. If you are unsure, ask the user with \`crtr human send\`.`, detail);
|
|
365
485
|
}
|
|
366
486
|
}
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
487
|
+
let stashNote;
|
|
488
|
+
if (baseCheckout !== null) {
|
|
489
|
+
const landed = landIntoBaseCheckout(baseCheckout, wt.branch, baseSha, landedSha);
|
|
490
|
+
if (!landed.ok) {
|
|
491
|
+
throw blocked(`could not fast-forward local base branch '${wt.base_ref}' to this worktree`, `Local base branch '${wt.base_ref}' is checked out at ${baseCheckout} in a state that blocks a fast-forward. Reconcile it there, then rerun \`crtr node worktree close\`. If you are unsure, ask the user with \`crtr human send\`.`, landed.detail);
|
|
492
|
+
}
|
|
493
|
+
stashNote = landed.stash;
|
|
494
|
+
}
|
|
495
|
+
else {
|
|
496
|
+
const land = gitSync(['update-ref', `refs/heads/${wt.base_ref}`, landedSha, baseSha], wt.path);
|
|
497
|
+
if (land.status !== 0) {
|
|
498
|
+
throw blocked(`could not fast-forward local base branch '${wt.base_ref}' to this worktree`, 'Ask the user with `crtr human send` how to land this managed worktree.', (land.stderr.trim() || land.stdout.trim()).trim());
|
|
499
|
+
}
|
|
375
500
|
}
|
|
376
501
|
// The caller broker still runs from wt.path after this API response. Do not
|
|
377
502
|
// invoke `git worktree remove`: it recursively deletes the caller's cwd and
|
|
@@ -385,6 +510,7 @@ function closeManagedWorktreeLocked(nodeId) {
|
|
|
385
510
|
worktree_path: wt.path,
|
|
386
511
|
landed_sha: landedSha,
|
|
387
512
|
...deferredCleanupResult(wt),
|
|
513
|
+
...(stashNote === undefined ? {} : { base_checkout_stash: stashNote }),
|
|
388
514
|
};
|
|
389
515
|
}
|
|
390
516
|
export function closeManagedWorktree(nodeId) {
|
|
@@ -467,7 +593,7 @@ export function abandonManagedWorktree(nodeId, by) {
|
|
|
467
593
|
* is still at the exact inspected SHA. `containingRef` can be the recorded
|
|
468
594
|
* local base or that base's remote-tracking upstream. */
|
|
469
595
|
function bestEffortDeleteBranchWithContainmentProof(wt, branchSha, containingRef, containingSha) {
|
|
470
|
-
const stdin =
|
|
596
|
+
const stdin = refDeleteTransactionStdin(wt.branch, branchSha, containingRef, containingSha);
|
|
471
597
|
const transaction = gitSync(['update-ref', '--stdin'], wt.repo_root, stdin);
|
|
472
598
|
const branchDeleted = transaction.status === 0;
|
|
473
599
|
if (branchDeleted)
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { test, beforeEach, afterEach } from 'node:test';
|
|
2
|
+
import assert from 'node:assert/strict';
|
|
3
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
4
|
+
import { tmpdir } from 'node:os';
|
|
5
|
+
import { join } from 'node:path';
|
|
6
|
+
import { blockedMigrationIn, clearDaemonStartupBlocked, daemonStartupBlocked, recordDaemonStartupBlocked, } from '../startup-block-marker.js';
|
|
7
|
+
import { exclusiveLockOwnerPid, withExclusiveDirectoryLockAsync } from '../../core/exclusive-lock.js';
|
|
8
|
+
import { MigrationBlockedError } from '../../migrations/activation.js';
|
|
9
|
+
// REGRESSION (daemon respawn storm, cross-process half): the first fix for the
|
|
10
|
+
// storm kept its suppression in module state, which caps exactly ONE process.
|
|
11
|
+
// The storm is made of many — every CLI invocation and every scheduled tick is
|
|
12
|
+
// its own process — so 10 separate invocations against a permanently blocked
|
|
13
|
+
// home still produced 10 spawns. Suppression has to outlive a pid, and it has
|
|
14
|
+
// to retire itself when the corpus is repaired or it wedges a fixed install.
|
|
15
|
+
let home;
|
|
16
|
+
const previousHome = process.env['CRTR_HOME'];
|
|
17
|
+
beforeEach(() => {
|
|
18
|
+
home = mkdtempSync(join(tmpdir(), 'crtr-block-'));
|
|
19
|
+
process.env['CRTR_HOME'] = home;
|
|
20
|
+
});
|
|
21
|
+
afterEach(() => {
|
|
22
|
+
if (previousHome === undefined)
|
|
23
|
+
delete process.env['CRTR_HOME'];
|
|
24
|
+
else
|
|
25
|
+
process.env['CRTR_HOME'] = previousHome;
|
|
26
|
+
rmSync(home, { recursive: true, force: true });
|
|
27
|
+
});
|
|
28
|
+
test('a recorded block is visible to a DIFFERENT process reading the same home', () => {
|
|
29
|
+
const blocker = join(home, 'INDEX.md');
|
|
30
|
+
writeFileSync(blocker, 'front door');
|
|
31
|
+
assert.equal(daemonStartupBlocked(), null, 'nothing recorded yet');
|
|
32
|
+
recordDaemonStartupBlocked('migration blocked\nRepair: crtr sys migrate', [blocker]);
|
|
33
|
+
// A fresh process shares no memory with the recorder — only this file.
|
|
34
|
+
assert.match(daemonStartupBlocked() ?? '', /Repair: crtr sys migrate/);
|
|
35
|
+
});
|
|
36
|
+
test('the marker retires itself once the corpus it named changes', () => {
|
|
37
|
+
const blocker = join(home, 'INDEX.md');
|
|
38
|
+
writeFileSync(blocker, 'front door');
|
|
39
|
+
recordDaemonStartupBlocked('blocked', [blocker]);
|
|
40
|
+
assert.equal(daemonStartupBlocked(), 'blocked');
|
|
41
|
+
// The repair — by `crtr sys migrate`, an editor, or a delete. Whatever did
|
|
42
|
+
// it, the user must never need to know this marker exists.
|
|
43
|
+
writeFileSync(blocker, 'front door removed, plus enough bytes to change size');
|
|
44
|
+
assert.equal(daemonStartupBlocked(), null, 'a changed blocker lifts suppression');
|
|
45
|
+
});
|
|
46
|
+
test('a deleted blocker also retires the marker', () => {
|
|
47
|
+
const blocker = join(home, 'INDEX.md');
|
|
48
|
+
writeFileSync(blocker, 'front door');
|
|
49
|
+
recordDaemonStartupBlocked('blocked', [blocker]);
|
|
50
|
+
rmSync(blocker);
|
|
51
|
+
assert.equal(daemonStartupBlocked(), null);
|
|
52
|
+
});
|
|
53
|
+
test('a marker naming no blockers cannot wedge autostart forever', () => {
|
|
54
|
+
// Nothing about it can be observed to change, so it is advisory only.
|
|
55
|
+
recordDaemonStartupBlocked('blocked with no paths', []);
|
|
56
|
+
assert.equal(daemonStartupBlocked(), null);
|
|
57
|
+
});
|
|
58
|
+
test('clearing is idempotent and safe on a home with no marker', () => {
|
|
59
|
+
clearDaemonStartupBlocked();
|
|
60
|
+
clearDaemonStartupBlocked();
|
|
61
|
+
assert.equal(daemonStartupBlocked(), null);
|
|
62
|
+
});
|
|
63
|
+
// REGRESSION (reap kills a healthy daemon mid-migration): the reap escalates to
|
|
64
|
+
// SIGKILL, so it must be able to tell a child that is DOING the migration from
|
|
65
|
+
// one merely queued behind it. The lock marker already names its owner's pid.
|
|
66
|
+
test('the migration lock names its holder, so a reap can spare the one doing the work', async () => {
|
|
67
|
+
const lock = join(home, 'lock');
|
|
68
|
+
assert.equal(exclusiveLockOwnerPid(lock), null, 'a free lock names nobody');
|
|
69
|
+
await withExclusiveDirectoryLockAsync(lock, async () => {
|
|
70
|
+
assert.equal(exclusiveLockOwnerPid(lock), process.pid, 'the holder is named while held');
|
|
71
|
+
});
|
|
72
|
+
assert.equal(exclusiveLockOwnerPid(lock), null, 'released again');
|
|
73
|
+
});
|
|
74
|
+
// REGRESSION (the aggregate swallows the one error that matters): startup
|
|
75
|
+
// releases its ownership claim before rethrowing a migration failure, and a
|
|
76
|
+
// release that ALSO fails is reported as an AggregateError. The blocked
|
|
77
|
+
// migration must still be recognised through that wrapper, or the daemon exits
|
|
78
|
+
// generically and every downstream suppression stops working.
|
|
79
|
+
test('a blocked migration is still recognised inside a cleanup aggregate', () => {
|
|
80
|
+
const blocked = new MigrationBlockedError('blocked', ['/some/INDEX.md']);
|
|
81
|
+
const aggregate = new AggregateError([blocked, new Error('cleanup also failed')], 'startup failed');
|
|
82
|
+
const found = blockedMigrationIn(aggregate);
|
|
83
|
+
assert.ok(found instanceof MigrationBlockedError, 'found through the wrapper');
|
|
84
|
+
assert.deepEqual(found.blockerPaths, ['/some/INDEX.md']);
|
|
85
|
+
assert.equal(blockedMigrationIn(new Error('unrelated')), null, 'an ordinary failure is not claimed');
|
|
86
|
+
});
|
|
@@ -13,6 +13,7 @@
|
|
|
13
13
|
// leaf can reconstruct the original agent-facing contract instead of collapsing
|
|
14
14
|
// to a generic status code.
|
|
15
15
|
import { abandonManagedWorktree, closeManagedWorktree } from '../../../core/worktree.js';
|
|
16
|
+
import { listQuarantinedManagedWorktrees } from '../../../core/worktree-quarantine.js';
|
|
16
17
|
import { usage } from '../../../core/errors.js';
|
|
17
18
|
function handleWorktreeClose(ctx) {
|
|
18
19
|
const id = ctx.params['id'];
|
|
@@ -27,7 +28,11 @@ function handleWorktreeAbandon(ctx) {
|
|
|
27
28
|
throw usage('abandon requires a non-empty invoking identity', { received: ctx.body, next: 'Pass the invoking node id or human identity.' });
|
|
28
29
|
return { status: 200, body: abandonManagedWorktree(ctx.params['id'], by) };
|
|
29
30
|
}
|
|
31
|
+
async function handleQuarantinedWorktrees() {
|
|
32
|
+
return { status: 200, body: await listQuarantinedManagedWorktrees() };
|
|
33
|
+
}
|
|
30
34
|
export const worktreeRoutes = [
|
|
31
35
|
{ method: 'POST', pattern: '/v1/nodes/:id/worktree/close', handler: handleWorktreeClose },
|
|
32
36
|
{ method: 'POST', pattern: '/v1/nodes/:id/worktree/abandon', handler: handleWorktreeAbandon },
|
|
37
|
+
{ method: 'GET', pattern: '/v1/worktrees/quarantined', handler: handleQuarantinedWorktrees },
|
|
33
38
|
];
|
package/dist/daemon/crtrd-cli.js
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
// raw fatal boundary; a winning daemon stays alive through its scheduled loop.
|
|
4
4
|
import '../suppress-experimental-warnings.js';
|
|
5
5
|
import { runDaemon } from './crtrd.js';
|
|
6
|
-
import { MigrationBlockedError } from '../migrations/activation.js';
|
|
7
6
|
import { DAEMON_EXIT_STARTUP_BLOCKED } from './startup-policy.js';
|
|
7
|
+
import { blockedMigrationIn, recordDaemonStartupBlocked } from './startup-block-marker.js';
|
|
8
8
|
import { envCrtrdTcp } from '../shared/env.js';
|
|
9
9
|
/** Extract `--tcp <host:port>` (or `--tcp=<host:port>`) from argv; falls back to
|
|
10
10
|
* `CRTRD_TCP`. Absent → unix socket only (§4.1). */
|
|
@@ -18,16 +18,21 @@ function parseTcpArg(argv) {
|
|
|
18
18
|
}
|
|
19
19
|
return envCrtrdTcp();
|
|
20
20
|
}
|
|
21
|
-
// A blocked migration is the one startup failure
|
|
22
|
-
//
|
|
23
|
-
//
|
|
24
|
-
// wall. Every other failure keeps reaching Node's raw fatal boundary.
|
|
21
|
+
// A blocked migration is the one startup failure guaranteed to recur, so it
|
|
22
|
+
// exits on its own code rather than reaching Node's raw fatal boundary like
|
|
23
|
+
// every other failure.
|
|
25
24
|
try {
|
|
26
25
|
await runDaemon({ tcp: parseTcpArg(process.argv.slice(2)) });
|
|
27
26
|
}
|
|
28
27
|
catch (error) {
|
|
29
|
-
|
|
28
|
+
// Record it in the canvas home BEFORE exiting: the exit code alone only
|
|
29
|
+
// reaches the one process that spawned us, and the processes that would spawn
|
|
30
|
+
// the next daemon are all different processes with no memory of this. The
|
|
31
|
+
// marker is what makes the suppression outlive this pid.
|
|
32
|
+
const blocked = blockedMigrationIn(error);
|
|
33
|
+
if (blocked === null)
|
|
30
34
|
throw error;
|
|
31
|
-
|
|
35
|
+
recordDaemonStartupBlocked(blocked.message, blocked.blockerPaths);
|
|
36
|
+
process.stderr.write(`[crtrd] startup blocked\n${blocked.message}\n`);
|
|
32
37
|
process.exit(DAEMON_EXIT_STARTUP_BLOCKED);
|
|
33
38
|
}
|
package/dist/daemon/crtrd.js
CHANGED
|
@@ -77,6 +77,7 @@ import { emitEvent } from '../core/events/emit.js';
|
|
|
77
77
|
import { operationIdContext } from '../core/events/operation-id.js';
|
|
78
78
|
import { bindDaemonEventSource } from '../core/events/source.js';
|
|
79
79
|
import { ensureOnDiskMigrations } from '../migrations/activation.js';
|
|
80
|
+
import { clearDaemonStartupBlocked } from './startup-block-marker.js';
|
|
80
81
|
// A broker DECLARES its owning canvas in argv at spawn (host.ts launch:
|
|
81
82
|
// `broker-cli.js --canvas-home <home> --epoch <id> <nodeId>`), so ownership is
|
|
82
83
|
// an exact match on this daemon's own canvas home — no path inference. An
|
|
@@ -303,7 +304,7 @@ export async function superviseTick(now = Date.now(), lifecycle = directTickLife
|
|
|
303
304
|
cronLane.run(now, { lifecycle });
|
|
304
305
|
humanDeliveryLane.run(now, { lifecycle });
|
|
305
306
|
pendingReviewSubmit.run(now, { lifecycle });
|
|
306
|
-
storageMaintenance.run(now, {});
|
|
307
|
+
storageMaintenance.run(now, { lifecycle, fleet });
|
|
307
308
|
await bashDeadline.run(now, { rows });
|
|
308
309
|
// Last: the saturation figure is only true once the freeze lane has spent
|
|
309
310
|
// this tick's free slots.
|
|
@@ -339,6 +340,10 @@ export async function runDaemon(opts = {}) {
|
|
|
339
340
|
const claim = ownership.claim;
|
|
340
341
|
try {
|
|
341
342
|
await ensureOnDiskMigrations();
|
|
343
|
+
// Migrations passed, so whatever once blocked startup here is resolved.
|
|
344
|
+
// Retire the suppression marker on the success path rather than making any
|
|
345
|
+
// repair route remember to: the daemon getting this far IS the proof.
|
|
346
|
+
clearDaemonStartupBlocked();
|
|
342
347
|
}
|
|
343
348
|
catch (migrationError) {
|
|
344
349
|
try {
|
package/dist/daemon/fleet.d.ts
CHANGED
|
@@ -59,6 +59,7 @@ export declare class DaemonFleet implements FleetRegistry {
|
|
|
59
59
|
epoch(): string;
|
|
60
60
|
register(nodeId: string, handle: HostHandle): void;
|
|
61
61
|
has(nodeId: string): boolean;
|
|
62
|
+
isClaimed(nodeId: string): boolean;
|
|
62
63
|
get(nodeId: string): FleetEntry | undefined;
|
|
63
64
|
size(): number;
|
|
64
65
|
occupancy(): number;
|
package/dist/daemon/fleet.js
CHANGED
package/dist/daemon/manage.js
CHANGED
|
@@ -15,6 +15,9 @@ import { hostExecPath } from '../core/runtime/branded-host.js';
|
|
|
15
15
|
// reaches openDb) so the CLI daemon front door stays canvas.db-free (plan B-0).
|
|
16
16
|
import { isDaemonRunning, readPidfile, isPidAlive } from './pidfile.js';
|
|
17
17
|
import { DAEMON_EXIT_STARTUP_BLOCKED, DAEMON_STARTUP_WINDOW_MS } from './startup-policy.js';
|
|
18
|
+
import { clearDaemonStartupBlocked, daemonStartupBlocked } from './startup-block-marker.js';
|
|
19
|
+
import { exclusiveLockOwnerPid } from '../core/exclusive-lock.js';
|
|
20
|
+
import { onDiskMigrationLockPath } from '../core/canvas/paths.js';
|
|
18
21
|
import { envNoDaemonAutostart } from '../shared/env.js';
|
|
19
22
|
import { CrtrClient } from '../api/index.js';
|
|
20
23
|
// Daemon env sanitization
|
|
@@ -421,7 +424,15 @@ export async function spawnDaemon() {
|
|
|
421
424
|
});
|
|
422
425
|
}
|
|
423
426
|
catch (error) {
|
|
424
|
-
|
|
427
|
+
// Reap ONLY a child that is not doing the work. A child holding the on-disk
|
|
428
|
+
// migration lock is mid-corpus-rewrite: the reap escalates to SIGKILL, and
|
|
429
|
+
// killing a migration partway through is far worse than the resident
|
|
430
|
+
// process the reap exists to avoid. Startup verification giving up is a
|
|
431
|
+
// statement about our patience, not proof the child is stuck — a large
|
|
432
|
+
// corpus can legitimately outlast the window, and that daemon should be
|
|
433
|
+
// allowed to finish and serve.
|
|
434
|
+
const childOwnsMigrationLock = exclusiveLockOwnerPid(onDiskMigrationLockPath()) === pid;
|
|
435
|
+
if (exitState === null && !childOwnsMigrationLock) {
|
|
425
436
|
try {
|
|
426
437
|
await stopDaemonProcess(pid, DAEMON_REAP_WINDOW_MS);
|
|
427
438
|
}
|
|
@@ -478,6 +489,7 @@ export function resetDaemonAutostart() {
|
|
|
478
489
|
autostartFailures = 0;
|
|
479
490
|
autostartNextAttemptAt = 0;
|
|
480
491
|
autostartBlocked = null;
|
|
492
|
+
clearDaemonStartupBlocked();
|
|
481
493
|
}
|
|
482
494
|
/** Start the daemon if it is not already running. No-op if already up, if
|
|
483
495
|
* autostart is suppressed for this invocation, while an attempt is in flight,
|
|
@@ -500,6 +512,15 @@ export function ensureDaemon(deps = {}) {
|
|
|
500
512
|
resetDaemonAutostart();
|
|
501
513
|
return;
|
|
502
514
|
}
|
|
515
|
+
// The durable half of the gate. Everything above is module state, which caps
|
|
516
|
+
// exactly one process; the storm is made of many. A daemon that already
|
|
517
|
+
// proved startup is blocked left this marker in the canvas home, and it
|
|
518
|
+
// retires itself as soon as the corpus it named changes.
|
|
519
|
+
const blocked = daemonStartupBlocked();
|
|
520
|
+
if (blocked !== null) {
|
|
521
|
+
autostartBlocked = blocked;
|
|
522
|
+
return;
|
|
523
|
+
}
|
|
503
524
|
autostartInFlight = true;
|
|
504
525
|
void spawn().then(() => {
|
|
505
526
|
autostartFailures = 0;
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import type { FleetRegistry } from '../../core/runtime/fleet.js';
|
|
2
|
+
import type { DetachedWorkLifecycle } from './broker-supervision.js';
|
|
3
|
+
export declare const WORKTREE_SWEEP_INTERVAL_MS: number;
|
|
4
|
+
/** Candidates reconciled per pass. Bounds the work a large backlog can demand. */
|
|
5
|
+
export declare const WORKTREE_SWEEP_BUDGET = 3;
|
|
6
|
+
export interface ManagedWorktreeSweepContext {
|
|
7
|
+
lifecycle: DetachedWorkLifecycle;
|
|
8
|
+
fleet: FleetRegistry;
|
|
9
|
+
}
|
|
10
|
+
export declare class ManagedWorktreeSweepReconciler {
|
|
11
|
+
private lastSweepAt;
|
|
12
|
+
private inFlight;
|
|
13
|
+
/** Round-robin cursor over candidate node ids, so a permanently stuck record
|
|
14
|
+
* at the head of the list cannot starve the ones behind it. */
|
|
15
|
+
private cursor;
|
|
16
|
+
run(now: number, ctx: ManagedWorktreeSweepContext): void;
|
|
17
|
+
/** Terminal nodes whose managed worktree is unreconciled and whose backoff has
|
|
18
|
+
* elapsed. Canvas reads only — engine liveness is classified later, for the
|
|
19
|
+
* budgeted few, from one shared process snapshot. Returns null when the
|
|
20
|
+
* canvas could not be read, which authorizes nothing. */
|
|
21
|
+
private candidates;
|
|
22
|
+
private sweep;
|
|
23
|
+
private reconcileOne;
|
|
24
|
+
}
|