@north-light/crouter 0.3.255 → 0.3.257

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. package/dist/api/client.d.ts +13 -1
  2. package/dist/api/client.js +17 -0
  3. package/dist/api/dto/worktree.d.ts +17 -0
  4. package/dist/api/routes.d.ts +1 -0
  5. package/dist/api/routes.js +1 -0
  6. package/dist/clients/attach/viewer.js +936 -797
  7. package/dist/commands/api-client.js +4 -0
  8. package/dist/commands/sys/worktrees.d.ts +1 -0
  9. package/dist/commands/sys/worktrees.js +53 -0
  10. package/dist/commands/sys.js +2 -1
  11. package/dist/core/__tests__/integration/worktree-land.test.js +71 -1
  12. package/dist/core/__tests__/integration/worktree-reap.test.js +101 -0
  13. package/dist/core/__tests__/worktree-landing.test.d.ts +1 -0
  14. package/dist/core/__tests__/worktree-landing.test.js +11 -0
  15. package/dist/core/canvas/canvas.js +24 -8
  16. package/dist/core/canvas/pid.d.ts +6 -0
  17. package/dist/core/canvas/pid.js +48 -3
  18. package/dist/core/canvas/types.d.ts +41 -1
  19. package/dist/core/exclusive-lock.d.ts +6 -0
  20. package/dist/core/exclusive-lock.js +12 -0
  21. package/dist/core/git.d.ts +1 -1
  22. package/dist/core/git.js +5 -1
  23. package/dist/core/runtime/fleet.d.ts +6 -0
  24. package/dist/core/worktree-landing.d.ts +44 -0
  25. package/dist/core/worktree-landing.js +56 -0
  26. package/dist/core/worktree-quarantine.d.ts +19 -0
  27. package/dist/core/worktree-quarantine.js +54 -0
  28. package/dist/core/worktree-sweep.d.ts +60 -0
  29. package/dist/core/worktree-sweep.js +493 -0
  30. package/dist/core/worktree.d.ts +37 -0
  31. package/dist/core/worktree.js +149 -23
  32. package/dist/daemon/__tests__/startup-block-marker.test.d.ts +1 -0
  33. package/dist/daemon/__tests__/startup-block-marker.test.js +86 -0
  34. package/dist/daemon/api/handlers/worktree.js +5 -0
  35. package/dist/daemon/crtrd-cli.js +12 -7
  36. package/dist/daemon/crtrd.js +6 -1
  37. package/dist/daemon/fleet.d.ts +1 -0
  38. package/dist/daemon/fleet.js +3 -0
  39. package/dist/daemon/manage.js +22 -1
  40. package/dist/daemon/reconcilers/managed-worktree-sweep.d.ts +24 -0
  41. package/dist/daemon/reconcilers/managed-worktree-sweep.js +190 -0
  42. package/dist/daemon/reconcilers/storage-maintenance.d.ts +11 -2
  43. package/dist/daemon/reconcilers/storage-maintenance.js +7 -2
  44. package/dist/daemon/startup-block-marker.d.ts +25 -0
  45. package/dist/daemon/startup-block-marker.js +99 -0
  46. package/dist/shared/generated-context.js +1 -1
  47. package/package.json +1 -1
  48. package/runtime.lock.json +2 -2
@@ -7,6 +7,7 @@ import { isPidAlive } from './canvas/pid.js';
7
7
  import { getNode, updateNode } from './canvas/index.js';
8
8
  import { CONFIG_FILE, CRTR_DIR_NAME, STATE_FILE } from '../types.js';
9
9
  import { withExclusiveDirectoryLock } from './exclusive-lock.js';
10
+ import { parseStatusPorcelain, planLandIntoCheckout, refDeleteTransactionStdin } from './worktree-landing.js';
10
11
  import { shellQuote } from '../shared/shell-quote.js';
11
12
  export class WorktreeError extends Error {
12
13
  code;
@@ -74,7 +75,7 @@ function checkoutPathForBranch(repoRoot, branch) {
74
75
  * callers that must not throw on a read failure (an absent-path proof that
75
76
  * needs to fail closed rather than error) call `gitSync` themselves and pass
76
77
  * the output here instead of going through `checkoutPathForBranch`. */
77
- function checkoutPathForBranchInList(porcelain, branch) {
78
+ export function checkoutPathForBranchInList(porcelain, branch) {
78
79
  let current = null;
79
80
  for (const line of porcelain.split('\n')) {
80
81
  if (line.startsWith('worktree '))
@@ -84,7 +85,36 @@ function checkoutPathForBranchInList(porcelain, branch) {
84
85
  }
85
86
  return null;
86
87
  }
87
- const WORKTREE_LOCK_WAIT_MS = 30_000;
88
+ /** Are these two spellings the same directory?
89
+ *
90
+ * Git canonicalizes every path it prints in `worktree list --porcelain`, while
91
+ * a recorded managed-worktree path preserves the spelling of `CRTR_HOME`. When
92
+ * the canvas home is reached through a symlink (macOS's `/var` → `/private/var`
93
+ * is the everyday case) the two forms name one directory but differ as
94
+ * strings, so a raw `===` reads a checkout crtr created as unowned. Resolution
95
+ * failures fall back to `resolve`, which is the pre-canonical comparison — no
96
+ * worse than the string compare it replaces. */
97
+ export function samePath(a, b) {
98
+ return a === b || canonicalPath(a) === canonicalPath(b);
99
+ }
100
+ function canonicalPath(path) {
101
+ try {
102
+ return realpathSync.native(path);
103
+ }
104
+ catch {
105
+ return resolve(path);
106
+ }
107
+ }
108
+ export const WORKTREE_LOCK_WAIT_MS = 30_000;
109
+ /** The lock directory naming one repository's common Git dir. Every process
110
+ * that mutates a managed worktree — the synchronous commands here and the
111
+ * daemon's asynchronous sweep — derives its lock path through this function,
112
+ * so the two locking styles contend for the SAME name. */
113
+ export function worktreeLockPathForCommonDir(commonDir) {
114
+ const lockRoot = `${crtrHome()}/worktree-locks`;
115
+ mkdirSync(lockRoot, { recursive: true });
116
+ return `${lockRoot}/${createHash('sha256').update(commonDir).digest('hex')}`;
117
+ }
88
118
  function commonGitDir(repoRoot) {
89
119
  const commonDir = runGit(repoRoot, ['rev-parse', '--git-common-dir'], 'git_common_dir_failed', 'Inspect the repository git metadata, then retry.');
90
120
  return realpathSync(resolve(repoRoot, commonDir));
@@ -94,10 +124,7 @@ function commonGitDir(repoRoot) {
94
124
  * add instead of waiting, so callers take this transaction lock first. */
95
125
  function withRepositoryWorktreeLock(repoRoot, action) {
96
126
  const commonDir = commonGitDir(repoRoot);
97
- const lockRoot = `${crtrHome()}/worktree-locks`;
98
- mkdirSync(lockRoot, { recursive: true });
99
- const lockPath = `${lockRoot}/${createHash('sha256').update(commonDir).digest('hex')}`;
100
- return withExclusiveDirectoryLock(lockPath, action, {
127
+ return withExclusiveDirectoryLock(worktreeLockPathForCommonDir(commonDir), action, {
101
128
  timeoutMs: WORKTREE_LOCK_WAIT_MS,
102
129
  timeoutError: () => new WorktreeError('worktree_lock_timeout', `timed out waiting to create a managed worktree in ${commonDir}`, 'Wait for the other managed-worktree operation to finish, then retry.'),
103
130
  });
@@ -110,7 +137,7 @@ function registeredWorktree(repoRoot, path, branch) {
110
137
  for (const line of list.stdout.split('\n')) {
111
138
  if (line.startsWith('worktree '))
112
139
  current = line.slice('worktree '.length).trim();
113
- else if (current === path && line === `branch refs/heads/${branch}`)
140
+ else if (current !== null && samePath(current, path) && line === `branch refs/heads/${branch}`)
114
141
  return true;
115
142
  }
116
143
  return false;
@@ -245,9 +272,10 @@ function gitPath(cwd, logicalPath) {
245
272
  export function isRebaseInProgress(cwd) {
246
273
  return existsSync(gitPath(cwd, 'rebase-merge')) || existsSync(gitPath(cwd, 'rebase-apply'));
247
274
  }
248
- /** The ONE authoritative closed-worktree state transition, shared by both
249
- * closing paths (explicit `closeManagedWorktreeLocked` land, and the clean
250
- * `autoDropCleanManagedWorktreeLocked` push-final drop): persist
275
+ /** The ONE authoritative closed-worktree state transition, shared by every
276
+ * closing path (explicit `closeManagedWorktreeLocked` land, the clean
277
+ * `autoDropCleanManagedWorktreeLocked` push-final drop, and the daemon sweep's
278
+ * `reconcileManagedWorktree`): persist
251
279
  * `managed_worktree.state: 'closed'`, name its cleanup state, and repoint the
252
280
  * node's launch `cwd` to the durable base checkout. The caller broker stays
253
281
  * alive and is Pi's sole transcript writer, so its session header cannot be
@@ -255,15 +283,100 @@ export function isRebaseInProgress(cwd) {
255
283
  * revive launch fresh instead. Both closing paths MUST go through this helper
256
284
  * instead of hand-rolling the `updateNode` call, so neither can drift out of
257
285
  * sync. */
258
- function persistClosedWorktreeState(nodeId, wt, cleanup = 'pending') {
286
+ export function persistClosedWorktreeState(nodeId, wt, cleanup = 'pending') {
259
287
  const node = getNode(nodeId);
260
288
  const sessionInvalidated = node?.pi_session_id != null || node?.pi_session_file != null;
289
+ // Every caller reaches this only after CONCLUDING the land — it succeeded, or
290
+ // the record was proven to have nothing to land. So a recorded land intent is
291
+ // discharged here, and a recorded refusal describes a state that no longer
292
+ // holds; dropping both leaves the sweep re-examining this record fresh.
293
+ const { land_intent: _landIntent, sweep: _sweep, ...retained } = wt;
261
294
  updateNode(nodeId, {
262
295
  cwd: wt.repo_root,
263
296
  ...(sessionInvalidated ? { pi_session_id: null, pi_session_file: null } : {}),
264
- managed_worktree: { ...wt, state: 'closed', cleanup, closed: new Date().toISOString() },
297
+ managed_worktree: { ...retained, state: 'closed', cleanup, closed: new Date().toISOString() },
265
298
  });
266
299
  }
300
+ /** Record that this node ASKED to land and was blocked, so the daemon sweep may
301
+ * later retry exactly that land once the base checkout stops blocking it. The
302
+ * sweep never lands commits for a node that did not get this far: a node that
303
+ * crashed mid-work and never called close has its worktree preserved for a
304
+ * human instead. Recording a fresh intent also clears any prior refusal state,
305
+ * so the sweep re-examines promptly rather than honoring an old backoff. */
306
+ function recordBlockedLandIntent(nodeId, blocked) {
307
+ const current = getNode(nodeId)?.managed_worktree;
308
+ if (current == null || current.state !== 'open')
309
+ return;
310
+ const { sweep: _sweep, ...retained } = current;
311
+ updateNode(nodeId, { managed_worktree: { ...retained, land_intent: { at: new Date().toISOString(), blocked } } });
312
+ }
313
+ /** Fast-forward a base branch that is CHECKED OUT somewhere, landing underneath
314
+ * whatever local changes that checkout holds.
315
+ *
316
+ * The base checkout is shared by every other node using this repository, and
317
+ * its local changes usually have nothing to do with the child that is closing.
318
+ * Disjoint changes ride along untouched; overlapping ones are stashed, the
319
+ * fast-forward runs underneath them, and they are re-applied. The re-apply is
320
+ * `stash apply`, never `stash pop`, so a conflict cannot consume the stash and
321
+ * lose the work: on conflict the checkout is reset to the landed commit and the
322
+ * stash is deliberately KEPT for a person to apply. Either way this never
323
+ * leaves conflict markers behind in a checkout other nodes are using. */
324
+ function landIntoBaseCheckout(baseCheckout, branch, baseSha, landedSha) {
325
+ const status = gitSync(['status', '--porcelain'], baseCheckout);
326
+ if (status.status !== 0)
327
+ return { ok: false, detail: (status.stderr.trim() || status.stdout.trim()).trim() };
328
+ const diff = gitSync(['diff', '--name-only', baseSha, landedSha], baseCheckout);
329
+ if (diff.status !== 0)
330
+ return { ok: false, detail: (diff.stderr.trim() || diff.stdout.trim()).trim() };
331
+ const incoming = diff.stdout.split('\n').map((l) => l.trim()).filter((l) => l !== '');
332
+ const plan = planLandIntoCheckout(parseStatusPorcelain(status.stdout), incoming);
333
+ if (plan.kind === 'refuse') {
334
+ return { ok: false, detail: `untracked files in ${baseCheckout} would be overwritten by the incoming commits: ${plan.collisions.join(', ')}` };
335
+ }
336
+ const merge = () => {
337
+ const res = gitSync(['merge', '--ff-only', landedSha], baseCheckout);
338
+ return res.status === 0 ? { ok: true } : { ok: false, detail: (res.stderr.trim() || res.stdout.trim()).trim() };
339
+ };
340
+ if (plan.kind === 'direct')
341
+ return merge();
342
+ // Identify the stash by the exact commit this push created. `refs/stash` can
343
+ // already name an UNRELATED stash the user made earlier, and a push that
344
+ // saved nothing leaves it untouched — applying or dropping that one would
345
+ // destroy work this code never owned.
346
+ const before = gitSync(['rev-parse', '--verify', '--quiet', 'refs/stash'], baseCheckout);
347
+ const beforeSha = before.status === 0 ? before.stdout.trim() : null;
348
+ const push = gitSync(['stash', 'push', '--message', `crtr landing ${branch}`], baseCheckout);
349
+ if (push.status !== 0)
350
+ return { ok: false, detail: (push.stderr.trim() || push.stdout.trim()).trim() };
351
+ const after = gitSync(['rev-parse', '--verify', '--quiet', 'refs/stash'], baseCheckout);
352
+ const stashSha = after.status === 0 ? after.stdout.trim() : null;
353
+ if (stashSha === null || stashSha === beforeSha)
354
+ return merge(); // nothing was saved
355
+ const landed = merge();
356
+ if (!landed.ok) {
357
+ // The failed merge left the tree exactly as the stash found it, so this
358
+ // re-apply is clean; restore before surfacing the refusal.
359
+ if (gitSync(['stash', 'apply', stashSha], baseCheckout).status === 0)
360
+ dropStashIfStillTip(baseCheckout, stashSha);
361
+ return landed;
362
+ }
363
+ if (gitSync(['stash', 'apply', stashSha], baseCheckout).status === 0) {
364
+ dropStashIfStillTip(baseCheckout, stashSha);
365
+ return { ok: true };
366
+ }
367
+ gitSync(['reset', '--hard', landedSha], baseCheckout);
368
+ return {
369
+ ok: true,
370
+ stash: `local changes in ${baseCheckout} conflicted with the landed commits and were PRESERVED as stash ${stashSha}; nothing was lost. Re-apply them with \`git -C ${shellQuote(baseCheckout)} stash apply ${stashSha}\` and resolve the conflict.`,
371
+ };
372
+ }
373
+ /** Drop a stash entry only while it is still the tip `stash@{0}` names, so a
374
+ * stash someone else pushed in between is never dropped instead. */
375
+ function dropStashIfStillTip(cwd, stashSha) {
376
+ const tip = gitSync(['rev-parse', '--verify', '--quiet', 'refs/stash'], cwd);
377
+ if (tip.status === 0 && tip.stdout.trim() === stashSha)
378
+ gitSync(['stash', 'drop'], cwd);
379
+ }
267
380
  function deferredCleanupResult(wt) {
268
381
  const remove = `git -C ${shellQuote(wt.repo_root)} worktree remove ${shellQuote(wt.path)}`;
269
382
  return {
@@ -349,9 +462,16 @@ function closeManagedWorktreeLocked(nodeId) {
349
462
  // captured baseSha — landing is purely local, nothing is ever pushed to
350
463
  // origin. Both landing paths require the base to still be exactly at that
351
464
  // captured tip, so neither can overwrite a concurrent base change.
465
+ // Every refusal below leaves a rebased, landable branch that this node ASKED
466
+ // to land. Recording that blocked intent is what later authorizes the daemon
467
+ // sweep to finish exactly this land once the blocker clears.
468
+ const blocked = (message, next, detail) => {
469
+ recordBlockedLandIntent(nodeId, message);
470
+ return new WorktreeError('land_base_failed', message, next, detail);
471
+ };
352
472
  const fastForward = gitSync(['merge-base', '--is-ancestor', baseSha, landedSha], wt.path);
353
473
  if (fastForward.status !== 0) {
354
- throw new WorktreeError('land_base_failed', `rebased worktree is not a fast-forward of local base branch '${wt.base_ref}'`, 'Ask the user with `crtr human send` how to land this managed worktree.', (fastForward.stderr.trim() || fastForward.stdout.trim()).trim());
474
+ throw blocked(`rebased worktree is not a fast-forward of local base branch '${wt.base_ref}'`, 'Ask the user with `crtr human send` how to land this managed worktree.', (fastForward.stderr.trim() || fastForward.stdout.trim()).trim());
355
475
  }
356
476
  const baseCheckout = checkoutPathForBranch(wt.path, wt.base_ref);
357
477
  if (baseCheckout !== null) {
@@ -361,17 +481,22 @@ function closeManagedWorktreeLocked(nodeId) {
361
481
  const detail = checkoutHead.status !== 0
362
482
  ? (checkoutHead.stderr.trim() || actualHead).trim()
363
483
  : `expected ${baseSha}, found ${actualHead}`;
364
- throw new WorktreeError('land_base_failed', `local base branch '${wt.base_ref}' changed while this worktree was closing`, `Reconcile the base checkout at ${baseCheckout}, then rerun \`crtr node worktree close\`. If you are unsure, ask the user with \`crtr human send\`.`, detail);
484
+ throw blocked(`local base branch '${wt.base_ref}' changed while this worktree was closing`, `Reconcile the base checkout at ${baseCheckout}, then rerun \`crtr node worktree close\`. If you are unsure, ask the user with \`crtr human send\`.`, detail);
365
485
  }
366
486
  }
367
- const land = baseCheckout !== null
368
- ? gitSync(['merge', '--ff-only', landedSha], baseCheckout)
369
- : gitSync(['update-ref', `refs/heads/${wt.base_ref}`, landedSha, baseSha], wt.path);
370
- if (land.status !== 0) {
371
- const detail = (land.stderr.trim() || land.stdout.trim()).trim();
372
- throw new WorktreeError('land_base_failed', `could not fast-forward local base branch '${wt.base_ref}' to this worktree`, baseCheckout !== null
373
- ? `Local base branch '${wt.base_ref}' is checked out at ${baseCheckout} in a state that blocks a fast-forward (uncommitted changes, or it has diverged). Reconcile it there, then rerun \`crtr node worktree close\`. If you are unsure, ask the user with \`crtr human send\`.`
374
- : 'Ask the user with `crtr human send` how to land this managed worktree.', detail);
487
+ let stashNote;
488
+ if (baseCheckout !== null) {
489
+ const landed = landIntoBaseCheckout(baseCheckout, wt.branch, baseSha, landedSha);
490
+ if (!landed.ok) {
491
+ throw blocked(`could not fast-forward local base branch '${wt.base_ref}' to this worktree`, `Local base branch '${wt.base_ref}' is checked out at ${baseCheckout} in a state that blocks a fast-forward. Reconcile it there, then rerun \`crtr node worktree close\`. If you are unsure, ask the user with \`crtr human send\`.`, landed.detail);
492
+ }
493
+ stashNote = landed.stash;
494
+ }
495
+ else {
496
+ const land = gitSync(['update-ref', `refs/heads/${wt.base_ref}`, landedSha, baseSha], wt.path);
497
+ if (land.status !== 0) {
498
+ throw blocked(`could not fast-forward local base branch '${wt.base_ref}' to this worktree`, 'Ask the user with `crtr human send` how to land this managed worktree.', (land.stderr.trim() || land.stdout.trim()).trim());
499
+ }
375
500
  }
376
501
  // The caller broker still runs from wt.path after this API response. Do not
377
502
  // invoke `git worktree remove`: it recursively deletes the caller's cwd and
@@ -385,6 +510,7 @@ function closeManagedWorktreeLocked(nodeId) {
385
510
  worktree_path: wt.path,
386
511
  landed_sha: landedSha,
387
512
  ...deferredCleanupResult(wt),
513
+ ...(stashNote === undefined ? {} : { base_checkout_stash: stashNote }),
388
514
  };
389
515
  }
390
516
  export function closeManagedWorktree(nodeId) {
@@ -467,7 +593,7 @@ export function abandonManagedWorktree(nodeId, by) {
467
593
  * is still at the exact inspected SHA. `containingRef` can be the recorded
468
594
  * local base or that base's remote-tracking upstream. */
469
595
  function bestEffortDeleteBranchWithContainmentProof(wt, branchSha, containingRef, containingSha) {
470
- const stdin = `verify ${containingRef} ${containingSha}\ndelete refs/heads/${wt.branch} ${branchSha}\n`;
596
+ const stdin = refDeleteTransactionStdin(wt.branch, branchSha, containingRef, containingSha);
471
597
  const transaction = gitSync(['update-ref', '--stdin'], wt.repo_root, stdin);
472
598
  const branchDeleted = transaction.status === 0;
473
599
  if (branchDeleted)
@@ -0,0 +1,86 @@
1
+ import { test, beforeEach, afterEach } from 'node:test';
2
+ import assert from 'node:assert/strict';
3
+ import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
4
+ import { tmpdir } from 'node:os';
5
+ import { join } from 'node:path';
6
+ import { blockedMigrationIn, clearDaemonStartupBlocked, daemonStartupBlocked, recordDaemonStartupBlocked, } from '../startup-block-marker.js';
7
+ import { exclusiveLockOwnerPid, withExclusiveDirectoryLockAsync } from '../../core/exclusive-lock.js';
8
+ import { MigrationBlockedError } from '../../migrations/activation.js';
9
+ // REGRESSION (daemon respawn storm, cross-process half): the first fix for the
10
+ // storm kept its suppression in module state, which caps exactly ONE process.
11
+ // The storm is made of many — every CLI invocation and every scheduled tick is
12
+ // its own process — so 10 separate invocations against a permanently blocked
13
+ // home still produced 10 spawns. Suppression has to outlive a pid, and it has
14
+ // to retire itself when the corpus is repaired or it wedges a fixed install.
15
+ let home;
16
+ const previousHome = process.env['CRTR_HOME'];
17
+ beforeEach(() => {
18
+ home = mkdtempSync(join(tmpdir(), 'crtr-block-'));
19
+ process.env['CRTR_HOME'] = home;
20
+ });
21
+ afterEach(() => {
22
+ if (previousHome === undefined)
23
+ delete process.env['CRTR_HOME'];
24
+ else
25
+ process.env['CRTR_HOME'] = previousHome;
26
+ rmSync(home, { recursive: true, force: true });
27
+ });
28
+ test('a recorded block is visible to a DIFFERENT process reading the same home', () => {
29
+ const blocker = join(home, 'INDEX.md');
30
+ writeFileSync(blocker, 'front door');
31
+ assert.equal(daemonStartupBlocked(), null, 'nothing recorded yet');
32
+ recordDaemonStartupBlocked('migration blocked\nRepair: crtr sys migrate', [blocker]);
33
+ // A fresh process shares no memory with the recorder — only this file.
34
+ assert.match(daemonStartupBlocked() ?? '', /Repair: crtr sys migrate/);
35
+ });
36
+ test('the marker retires itself once the corpus it named changes', () => {
37
+ const blocker = join(home, 'INDEX.md');
38
+ writeFileSync(blocker, 'front door');
39
+ recordDaemonStartupBlocked('blocked', [blocker]);
40
+ assert.equal(daemonStartupBlocked(), 'blocked');
41
+ // The repair — by `crtr sys migrate`, an editor, or a delete. Whatever did
42
+ // it, the user must never need to know this marker exists.
43
+ writeFileSync(blocker, 'front door removed, plus enough bytes to change size');
44
+ assert.equal(daemonStartupBlocked(), null, 'a changed blocker lifts suppression');
45
+ });
46
+ test('a deleted blocker also retires the marker', () => {
47
+ const blocker = join(home, 'INDEX.md');
48
+ writeFileSync(blocker, 'front door');
49
+ recordDaemonStartupBlocked('blocked', [blocker]);
50
+ rmSync(blocker);
51
+ assert.equal(daemonStartupBlocked(), null);
52
+ });
53
+ test('a marker naming no blockers cannot wedge autostart forever', () => {
54
+ // Nothing about it can be observed to change, so it is advisory only.
55
+ recordDaemonStartupBlocked('blocked with no paths', []);
56
+ assert.equal(daemonStartupBlocked(), null);
57
+ });
58
+ test('clearing is idempotent and safe on a home with no marker', () => {
59
+ clearDaemonStartupBlocked();
60
+ clearDaemonStartupBlocked();
61
+ assert.equal(daemonStartupBlocked(), null);
62
+ });
63
+ // REGRESSION (reap kills a healthy daemon mid-migration): the reap escalates to
64
+ // SIGKILL, so it must be able to tell a child that is DOING the migration from
65
+ // one merely queued behind it. The lock marker already names its owner's pid.
66
+ test('the migration lock names its holder, so a reap can spare the one doing the work', async () => {
67
+ const lock = join(home, 'lock');
68
+ assert.equal(exclusiveLockOwnerPid(lock), null, 'a free lock names nobody');
69
+ await withExclusiveDirectoryLockAsync(lock, async () => {
70
+ assert.equal(exclusiveLockOwnerPid(lock), process.pid, 'the holder is named while held');
71
+ });
72
+ assert.equal(exclusiveLockOwnerPid(lock), null, 'released again');
73
+ });
74
+ // REGRESSION (the aggregate swallows the one error that matters): startup
75
+ // releases its ownership claim before rethrowing a migration failure, and a
76
+ // release that ALSO fails is reported as an AggregateError. The blocked
77
+ // migration must still be recognised through that wrapper, or the daemon exits
78
+ // generically and every downstream suppression stops working.
79
+ test('a blocked migration is still recognised inside a cleanup aggregate', () => {
80
+ const blocked = new MigrationBlockedError('blocked', ['/some/INDEX.md']);
81
+ const aggregate = new AggregateError([blocked, new Error('cleanup also failed')], 'startup failed');
82
+ const found = blockedMigrationIn(aggregate);
83
+ assert.ok(found instanceof MigrationBlockedError, 'found through the wrapper');
84
+ assert.deepEqual(found.blockerPaths, ['/some/INDEX.md']);
85
+ assert.equal(blockedMigrationIn(new Error('unrelated')), null, 'an ordinary failure is not claimed');
86
+ });
@@ -13,6 +13,7 @@
13
13
  // leaf can reconstruct the original agent-facing contract instead of collapsing
14
14
  // to a generic status code.
15
15
  import { abandonManagedWorktree, closeManagedWorktree } from '../../../core/worktree.js';
16
+ import { listQuarantinedManagedWorktrees } from '../../../core/worktree-quarantine.js';
16
17
  import { usage } from '../../../core/errors.js';
17
18
  function handleWorktreeClose(ctx) {
18
19
  const id = ctx.params['id'];
@@ -27,7 +28,11 @@ function handleWorktreeAbandon(ctx) {
27
28
  throw usage('abandon requires a non-empty invoking identity', { received: ctx.body, next: 'Pass the invoking node id or human identity.' });
28
29
  return { status: 200, body: abandonManagedWorktree(ctx.params['id'], by) };
29
30
  }
31
+ async function handleQuarantinedWorktrees() {
32
+ return { status: 200, body: await listQuarantinedManagedWorktrees() };
33
+ }
30
34
  export const worktreeRoutes = [
31
35
  { method: 'POST', pattern: '/v1/nodes/:id/worktree/close', handler: handleWorktreeClose },
32
36
  { method: 'POST', pattern: '/v1/nodes/:id/worktree/abandon', handler: handleWorktreeAbandon },
37
+ { method: 'GET', pattern: '/v1/worktrees/quarantined', handler: handleQuarantinedWorktrees },
33
38
  ];
@@ -3,8 +3,8 @@
3
3
  // raw fatal boundary; a winning daemon stays alive through its scheduled loop.
4
4
  import '../suppress-experimental-warnings.js';
5
5
  import { runDaemon } from './crtrd.js';
6
- import { MigrationBlockedError } from '../migrations/activation.js';
7
6
  import { DAEMON_EXIT_STARTUP_BLOCKED } from './startup-policy.js';
7
+ import { blockedMigrationIn, recordDaemonStartupBlocked } from './startup-block-marker.js';
8
8
  import { envCrtrdTcp } from '../shared/env.js';
9
9
  /** Extract `--tcp <host:port>` (or `--tcp=<host:port>`) from argv; falls back to
10
10
  * `CRTRD_TCP`. Absent → unix socket only (§4.1). */
@@ -18,16 +18,21 @@ function parseTcpArg(argv) {
18
18
  }
19
19
  return envCrtrdTcp();
20
20
  }
21
- // A blocked migration is the one startup failure that is guaranteed to recur.
22
- // Exit on its own code — with the repair line intact on stderr — so the
23
- // spawner suppresses further autostarts rather than respawning into the same
24
- // wall. Every other failure keeps reaching Node's raw fatal boundary.
21
+ // A blocked migration is the one startup failure guaranteed to recur, so it
22
+ // exits on its own code rather than reaching Node's raw fatal boundary like
23
+ // every other failure.
25
24
  try {
26
25
  await runDaemon({ tcp: parseTcpArg(process.argv.slice(2)) });
27
26
  }
28
27
  catch (error) {
29
- if (!(error instanceof MigrationBlockedError))
28
+ // Record it in the canvas home BEFORE exiting: the exit code alone only
29
+ // reaches the one process that spawned us, and the processes that would spawn
30
+ // the next daemon are all different processes with no memory of this. The
31
+ // marker is what makes the suppression outlive this pid.
32
+ const blocked = blockedMigrationIn(error);
33
+ if (blocked === null)
30
34
  throw error;
31
- process.stderr.write(`[crtrd] startup blocked\n${error.message}\n`);
35
+ recordDaemonStartupBlocked(blocked.message, blocked.blockerPaths);
36
+ process.stderr.write(`[crtrd] startup blocked\n${blocked.message}\n`);
32
37
  process.exit(DAEMON_EXIT_STARTUP_BLOCKED);
33
38
  }
@@ -77,6 +77,7 @@ import { emitEvent } from '../core/events/emit.js';
77
77
  import { operationIdContext } from '../core/events/operation-id.js';
78
78
  import { bindDaemonEventSource } from '../core/events/source.js';
79
79
  import { ensureOnDiskMigrations } from '../migrations/activation.js';
80
+ import { clearDaemonStartupBlocked } from './startup-block-marker.js';
80
81
  // A broker DECLARES its owning canvas in argv at spawn (host.ts launch:
81
82
  // `broker-cli.js --canvas-home <home> --epoch <id> <nodeId>`), so ownership is
82
83
  // an exact match on this daemon's own canvas home — no path inference. An
@@ -303,7 +304,7 @@ export async function superviseTick(now = Date.now(), lifecycle = directTickLife
303
304
  cronLane.run(now, { lifecycle });
304
305
  humanDeliveryLane.run(now, { lifecycle });
305
306
  pendingReviewSubmit.run(now, { lifecycle });
306
- storageMaintenance.run(now, {});
307
+ storageMaintenance.run(now, { lifecycle, fleet });
307
308
  await bashDeadline.run(now, { rows });
308
309
  // Last: the saturation figure is only true once the freeze lane has spent
309
310
  // this tick's free slots.
@@ -339,6 +340,10 @@ export async function runDaemon(opts = {}) {
339
340
  const claim = ownership.claim;
340
341
  try {
341
342
  await ensureOnDiskMigrations();
343
+ // Migrations passed, so whatever once blocked startup here is resolved.
344
+ // Retire the suppression marker on the success path rather than making any
345
+ // repair route remember to: the daemon getting this far IS the proof.
346
+ clearDaemonStartupBlocked();
342
347
  }
343
348
  catch (migrationError) {
344
349
  try {
@@ -59,6 +59,7 @@ export declare class DaemonFleet implements FleetRegistry {
59
59
  epoch(): string;
60
60
  register(nodeId: string, handle: HostHandle): void;
61
61
  has(nodeId: string): boolean;
62
+ isClaimed(nodeId: string): boolean;
62
63
  get(nodeId: string): FleetEntry | undefined;
63
64
  size(): number;
64
65
  occupancy(): number;
@@ -149,6 +149,9 @@ export class DaemonFleet {
149
149
  has(nodeId) {
150
150
  return this.#map.has(nodeId);
151
151
  }
152
+ isClaimed(nodeId) {
153
+ return this.#map.has(nodeId) || this.#reservations.has(nodeId);
154
+ }
152
155
  get(nodeId) {
153
156
  return this.#map.get(nodeId);
154
157
  }
@@ -15,6 +15,9 @@ import { hostExecPath } from '../core/runtime/branded-host.js';
15
15
  // reaches openDb) so the CLI daemon front door stays canvas.db-free (plan B-0).
16
16
  import { isDaemonRunning, readPidfile, isPidAlive } from './pidfile.js';
17
17
  import { DAEMON_EXIT_STARTUP_BLOCKED, DAEMON_STARTUP_WINDOW_MS } from './startup-policy.js';
18
+ import { clearDaemonStartupBlocked, daemonStartupBlocked } from './startup-block-marker.js';
19
+ import { exclusiveLockOwnerPid } from '../core/exclusive-lock.js';
20
+ import { onDiskMigrationLockPath } from '../core/canvas/paths.js';
18
21
  import { envNoDaemonAutostart } from '../shared/env.js';
19
22
  import { CrtrClient } from '../api/index.js';
20
23
  // Daemon env sanitization
@@ -421,7 +424,15 @@ export async function spawnDaemon() {
421
424
  });
422
425
  }
423
426
  catch (error) {
424
- if (exitState === null) {
427
+ // Reap ONLY a child that is not doing the work. A child holding the on-disk
428
+ // migration lock is mid-corpus-rewrite: the reap escalates to SIGKILL, and
429
+ // killing a migration partway through is far worse than the resident
430
+ // process the reap exists to avoid. Startup verification giving up is a
431
+ // statement about our patience, not proof the child is stuck — a large
432
+ // corpus can legitimately outlast the window, and that daemon should be
433
+ // allowed to finish and serve.
434
+ const childOwnsMigrationLock = exclusiveLockOwnerPid(onDiskMigrationLockPath()) === pid;
435
+ if (exitState === null && !childOwnsMigrationLock) {
425
436
  try {
426
437
  await stopDaemonProcess(pid, DAEMON_REAP_WINDOW_MS);
427
438
  }
@@ -478,6 +489,7 @@ export function resetDaemonAutostart() {
478
489
  autostartFailures = 0;
479
490
  autostartNextAttemptAt = 0;
480
491
  autostartBlocked = null;
492
+ clearDaemonStartupBlocked();
481
493
  }
482
494
  /** Start the daemon if it is not already running. No-op if already up, if
483
495
  * autostart is suppressed for this invocation, while an attempt is in flight,
@@ -500,6 +512,15 @@ export function ensureDaemon(deps = {}) {
500
512
  resetDaemonAutostart();
501
513
  return;
502
514
  }
515
+ // The durable half of the gate. Everything above is module state, which caps
516
+ // exactly one process; the storm is made of many. A daemon that already
517
+ // proved startup is blocked left this marker in the canvas home, and it
518
+ // retires itself as soon as the corpus it named changes.
519
+ const blocked = daemonStartupBlocked();
520
+ if (blocked !== null) {
521
+ autostartBlocked = blocked;
522
+ return;
523
+ }
503
524
  autostartInFlight = true;
504
525
  void spawn().then(() => {
505
526
  autostartFailures = 0;
@@ -0,0 +1,24 @@
1
+ import type { FleetRegistry } from '../../core/runtime/fleet.js';
2
+ import type { DetachedWorkLifecycle } from './broker-supervision.js';
3
+ export declare const WORKTREE_SWEEP_INTERVAL_MS: number;
4
+ /** Candidates reconciled per pass. Bounds the work a large backlog can demand. */
5
+ export declare const WORKTREE_SWEEP_BUDGET = 3;
6
+ export interface ManagedWorktreeSweepContext {
7
+ lifecycle: DetachedWorkLifecycle;
8
+ fleet: FleetRegistry;
9
+ }
10
+ export declare class ManagedWorktreeSweepReconciler {
11
+ private lastSweepAt;
12
+ private inFlight;
13
+ /** Round-robin cursor over candidate node ids, so a permanently stuck record
14
+ * at the head of the list cannot starve the ones behind it. */
15
+ private cursor;
16
+ run(now: number, ctx: ManagedWorktreeSweepContext): void;
17
+ /** Terminal nodes whose managed worktree is unreconciled and whose backoff has
18
+ * elapsed. Canvas reads only — engine liveness is classified later, for the
19
+ * budgeted few, from one shared process snapshot. Returns null when the
20
+ * canvas could not be read, which authorizes nothing. */
21
+ private candidates;
22
+ private sweep;
23
+ private reconcileOne;
24
+ }