@north-light/crouter 0.3.304 → 0.3.305

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (39) hide show
  1. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/integration/provider-rotation.test.ts +87 -0
  2. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +31 -26
  3. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +33 -26
  4. package/dist/clients/attach/viewer.js +581 -581
  5. package/dist/commands/node/create.js +7 -1
  6. package/dist/commands/node/wait.js +1 -1
  7. package/dist/commands/node-worktree.js +5 -1
  8. package/dist/core/__tests__/fixtures/fake-engine.js +14 -15
  9. package/dist/core/__tests__/integration/tmux-surface.test.js +53 -0
  10. package/dist/core/__tests__/kickoff.test.js +40 -0
  11. package/dist/core/__tests__/seam/broker-provider-retry.test.js +25 -0
  12. package/dist/core/__tests__/seam/deadline-wake-delivery.test.d.ts +1 -0
  13. package/dist/core/__tests__/seam/deadline-wake-delivery.test.js +138 -0
  14. package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.d.ts +1 -0
  15. package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.js +60 -0
  16. package/dist/core/fault-classifier.d.ts +6 -0
  17. package/dist/core/fault-classifier.js +9 -4
  18. package/dist/core/provider-failure.d.ts +30 -0
  19. package/dist/core/provider-failure.js +86 -0
  20. package/dist/core/runtime/broker/fault-retry.js +8 -0
  21. package/dist/core/runtime/fault.d.ts +2 -1
  22. package/dist/core/runtime/fault.js +38 -0
  23. package/dist/core/runtime/kickoff.d.ts +3 -0
  24. package/dist/core/runtime/kickoff.js +24 -4
  25. package/dist/core/runtime/reopen.d.ts +2 -2
  26. package/dist/core/runtime/reopen.js +6 -4
  27. package/dist/core/runtime/revive.js +3 -3
  28. package/dist/core/runtime/tmux-bindings.js +14 -3
  29. package/dist/core/termrender/version.d.ts +1 -1
  30. package/dist/core/termrender/version.js +1 -1
  31. package/dist/core/worktree-close.js +6 -3
  32. package/dist/core/worktree-sweep.js +7 -3
  33. package/dist/core/worktree.js +6 -3
  34. package/dist/daemon/api/__tests__/reopen-delivery.test.js +31 -0
  35. package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +36 -4
  36. package/dist/daemon/api/handlers/messages.js +26 -9
  37. package/dist/daemon/reconcilers/broker-supervision.js +10 -1
  38. package/package.json +1 -1
  39. package/runtime.lock.json +5 -5
@@ -82,8 +82,9 @@ function nodeNewParams() {
82
82
  { kind: 'flag', name: 'name', type: 'string', required: false, constraint: 'Display name.' },
83
83
  { kind: 'flag', name: 'parent', type: 'string', required: false, constraint: 'Parent node id. Defaults to the calling node.' },
84
84
  { kind: 'flag', name: 'node-id', type: 'string', required: false, constraint: 'Exact id for the new node; lowercase letters, digits, and hyphens only, at most 128 characters. Rejected if already present.' },
85
- { kind: 'flag', name: 'root', type: 'bool', required: false, constraint: 'Spawn an independent resident root with no parent or subscription. Mutually exclusive with --worktree.' },
85
+ { kind: 'flag', name: 'root', type: 'bool', required: false, constraint: 'Spawn an independent root with no parent or subscription. Mutually exclusive with --worktree.' },
86
86
  { kind: 'flag', name: 'resident', type: 'bool', required: false, constraint: 'Birth the node as resident: it parks and stays wakeable without being forced to submit a final. Pass it ONLY when the user will open this node’s viewer pane and work with it back and forth; a child that only other nodes message stays terminal. Applies to managed children and roots; roots default to resident.' },
87
+ { kind: 'flag', name: 'terminal', type: 'bool', required: false, constraint: 'Birth the node as terminal: it finalizes when its work is done and leaves nothing wakeable behind. Only meaningful with --root, whose default is resident; managed children are already terminal. Pass it for bounded programmatic work no one will open a viewer on. Mutually exclusive with --resident.' },
87
88
  { kind: 'flag', name: 'worktree', type: 'string', required: false, constraint: 'Local base branch for a crouter-managed worktree for this child. Mutually exclusive with --root.' },
88
89
  { kind: 'flag', name: 'fork-from', type: 'string', required: false, constraint: 'Fork the new node from an existing node id, absolute session path, or partial pi session uuid.' },
89
90
  {
@@ -190,6 +191,10 @@ async function runNodeCreation(input) {
190
191
  const parent = input['parent'];
191
192
  const root = input['root'] === true;
192
193
  const resident = input['resident'] === true;
194
+ const terminal = input['terminal'] === true;
195
+ if (resident && terminal) {
196
+ throw new InputError({ error: 'lifecycle_conflict', message: '--resident and --terminal cannot both be passed', field: 'terminal', next: 'Pass the one birth lifecycle you want, or neither to take the default (resident for --root, terminal for a managed child).' });
197
+ }
193
198
  const worktreeBaseRaw = input['worktree'];
194
199
  if (worktreeBaseRaw !== undefined && worktreeBaseRaw.trim() === '') {
195
200
  throw new InputError({ error: 'worktree_base_required', message: '--worktree requires a base branch', field: 'worktree', next: 'Pass an existing local branch that landing can fast-forward.' });
@@ -257,6 +262,7 @@ async function runNodeCreation(input) {
257
262
  ...(root ? { root: true } : { parent: parentId ?? null }),
258
263
  ...(creator !== undefined ? { creator } : {}),
259
264
  ...(resident ? { root_lifecycle: 'resident' } : {}),
265
+ ...(terminal ? { root_lifecycle: 'terminal' } : {}),
260
266
  ...(worktreeBase !== undefined ? { worktree: worktreeBase } : {}),
261
267
  ...(forkFrom !== undefined ? { fork_from: forkFrom } : {}),
262
268
  ...(modelSpec !== undefined ? { model: modelSpec } : {}),
@@ -46,7 +46,7 @@ function deadlineHeredoc(body, nodeId) {
46
46
  const lines = new Set(body.split('\n'));
47
47
  while (lines.has(delimiter))
48
48
  delimiter = `CRTR_DEADLINE_${++suffix}`;
49
- return `crtr cron cancel "$CRTR_CRON_ID" && crtr node message send --to ${shellQuote(nodeId)} --tier urgent <<'${delimiter}'\n${body}\n${delimiter}`;
49
+ return `crtr cron cancel "$CRTR_CRON_ID"\ncrtr node message send --to ${shellQuote(nodeId)} --tier urgent <<'${delimiter}'\n${body}\n${delimiter}`;
50
50
  }
51
51
  const nodeWaitDeadline = defineLeaf({
52
52
  name: 'deadline',
@@ -85,7 +85,8 @@ const worktreeClose = defineLeaf({
85
85
  { name: 'worktree_removed', type: 'boolean', required: true, constraint: 'Always false for an active caller: its checkout remains intact rather than being removed beneath the node.' },
86
86
  { name: 'worktree_remove_error', type: 'string', required: true, constraint: 'The named deferred-cleanup state and safe manual follow-up after the caller exits.' },
87
87
  { name: 'branch_deleted', type: 'boolean', required: true, constraint: 'False while the deferred-cleanup checkout still has this branch checked out.' },
88
- { name: 'branch_delete_error', type: 'string', required: true, constraint: 'The safe branch-cleanup follow-up after the checkout is removed.' }
88
+ { name: 'branch_delete_error', type: 'string', required: true, constraint: 'The safe branch-cleanup follow-up after the checkout is removed.' },
89
+ { name: 'base_checkout_stash', type: 'string', required: false, constraint: 'Present when conflicting base-checkout changes were retained in a stash; names the exact stash SHA and command required to recover them.' }
89
90
  ],
90
91
  outputKind: 'object',
91
92
  effects: [
@@ -116,6 +117,9 @@ const worktreeClose = defineLeaf({
116
117
  if (r['branch_deleted'] === false) {
117
118
  notes.push(String(r['branch_delete_error'] ?? `local branch ${branch} could not be deleted automatically.`));
118
119
  }
120
+ if (typeof r['base_checkout_stash'] === 'string') {
121
+ notes.push(`Warning: ${r['base_checkout_stash']}`);
122
+ }
119
123
  if (notes.length > 0)
120
124
  return `${base} Note: ${notes.join(' ')}`;
121
125
  return base;
@@ -35,6 +35,7 @@ import { join, dirname } from 'node:path';
35
35
  import { randomUUID } from 'node:crypto';
36
36
  import { DefaultResourceLoader, getAgentDir } from '@earendil-works/pi-coding-agent';
37
37
  import { MANAGED_PROVIDER_COOLING_DIAGNOSTIC } from '../../runtime/managed-provider-cooling.js';
38
+ import { PROVIDER_FAILURE_DIAGNOSTIC } from '../../provider-failure.js';
38
39
  import registerCanvasBashValve, { createValveOperations } from '../../../pi-extensions/canvas-bash-valve.js';
39
40
  /** Test seams for prompt-scoped failures before session_start. The one-shot
40
41
  * variant records its first failure per node so the next cycling retry can
@@ -1085,6 +1086,18 @@ class FakeSession {
1085
1086
  // before the stophook sees agent_end while the run is active; the
1086
1087
  // subscriber receives agent_settled after it clears.
1087
1088
  this.streaming = true;
1089
+ const diagnostics = [
1090
+ ...(cmd.deadline === undefined ? [] : [{
1091
+ type: MANAGED_PROVIDER_COOLING_DIAGNOSTIC,
1092
+ timestamp: Date.now(),
1093
+ details: { rateLimitedUntil: cmd.deadline },
1094
+ }]),
1095
+ ...(cmd.providerFailure === undefined ? [] : [{
1096
+ type: PROVIDER_FAILURE_DIAGNOSTIC,
1097
+ timestamp: Date.now(),
1098
+ details: cmd.providerFailure,
1099
+ }]),
1100
+ ];
1088
1101
  const stopMessages = [
1089
1102
  {
1090
1103
  role: 'assistant',
@@ -1093,21 +1106,7 @@ class FakeSession {
1093
1106
  // turn; the stophook classifies it for the active-fault marker.
1094
1107
  // Forward it so a test can drive a CONNECTION stall faithfully.
1095
1108
  ...(cmd.errorMessage === undefined ? {} : { errorMessage: cmd.errorMessage }),
1096
- // A managed-provider cooling terminal carries the earliest pool
1097
- // deadline as a typed diagnostic that survives verbatim into
1098
- // agent_end.messages; the stophook reads it (extractCoolingDeadline)
1099
- // to stamp retry.nextAt. Attached only when a test drives `deadline`.
1100
- ...(cmd.deadline === undefined
1101
- ? {}
1102
- : {
1103
- diagnostics: [
1104
- {
1105
- type: MANAGED_PROVIDER_COOLING_DIAGNOSTIC,
1106
- timestamp: Date.now(),
1107
- details: { rateLimitedUntil: cmd.deadline },
1108
- },
1109
- ],
1110
- }),
1109
+ ...(diagnostics.length === 0 ? {} : { diagnostics }),
1111
1110
  content: [{ type: 'text', text: cmd.text ?? '' }],
1112
1111
  },
1113
1112
  ];
@@ -275,3 +275,56 @@ test('tmux installer reads pair-array manifests and replaces owned pairs', (t) =
275
275
  isolatedTmux(socket, ['kill-server']);
276
276
  }
277
277
  });
278
+ // Regression: a passthrough wrapper survives the user re-sourcing their own tmux
279
+ // config over it. Crouter wraps an occupied root key and fingerprints the result;
280
+ // every `source-file` (and every fresh tmux server) re-installs the user's own
281
+ // bind-key line over that wrapper. Reading the mismatch as a deliberate rebind
282
+ // relinquished the key permanently, so the attach viewer silently stopped
283
+ // receiving its own scroll keys and no reconciliation ever restored them.
284
+ test('tmux passthrough re-wraps a re-sourced config but yields to a real rebind', (t) => {
285
+ if (spawnSync('tmux', ['-V']).status !== 0) {
286
+ t.skip('tmux is unavailable');
287
+ return;
288
+ }
289
+ const name = `crtr-passthrough-${process.pid}-${Date.now()}`;
290
+ const boot = spawnSync('tmux', ['-L', name, '-f', '/dev/null', 'new-session', '-d', '-s', 'test'], { encoding: 'utf8' });
291
+ assert.equal(boot.status, 0, boot.stderr);
292
+ const socket = spawnSync('tmux', ['-L', name, 'display-message', '-p', '#{socket_path}'], { encoding: 'utf8' }).stdout.trim();
293
+ assert.notEqual(socket, '');
294
+ const priorTmux = process.env['TMUX'];
295
+ process.env['TMUX'] = `${socket},1,0`;
296
+ const userBinding = ['bind-key', '-T', 'root', 'C-k', 'copy-mode', '-e'];
297
+ const bindings = resolveKeybindings({ 'crtr.attach.scroll.up': ['ctrl+k'] });
298
+ const passthrough = ['crtr.attach.scroll.up'];
299
+ try {
300
+ isolatedTmux(socket, userBinding);
301
+ const original = tmuxBinding(socket, 'root', 'C-k');
302
+ assert.match(original ?? '', /copy-mode/);
303
+ const wrapped = installTmuxBindings(bindings, passthrough);
304
+ assert.equal(wrapped.ok, true, JSON.stringify(wrapped.diagnostics));
305
+ assert.match(tmuxBinding(socket, 'root', 'C-k') ?? '', /@crtr_attach/, 'occupied key should be wrapped');
306
+ // The user reloads their tmux config: their own line lands over the wrapper.
307
+ isolatedTmux(socket, userBinding);
308
+ assert.equal(tmuxBinding(socket, 'root', 'C-k'), original);
309
+ const resourced = installTmuxBindings(bindings, passthrough);
310
+ assert.equal(resourced.ok, true, JSON.stringify(resourced.diagnostics));
311
+ assert.match(tmuxBinding(socket, 'root', 'C-k') ?? '', /@crtr_attach/, 're-sourced config should be re-wrapped, not relinquished');
312
+ // A line matching neither the wrapper nor the recorded original is a real
313
+ // rebind and stays the user's, here and on every later reconciliation.
314
+ isolatedTmux(socket, ['bind-key', '-T', 'root', 'C-k', 'display-message', 'user-owned']);
315
+ for (const pass of ['first', 'second']) {
316
+ const rebound = installTmuxBindings(bindings, passthrough);
317
+ assert.equal(rebound.ok, true, JSON.stringify(rebound.diagnostics));
318
+ const live = tmuxBinding(socket, 'root', 'C-k') ?? '';
319
+ assert.match(live, /user-owned/, `${pass} reconciliation should leave a real rebind alone`);
320
+ assert.doesNotMatch(live, /@crtr_attach/);
321
+ }
322
+ }
323
+ finally {
324
+ if (priorTmux === undefined)
325
+ delete process.env['TMUX'];
326
+ else
327
+ process.env['TMUX'] = priorTmux;
328
+ isolatedTmux(socket, ['kill-server']);
329
+ }
330
+ });
@@ -62,6 +62,46 @@ test('drainBearings preserves the yield note until explicit boot confirmation',
62
62
  assert.equal(readYieldMessage(id), null, 'confirmation clear is idempotent');
63
63
  assert.equal(existsSync(yieldMessagePath(id)), false, 'confirmed handoff removes the note');
64
64
  });
65
+ test('fresh kickoff keeps each inbox card directly before its own message', () => {
66
+ const id = 'ordered-cards';
67
+ const meta = createNode(node(id));
68
+ appendInbox(id, {
69
+ from: 'sender',
70
+ tier: 'normal',
71
+ kind: 'message',
72
+ label: 'FIRST-BODY',
73
+ data: { body: 'FIRST-BODY', cards: ['<runtime kind="test-card">FIRST-CARD</runtime>'] },
74
+ });
75
+ appendInbox(id, {
76
+ from: 'sender',
77
+ tier: 'normal',
78
+ kind: 'message',
79
+ label: 'SECOND-BODY',
80
+ data: { body: 'SECOND-BODY', cards: ['<runtime kind="test-card">SECOND-CARD</runtime>'] },
81
+ });
82
+ for (const body of ['GROUPED-BODY-1', 'GROUPED-BODY-2', 'GROUPED-BODY-3']) {
83
+ appendInbox(id, {
84
+ from: 'card-free-sender',
85
+ tier: 'normal',
86
+ kind: 'message',
87
+ label: body,
88
+ data: { body },
89
+ });
90
+ }
91
+ const kickoff = buildReviveKickoff(meta, drainBearings(meta));
92
+ const firstCard = kickoff.indexOf('FIRST-CARD');
93
+ const firstBody = kickoff.indexOf('FIRST-BODY');
94
+ const secondCard = kickoff.indexOf('SECOND-CARD');
95
+ const secondBody = kickoff.indexOf('SECOND-BODY');
96
+ assert.ok(firstCard >= 0 && firstCard < firstBody, 'the first card stays ahead of its own message');
97
+ assert.ok(firstBody < secondCard, 'the second card cannot leap ahead of the first message');
98
+ assert.ok(secondCard < secondBody, 'the second card stays ahead of its own message');
99
+ const groupedSections = kickoff.match(/<from id="card-free-sender">([\s\S]*?)<\/from>/g) ?? [];
100
+ assert.equal(groupedSections.length, 1, 'card-free entries from one sender remain one delivery');
101
+ for (const body of ['GROUPED-BODY-1', 'GROUPED-BODY-2', 'GROUPED-BODY-3']) {
102
+ assert.ok(groupedSections[0].includes(body), `${body} remains in the sender's single delivery`);
103
+ }
104
+ });
65
105
  test('drainBearings leaves persona drift pending until the replacement context exists', () => {
66
106
  const oldPersona = { kind: 'developer', mode: 'base', lifecycle: 'terminal' };
67
107
  const meta = createNode(node('pending-persona', {
@@ -117,6 +117,31 @@ after(async () => {
117
117
  afterEach(() => {
118
118
  kit.closeAll();
119
119
  });
120
+ test('a normalized 503 survives into a durable provider fault and re-drives the turn', { timeout: 30_000 }, async () => {
121
+ const parent = h.spawnRoot('structured provider failure parent');
122
+ seedPool('anthropic', false);
123
+ seedPool('openai-codex', false);
124
+ const child = await h.spawnHeadlessChild(parent, 'structured provider failure', { model: OPENAI_STRONG.spec });
125
+ await h.waitFor(() => agentStartCount(child) >= 1, { label: 'initial structured-failure fixture turn started' });
126
+ const baselineStarts = agentStartCount(child);
127
+ h.fakeCmd(child, {
128
+ cmd: 'stop',
129
+ reason: 'error',
130
+ errorMessage: 'Request failed',
131
+ providerFailure: { status: 503 },
132
+ });
133
+ const fault = await h.waitFor(() => {
134
+ const current = readFault(child);
135
+ return current?.kind === 'overloaded' && current.retry.disposition === 'auto' ? current : undefined;
136
+ }, { label: 'structured 503 published as an automatic provider fault' });
137
+ assert.equal(fault.providerError?.provider, OPENAI_STRONG.model.provider);
138
+ assert.equal(fault.providerError?.model, OPENAI_STRONG.model.id);
139
+ assert.equal(fault.providerError?.status, 503);
140
+ await h.waitFor(() => agentStartCount(child) >= baselineStarts + 1, {
141
+ label: 'structured 503 automatic recovery turn started',
142
+ });
143
+ await h.waitFor(() => readFault(child) === null, { label: 'successful recovery clears the structured provider fault' });
144
+ });
120
145
  test('a managed-cooling terminal authors a rate-limit auto fault whose deadline drives exactly one broker re-drive', { timeout: 30_000 }, async () => {
121
146
  const parent = h.spawnRoot('cooling terminal parent');
122
147
  // The terminal fault path is pool-independent — the stophook classifies purely
@@ -0,0 +1,138 @@
1
+ // Run with: npm run build && node --test --test-timeout=180000 dist/core/__tests__/seam/deadline-wake-delivery.test.js
2
+ //
3
+ // Pins the compiled deadline command: self-cancel cleanup runs before its urgent
4
+ // inbox wake, but a nonzero cleanup result cannot suppress that durable wake.
5
+ import { after, before, test } from 'node:test';
6
+ import assert from 'node:assert/strict';
7
+ import { chmodSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
8
+ import { tmpdir } from 'node:os';
9
+ import { join } from 'node:path';
10
+ import { getCron } from '../../canvas/crons.js';
11
+ import { ensureRootProfile } from '../../profiles/manifest.js';
12
+ import { executeCron } from '../../../daemon/cron-run.js';
13
+ import { createHeadlessHarness } from '../helpers/harness.js';
14
+ let h;
15
+ let root;
16
+ let fakeBin;
17
+ let journal;
18
+ let argsPath;
19
+ let bodyPath;
20
+ let cancelCodePath;
21
+ let sendCodePath;
22
+ let savedPath;
23
+ let savedProfileId;
24
+ let profileId;
25
+ function shellQuote(value) {
26
+ return `'${value.replaceAll("'", "'\\''")}'`;
27
+ }
28
+ function writeFakeFrontDoor() {
29
+ writeFileSync(join(fakeBin, 'crtr'), `#!/bin/sh
30
+ journal=${shellQuote(journal)}
31
+ args=${shellQuote(argsPath)}
32
+ body=${shellQuote(bodyPath)}
33
+ cancel_code=${shellQuote(cancelCodePath)}
34
+ send_code=${shellQuote(sendCodePath)}
35
+
36
+ if [ "$1" = cron ] && [ "$2" = cancel ]; then
37
+ printf '%s\\n' cancel >> "$journal"
38
+ ${shellQuote(process.execPath)} --input-type=module -e ${shellQuote(`import { cancelCron } from ${JSON.stringify(join(process.cwd(), 'dist', 'core', 'canvas', 'crons.js'))}; cancelCron(process.argv[1], { preserveRunningProcess: true });`)} "$3" || exit $?
39
+ code=$(cat "$cancel_code")
40
+ if [ "$code" -ne 0 ]; then
41
+ printf '%s\\n' 'cancel failed' >&2
42
+ fi
43
+ exit "$code"
44
+ fi
45
+
46
+ if [ "$1" = node ] && [ "$2" = message ]; then
47
+ printf '%s\\n' message >> "$journal"
48
+ printf '%s\\n' "$@" > "$args"
49
+ cat > "$body"
50
+ code=$(cat "$send_code")
51
+ if [ "$code" -ne 0 ]; then
52
+ printf '%s\\n' 'send failed' >&2
53
+ exit "$code"
54
+ fi
55
+ exec ${shellQuote(process.execPath)} ${shellQuote(join(process.cwd(), 'dist', 'cli.js'))} "$@" < "$body"
56
+ fi
57
+
58
+ printf '%s\\n' unexpected >> "$journal"
59
+ exit 99
60
+ `, { mode: 0o755 });
61
+ chmodSync(join(fakeBin, 'crtr'), 0o755);
62
+ }
63
+ async function armDeadline(body) {
64
+ const target = await h.spawnHeadlessChild(root, 'deadline target', { profile: profileId });
65
+ const armed = h.cli(target, ['--json', 'node', 'wait', 'deadline', body, '--until', '1m']);
66
+ assert.equal(armed.code, 0, `node wait deadline: ${armed.stderr}`);
67
+ const id = armed.json?.id;
68
+ assert.equal(typeof id, 'string', 'the compiled deadline CLI returns its cron id');
69
+ const cron = getCron(id);
70
+ assert.notEqual(cron, null, 'the compiled deadline CLI persisted its cron command');
71
+ return { target, cron: cron };
72
+ }
73
+ async function runDeadline(cancelCode, sendCode, body) {
74
+ writeFileSync(journal, '');
75
+ writeFileSync(argsPath, '');
76
+ writeFileSync(bodyPath, '');
77
+ writeFileSync(cancelCodePath, String(cancelCode));
78
+ writeFileSync(sendCodePath, String(sendCode));
79
+ const { target, cron } = await armDeadline(body);
80
+ process.env['PATH'] = `${fakeBin}:${savedPath ?? ''}`;
81
+ try {
82
+ const record = await executeCron(cron, { outOfBand: false });
83
+ return { target, record: record ?? null };
84
+ }
85
+ finally {
86
+ if (savedPath === undefined)
87
+ delete process.env['PATH'];
88
+ else
89
+ process.env['PATH'] = savedPath;
90
+ }
91
+ }
92
+ before(async () => {
93
+ h = await createHeadlessHarness({ sessionPrefix: 'crtr-deadline-wake' });
94
+ profileId = ensureRootProfile().profileId;
95
+ savedProfileId = process.env['CRTR_PROFILE_ID'];
96
+ process.env['CRTR_PROFILE_ID'] = profileId;
97
+ root = h.spawnRoot('deadline wake seam root');
98
+ fakeBin = mkdtempSync(join(tmpdir(), 'crtr-deadline-front-door-'));
99
+ journal = join(fakeBin, 'journal');
100
+ argsPath = join(fakeBin, 'args');
101
+ bodyPath = join(fakeBin, 'body');
102
+ cancelCodePath = join(fakeBin, 'cancel-code');
103
+ sendCodePath = join(fakeBin, 'send-code');
104
+ writeFakeFrontDoor();
105
+ savedPath = process.env['PATH'];
106
+ });
107
+ after(async () => {
108
+ if (savedPath === undefined)
109
+ delete process.env['PATH'];
110
+ else
111
+ process.env['PATH'] = savedPath;
112
+ if (savedProfileId === undefined)
113
+ delete process.env['CRTR_PROFILE_ID'];
114
+ else
115
+ process.env['CRTR_PROFILE_ID'] = savedProfileId;
116
+ rmSync(fakeBin, { recursive: true, force: true });
117
+ if (h !== undefined)
118
+ await h.dispose();
119
+ });
120
+ test('compiled deadline wakes once after cancel failure, retains cancel-first ordering, and propagates send failure', { timeout: 60_000 }, async () => {
121
+ const failedCancelBody = 'deadline wake after failed cancel\nbody remains intact';
122
+ const failedCancel = await runDeadline(2, 0, failedCancelBody);
123
+ assert.equal(failedCancel.record?.exit_code, 0, 'a successful send is the cron command result after failed cleanup');
124
+ assert.match(failedCancel.record?.stderr_head ?? '', /cancel failed/, 'cancel diagnostics remain on stderr');
125
+ assert.deepEqual(readFileSync(journal, 'utf8').trim().split('\n'), ['cancel', 'message'], 'cancel runs before exactly one message send');
126
+ assert.deepEqual(readFileSync(argsPath, 'utf8').trim().split('\n'), ['node', 'message', 'send', '--to', failedCancel.target, '--tier', 'urgent'], 'the generated urgent send receives its target and tier');
127
+ assert.equal(readFileSync(bodyPath, 'utf8'), `${failedCancelBody}\n`, 'the generated heredoc body reaches the front door byte-for-byte');
128
+ await h.awaitWake(failedCancel.target, { match: /deadline wake after failed cancel/ });
129
+ assert.ok(h.inbox(failedCancel.target).some((entry) => entry.tier === 'urgent' && entry.data?.['body'] === failedCancelBody), 'the successful send durably appends the urgent deadline message to the target inbox');
130
+ const successfulCancel = await runDeadline(0, 0, 'deadline wake after successful cancel');
131
+ assert.equal(successfulCancel.record?.exit_code, 0, 'the ordinary successful-cancel path still succeeds');
132
+ assert.deepEqual(readFileSync(journal, 'utf8').trim().split('\n'), ['cancel', 'message'], 'successful cleanup also precedes one message send');
133
+ await h.awaitWake(successfulCancel.target, { match: /deadline wake after successful cancel/ });
134
+ const failedSend = await runDeadline(2, 23, 'deadline send failure');
135
+ assert.equal(failedSend.record?.exit_code, 23, 'the final message-send status remains the cron failure status');
136
+ assert.match(failedSend.record?.stderr_head ?? '', /cancel failed[\s\S]*send failed/, 'both cleanup and send diagnostics remain visible');
137
+ assert.deepEqual(readFileSync(journal, 'utf8').trim().split('\n'), ['cancel', 'message'], 'a failed send is still attempted once after failed cleanup');
138
+ });
@@ -0,0 +1,60 @@
1
+ // Run with: npm run build && node --test --test-timeout=180000 dist/core/__tests__/seam/reopen-invalidated-worktree.test.js
2
+ //
3
+ // Regression for #1070 and #1064: a completed managed-worktree node loses its
4
+ // session when the worktree closes. Atomic message reopen must start from the
5
+ // retained final report and the new instruction, never replay its birth goal.
6
+ import { after, before, test } from 'node:test';
7
+ import assert from 'node:assert/strict';
8
+ import { existsSync } from 'node:fs';
9
+ import { join } from 'node:path';
10
+ import { createHeadlessHarness } from '../helpers/harness.js';
11
+ import { pushFinal } from '../../feed/feed.js';
12
+ import { persistClosedWorktreeState } from '../../worktree.js';
13
+ import { readGoal } from '../../runtime/kickoff.js';
14
+ let h;
15
+ before(async () => {
16
+ h = await createHeadlessHarness({ sessionPrefix: 'crtr-reopen-invalidated-worktree' });
17
+ });
18
+ after(async () => {
19
+ if (h !== undefined)
20
+ await h.dispose();
21
+ });
22
+ test('atomic message reopen of a closed worktree starts fresh from the prior final and follow-up, not the birth goal', { timeout: 30_000 }, async () => {
23
+ const root = h.spawnRoot('reopen invalidated-worktree manager');
24
+ const originalGoal = 'ORIGINAL-MANDATE: do not replay this completed implementation task';
25
+ const followUp = 'FOLLOW-UP-MANDATE: inspect the completed work without changing it';
26
+ const child = await h.spawnHeadlessChild(root, originalGoal);
27
+ await pushFinal(child, 'COMPLETED-FINAL: the original implementation is complete');
28
+ const finalReport = h.node(child)?.final_report;
29
+ assert.ok(finalReport, 'the completed worker has a canonical final report');
30
+ const finalPath = join(h.home, 'nodes', child, 'reports', finalReport);
31
+ assert.equal(existsSync(finalPath), true, 'the canonical final exists before reopening');
32
+ persistClosedWorktreeState(child, {
33
+ state: 'open',
34
+ path: join(h.home, 'removed-worktree'),
35
+ branch: 'crtr/reopen-invalidated-worktree',
36
+ repo_root: process.cwd(),
37
+ base_ref: 'main',
38
+ base_sha: 'test',
39
+ created: new Date().toISOString(),
40
+ });
41
+ assert.equal(h.node(child)?.pi_session_file ?? null, null, 'closed worktree invalidates the old session coordinate');
42
+ const oldPid = h.node(child)?.pi_pid;
43
+ assert.ok(oldPid != null, 'the original broker is still live until its completed turn exits');
44
+ process.kill(oldPid, 'SIGKILL');
45
+ await h.awaitFleetExit(child);
46
+ const result = h.cli(root, ['node', 'message', 'send', '--to', child, '--reopen', followUp]);
47
+ assert.equal(result.code, 0, `atomic reopen message should succeed\n--stderr--\n${result.stderr}`);
48
+ const boot = await h.awaitBoot(child, { minCount: 2 });
49
+ assert.equal(boot.resuming, false, 'the cleared session forces a fresh broker boot');
50
+ assert.equal(boot.prompt?.includes(originalGoal), false, 'a completed node never replays its birth mandate');
51
+ const priorFinalIndex = boot.prompt?.indexOf('reopen-final-report') ?? -1;
52
+ const followUpIndex = boot.prompt?.indexOf(followUp) ?? -1;
53
+ assert.ok(priorFinalIndex >= 0 && boot.prompt?.includes(finalPath), 'the prior final report path is delivered as context');
54
+ assert.ok(followUpIndex >= 0, 'the new instruction is delivered in the fresh kickoff');
55
+ assert.ok(priorFinalIndex < followUpIndex, 'the prior final report context arrives before the new instruction');
56
+ assert.equal(readGoal(child)?.trim(), originalGoal, 'the birth goal remains on disk as historical data');
57
+ assert.equal(existsSync(finalPath), true, 'reopen does not delete the prior final report');
58
+ assert.equal(h.node(child)?.lifecycle, 'resident', 'atomic reopen restores resident lifecycle before the fresh broker starts');
59
+ assert.equal(h.node(child)?.final_report ?? null, null, 'atomic reopen clears only the live finalization latch');
60
+ });
@@ -1,4 +1,5 @@
1
1
  import type { ErrorClass, OperationId } from './events/types.js';
2
+ import { type ProviderFailureDetails } from './provider-failure.js';
2
3
  export type FaultLink = 'pi→provider' | 'viewer↔broker' | 'relay↔broker' | 'viewer↔crtrd' | 'broker↔crtrd' | 'daemon→node' | 'crtr→pi';
3
4
  export type FaultKind = 'rate-limit' | 'overloaded' | 'connection' | 'auth' | 'protocol' | 'context-overflow' | 'other' | 'wedged' | 'model-not-found';
4
5
  export type FaultRetryDisposition = 'auto' | 'manual' | 'fatal';
@@ -10,6 +11,10 @@ export interface FaultRetry {
10
11
  max?: number;
11
12
  nextAt?: string;
12
13
  }
14
+ export interface FaultProviderError extends ProviderFailureDetails {
15
+ provider: string;
16
+ model: string;
17
+ }
13
18
  export interface Fault {
14
19
  link: FaultLink;
15
20
  op: string;
@@ -19,6 +24,7 @@ export interface Fault {
19
24
  since: string;
20
25
  operation_id: OperationId;
21
26
  anchorEntryId?: string;
27
+ providerError?: FaultProviderError;
22
28
  }
23
29
  export interface FaultClassification {
24
30
  kind: FaultKind;
@@ -1,5 +1,6 @@
1
1
  import { hasManagedProviderCoolingDiagnostic } from './runtime/managed-provider-cooling.js';
2
2
  import { recognizeProviderError } from './provider-error-recognition.js';
3
+ import { extractProviderFailureDetails } from './provider-failure.js';
3
4
  import { isRecord } from '../shared/predicates.js';
4
5
  const errorClassByFaultKind = {
5
6
  'rate-limit': 'rate_limit',
@@ -78,13 +79,17 @@ export function classify(link, raw) {
78
79
  return { kind: 'rate-limit', disposition: 'auto' };
79
80
  }
80
81
  const value = isRecord(raw) ? raw : {};
82
+ const failure = extractProviderFailureDetails(raw);
83
+ const nested = failure?.causes ?? [];
81
84
  const signal = recognizeProviderError({
82
- code: value['code'],
83
- errno: value['errno'],
84
- status: value['status'],
85
+ code: value['code'] ?? failure?.code ?? nested.find((cause) => cause.code !== undefined)?.code,
86
+ errno: value['errno'] ?? failure?.errno ?? nested.find((cause) => cause.errno !== undefined)?.errno,
87
+ status: value['status'] ?? failure?.status,
85
88
  wsCloseCode: value['code'],
86
89
  wsCloseReason: value['reason'],
87
- message: rawText(raw),
90
+ message: [rawText(raw), failure?.body, ...nested.flatMap((cause) => [cause.message, cause.body])]
91
+ .filter((part) => typeof part === 'string' && part !== '')
92
+ .join(' '),
88
93
  });
89
94
  const kind = faultKindFromProviderSignal(signal);
90
95
  return { kind, disposition: classifyDisposition(link, kind) };
@@ -0,0 +1,30 @@
1
+ export declare const PROVIDER_FAILURE_DIAGNOSTIC = "crtr:provider-failure";
2
+ export interface ProviderFailureCause {
3
+ message?: string;
4
+ status?: number;
5
+ code?: string | number;
6
+ errno?: string;
7
+ body?: string;
8
+ }
9
+ export interface ProviderFailureDetails {
10
+ status?: number;
11
+ code?: string | number;
12
+ errno?: string;
13
+ body?: string;
14
+ causes?: ProviderFailureCause[];
15
+ }
16
+ export interface NormalizedProviderFailure extends ProviderFailureDetails {
17
+ message: string;
18
+ }
19
+ /** Converts an arbitrary provider failure into serializable facts before an SDK stream flattens it to `errorMessage`. */
20
+ export declare function normalizeProviderFailure(error: unknown, observed?: {
21
+ status?: number;
22
+ }): NormalizedProviderFailure;
23
+ /** The stable diagnostic carrier for normalized provider error facts on an assistant error message. */
24
+ export declare function providerFailureDiagnostic(details: ProviderFailureDetails): {
25
+ type: string;
26
+ timestamp: number;
27
+ details: Record<string, unknown>;
28
+ };
29
+ /** Reads provider facts from the assistant-message diagnostic without trusting arbitrary nested error objects. */
30
+ export declare function extractProviderFailureDetails(message: unknown): ProviderFailureDetails | undefined;
@@ -0,0 +1,86 @@
1
+ import { isRecord } from '../shared/predicates.js';
2
+ export const PROVIDER_FAILURE_DIAGNOSTIC = 'crtr:provider-failure';
3
+ const CAUSE_LIMIT = 8;
4
+ function text(value) {
5
+ return typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined;
6
+ }
7
+ function status(value) {
8
+ return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
9
+ }
10
+ function code(value) {
11
+ return typeof value === 'string' || typeof value === 'number' ? value : undefined;
12
+ }
13
+ function record(value) {
14
+ return isRecord(value) ? value : undefined;
15
+ }
16
+ function failureCause(value) {
17
+ const message = text(value['errorMessage']) ?? text(value['message']) ?? text(value['reason']) ?? text(value['statusText']);
18
+ const cause = {
19
+ ...(message === undefined ? {} : { message }),
20
+ ...(status(value['status']) === undefined ? {} : { status: status(value['status']) }),
21
+ ...(code(value['code']) === undefined ? {} : { code: code(value['code']) }),
22
+ ...(text(value['errno']) === undefined ? {} : { errno: text(value['errno']) }),
23
+ ...(text(value['body']) === undefined ? {} : { body: text(value['body']) }),
24
+ };
25
+ return Object.keys(cause).length === 0 ? undefined : cause;
26
+ }
27
+ /** Converts an arbitrary provider failure into serializable facts before an SDK stream flattens it to `errorMessage`. */
28
+ export function normalizeProviderFailure(error, observed = {}) {
29
+ const root = record(error);
30
+ const rootCause = root === undefined ? undefined : failureCause(root);
31
+ const causes = [];
32
+ const seen = new Set([error]);
33
+ let current = root === undefined ? undefined : record(root['cause']);
34
+ while (current !== undefined && !seen.has(current) && causes.length < CAUSE_LIMIT) {
35
+ seen.add(current);
36
+ const cause = failureCause(current);
37
+ if (cause !== undefined)
38
+ causes.push(cause);
39
+ current = record(current['cause']);
40
+ }
41
+ const message = rootCause?.message ?? (error instanceof Error ? error.message : undefined) ?? String(error ?? 'Provider request failed');
42
+ return {
43
+ message,
44
+ ...((observed.status ?? rootCause?.status) === undefined ? {} : { status: observed.status ?? rootCause?.status }),
45
+ ...(rootCause?.code === undefined ? {} : { code: rootCause.code }),
46
+ ...(rootCause?.errno === undefined ? {} : { errno: rootCause.errno }),
47
+ ...(rootCause?.body === undefined ? {} : { body: rootCause.body }),
48
+ ...(causes.length === 0 ? {} : { causes }),
49
+ };
50
+ }
51
+ /** The stable diagnostic carrier for normalized provider error facts on an assistant error message. */
52
+ export function providerFailureDiagnostic(details) {
53
+ return { type: PROVIDER_FAILURE_DIAGNOSTIC, timestamp: Date.now(), details: { ...details } };
54
+ }
55
+ function providerFailureDetails(value) {
56
+ const diagnostics = record(value)?.['diagnostics'];
57
+ if (!Array.isArray(diagnostics))
58
+ return undefined;
59
+ for (const diagnostic of diagnostics) {
60
+ const diagnosticRecord = record(diagnostic);
61
+ if (diagnosticRecord?.['type'] !== PROVIDER_FAILURE_DIAGNOSTIC)
62
+ continue;
63
+ const details = record(diagnosticRecord['details']);
64
+ if (details === undefined)
65
+ continue;
66
+ const causes = Array.isArray(details['causes'])
67
+ ? details['causes'].flatMap((item) => {
68
+ const cause = record(item);
69
+ const normalized = cause === undefined ? undefined : failureCause(cause);
70
+ return normalized === undefined ? [] : [normalized];
71
+ })
72
+ : undefined;
73
+ return {
74
+ ...(status(details['status']) === undefined ? {} : { status: status(details['status']) }),
75
+ ...(code(details['code']) === undefined ? {} : { code: code(details['code']) }),
76
+ ...(text(details['errno']) === undefined ? {} : { errno: text(details['errno']) }),
77
+ ...(text(details['body']) === undefined ? {} : { body: text(details['body']) }),
78
+ ...(causes === undefined || causes.length === 0 ? {} : { causes }),
79
+ };
80
+ }
81
+ return undefined;
82
+ }
83
+ /** Reads provider facts from the assistant-message diagnostic without trusting arbitrary nested error objects. */
84
+ export function extractProviderFailureDetails(message) {
85
+ return providerFailureDetails(message);
86
+ }