@north-light/crouter 0.3.303 → 0.3.305

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (102) hide show
  1. package/dist/api/__tests__/integration/client.test.js +37 -36
  2. package/dist/api/client.d.ts +32 -55
  3. package/dist/api/client.js +102 -98
  4. package/dist/api/dto/messages.d.ts +6 -5
  5. package/dist/api/errors.d.ts +3 -0
  6. package/dist/api/errors.js +10 -0
  7. package/dist/api/index.d.ts +1 -1
  8. package/dist/api/index.js +1 -1
  9. package/dist/builtin-memory/internal/plugins.md +1 -1
  10. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/integration/provider-rotation.test.ts +87 -0
  11. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +31 -26
  12. package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +33 -26
  13. package/dist/clients/attach/viewer.js +584 -584
  14. package/dist/commands/__tests__/seam/daemon-status.test.d.ts +1 -0
  15. package/dist/commands/__tests__/seam/daemon-status.test.js +29 -0
  16. package/dist/commands/api-client.d.ts +3 -3
  17. package/dist/commands/api-client.js +3 -3
  18. package/dist/commands/memory/read.js +2 -1
  19. package/dist/commands/node/bash.js +6 -6
  20. package/dist/commands/node/create.js +7 -1
  21. package/dist/commands/node/message.js +3 -3
  22. package/dist/commands/node/wait.js +1 -1
  23. package/dist/commands/node-worktree.js +5 -1
  24. package/dist/commands/sys/daemon.js +11 -13
  25. package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +0 -2
  26. package/dist/core/__tests__/daemon-boot.test.js +1 -1
  27. package/dist/core/__tests__/fixtures/fake-engine.js +14 -15
  28. package/dist/core/__tests__/helpers/harness.d.ts +2 -0
  29. package/dist/core/__tests__/helpers/harness.js +9 -0
  30. package/dist/core/__tests__/integration/tmux-surface.test.js +53 -0
  31. package/dist/core/__tests__/integration/worktree-land.test.js +50 -0
  32. package/dist/core/__tests__/integration/worktree-reap.test.js +184 -5
  33. package/dist/core/__tests__/kickoff.test.js +40 -0
  34. package/dist/core/__tests__/parse-argv-stdin-secret.test.js +14 -0
  35. package/dist/core/__tests__/seam/broker-attach-stream.test.js +13 -0
  36. package/dist/core/__tests__/seam/broker-provider-retry.test.js +25 -0
  37. package/dist/core/__tests__/seam/broker-startup-diagnostics.test.d.ts +1 -0
  38. package/dist/core/__tests__/seam/broker-startup-diagnostics.test.js +82 -0
  39. package/dist/core/__tests__/seam/deadline-wake-delivery.test.d.ts +1 -0
  40. package/dist/core/__tests__/seam/deadline-wake-delivery.test.js +138 -0
  41. package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.d.ts +1 -0
  42. package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.js +60 -0
  43. package/dist/core/bash-jobs.d.ts +20 -8
  44. package/dist/core/bash-jobs.js +40 -21
  45. package/dist/core/canvas/types.d.ts +4 -2
  46. package/dist/core/command.js +1 -1
  47. package/dist/core/fault-classifier.d.ts +7 -1
  48. package/dist/core/fault-classifier.js +9 -4
  49. package/dist/core/provider-failure.d.ts +30 -0
  50. package/dist/core/provider-failure.js +86 -0
  51. package/dist/core/runtime/broker/event-projection.d.ts +0 -3
  52. package/dist/core/runtime/broker/event-projection.js +2 -15
  53. package/dist/core/runtime/broker/fault-retry.js +16 -0
  54. package/dist/core/runtime/broker-persona-guidance.js +12 -0
  55. package/dist/core/runtime/broker.js +0 -5
  56. package/dist/core/runtime/fault.d.ts +2 -1
  57. package/dist/core/runtime/fault.js +39 -1
  58. package/dist/core/runtime/host.js +10 -1
  59. package/dist/core/runtime/kickoff.d.ts +3 -0
  60. package/dist/core/runtime/kickoff.js +24 -4
  61. package/dist/core/runtime/reopen.d.ts +2 -2
  62. package/dist/core/runtime/reopen.js +6 -4
  63. package/dist/core/runtime/revive.js +3 -3
  64. package/dist/core/runtime/spawn.js +5 -6
  65. package/dist/core/runtime/tmux-bindings.js +14 -3
  66. package/dist/core/termrender/version.d.ts +1 -1
  67. package/dist/core/termrender/version.js +1 -1
  68. package/dist/core/worktree-close.d.ts +4 -0
  69. package/dist/core/worktree-close.js +228 -0
  70. package/dist/core/worktree-containment.d.ts +14 -0
  71. package/dist/core/worktree-containment.js +48 -0
  72. package/dist/core/worktree-mutation-async.d.ts +13 -0
  73. package/dist/core/worktree-mutation-async.js +281 -0
  74. package/dist/core/worktree-sweep.js +24 -82
  75. package/dist/core/worktree.d.ts +1 -0
  76. package/dist/core/worktree.js +7 -4
  77. package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +10 -8
  78. package/dist/daemon/api/__tests__/reopen-delivery.test.js +31 -0
  79. package/dist/daemon/api/__tests__/seam/api-server.test.js +70 -1
  80. package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +36 -4
  81. package/dist/daemon/api/handlers/bash-jobs.js +1 -1
  82. package/dist/daemon/api/handlers/messages.js +33 -14
  83. package/dist/daemon/api/handlers/reports.js +3 -2
  84. package/dist/daemon/api/handlers/worktree.js +7 -6
  85. package/dist/daemon/fleet.d.ts +1 -1
  86. package/dist/daemon/fleet.js +30 -12
  87. package/dist/daemon/manage.d.ts +8 -5
  88. package/dist/daemon/manage.js +63 -40
  89. package/dist/daemon/reconcilers/broker-supervision.js +10 -1
  90. package/dist/daemon/reconcilers/managed-worktree-sweep.js +12 -0
  91. package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +0 -6
  92. package/dist/daemon/reconcilers/node-lifecycle/tick.js +1 -13
  93. package/dist/daemon/reconcilers/node-lifecycle/wants-execution.d.ts +5 -0
  94. package/dist/daemon/reconcilers/node-lifecycle/wants-execution.js +11 -0
  95. package/dist/pi-extensions/__tests__/canvas-context-intro.test.js +83 -0
  96. package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.d.ts +1 -0
  97. package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.js +133 -0
  98. package/dist/pi-extensions/canvas-bash-valve.d.ts +2 -0
  99. package/dist/pi-extensions/canvas-bash-valve.js +73 -43
  100. package/dist/pi-extensions/canvas-inbox-watcher.js +0 -2
  101. package/package.json +1 -1
  102. package/runtime.lock.json +5 -5
@@ -12,6 +12,9 @@ export interface BashJobPaths {
12
12
  * Inspector's cancel) — without it on disk, only the agent's own handoff
13
13
  * notice knows the group to signal. */
14
14
  jobPgid: string;
15
+ /** Launch-time identity of the detached supervisor, used to refuse a PID that
16
+ * has since been reused by an unrelated process. */
17
+ jobPgidIdentity: string;
15
18
  /** Human-readable label supplied with the bash call, when any. */
16
19
  jobPurpose: string;
17
20
  /** Epoch-ms cancel deadline for an AUTO-backgrounded job. Present only when
@@ -33,6 +36,9 @@ export declare function bashJobPaths(contextDir: string, jobId: string): BashJob
33
36
  /** The job's supervisor process group, or undefined for a job started before
34
37
  * pgid was persisted (or a partially-written dir). */
35
38
  export declare function readJobPgid(contextDir: string, jobId: string): number | undefined;
39
+ /** Launch-time identity of the supervisor. Missing for jobs created before
40
+ * process-identity persistence, which are deliberately not safe to signal. */
41
+ export declare function readJobPgidIdentity(contextDir: string, jobId: string): string | undefined;
36
42
  /** The optional human-readable job label. Jobs created before labels existed
37
43
  * have no file and deliberately project null. */
38
44
  export declare function readJobPurpose(contextDir: string, jobId: string): string | null;
@@ -73,6 +79,12 @@ export declare function clearJobDeadline(paths: BashJobPaths): void;
73
79
  * means the group exists but cannot be signaled by this process; it is still
74
80
  * live for the status surface. */
75
81
  export declare function isBashJobProcessGroupAlive(pgid: number): boolean;
82
+ /** Best-effort cancellation for a detached bash supervisor. It snapshots the
83
+ * observable descendant tree while the supervisor is still alive, signals its
84
+ * leaves before their parents, then uses the shared identity-guarded KILL
85
+ * escalation. A descendant that detached and reparented before this snapshot
86
+ * is outside this guarantee. */
87
+ export declare function cancelBashJobProcessTree(pgid: number, expectedIdentity: string): Promise<boolean>;
76
88
  export type StopBackgroundBashJobResult = {
77
89
  kind: 'stopped';
78
90
  signaled: boolean;
@@ -87,15 +99,15 @@ export type StopBackgroundBashJobResult = {
87
99
  kind: 'not-stoppable';
88
100
  logPath: string;
89
101
  };
90
- /** Stop one live background job using the same process-group and sentinel
91
- * semantics as `crtr node bash kill`. The caller owns the user-facing
92
- * notification; this primitive owns only the job control-plane mutation.
102
+ /** Stop one live background job using the same best-effort tree cancellation
103
+ * semantics as inline abort and timeout. The caller owns the user-facing
104
+ * notification; this primitive owns only the job control-plane mutation.
93
105
  *
94
- * Only a job in the live-background state the roster projects (job.bg
95
- * present, job.exit absent) may be signaled. Job directories persist after
96
- * the supervisor exits, so a retired, still-foreground, or already-stopped
97
- * job would otherwise have its recorded pgid signaled long after the OS
98
- * could have reused that id for an unrelated process group. */
106
+ * Only a job in the live-background state the roster projects (job.bg present,
107
+ * job.exit absent) may be signaled. Job directories persist after the
108
+ * supervisor exits, so a retired, still-foreground, or already-stopped job
109
+ * would otherwise have its recorded pgid signaled long after the OS could have
110
+ * reused that id for an unrelated process group. */
99
111
  export declare function stopBackgroundBashJob(contextDir: string, jobId: string): Promise<StopBackgroundBashJobResult>;
100
112
  /** Last `count` lines of a job log, cheaply: read at most the trailing 64 KiB
101
113
  * rather than the whole file, which for a long-running job can be enormous. */
@@ -5,6 +5,7 @@
5
5
  import { randomBytes } from 'node:crypto';
6
6
  import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
7
7
  import { join } from 'node:path';
8
+ import { captureTeardownSnapshot, isAnyPidAlive, killProcessTreePids } from './canvas/pid.js';
8
9
  const MAX_BASH_JOB_PURPOSE_BYTES = 280;
9
10
  /** A purpose is display text, never shell input. Keep the file-backed control
10
11
  * plane to one bounded, visible line so old or hand-written job directories
@@ -41,6 +42,7 @@ export function bashJobPaths(contextDir, jobId) {
41
42
  jobBg: join(dir, 'job.bg'),
42
43
  jobDone: join(dir, 'job.done'),
43
44
  jobPgid: join(dir, 'job.pgid'),
45
+ jobPgidIdentity: join(dir, 'job.pgid-identity'),
44
46
  jobPurpose: join(dir, 'purpose'),
45
47
  jobDeadline: join(dir, 'job.deadline'),
46
48
  };
@@ -56,6 +58,17 @@ export function readJobPgid(contextDir, jobId) {
56
58
  return undefined;
57
59
  }
58
60
  }
61
+ /** Launch-time identity of the supervisor. Missing for jobs created before
62
+ * process-identity persistence, which are deliberately not safe to signal. */
63
+ export function readJobPgidIdentity(contextDir, jobId) {
64
+ try {
65
+ const identity = readFileSync(bashJobPaths(contextDir, jobId).jobPgidIdentity, 'utf8').trim();
66
+ return identity === '' ? undefined : identity;
67
+ }
68
+ catch {
69
+ return undefined;
70
+ }
71
+ }
59
72
  /** The optional human-readable job label. Jobs created before labels existed
60
73
  * have no file and deliberately project null. */
61
74
  export function readJobPurpose(contextDir, jobId) {
@@ -143,15 +156,31 @@ export function isBashJobProcessGroupAlive(pgid) {
143
156
  return err.code === 'EPERM';
144
157
  }
145
158
  }
146
- /** Stop one live background job using the same process-group and sentinel
147
- * semantics as `crtr node bash kill`. The caller owns the user-facing
148
- * notification; this primitive owns only the job control-plane mutation.
159
+ /** Best-effort cancellation for a detached bash supervisor. It snapshots the
160
+ * observable descendant tree while the supervisor is still alive, signals its
161
+ * leaves before their parents, then uses the shared identity-guarded KILL
162
+ * escalation. A descendant that detached and reparented before this snapshot
163
+ * is outside this guarantee. */
164
+ export async function cancelBashJobProcessTree(pgid, expectedIdentity) {
165
+ const snapshot = captureTeardownSnapshot(pgid, expectedIdentity);
166
+ if (snapshot.reused || snapshot.tree.length === 0)
167
+ return false;
168
+ const tree = [...snapshot.tree].reverse();
169
+ const signaled = isAnyPidAlive(tree);
170
+ killProcessTreePids(tree, 'SIGTERM', snapshot.identities);
171
+ await new Promise((resolve) => setTimeout(resolve, 400));
172
+ killProcessTreePids(tree, 'SIGKILL', snapshot.identities);
173
+ return signaled;
174
+ }
175
+ /** Stop one live background job using the same best-effort tree cancellation
176
+ * semantics as inline abort and timeout. The caller owns the user-facing
177
+ * notification; this primitive owns only the job control-plane mutation.
149
178
  *
150
- * Only a job in the live-background state the roster projects (job.bg
151
- * present, job.exit absent) may be signaled. Job directories persist after
152
- * the supervisor exits, so a retired, still-foreground, or already-stopped
153
- * job would otherwise have its recorded pgid signaled long after the OS
154
- * could have reused that id for an unrelated process group. */
179
+ * Only a job in the live-background state the roster projects (job.bg present,
180
+ * job.exit absent) may be signaled. Job directories persist after the
181
+ * supervisor exits, so a retired, still-foreground, or already-stopped job
182
+ * would otherwise have its recorded pgid signaled long after the OS could have
183
+ * reused that id for an unrelated process group. */
155
184
  export async function stopBackgroundBashJob(contextDir, jobId) {
156
185
  const paths = bashJobPaths(contextDir, jobId);
157
186
  if (!existsSync(paths.dir))
@@ -159,7 +188,8 @@ export async function stopBackgroundBashJob(contextDir, jobId) {
159
188
  if (!existsSync(paths.jobBg) || existsSync(paths.jobExit))
160
189
  return { kind: 'retired', logPath: paths.jobLog };
161
190
  const pgid = readJobPgid(contextDir, jobId);
162
- if (pgid === undefined)
191
+ const pgidIdentity = readJobPgidIdentity(contextDir, jobId);
192
+ if (pgid === undefined || pgidIdentity === undefined)
163
193
  return { kind: 'not-stoppable', logPath: paths.jobLog };
164
194
  // Claim the stop by creating job.exit exclusively, before signaling. The
165
195
  // loser of two concurrent stops — and a stop that races the supervisor's own
@@ -179,18 +209,7 @@ export async function stopBackgroundBashJob(contextDir, jobId) {
179
209
  return { kind: 'retired', logPath: paths.jobLog };
180
210
  throw err;
181
211
  }
182
- let signaled = true;
183
- try {
184
- process.kill(-pgid, 'SIGTERM');
185
- }
186
- catch {
187
- signaled = false;
188
- }
189
- await new Promise((resolve) => setTimeout(resolve, 400));
190
- try {
191
- process.kill(-pgid, 'SIGKILL');
192
- }
193
- catch { /* already gone */ }
212
+ const signaled = await cancelBashJobProcessTree(pgid, pgidIdentity);
194
213
  return { kind: 'stopped', signaled, pgid, logPath: paths.jobLog };
195
214
  }
196
215
  /** Last `count` lines of a job log, cheaply: read at most the trailing 64 KiB
@@ -76,11 +76,13 @@ export interface LaunchSpec {
76
76
  env: Record<string, string>;
77
77
  }
78
78
  /** The repository fingerprint a refusal was decided against. An examination
79
- * that reads the same three values can skip the whole proof cascade: nothing
80
- * about the refusal can have changed. */
79
+ * that reads the same values can skip the whole proof cascade: nothing about
80
+ * the refusal can have changed. */
81
81
  export interface ManagedWorktreeObservation {
82
82
  /** Tip of `refs/heads/<base_ref>`, or null when it no longer resolves. */
83
83
  base: string | null;
84
+ /** Tip of the base branch's upstream (or `origin/<base_ref>` fallback). */
85
+ upstream: string | null;
84
86
  /** Tip of `refs/heads/<branch>`, or null when the branch is gone. */
85
87
  branch: string | null;
86
88
  /** `absent`, or a hash of the checkout's HEAD plus `git status --porcelain`. */
@@ -403,7 +403,7 @@ export async function parseArgv(params, tokens, options) {
403
403
  continue;
404
404
  }
405
405
  const rawVal = inlineValue !== undefined ? inlineValue : tokens[++i];
406
- if (rawVal === undefined || rawVal.startsWith('--')) {
406
+ if (rawVal === undefined || (inlineValue === undefined && rawVal.startsWith('--'))) {
407
407
  const focusedNext = flagDef.focusedHelp !== undefined && (options?.leafPath?.length ?? 0) > 0
408
408
  ? `Run \`crtr ${options.leafPath.join(' ')} --${flagName} -h\` and read the focused value contract before retrying.`
409
409
  : undefined;
@@ -1,5 +1,6 @@
1
1
  import type { ErrorClass, OperationId } from './events/types.js';
2
- export type FaultLink = 'pi→provider' | 'viewer↔broker' | 'relay↔broker' | 'viewer↔crtrd' | 'daemon→node' | 'crtr→pi';
2
+ import { type ProviderFailureDetails } from './provider-failure.js';
3
+ export type FaultLink = 'pi→provider' | 'viewer↔broker' | 'relay↔broker' | 'viewer↔crtrd' | 'broker↔crtrd' | 'daemon→node' | 'crtr→pi';
3
4
  export type FaultKind = 'rate-limit' | 'overloaded' | 'connection' | 'auth' | 'protocol' | 'context-overflow' | 'other' | 'wedged' | 'model-not-found';
4
5
  export type FaultRetryDisposition = 'auto' | 'manual' | 'fatal';
5
6
  export type FaultRetryOwner = 'sdk' | 'daemon' | 'client';
@@ -10,6 +11,10 @@ export interface FaultRetry {
10
11
  max?: number;
11
12
  nextAt?: string;
12
13
  }
14
+ export interface FaultProviderError extends ProviderFailureDetails {
15
+ provider: string;
16
+ model: string;
17
+ }
13
18
  export interface Fault {
14
19
  link: FaultLink;
15
20
  op: string;
@@ -19,6 +24,7 @@ export interface Fault {
19
24
  since: string;
20
25
  operation_id: OperationId;
21
26
  anchorEntryId?: string;
27
+ providerError?: FaultProviderError;
22
28
  }
23
29
  export interface FaultClassification {
24
30
  kind: FaultKind;
@@ -1,5 +1,6 @@
1
1
  import { hasManagedProviderCoolingDiagnostic } from './runtime/managed-provider-cooling.js';
2
2
  import { recognizeProviderError } from './provider-error-recognition.js';
3
+ import { extractProviderFailureDetails } from './provider-failure.js';
3
4
  import { isRecord } from '../shared/predicates.js';
4
5
  const errorClassByFaultKind = {
5
6
  'rate-limit': 'rate_limit',
@@ -78,13 +79,17 @@ export function classify(link, raw) {
78
79
  return { kind: 'rate-limit', disposition: 'auto' };
79
80
  }
80
81
  const value = isRecord(raw) ? raw : {};
82
+ const failure = extractProviderFailureDetails(raw);
83
+ const nested = failure?.causes ?? [];
81
84
  const signal = recognizeProviderError({
82
- code: value['code'],
83
- errno: value['errno'],
84
- status: value['status'],
85
+ code: value['code'] ?? failure?.code ?? nested.find((cause) => cause.code !== undefined)?.code,
86
+ errno: value['errno'] ?? failure?.errno ?? nested.find((cause) => cause.errno !== undefined)?.errno,
87
+ status: value['status'] ?? failure?.status,
85
88
  wsCloseCode: value['code'],
86
89
  wsCloseReason: value['reason'],
87
- message: rawText(raw),
90
+ message: [rawText(raw), failure?.body, ...nested.flatMap((cause) => [cause.message, cause.body])]
91
+ .filter((part) => typeof part === 'string' && part !== '')
92
+ .join(' '),
88
93
  });
89
94
  const kind = faultKindFromProviderSignal(signal);
90
95
  return { kind, disposition: classifyDisposition(link, kind) };
@@ -0,0 +1,30 @@
1
+ export declare const PROVIDER_FAILURE_DIAGNOSTIC = "crtr:provider-failure";
2
+ export interface ProviderFailureCause {
3
+ message?: string;
4
+ status?: number;
5
+ code?: string | number;
6
+ errno?: string;
7
+ body?: string;
8
+ }
9
+ export interface ProviderFailureDetails {
10
+ status?: number;
11
+ code?: string | number;
12
+ errno?: string;
13
+ body?: string;
14
+ causes?: ProviderFailureCause[];
15
+ }
16
+ export interface NormalizedProviderFailure extends ProviderFailureDetails {
17
+ message: string;
18
+ }
19
+ /** Converts an arbitrary provider failure into serializable facts before an SDK stream flattens it to `errorMessage`. */
20
+ export declare function normalizeProviderFailure(error: unknown, observed?: {
21
+ status?: number;
22
+ }): NormalizedProviderFailure;
23
+ /** The stable diagnostic carrier for normalized provider error facts on an assistant error message. */
24
+ export declare function providerFailureDiagnostic(details: ProviderFailureDetails): {
25
+ type: string;
26
+ timestamp: number;
27
+ details: Record<string, unknown>;
28
+ };
29
+ /** Reads provider facts from the assistant-message diagnostic without trusting arbitrary nested error objects. */
30
+ export declare function extractProviderFailureDetails(message: unknown): ProviderFailureDetails | undefined;
@@ -0,0 +1,86 @@
1
+ import { isRecord } from '../shared/predicates.js';
2
+ export const PROVIDER_FAILURE_DIAGNOSTIC = 'crtr:provider-failure';
3
+ const CAUSE_LIMIT = 8;
4
+ function text(value) {
5
+ return typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined;
6
+ }
7
+ function status(value) {
8
+ return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
9
+ }
10
+ function code(value) {
11
+ return typeof value === 'string' || typeof value === 'number' ? value : undefined;
12
+ }
13
+ function record(value) {
14
+ return isRecord(value) ? value : undefined;
15
+ }
16
+ function failureCause(value) {
17
+ const message = text(value['errorMessage']) ?? text(value['message']) ?? text(value['reason']) ?? text(value['statusText']);
18
+ const cause = {
19
+ ...(message === undefined ? {} : { message }),
20
+ ...(status(value['status']) === undefined ? {} : { status: status(value['status']) }),
21
+ ...(code(value['code']) === undefined ? {} : { code: code(value['code']) }),
22
+ ...(text(value['errno']) === undefined ? {} : { errno: text(value['errno']) }),
23
+ ...(text(value['body']) === undefined ? {} : { body: text(value['body']) }),
24
+ };
25
+ return Object.keys(cause).length === 0 ? undefined : cause;
26
+ }
27
+ /** Converts an arbitrary provider failure into serializable facts before an SDK stream flattens it to `errorMessage`. */
28
+ export function normalizeProviderFailure(error, observed = {}) {
29
+ const root = record(error);
30
+ const rootCause = root === undefined ? undefined : failureCause(root);
31
+ const causes = [];
32
+ const seen = new Set([error]);
33
+ let current = root === undefined ? undefined : record(root['cause']);
34
+ while (current !== undefined && !seen.has(current) && causes.length < CAUSE_LIMIT) {
35
+ seen.add(current);
36
+ const cause = failureCause(current);
37
+ if (cause !== undefined)
38
+ causes.push(cause);
39
+ current = record(current['cause']);
40
+ }
41
+ const message = rootCause?.message ?? (error instanceof Error ? error.message : undefined) ?? String(error ?? 'Provider request failed');
42
+ return {
43
+ message,
44
+ ...((observed.status ?? rootCause?.status) === undefined ? {} : { status: observed.status ?? rootCause?.status }),
45
+ ...(rootCause?.code === undefined ? {} : { code: rootCause.code }),
46
+ ...(rootCause?.errno === undefined ? {} : { errno: rootCause.errno }),
47
+ ...(rootCause?.body === undefined ? {} : { body: rootCause.body }),
48
+ ...(causes.length === 0 ? {} : { causes }),
49
+ };
50
+ }
51
+ /** The stable diagnostic carrier for normalized provider error facts on an assistant error message. */
52
+ export function providerFailureDiagnostic(details) {
53
+ return { type: PROVIDER_FAILURE_DIAGNOSTIC, timestamp: Date.now(), details: { ...details } };
54
+ }
55
+ function providerFailureDetails(value) {
56
+ const diagnostics = record(value)?.['diagnostics'];
57
+ if (!Array.isArray(diagnostics))
58
+ return undefined;
59
+ for (const diagnostic of diagnostics) {
60
+ const diagnosticRecord = record(diagnostic);
61
+ if (diagnosticRecord?.['type'] !== PROVIDER_FAILURE_DIAGNOSTIC)
62
+ continue;
63
+ const details = record(diagnosticRecord['details']);
64
+ if (details === undefined)
65
+ continue;
66
+ const causes = Array.isArray(details['causes'])
67
+ ? details['causes'].flatMap((item) => {
68
+ const cause = record(item);
69
+ const normalized = cause === undefined ? undefined : failureCause(cause);
70
+ return normalized === undefined ? [] : [normalized];
71
+ })
72
+ : undefined;
73
+ return {
74
+ ...(status(details['status']) === undefined ? {} : { status: status(details['status']) }),
75
+ ...(code(details['code']) === undefined ? {} : { code: code(details['code']) }),
76
+ ...(text(details['errno']) === undefined ? {} : { errno: text(details['errno']) }),
77
+ ...(text(details['body']) === undefined ? {} : { body: text(details['body']) }),
78
+ ...(causes === undefined || causes.length === 0 ? {} : { causes }),
79
+ };
80
+ }
81
+ return undefined;
82
+ }
83
+ /** Reads provider facts from the assistant-message diagnostic without trusting arbitrary nested error objects. */
84
+ export function extractProviderFailureDetails(message) {
85
+ return providerFailureDetails(message);
86
+ }
@@ -11,8 +11,6 @@ type EventProjectionDeps = {
11
11
  notifyTurnAccepted: () => void;
12
12
  emitStartupMilestone: (event: string, model: string | null) => void;
13
13
  formatModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
14
- refreshIntent: () => Promise<boolean>;
15
- onRefreshIntentError: (error: unknown) => void;
16
14
  };
17
15
  /** Projects one live engine event stream to broker viewers and owns its relay timers. */
18
16
  export declare class EventProjection {
@@ -40,7 +38,6 @@ export declare class EventProjection {
40
38
  private flushPendingUpdate;
41
39
  private invalidatePendingBroadcasts;
42
40
  private broadcast;
43
- private broadcastMessageEnd;
44
41
  private setPendingBroadcast;
45
42
  }
46
43
  export {};
@@ -128,8 +128,8 @@ export class EventProjection {
128
128
  return;
129
129
  }
130
130
  this.flushPendingUpdate();
131
- if (type === 'message_end') {
132
- this.broadcastMessageEnd(event);
131
+ if (type === 'message_end' && generation.stagedRefreshAbort) {
132
+ this.broadcast({ ...event, message: withoutYieldAbort(event.message) });
133
133
  return;
134
134
  }
135
135
  this.broadcast(event);
@@ -183,19 +183,6 @@ export class EventProjection {
183
183
  this.deps.registry.broadcast(frame);
184
184
  }));
185
185
  }
186
- broadcastMessageEnd(event) {
187
- const preceding = this.pendingBroadcast ?? Promise.resolve();
188
- const generation = this.broadcastGeneration;
189
- this.setPendingBroadcast(preceding.then(async () => {
190
- const refresh = await this.deps.refreshIntent();
191
- if (this.broadcastGeneration !== generation)
192
- return;
193
- this.deps.registry.broadcast(refresh ? { ...event, message: withoutYieldAbort(event.message) } : event);
194
- }).catch((error) => {
195
- if (this.broadcastGeneration === generation)
196
- this.deps.onRefreshIntentError(error);
197
- }));
198
- }
199
186
  setPendingBroadcast(work) {
200
187
  this.pendingBroadcast = work;
201
188
  void work.then(() => {
@@ -4,6 +4,7 @@ import { isContextOverflow } from '@earendil-works/pi-ai';
4
4
  import { classify } from '../../fault-classifier.js';
5
5
  import { admitProviderRetryEpisode, clearFault, clearProviderRetryEpisode, invalidateProviderRetryEpisode, isModelNotFoundError, isProviderUnconfiguredError, readFault, readProviderRetryEpisode, recordFault, recordPendingProviderRetryFault, reopenProviderRetryEpisode, } from '../fault.js';
6
6
  import { extractCoolingDeadline } from '../managed-provider-cooling.js';
7
+ import { extractProviderFailureDetails } from '../../provider-failure.js';
7
8
  import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from '../../model-routes.js';
8
9
  import { operationIdContext } from '../../events/operation-id.js';
9
10
  import { emitEvent } from '../../events/emit.js';
@@ -71,6 +72,14 @@ export class FaultRetry {
71
72
  // admitted retry therefore carries from its durable episode, while an
72
73
  // initial provider failure still reads from the ordinary marker.
73
74
  const prior = readFault(this.deps.nodeId);
75
+ // A persona gate runs before Pi contacts a provider. It records a typed
76
+ // local API transport failure, then Pi represents that thrown error as a
77
+ // generic provider-looking agent_end. Keep its actual boundary instead of
78
+ // turning it into a fatal pi→provider `other` fault.
79
+ if (prior?.link === 'broker↔crtrd' && prior.kind === 'connection') {
80
+ clearProviderRetryEpisode(this.deps.nodeId);
81
+ return;
82
+ }
74
83
  const durableEpisode = readProviderRetryEpisode(this.deps.nodeId);
75
84
  const episodeFault = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
76
85
  ? prior
@@ -123,6 +132,12 @@ export class FaultRetry {
123
132
  const uncompactedOverflow = isContextOverflow(last, settledSession.model?.contextWindow);
124
133
  const classified = classify('pi→provider', last);
125
134
  const coolingDeadline = extractCoolingDeadline(last);
135
+ const failure = extractProviderFailureDetails(last);
136
+ const provider = typeof last.provider === 'string' && last.provider !== '' ? last.provider : settledSession.model?.provider;
137
+ const model = typeof last.model === 'string' && last.model !== '' ? last.model : settledSession.model?.id;
138
+ const providerError = failure === undefined || provider === undefined || model === undefined
139
+ ? undefined
140
+ : { provider, model, ...failure };
126
141
  if (uncompactedOverflow) {
127
142
  clearFault(this.deps.nodeId, { link: 'pi→provider' });
128
143
  clearProviderRetryEpisode(this.deps.nodeId);
@@ -143,6 +158,7 @@ export class FaultRetry {
143
158
  anchorEntryId: !uncompactedOverflow && classified.disposition === 'auto' && episode?.anchorEntryId !== undefined
144
159
  ? episode.anchorEntryId
145
160
  : settledSession.sessionManager.getLeafId?.() ?? undefined,
161
+ ...(providerError === undefined ? {} : { providerError }),
146
162
  };
147
163
  if (faultInput.retry.disposition === 'auto')
148
164
  this.recordPendingProviderFault(settledSession, faultInput);
@@ -1,5 +1,7 @@
1
1
  import { contextDir } from '../canvas/paths.js';
2
+ import { isDaemonTransportApiError } from '../../api/errors.js';
2
3
  import { emitEvent } from '../events/emit.js';
4
+ import { clearFault, recordFault } from './fault.js';
3
5
  import { brokerExtensionState, commitBrokerPersonaAck } from './broker/daemon-ops.js';
4
6
  import { readRoadmap, roadmapPath } from './roadmap.js';
5
7
  import { buildSubPersonaMenu, renderConfigChangeDelta } from '../substrate/render.js';
@@ -137,9 +139,19 @@ export function installPersonaTransitionGate(nodeId, session) {
137
139
  for (const context of new Set([active, messages, transformed]))
138
140
  context.push(message);
139
141
  });
142
+ clearFault(nodeId, { link: 'broker↔crtrd' });
140
143
  return transformed;
141
144
  }
142
145
  catch (error) {
146
+ if (isDaemonTransportApiError(error)) {
147
+ recordFault(nodeId, {
148
+ link: 'broker↔crtrd',
149
+ op: 'persona transition',
150
+ kind: 'connection',
151
+ retry: { disposition: 'manual' },
152
+ message: error.message,
153
+ });
154
+ }
143
155
  emitEvent({
144
156
  event: 'broker.daemon.operation.failed',
145
157
  level: 'error',
@@ -375,11 +375,6 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
375
375
  notifyTurnAccepted: () => notifyTurnAccepted(),
376
376
  emitStartupMilestone,
377
377
  formatModelSpec,
378
- refreshIntent: async () => (await brokerExtensionState(nodeId)).node.intent === 'refresh',
379
- onRefreshIntentError: (error) => {
380
- emitEvent({ level: 'error', event: 'broker.daemon_state.read_failed', error });
381
- disposeAndExit('daemon-state-read-failed', 1);
382
- },
383
378
  });
384
379
  const buildSnapshot = (client) => {
385
380
  const liveSession = rebind.session();
@@ -1,4 +1,4 @@
1
- import { type Fault, type FaultKind, type FaultLink, type FaultRetry } from '../fault-classifier.js';
1
+ import { type Fault, type FaultKind, type FaultLink, type FaultProviderError, type FaultRetry } from '../fault-classifier.js';
2
2
  export type { Fault, FaultKind, FaultLink, FaultRetry, FaultRetryDisposition, FaultRetryOwner } from '../fault-classifier.js';
3
3
  export declare const FAULT_QUIET_MS: number;
4
4
  export interface FaultRecordInput {
@@ -10,6 +10,7 @@ export interface FaultRecordInput {
10
10
  operation_id?: string;
11
11
  since?: string;
12
12
  anchorEntryId?: string;
13
+ providerError?: FaultProviderError;
13
14
  }
14
15
  /** A provider-retry episode outlives its broker-local timer. Its transcript
15
16
  * path and anchor are explicit coordinates, so replacement invalidation never
@@ -1,4 +1,5 @@
1
1
  import { envNodeId } from '../../shared/env.js';
2
+ import { isRecord } from '../../shared/predicates.js';
2
3
  import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
3
4
  import { join } from 'node:path';
4
5
  import { jobDir } from '../canvas/paths.js';
@@ -31,8 +32,39 @@ function faultPriority(fault) {
31
32
  return 2;
32
33
  return 1;
33
34
  }
34
- const faultLinks = ['pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'daemon→node', 'crtr→pi'];
35
+ const faultLinks = ['pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'broker↔crtrd', 'daemon→node', 'crtr→pi'];
35
36
  const faultKinds = ['rate-limit', 'overloaded', 'connection', 'auth', 'protocol', 'context-overflow', 'other', 'wedged', 'model-not-found'];
37
+ function parseProviderError(value) {
38
+ if (!isRecord(value) || typeof value.provider !== 'string' || value.provider === '' || typeof value.model !== 'string' || value.model === '')
39
+ return undefined;
40
+ const status = typeof value.status === 'number' && Number.isFinite(value.status) ? value.status : undefined;
41
+ const code = typeof value.code === 'string' || typeof value.code === 'number' ? value.code : undefined;
42
+ const errno = typeof value.errno === 'string' ? value.errno : undefined;
43
+ const body = typeof value.body === 'string' ? value.body : undefined;
44
+ const causes = Array.isArray(value.causes)
45
+ ? value.causes.flatMap((item) => {
46
+ if (!isRecord(item))
47
+ return [];
48
+ const cause = {
49
+ ...(typeof item.message === 'string' ? { message: item.message } : {}),
50
+ ...(typeof item.status === 'number' && Number.isFinite(item.status) ? { status: item.status } : {}),
51
+ ...(typeof item.code === 'string' || typeof item.code === 'number' ? { code: item.code } : {}),
52
+ ...(typeof item.errno === 'string' ? { errno: item.errno } : {}),
53
+ ...(typeof item.body === 'string' ? { body: item.body } : {}),
54
+ };
55
+ return Object.keys(cause).length === 0 ? [] : [cause];
56
+ })
57
+ : undefined;
58
+ return {
59
+ provider: value.provider,
60
+ model: value.model,
61
+ ...(status === undefined ? {} : { status }),
62
+ ...(code === undefined ? {} : { code }),
63
+ ...(errno === undefined ? {} : { errno }),
64
+ ...(body === undefined ? {} : { body }),
65
+ ...(causes === undefined || causes.length === 0 ? {} : { causes }),
66
+ };
67
+ }
36
68
  function parseFault(raw) {
37
69
  try {
38
70
  const parsed = JSON.parse(raw);
@@ -60,6 +92,9 @@ function parseFault(raw) {
60
92
  return null;
61
93
  if (parsed.anchorEntryId !== undefined && (typeof parsed.anchorEntryId !== 'string' || parsed.anchorEntryId === ''))
62
94
  return null;
95
+ const providerError = parsed.providerError === undefined ? undefined : parseProviderError(parsed.providerError);
96
+ if (parsed.providerError !== undefined && providerError === undefined)
97
+ return null;
63
98
  return {
64
99
  link: parsed.link,
65
100
  op: parsed.op,
@@ -75,6 +110,7 @@ function parseFault(raw) {
75
110
  since: parsed.since,
76
111
  operation_id: operationId,
77
112
  ...(parsed.anchorEntryId === undefined ? {} : { anchorEntryId: parsed.anchorEntryId }),
113
+ ...(providerError === undefined ? {} : { providerError }),
78
114
  };
79
115
  }
80
116
  catch {
@@ -166,6 +202,7 @@ function storedFault(input) {
166
202
  since: input.since ?? new Date().toISOString(),
167
203
  operation_id: resolveOperationId(input.operation_id),
168
204
  ...(typeof input.anchorEntryId === 'string' && input.anchorEntryId !== '' ? { anchorEntryId: input.anchorEntryId } : {}),
205
+ ...(input.providerError === undefined ? {} : { providerError: input.providerError }),
169
206
  };
170
207
  }
171
208
  function emitRecordedFault(nodeId, fault) {
@@ -182,6 +219,7 @@ function emitRecordedFault(nodeId, fault) {
182
219
  retry: fault.retry,
183
220
  message: fault.message,
184
221
  ...(fault.anchorEntryId === undefined ? {} : { anchor_entry_id: fault.anchorEntryId }),
222
+ ...(fault.providerError === undefined ? {} : { provider_error: fault.providerError }),
185
223
  },
186
224
  });
187
225
  }
@@ -319,7 +319,16 @@ export const headlessBrokerHost = {
319
319
  });
320
320
  finish({ code, signal });
321
321
  });
322
- spawned.once('error', () => finish({ code: null, signal: null }));
322
+ spawned.once('error', (error) => {
323
+ emitEvent({
324
+ level: 'error',
325
+ event: 'broker.launch.failed',
326
+ node_id: nodeId,
327
+ error,
328
+ fields: { uptime_ms: Date.now() - launchedAt },
329
+ });
330
+ finish({ code: null, signal: null });
331
+ });
323
332
  });
324
333
  spawned.unref();
325
334
  const handle = { pid: spawned.pid ?? null, exited };
@@ -26,6 +26,9 @@ export declare function readYieldMessage(nodeId: string): string | null;
26
26
  export declare function clearYieldMessage(nodeId: string): void;
27
27
  /** List the node's context/ dir (filenames, sorted). Empty when absent. */
28
28
  export declare function listContextDir(nodeId: string): string[];
29
+ /** A retained report proves this node has already done work. It must restart
30
+ * from current context, never by replaying its birth mandate. */
31
+ export declare function hasRetainedReports(nodeId: string): boolean;
29
32
  export interface ReviveBearings {
30
33
  /** The one-shot yield note left by the prior self, pending boot confirmation. */
31
34
  yieldMsg: string | null;