@north-light/crouter 0.3.303 → 0.3.305
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/__tests__/integration/client.test.js +37 -36
- package/dist/api/client.d.ts +32 -55
- package/dist/api/client.js +102 -98
- package/dist/api/dto/messages.d.ts +6 -5
- package/dist/api/errors.d.ts +3 -0
- package/dist/api/errors.js +10 -0
- package/dist/api/index.d.ts +1 -1
- package/dist/api/index.js +1 -1
- package/dist/builtin-memory/internal/plugins.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/__tests__/integration/provider-rotation.test.ts +87 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +31 -26
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +33 -26
- package/dist/clients/attach/viewer.js +584 -584
- package/dist/commands/__tests__/seam/daemon-status.test.d.ts +1 -0
- package/dist/commands/__tests__/seam/daemon-status.test.js +29 -0
- package/dist/commands/api-client.d.ts +3 -3
- package/dist/commands/api-client.js +3 -3
- package/dist/commands/memory/read.js +2 -1
- package/dist/commands/node/bash.js +6 -6
- package/dist/commands/node/create.js +7 -1
- package/dist/commands/node/message.js +3 -3
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/node-worktree.js +5 -1
- package/dist/commands/sys/daemon.js +11 -13
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +0 -2
- package/dist/core/__tests__/daemon-boot.test.js +1 -1
- package/dist/core/__tests__/fixtures/fake-engine.js +14 -15
- package/dist/core/__tests__/helpers/harness.d.ts +2 -0
- package/dist/core/__tests__/helpers/harness.js +9 -0
- package/dist/core/__tests__/integration/tmux-surface.test.js +53 -0
- package/dist/core/__tests__/integration/worktree-land.test.js +50 -0
- package/dist/core/__tests__/integration/worktree-reap.test.js +184 -5
- package/dist/core/__tests__/kickoff.test.js +40 -0
- package/dist/core/__tests__/parse-argv-stdin-secret.test.js +14 -0
- package/dist/core/__tests__/seam/broker-attach-stream.test.js +13 -0
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +25 -0
- package/dist/core/__tests__/seam/broker-startup-diagnostics.test.d.ts +1 -0
- package/dist/core/__tests__/seam/broker-startup-diagnostics.test.js +82 -0
- package/dist/core/__tests__/seam/deadline-wake-delivery.test.d.ts +1 -0
- package/dist/core/__tests__/seam/deadline-wake-delivery.test.js +138 -0
- package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.d.ts +1 -0
- package/dist/core/__tests__/seam/reopen-invalidated-worktree.test.js +60 -0
- package/dist/core/bash-jobs.d.ts +20 -8
- package/dist/core/bash-jobs.js +40 -21
- package/dist/core/canvas/types.d.ts +4 -2
- package/dist/core/command.js +1 -1
- package/dist/core/fault-classifier.d.ts +7 -1
- package/dist/core/fault-classifier.js +9 -4
- package/dist/core/provider-failure.d.ts +30 -0
- package/dist/core/provider-failure.js +86 -0
- package/dist/core/runtime/broker/event-projection.d.ts +0 -3
- package/dist/core/runtime/broker/event-projection.js +2 -15
- package/dist/core/runtime/broker/fault-retry.js +16 -0
- package/dist/core/runtime/broker-persona-guidance.js +12 -0
- package/dist/core/runtime/broker.js +0 -5
- package/dist/core/runtime/fault.d.ts +2 -1
- package/dist/core/runtime/fault.js +39 -1
- package/dist/core/runtime/host.js +10 -1
- package/dist/core/runtime/kickoff.d.ts +3 -0
- package/dist/core/runtime/kickoff.js +24 -4
- package/dist/core/runtime/reopen.d.ts +2 -2
- package/dist/core/runtime/reopen.js +6 -4
- package/dist/core/runtime/revive.js +3 -3
- package/dist/core/runtime/spawn.js +5 -6
- package/dist/core/runtime/tmux-bindings.js +14 -3
- package/dist/core/termrender/version.d.ts +1 -1
- package/dist/core/termrender/version.js +1 -1
- package/dist/core/worktree-close.d.ts +4 -0
- package/dist/core/worktree-close.js +228 -0
- package/dist/core/worktree-containment.d.ts +14 -0
- package/dist/core/worktree-containment.js +48 -0
- package/dist/core/worktree-mutation-async.d.ts +13 -0
- package/dist/core/worktree-mutation-async.js +281 -0
- package/dist/core/worktree-sweep.js +24 -82
- package/dist/core/worktree.d.ts +1 -0
- package/dist/core/worktree.js +7 -4
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +10 -8
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +31 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +70 -1
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +36 -4
- package/dist/daemon/api/handlers/bash-jobs.js +1 -1
- package/dist/daemon/api/handlers/messages.js +33 -14
- package/dist/daemon/api/handlers/reports.js +3 -2
- package/dist/daemon/api/handlers/worktree.js +7 -6
- package/dist/daemon/fleet.d.ts +1 -1
- package/dist/daemon/fleet.js +30 -12
- package/dist/daemon/manage.d.ts +8 -5
- package/dist/daemon/manage.js +63 -40
- package/dist/daemon/reconcilers/broker-supervision.js +10 -1
- package/dist/daemon/reconcilers/managed-worktree-sweep.js +12 -0
- package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +0 -6
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +1 -13
- package/dist/daemon/reconcilers/node-lifecycle/wants-execution.d.ts +5 -0
- package/dist/daemon/reconcilers/node-lifecycle/wants-execution.js +11 -0
- package/dist/pi-extensions/__tests__/canvas-context-intro.test.js +83 -0
- package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.d.ts +1 -0
- package/dist/pi-extensions/__tests__/integration/canvas-bash-valve.test.js +133 -0
- package/dist/pi-extensions/canvas-bash-valve.d.ts +2 -0
- package/dist/pi-extensions/canvas-bash-valve.js +73 -43
- package/dist/pi-extensions/canvas-inbox-watcher.js +0 -2
- package/package.json +1 -1
- package/runtime.lock.json +5 -5
package/dist/core/bash-jobs.d.ts
CHANGED
|
@@ -12,6 +12,9 @@ export interface BashJobPaths {
|
|
|
12
12
|
* Inspector's cancel) — without it on disk, only the agent's own handoff
|
|
13
13
|
* notice knows the group to signal. */
|
|
14
14
|
jobPgid: string;
|
|
15
|
+
/** Launch-time identity of the detached supervisor, used to refuse a PID that
|
|
16
|
+
* has since been reused by an unrelated process. */
|
|
17
|
+
jobPgidIdentity: string;
|
|
15
18
|
/** Human-readable label supplied with the bash call, when any. */
|
|
16
19
|
jobPurpose: string;
|
|
17
20
|
/** Epoch-ms cancel deadline for an AUTO-backgrounded job. Present only when
|
|
@@ -33,6 +36,9 @@ export declare function bashJobPaths(contextDir: string, jobId: string): BashJob
|
|
|
33
36
|
/** The job's supervisor process group, or undefined for a job started before
|
|
34
37
|
* pgid was persisted (or a partially-written dir). */
|
|
35
38
|
export declare function readJobPgid(contextDir: string, jobId: string): number | undefined;
|
|
39
|
+
/** Launch-time identity of the supervisor. Missing for jobs created before
|
|
40
|
+
* process-identity persistence, which are deliberately not safe to signal. */
|
|
41
|
+
export declare function readJobPgidIdentity(contextDir: string, jobId: string): string | undefined;
|
|
36
42
|
/** The optional human-readable job label. Jobs created before labels existed
|
|
37
43
|
* have no file and deliberately project null. */
|
|
38
44
|
export declare function readJobPurpose(contextDir: string, jobId: string): string | null;
|
|
@@ -73,6 +79,12 @@ export declare function clearJobDeadline(paths: BashJobPaths): void;
|
|
|
73
79
|
* means the group exists but cannot be signaled by this process; it is still
|
|
74
80
|
* live for the status surface. */
|
|
75
81
|
export declare function isBashJobProcessGroupAlive(pgid: number): boolean;
|
|
82
|
+
/** Best-effort cancellation for a detached bash supervisor. It snapshots the
|
|
83
|
+
* observable descendant tree while the supervisor is still alive, signals its
|
|
84
|
+
* leaves before their parents, then uses the shared identity-guarded KILL
|
|
85
|
+
* escalation. A descendant that detached and reparented before this snapshot
|
|
86
|
+
* is outside this guarantee. */
|
|
87
|
+
export declare function cancelBashJobProcessTree(pgid: number, expectedIdentity: string): Promise<boolean>;
|
|
76
88
|
export type StopBackgroundBashJobResult = {
|
|
77
89
|
kind: 'stopped';
|
|
78
90
|
signaled: boolean;
|
|
@@ -87,15 +99,15 @@ export type StopBackgroundBashJobResult = {
|
|
|
87
99
|
kind: 'not-stoppable';
|
|
88
100
|
logPath: string;
|
|
89
101
|
};
|
|
90
|
-
/** Stop one live background job using the same
|
|
91
|
-
*
|
|
92
|
-
*
|
|
102
|
+
/** Stop one live background job using the same best-effort tree cancellation
|
|
103
|
+
* semantics as inline abort and timeout. The caller owns the user-facing
|
|
104
|
+
* notification; this primitive owns only the job control-plane mutation.
|
|
93
105
|
*
|
|
94
|
-
*
|
|
95
|
-
*
|
|
96
|
-
*
|
|
97
|
-
*
|
|
98
|
-
*
|
|
106
|
+
* Only a job in the live-background state the roster projects (job.bg present,
|
|
107
|
+
* job.exit absent) may be signaled. Job directories persist after the
|
|
108
|
+
* supervisor exits, so a retired, still-foreground, or already-stopped job
|
|
109
|
+
* would otherwise have its recorded pgid signaled long after the OS could have
|
|
110
|
+
* reused that id for an unrelated process group. */
|
|
99
111
|
export declare function stopBackgroundBashJob(contextDir: string, jobId: string): Promise<StopBackgroundBashJobResult>;
|
|
100
112
|
/** Last `count` lines of a job log, cheaply: read at most the trailing 64 KiB
|
|
101
113
|
* rather than the whole file, which for a long-running job can be enormous. */
|
package/dist/core/bash-jobs.js
CHANGED
|
@@ -5,6 +5,7 @@
|
|
|
5
5
|
import { randomBytes } from 'node:crypto';
|
|
6
6
|
import { closeSync, existsSync, mkdirSync, openSync, readdirSync, readFileSync, readSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
7
7
|
import { join } from 'node:path';
|
|
8
|
+
import { captureTeardownSnapshot, isAnyPidAlive, killProcessTreePids } from './canvas/pid.js';
|
|
8
9
|
const MAX_BASH_JOB_PURPOSE_BYTES = 280;
|
|
9
10
|
/** A purpose is display text, never shell input. Keep the file-backed control
|
|
10
11
|
* plane to one bounded, visible line so old or hand-written job directories
|
|
@@ -41,6 +42,7 @@ export function bashJobPaths(contextDir, jobId) {
|
|
|
41
42
|
jobBg: join(dir, 'job.bg'),
|
|
42
43
|
jobDone: join(dir, 'job.done'),
|
|
43
44
|
jobPgid: join(dir, 'job.pgid'),
|
|
45
|
+
jobPgidIdentity: join(dir, 'job.pgid-identity'),
|
|
44
46
|
jobPurpose: join(dir, 'purpose'),
|
|
45
47
|
jobDeadline: join(dir, 'job.deadline'),
|
|
46
48
|
};
|
|
@@ -56,6 +58,17 @@ export function readJobPgid(contextDir, jobId) {
|
|
|
56
58
|
return undefined;
|
|
57
59
|
}
|
|
58
60
|
}
|
|
61
|
+
/** Launch-time identity of the supervisor. Missing for jobs created before
|
|
62
|
+
* process-identity persistence, which are deliberately not safe to signal. */
|
|
63
|
+
export function readJobPgidIdentity(contextDir, jobId) {
|
|
64
|
+
try {
|
|
65
|
+
const identity = readFileSync(bashJobPaths(contextDir, jobId).jobPgidIdentity, 'utf8').trim();
|
|
66
|
+
return identity === '' ? undefined : identity;
|
|
67
|
+
}
|
|
68
|
+
catch {
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
}
|
|
59
72
|
/** The optional human-readable job label. Jobs created before labels existed
|
|
60
73
|
* have no file and deliberately project null. */
|
|
61
74
|
export function readJobPurpose(contextDir, jobId) {
|
|
@@ -143,15 +156,31 @@ export function isBashJobProcessGroupAlive(pgid) {
|
|
|
143
156
|
return err.code === 'EPERM';
|
|
144
157
|
}
|
|
145
158
|
}
|
|
146
|
-
/**
|
|
147
|
-
*
|
|
148
|
-
*
|
|
159
|
+
/** Best-effort cancellation for a detached bash supervisor. It snapshots the
|
|
160
|
+
* observable descendant tree while the supervisor is still alive, signals its
|
|
161
|
+
* leaves before their parents, then uses the shared identity-guarded KILL
|
|
162
|
+
* escalation. A descendant that detached and reparented before this snapshot
|
|
163
|
+
* is outside this guarantee. */
|
|
164
|
+
export async function cancelBashJobProcessTree(pgid, expectedIdentity) {
|
|
165
|
+
const snapshot = captureTeardownSnapshot(pgid, expectedIdentity);
|
|
166
|
+
if (snapshot.reused || snapshot.tree.length === 0)
|
|
167
|
+
return false;
|
|
168
|
+
const tree = [...snapshot.tree].reverse();
|
|
169
|
+
const signaled = isAnyPidAlive(tree);
|
|
170
|
+
killProcessTreePids(tree, 'SIGTERM', snapshot.identities);
|
|
171
|
+
await new Promise((resolve) => setTimeout(resolve, 400));
|
|
172
|
+
killProcessTreePids(tree, 'SIGKILL', snapshot.identities);
|
|
173
|
+
return signaled;
|
|
174
|
+
}
|
|
175
|
+
/** Stop one live background job using the same best-effort tree cancellation
|
|
176
|
+
* semantics as inline abort and timeout. The caller owns the user-facing
|
|
177
|
+
* notification; this primitive owns only the job control-plane mutation.
|
|
149
178
|
*
|
|
150
|
-
*
|
|
151
|
-
*
|
|
152
|
-
*
|
|
153
|
-
*
|
|
154
|
-
*
|
|
179
|
+
* Only a job in the live-background state the roster projects (job.bg present,
|
|
180
|
+
* job.exit absent) may be signaled. Job directories persist after the
|
|
181
|
+
* supervisor exits, so a retired, still-foreground, or already-stopped job
|
|
182
|
+
* would otherwise have its recorded pgid signaled long after the OS could have
|
|
183
|
+
* reused that id for an unrelated process group. */
|
|
155
184
|
export async function stopBackgroundBashJob(contextDir, jobId) {
|
|
156
185
|
const paths = bashJobPaths(contextDir, jobId);
|
|
157
186
|
if (!existsSync(paths.dir))
|
|
@@ -159,7 +188,8 @@ export async function stopBackgroundBashJob(contextDir, jobId) {
|
|
|
159
188
|
if (!existsSync(paths.jobBg) || existsSync(paths.jobExit))
|
|
160
189
|
return { kind: 'retired', logPath: paths.jobLog };
|
|
161
190
|
const pgid = readJobPgid(contextDir, jobId);
|
|
162
|
-
|
|
191
|
+
const pgidIdentity = readJobPgidIdentity(contextDir, jobId);
|
|
192
|
+
if (pgid === undefined || pgidIdentity === undefined)
|
|
163
193
|
return { kind: 'not-stoppable', logPath: paths.jobLog };
|
|
164
194
|
// Claim the stop by creating job.exit exclusively, before signaling. The
|
|
165
195
|
// loser of two concurrent stops — and a stop that races the supervisor's own
|
|
@@ -179,18 +209,7 @@ export async function stopBackgroundBashJob(contextDir, jobId) {
|
|
|
179
209
|
return { kind: 'retired', logPath: paths.jobLog };
|
|
180
210
|
throw err;
|
|
181
211
|
}
|
|
182
|
-
|
|
183
|
-
try {
|
|
184
|
-
process.kill(-pgid, 'SIGTERM');
|
|
185
|
-
}
|
|
186
|
-
catch {
|
|
187
|
-
signaled = false;
|
|
188
|
-
}
|
|
189
|
-
await new Promise((resolve) => setTimeout(resolve, 400));
|
|
190
|
-
try {
|
|
191
|
-
process.kill(-pgid, 'SIGKILL');
|
|
192
|
-
}
|
|
193
|
-
catch { /* already gone */ }
|
|
212
|
+
const signaled = await cancelBashJobProcessTree(pgid, pgidIdentity);
|
|
194
213
|
return { kind: 'stopped', signaled, pgid, logPath: paths.jobLog };
|
|
195
214
|
}
|
|
196
215
|
/** Last `count` lines of a job log, cheaply: read at most the trailing 64 KiB
|
|
@@ -76,11 +76,13 @@ export interface LaunchSpec {
|
|
|
76
76
|
env: Record<string, string>;
|
|
77
77
|
}
|
|
78
78
|
/** The repository fingerprint a refusal was decided against. An examination
|
|
79
|
-
* that reads the same
|
|
80
|
-
*
|
|
79
|
+
* that reads the same values can skip the whole proof cascade: nothing about
|
|
80
|
+
* the refusal can have changed. */
|
|
81
81
|
export interface ManagedWorktreeObservation {
|
|
82
82
|
/** Tip of `refs/heads/<base_ref>`, or null when it no longer resolves. */
|
|
83
83
|
base: string | null;
|
|
84
|
+
/** Tip of the base branch's upstream (or `origin/<base_ref>` fallback). */
|
|
85
|
+
upstream: string | null;
|
|
84
86
|
/** Tip of `refs/heads/<branch>`, or null when the branch is gone. */
|
|
85
87
|
branch: string | null;
|
|
86
88
|
/** `absent`, or a hash of the checkout's HEAD plus `git status --porcelain`. */
|
package/dist/core/command.js
CHANGED
|
@@ -403,7 +403,7 @@ export async function parseArgv(params, tokens, options) {
|
|
|
403
403
|
continue;
|
|
404
404
|
}
|
|
405
405
|
const rawVal = inlineValue !== undefined ? inlineValue : tokens[++i];
|
|
406
|
-
if (rawVal === undefined || rawVal.startsWith('--')) {
|
|
406
|
+
if (rawVal === undefined || (inlineValue === undefined && rawVal.startsWith('--'))) {
|
|
407
407
|
const focusedNext = flagDef.focusedHelp !== undefined && (options?.leafPath?.length ?? 0) > 0
|
|
408
408
|
? `Run \`crtr ${options.leafPath.join(' ')} --${flagName} -h\` and read the focused value contract before retrying.`
|
|
409
409
|
: undefined;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { ErrorClass, OperationId } from './events/types.js';
|
|
2
|
-
|
|
2
|
+
import { type ProviderFailureDetails } from './provider-failure.js';
|
|
3
|
+
export type FaultLink = 'pi→provider' | 'viewer↔broker' | 'relay↔broker' | 'viewer↔crtrd' | 'broker↔crtrd' | 'daemon→node' | 'crtr→pi';
|
|
3
4
|
export type FaultKind = 'rate-limit' | 'overloaded' | 'connection' | 'auth' | 'protocol' | 'context-overflow' | 'other' | 'wedged' | 'model-not-found';
|
|
4
5
|
export type FaultRetryDisposition = 'auto' | 'manual' | 'fatal';
|
|
5
6
|
export type FaultRetryOwner = 'sdk' | 'daemon' | 'client';
|
|
@@ -10,6 +11,10 @@ export interface FaultRetry {
|
|
|
10
11
|
max?: number;
|
|
11
12
|
nextAt?: string;
|
|
12
13
|
}
|
|
14
|
+
export interface FaultProviderError extends ProviderFailureDetails {
|
|
15
|
+
provider: string;
|
|
16
|
+
model: string;
|
|
17
|
+
}
|
|
13
18
|
export interface Fault {
|
|
14
19
|
link: FaultLink;
|
|
15
20
|
op: string;
|
|
@@ -19,6 +24,7 @@ export interface Fault {
|
|
|
19
24
|
since: string;
|
|
20
25
|
operation_id: OperationId;
|
|
21
26
|
anchorEntryId?: string;
|
|
27
|
+
providerError?: FaultProviderError;
|
|
22
28
|
}
|
|
23
29
|
export interface FaultClassification {
|
|
24
30
|
kind: FaultKind;
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { hasManagedProviderCoolingDiagnostic } from './runtime/managed-provider-cooling.js';
|
|
2
2
|
import { recognizeProviderError } from './provider-error-recognition.js';
|
|
3
|
+
import { extractProviderFailureDetails } from './provider-failure.js';
|
|
3
4
|
import { isRecord } from '../shared/predicates.js';
|
|
4
5
|
const errorClassByFaultKind = {
|
|
5
6
|
'rate-limit': 'rate_limit',
|
|
@@ -78,13 +79,17 @@ export function classify(link, raw) {
|
|
|
78
79
|
return { kind: 'rate-limit', disposition: 'auto' };
|
|
79
80
|
}
|
|
80
81
|
const value = isRecord(raw) ? raw : {};
|
|
82
|
+
const failure = extractProviderFailureDetails(raw);
|
|
83
|
+
const nested = failure?.causes ?? [];
|
|
81
84
|
const signal = recognizeProviderError({
|
|
82
|
-
code: value['code'],
|
|
83
|
-
errno: value['errno'],
|
|
84
|
-
status: value['status'],
|
|
85
|
+
code: value['code'] ?? failure?.code ?? nested.find((cause) => cause.code !== undefined)?.code,
|
|
86
|
+
errno: value['errno'] ?? failure?.errno ?? nested.find((cause) => cause.errno !== undefined)?.errno,
|
|
87
|
+
status: value['status'] ?? failure?.status,
|
|
85
88
|
wsCloseCode: value['code'],
|
|
86
89
|
wsCloseReason: value['reason'],
|
|
87
|
-
message: rawText(raw),
|
|
90
|
+
message: [rawText(raw), failure?.body, ...nested.flatMap((cause) => [cause.message, cause.body])]
|
|
91
|
+
.filter((part) => typeof part === 'string' && part !== '')
|
|
92
|
+
.join(' '),
|
|
88
93
|
});
|
|
89
94
|
const kind = faultKindFromProviderSignal(signal);
|
|
90
95
|
return { kind, disposition: classifyDisposition(link, kind) };
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export declare const PROVIDER_FAILURE_DIAGNOSTIC = "crtr:provider-failure";
|
|
2
|
+
export interface ProviderFailureCause {
|
|
3
|
+
message?: string;
|
|
4
|
+
status?: number;
|
|
5
|
+
code?: string | number;
|
|
6
|
+
errno?: string;
|
|
7
|
+
body?: string;
|
|
8
|
+
}
|
|
9
|
+
export interface ProviderFailureDetails {
|
|
10
|
+
status?: number;
|
|
11
|
+
code?: string | number;
|
|
12
|
+
errno?: string;
|
|
13
|
+
body?: string;
|
|
14
|
+
causes?: ProviderFailureCause[];
|
|
15
|
+
}
|
|
16
|
+
export interface NormalizedProviderFailure extends ProviderFailureDetails {
|
|
17
|
+
message: string;
|
|
18
|
+
}
|
|
19
|
+
/** Converts an arbitrary provider failure into serializable facts before an SDK stream flattens it to `errorMessage`. */
|
|
20
|
+
export declare function normalizeProviderFailure(error: unknown, observed?: {
|
|
21
|
+
status?: number;
|
|
22
|
+
}): NormalizedProviderFailure;
|
|
23
|
+
/** The stable diagnostic carrier for normalized provider error facts on an assistant error message. */
|
|
24
|
+
export declare function providerFailureDiagnostic(details: ProviderFailureDetails): {
|
|
25
|
+
type: string;
|
|
26
|
+
timestamp: number;
|
|
27
|
+
details: Record<string, unknown>;
|
|
28
|
+
};
|
|
29
|
+
/** Reads provider facts from the assistant-message diagnostic without trusting arbitrary nested error objects. */
|
|
30
|
+
export declare function extractProviderFailureDetails(message: unknown): ProviderFailureDetails | undefined;
|
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
import { isRecord } from '../shared/predicates.js';
|
|
2
|
+
export const PROVIDER_FAILURE_DIAGNOSTIC = 'crtr:provider-failure';
|
|
3
|
+
const CAUSE_LIMIT = 8;
|
|
4
|
+
function text(value) {
|
|
5
|
+
return typeof value === 'string' && value.trim() !== '' ? value.trim() : undefined;
|
|
6
|
+
}
|
|
7
|
+
function status(value) {
|
|
8
|
+
return typeof value === 'number' && Number.isFinite(value) ? value : undefined;
|
|
9
|
+
}
|
|
10
|
+
function code(value) {
|
|
11
|
+
return typeof value === 'string' || typeof value === 'number' ? value : undefined;
|
|
12
|
+
}
|
|
13
|
+
function record(value) {
|
|
14
|
+
return isRecord(value) ? value : undefined;
|
|
15
|
+
}
|
|
16
|
+
function failureCause(value) {
|
|
17
|
+
const message = text(value['errorMessage']) ?? text(value['message']) ?? text(value['reason']) ?? text(value['statusText']);
|
|
18
|
+
const cause = {
|
|
19
|
+
...(message === undefined ? {} : { message }),
|
|
20
|
+
...(status(value['status']) === undefined ? {} : { status: status(value['status']) }),
|
|
21
|
+
...(code(value['code']) === undefined ? {} : { code: code(value['code']) }),
|
|
22
|
+
...(text(value['errno']) === undefined ? {} : { errno: text(value['errno']) }),
|
|
23
|
+
...(text(value['body']) === undefined ? {} : { body: text(value['body']) }),
|
|
24
|
+
};
|
|
25
|
+
return Object.keys(cause).length === 0 ? undefined : cause;
|
|
26
|
+
}
|
|
27
|
+
/** Converts an arbitrary provider failure into serializable facts before an SDK stream flattens it to `errorMessage`. */
|
|
28
|
+
export function normalizeProviderFailure(error, observed = {}) {
|
|
29
|
+
const root = record(error);
|
|
30
|
+
const rootCause = root === undefined ? undefined : failureCause(root);
|
|
31
|
+
const causes = [];
|
|
32
|
+
const seen = new Set([error]);
|
|
33
|
+
let current = root === undefined ? undefined : record(root['cause']);
|
|
34
|
+
while (current !== undefined && !seen.has(current) && causes.length < CAUSE_LIMIT) {
|
|
35
|
+
seen.add(current);
|
|
36
|
+
const cause = failureCause(current);
|
|
37
|
+
if (cause !== undefined)
|
|
38
|
+
causes.push(cause);
|
|
39
|
+
current = record(current['cause']);
|
|
40
|
+
}
|
|
41
|
+
const message = rootCause?.message ?? (error instanceof Error ? error.message : undefined) ?? String(error ?? 'Provider request failed');
|
|
42
|
+
return {
|
|
43
|
+
message,
|
|
44
|
+
...((observed.status ?? rootCause?.status) === undefined ? {} : { status: observed.status ?? rootCause?.status }),
|
|
45
|
+
...(rootCause?.code === undefined ? {} : { code: rootCause.code }),
|
|
46
|
+
...(rootCause?.errno === undefined ? {} : { errno: rootCause.errno }),
|
|
47
|
+
...(rootCause?.body === undefined ? {} : { body: rootCause.body }),
|
|
48
|
+
...(causes.length === 0 ? {} : { causes }),
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
/** The stable diagnostic carrier for normalized provider error facts on an assistant error message. */
|
|
52
|
+
export function providerFailureDiagnostic(details) {
|
|
53
|
+
return { type: PROVIDER_FAILURE_DIAGNOSTIC, timestamp: Date.now(), details: { ...details } };
|
|
54
|
+
}
|
|
55
|
+
function providerFailureDetails(value) {
|
|
56
|
+
const diagnostics = record(value)?.['diagnostics'];
|
|
57
|
+
if (!Array.isArray(diagnostics))
|
|
58
|
+
return undefined;
|
|
59
|
+
for (const diagnostic of diagnostics) {
|
|
60
|
+
const diagnosticRecord = record(diagnostic);
|
|
61
|
+
if (diagnosticRecord?.['type'] !== PROVIDER_FAILURE_DIAGNOSTIC)
|
|
62
|
+
continue;
|
|
63
|
+
const details = record(diagnosticRecord['details']);
|
|
64
|
+
if (details === undefined)
|
|
65
|
+
continue;
|
|
66
|
+
const causes = Array.isArray(details['causes'])
|
|
67
|
+
? details['causes'].flatMap((item) => {
|
|
68
|
+
const cause = record(item);
|
|
69
|
+
const normalized = cause === undefined ? undefined : failureCause(cause);
|
|
70
|
+
return normalized === undefined ? [] : [normalized];
|
|
71
|
+
})
|
|
72
|
+
: undefined;
|
|
73
|
+
return {
|
|
74
|
+
...(status(details['status']) === undefined ? {} : { status: status(details['status']) }),
|
|
75
|
+
...(code(details['code']) === undefined ? {} : { code: code(details['code']) }),
|
|
76
|
+
...(text(details['errno']) === undefined ? {} : { errno: text(details['errno']) }),
|
|
77
|
+
...(text(details['body']) === undefined ? {} : { body: text(details['body']) }),
|
|
78
|
+
...(causes === undefined || causes.length === 0 ? {} : { causes }),
|
|
79
|
+
};
|
|
80
|
+
}
|
|
81
|
+
return undefined;
|
|
82
|
+
}
|
|
83
|
+
/** Reads provider facts from the assistant-message diagnostic without trusting arbitrary nested error objects. */
|
|
84
|
+
export function extractProviderFailureDetails(message) {
|
|
85
|
+
return providerFailureDetails(message);
|
|
86
|
+
}
|
|
@@ -11,8 +11,6 @@ type EventProjectionDeps = {
|
|
|
11
11
|
notifyTurnAccepted: () => void;
|
|
12
12
|
emitStartupMilestone: (event: string, model: string | null) => void;
|
|
13
13
|
formatModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
|
|
14
|
-
refreshIntent: () => Promise<boolean>;
|
|
15
|
-
onRefreshIntentError: (error: unknown) => void;
|
|
16
14
|
};
|
|
17
15
|
/** Projects one live engine event stream to broker viewers and owns its relay timers. */
|
|
18
16
|
export declare class EventProjection {
|
|
@@ -40,7 +38,6 @@ export declare class EventProjection {
|
|
|
40
38
|
private flushPendingUpdate;
|
|
41
39
|
private invalidatePendingBroadcasts;
|
|
42
40
|
private broadcast;
|
|
43
|
-
private broadcastMessageEnd;
|
|
44
41
|
private setPendingBroadcast;
|
|
45
42
|
}
|
|
46
43
|
export {};
|
|
@@ -128,8 +128,8 @@ export class EventProjection {
|
|
|
128
128
|
return;
|
|
129
129
|
}
|
|
130
130
|
this.flushPendingUpdate();
|
|
131
|
-
if (type === 'message_end') {
|
|
132
|
-
this.
|
|
131
|
+
if (type === 'message_end' && generation.stagedRefreshAbort) {
|
|
132
|
+
this.broadcast({ ...event, message: withoutYieldAbort(event.message) });
|
|
133
133
|
return;
|
|
134
134
|
}
|
|
135
135
|
this.broadcast(event);
|
|
@@ -183,19 +183,6 @@ export class EventProjection {
|
|
|
183
183
|
this.deps.registry.broadcast(frame);
|
|
184
184
|
}));
|
|
185
185
|
}
|
|
186
|
-
broadcastMessageEnd(event) {
|
|
187
|
-
const preceding = this.pendingBroadcast ?? Promise.resolve();
|
|
188
|
-
const generation = this.broadcastGeneration;
|
|
189
|
-
this.setPendingBroadcast(preceding.then(async () => {
|
|
190
|
-
const refresh = await this.deps.refreshIntent();
|
|
191
|
-
if (this.broadcastGeneration !== generation)
|
|
192
|
-
return;
|
|
193
|
-
this.deps.registry.broadcast(refresh ? { ...event, message: withoutYieldAbort(event.message) } : event);
|
|
194
|
-
}).catch((error) => {
|
|
195
|
-
if (this.broadcastGeneration === generation)
|
|
196
|
-
this.deps.onRefreshIntentError(error);
|
|
197
|
-
}));
|
|
198
|
-
}
|
|
199
186
|
setPendingBroadcast(work) {
|
|
200
187
|
this.pendingBroadcast = work;
|
|
201
188
|
void work.then(() => {
|
|
@@ -4,6 +4,7 @@ import { isContextOverflow } from '@earendil-works/pi-ai';
|
|
|
4
4
|
import { classify } from '../../fault-classifier.js';
|
|
5
5
|
import { admitProviderRetryEpisode, clearFault, clearProviderRetryEpisode, invalidateProviderRetryEpisode, isModelNotFoundError, isProviderUnconfiguredError, readFault, readProviderRetryEpisode, recordFault, recordPendingProviderRetryFault, reopenProviderRetryEpisode, } from '../fault.js';
|
|
6
6
|
import { extractCoolingDeadline } from '../managed-provider-cooling.js';
|
|
7
|
+
import { extractProviderFailureDetails } from '../../provider-failure.js';
|
|
7
8
|
import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from '../../model-routes.js';
|
|
8
9
|
import { operationIdContext } from '../../events/operation-id.js';
|
|
9
10
|
import { emitEvent } from '../../events/emit.js';
|
|
@@ -71,6 +72,14 @@ export class FaultRetry {
|
|
|
71
72
|
// admitted retry therefore carries from its durable episode, while an
|
|
72
73
|
// initial provider failure still reads from the ordinary marker.
|
|
73
74
|
const prior = readFault(this.deps.nodeId);
|
|
75
|
+
// A persona gate runs before Pi contacts a provider. It records a typed
|
|
76
|
+
// local API transport failure, then Pi represents that thrown error as a
|
|
77
|
+
// generic provider-looking agent_end. Keep its actual boundary instead of
|
|
78
|
+
// turning it into a fatal pi→provider `other` fault.
|
|
79
|
+
if (prior?.link === 'broker↔crtrd' && prior.kind === 'connection') {
|
|
80
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
81
|
+
return;
|
|
82
|
+
}
|
|
74
83
|
const durableEpisode = readProviderRetryEpisode(this.deps.nodeId);
|
|
75
84
|
const episodeFault = this.isActiveAutoFault(prior) && prior.link === 'pi→provider'
|
|
76
85
|
? prior
|
|
@@ -123,6 +132,12 @@ export class FaultRetry {
|
|
|
123
132
|
const uncompactedOverflow = isContextOverflow(last, settledSession.model?.contextWindow);
|
|
124
133
|
const classified = classify('pi→provider', last);
|
|
125
134
|
const coolingDeadline = extractCoolingDeadline(last);
|
|
135
|
+
const failure = extractProviderFailureDetails(last);
|
|
136
|
+
const provider = typeof last.provider === 'string' && last.provider !== '' ? last.provider : settledSession.model?.provider;
|
|
137
|
+
const model = typeof last.model === 'string' && last.model !== '' ? last.model : settledSession.model?.id;
|
|
138
|
+
const providerError = failure === undefined || provider === undefined || model === undefined
|
|
139
|
+
? undefined
|
|
140
|
+
: { provider, model, ...failure };
|
|
126
141
|
if (uncompactedOverflow) {
|
|
127
142
|
clearFault(this.deps.nodeId, { link: 'pi→provider' });
|
|
128
143
|
clearProviderRetryEpisode(this.deps.nodeId);
|
|
@@ -143,6 +158,7 @@ export class FaultRetry {
|
|
|
143
158
|
anchorEntryId: !uncompactedOverflow && classified.disposition === 'auto' && episode?.anchorEntryId !== undefined
|
|
144
159
|
? episode.anchorEntryId
|
|
145
160
|
: settledSession.sessionManager.getLeafId?.() ?? undefined,
|
|
161
|
+
...(providerError === undefined ? {} : { providerError }),
|
|
146
162
|
};
|
|
147
163
|
if (faultInput.retry.disposition === 'auto')
|
|
148
164
|
this.recordPendingProviderFault(settledSession, faultInput);
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { contextDir } from '../canvas/paths.js';
|
|
2
|
+
import { isDaemonTransportApiError } from '../../api/errors.js';
|
|
2
3
|
import { emitEvent } from '../events/emit.js';
|
|
4
|
+
import { clearFault, recordFault } from './fault.js';
|
|
3
5
|
import { brokerExtensionState, commitBrokerPersonaAck } from './broker/daemon-ops.js';
|
|
4
6
|
import { readRoadmap, roadmapPath } from './roadmap.js';
|
|
5
7
|
import { buildSubPersonaMenu, renderConfigChangeDelta } from '../substrate/render.js';
|
|
@@ -137,9 +139,19 @@ export function installPersonaTransitionGate(nodeId, session) {
|
|
|
137
139
|
for (const context of new Set([active, messages, transformed]))
|
|
138
140
|
context.push(message);
|
|
139
141
|
});
|
|
142
|
+
clearFault(nodeId, { link: 'broker↔crtrd' });
|
|
140
143
|
return transformed;
|
|
141
144
|
}
|
|
142
145
|
catch (error) {
|
|
146
|
+
if (isDaemonTransportApiError(error)) {
|
|
147
|
+
recordFault(nodeId, {
|
|
148
|
+
link: 'broker↔crtrd',
|
|
149
|
+
op: 'persona transition',
|
|
150
|
+
kind: 'connection',
|
|
151
|
+
retry: { disposition: 'manual' },
|
|
152
|
+
message: error.message,
|
|
153
|
+
});
|
|
154
|
+
}
|
|
143
155
|
emitEvent({
|
|
144
156
|
event: 'broker.daemon.operation.failed',
|
|
145
157
|
level: 'error',
|
|
@@ -375,11 +375,6 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
|
|
|
375
375
|
notifyTurnAccepted: () => notifyTurnAccepted(),
|
|
376
376
|
emitStartupMilestone,
|
|
377
377
|
formatModelSpec,
|
|
378
|
-
refreshIntent: async () => (await brokerExtensionState(nodeId)).node.intent === 'refresh',
|
|
379
|
-
onRefreshIntentError: (error) => {
|
|
380
|
-
emitEvent({ level: 'error', event: 'broker.daemon_state.read_failed', error });
|
|
381
|
-
disposeAndExit('daemon-state-read-failed', 1);
|
|
382
|
-
},
|
|
383
378
|
});
|
|
384
379
|
const buildSnapshot = (client) => {
|
|
385
380
|
const liveSession = rebind.session();
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type Fault, type FaultKind, type FaultLink, type FaultRetry } from '../fault-classifier.js';
|
|
1
|
+
import { type Fault, type FaultKind, type FaultLink, type FaultProviderError, type FaultRetry } from '../fault-classifier.js';
|
|
2
2
|
export type { Fault, FaultKind, FaultLink, FaultRetry, FaultRetryDisposition, FaultRetryOwner } from '../fault-classifier.js';
|
|
3
3
|
export declare const FAULT_QUIET_MS: number;
|
|
4
4
|
export interface FaultRecordInput {
|
|
@@ -10,6 +10,7 @@ export interface FaultRecordInput {
|
|
|
10
10
|
operation_id?: string;
|
|
11
11
|
since?: string;
|
|
12
12
|
anchorEntryId?: string;
|
|
13
|
+
providerError?: FaultProviderError;
|
|
13
14
|
}
|
|
14
15
|
/** A provider-retry episode outlives its broker-local timer. Its transcript
|
|
15
16
|
* path and anchor are explicit coordinates, so replacement invalidation never
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { envNodeId } from '../../shared/env.js';
|
|
2
|
+
import { isRecord } from '../../shared/predicates.js';
|
|
2
3
|
import { existsSync, mkdirSync, readFileSync, realpathSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
4
|
import { join } from 'node:path';
|
|
4
5
|
import { jobDir } from '../canvas/paths.js';
|
|
@@ -31,8 +32,39 @@ function faultPriority(fault) {
|
|
|
31
32
|
return 2;
|
|
32
33
|
return 1;
|
|
33
34
|
}
|
|
34
|
-
const faultLinks = ['pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'daemon→node', 'crtr→pi'];
|
|
35
|
+
const faultLinks = ['pi→provider', 'viewer↔broker', 'relay↔broker', 'viewer↔crtrd', 'broker↔crtrd', 'daemon→node', 'crtr→pi'];
|
|
35
36
|
const faultKinds = ['rate-limit', 'overloaded', 'connection', 'auth', 'protocol', 'context-overflow', 'other', 'wedged', 'model-not-found'];
|
|
37
|
+
function parseProviderError(value) {
|
|
38
|
+
if (!isRecord(value) || typeof value.provider !== 'string' || value.provider === '' || typeof value.model !== 'string' || value.model === '')
|
|
39
|
+
return undefined;
|
|
40
|
+
const status = typeof value.status === 'number' && Number.isFinite(value.status) ? value.status : undefined;
|
|
41
|
+
const code = typeof value.code === 'string' || typeof value.code === 'number' ? value.code : undefined;
|
|
42
|
+
const errno = typeof value.errno === 'string' ? value.errno : undefined;
|
|
43
|
+
const body = typeof value.body === 'string' ? value.body : undefined;
|
|
44
|
+
const causes = Array.isArray(value.causes)
|
|
45
|
+
? value.causes.flatMap((item) => {
|
|
46
|
+
if (!isRecord(item))
|
|
47
|
+
return [];
|
|
48
|
+
const cause = {
|
|
49
|
+
...(typeof item.message === 'string' ? { message: item.message } : {}),
|
|
50
|
+
...(typeof item.status === 'number' && Number.isFinite(item.status) ? { status: item.status } : {}),
|
|
51
|
+
...(typeof item.code === 'string' || typeof item.code === 'number' ? { code: item.code } : {}),
|
|
52
|
+
...(typeof item.errno === 'string' ? { errno: item.errno } : {}),
|
|
53
|
+
...(typeof item.body === 'string' ? { body: item.body } : {}),
|
|
54
|
+
};
|
|
55
|
+
return Object.keys(cause).length === 0 ? [] : [cause];
|
|
56
|
+
})
|
|
57
|
+
: undefined;
|
|
58
|
+
return {
|
|
59
|
+
provider: value.provider,
|
|
60
|
+
model: value.model,
|
|
61
|
+
...(status === undefined ? {} : { status }),
|
|
62
|
+
...(code === undefined ? {} : { code }),
|
|
63
|
+
...(errno === undefined ? {} : { errno }),
|
|
64
|
+
...(body === undefined ? {} : { body }),
|
|
65
|
+
...(causes === undefined || causes.length === 0 ? {} : { causes }),
|
|
66
|
+
};
|
|
67
|
+
}
|
|
36
68
|
function parseFault(raw) {
|
|
37
69
|
try {
|
|
38
70
|
const parsed = JSON.parse(raw);
|
|
@@ -60,6 +92,9 @@ function parseFault(raw) {
|
|
|
60
92
|
return null;
|
|
61
93
|
if (parsed.anchorEntryId !== undefined && (typeof parsed.anchorEntryId !== 'string' || parsed.anchorEntryId === ''))
|
|
62
94
|
return null;
|
|
95
|
+
const providerError = parsed.providerError === undefined ? undefined : parseProviderError(parsed.providerError);
|
|
96
|
+
if (parsed.providerError !== undefined && providerError === undefined)
|
|
97
|
+
return null;
|
|
63
98
|
return {
|
|
64
99
|
link: parsed.link,
|
|
65
100
|
op: parsed.op,
|
|
@@ -75,6 +110,7 @@ function parseFault(raw) {
|
|
|
75
110
|
since: parsed.since,
|
|
76
111
|
operation_id: operationId,
|
|
77
112
|
...(parsed.anchorEntryId === undefined ? {} : { anchorEntryId: parsed.anchorEntryId }),
|
|
113
|
+
...(providerError === undefined ? {} : { providerError }),
|
|
78
114
|
};
|
|
79
115
|
}
|
|
80
116
|
catch {
|
|
@@ -166,6 +202,7 @@ function storedFault(input) {
|
|
|
166
202
|
since: input.since ?? new Date().toISOString(),
|
|
167
203
|
operation_id: resolveOperationId(input.operation_id),
|
|
168
204
|
...(typeof input.anchorEntryId === 'string' && input.anchorEntryId !== '' ? { anchorEntryId: input.anchorEntryId } : {}),
|
|
205
|
+
...(input.providerError === undefined ? {} : { providerError: input.providerError }),
|
|
169
206
|
};
|
|
170
207
|
}
|
|
171
208
|
function emitRecordedFault(nodeId, fault) {
|
|
@@ -182,6 +219,7 @@ function emitRecordedFault(nodeId, fault) {
|
|
|
182
219
|
retry: fault.retry,
|
|
183
220
|
message: fault.message,
|
|
184
221
|
...(fault.anchorEntryId === undefined ? {} : { anchor_entry_id: fault.anchorEntryId }),
|
|
222
|
+
...(fault.providerError === undefined ? {} : { provider_error: fault.providerError }),
|
|
185
223
|
},
|
|
186
224
|
});
|
|
187
225
|
}
|
|
@@ -319,7 +319,16 @@ export const headlessBrokerHost = {
|
|
|
319
319
|
});
|
|
320
320
|
finish({ code, signal });
|
|
321
321
|
});
|
|
322
|
-
spawned.once('error', () =>
|
|
322
|
+
spawned.once('error', (error) => {
|
|
323
|
+
emitEvent({
|
|
324
|
+
level: 'error',
|
|
325
|
+
event: 'broker.launch.failed',
|
|
326
|
+
node_id: nodeId,
|
|
327
|
+
error,
|
|
328
|
+
fields: { uptime_ms: Date.now() - launchedAt },
|
|
329
|
+
});
|
|
330
|
+
finish({ code: null, signal: null });
|
|
331
|
+
});
|
|
323
332
|
});
|
|
324
333
|
spawned.unref();
|
|
325
334
|
const handle = { pid: spawned.pid ?? null, exited };
|
|
@@ -26,6 +26,9 @@ export declare function readYieldMessage(nodeId: string): string | null;
|
|
|
26
26
|
export declare function clearYieldMessage(nodeId: string): void;
|
|
27
27
|
/** List the node's context/ dir (filenames, sorted). Empty when absent. */
|
|
28
28
|
export declare function listContextDir(nodeId: string): string[];
|
|
29
|
+
/** A retained report proves this node has already done work. It must restart
|
|
30
|
+
* from current context, never by replaying its birth mandate. */
|
|
31
|
+
export declare function hasRetainedReports(nodeId: string): boolean;
|
|
29
32
|
export interface ReviveBearings {
|
|
30
33
|
/** The one-shot yield note left by the prior self, pending boot confirmation. */
|
|
31
34
|
yieldMsg: string | null;
|