@north-light/crouter 0.3.249 → 0.3.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/__tests__/integration/client.test.js +28 -7
- package/dist/api/client.d.ts +1 -1
- package/dist/api/client.js +2 -2
- package/dist/api/dto/lifecycle.d.ts +3 -0
- package/dist/api/dto/messages.d.ts +4 -0
- package/dist/api/dto/nodes.d.ts +4 -0
- package/dist/api/dto/reports.d.ts +7 -2
- package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
- package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
- package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +1 -0
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
- package/dist/clients/inbox/controller.js +32 -5
- package/dist/clients/inbox/tui/ansi.d.ts +3 -1
- package/dist/clients/inbox/tui/ansi.js +14 -10
- package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
- package/dist/commands/api-client.d.ts +2 -2
- package/dist/commands/api-client.js +5 -5
- package/dist/commands/canvas-browse.js +1 -1
- package/dist/commands/node/create.js +3 -1
- package/dist/commands/node/inspect.js +1 -0
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node/message.js +6 -4
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/push.js +36 -37
- package/dist/commands/sys/daemon.js +21 -8
- package/dist/commands/sys/setup-core.js +2 -2
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
- package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
- package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +100 -1
- package/dist/core/__tests__/child-death-wake.test.js +11 -7
- package/dist/core/__tests__/collectible.test.d.ts +1 -0
- package/dist/core/__tests__/collectible.test.js +88 -0
- package/dist/core/__tests__/context-intro.test.js +4 -41
- package/dist/core/__tests__/daemon-boot.test.js +30 -14
- package/dist/core/__tests__/daemon-wedge.test.js +25 -19
- package/dist/core/__tests__/fixtures/fake-engine.js +20 -2
- package/dist/core/__tests__/human-deliver.test.js +5 -5
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
- package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
- package/dist/core/__tests__/integration/revive.test.js +48 -0
- package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
- package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
- package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
- package/dist/core/__tests__/migration.test.js +30 -0
- package/dist/core/__tests__/passive-subscription.test.js +39 -1
- package/dist/core/__tests__/prune.test.js +12 -3
- package/dist/core/__tests__/relaunch-root.test.js +1 -1
- package/dist/core/__tests__/respawn-throttle.test.js +16 -146
- package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
- package/dist/core/canvas/canvas.d.ts +9 -15
- package/dist/core/canvas/canvas.js +41 -47
- package/dist/core/canvas/crons.d.ts +5 -8
- package/dist/core/canvas/crons.js +5 -8
- package/dist/core/canvas/history.d.ts +5 -0
- package/dist/core/canvas/history.js +12 -1
- package/dist/core/canvas/migrations.js +58 -1
- package/dist/core/canvas/pid.d.ts +1 -1
- package/dist/core/canvas/pid.js +5 -2
- package/dist/core/canvas/types.d.ts +4 -0
- package/dist/core/feed/feed.d.ts +4 -0
- package/dist/core/feed/feed.js +13 -8
- package/dist/core/feed/inbox.d.ts +2 -1
- package/dist/core/feed/inbox.js +22 -3
- package/dist/core/git.d.ts +3 -0
- package/dist/core/git.js +3 -1
- package/dist/core/human/feedback-companion.js +1 -0
- package/dist/core/preview-registry.js +0 -1
- package/dist/core/review/companion.js +1 -0
- package/dist/core/review/realize.js +1 -0
- package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
- package/dist/core/runtime/broker/auth-reload.js +9 -3
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
- package/dist/core/runtime/broker/engine-drive.js +39 -23
- package/dist/core/runtime/broker/event-projection.d.ts +4 -0
- package/dist/core/runtime/broker/event-projection.js +43 -23
- package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
- package/dist/core/runtime/broker/fault-retry.js +94 -8
- package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
- package/dist/core/runtime/broker/frame-dispatch.js +4 -1
- package/dist/core/runtime/broker/held-deferred-inbox.d.ts +9 -0
- package/dist/core/runtime/broker/held-deferred-inbox.js +8 -0
- package/dist/core/runtime/broker/inbox.js +4 -0
- package/dist/core/runtime/broker/rebind.js +18 -0
- package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
- package/dist/core/runtime/broker/turn-admission.js +31 -0
- package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
- package/dist/core/runtime/broker/turn-ignition.js +25 -15
- package/dist/core/runtime/broker.js +21 -2
- package/dist/core/runtime/close.d.ts +11 -9
- package/dist/core/runtime/close.js +23 -19
- package/dist/core/runtime/fault.d.ts +21 -0
- package/dist/core/runtime/fault.js +129 -19
- package/dist/core/runtime/fleet.d.ts +0 -6
- package/dist/core/runtime/host.d.ts +7 -1
- package/dist/core/runtime/host.js +20 -2
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +24 -1
- package/dist/core/runtime/placement.d.ts +6 -6
- package/dist/core/runtime/placement.js +15 -8
- package/dist/core/runtime/reset.js +2 -2
- package/dist/core/runtime/revive.js +40 -4
- package/dist/core/runtime/spawn.d.ts +3 -0
- package/dist/core/runtime/spawn.js +1 -0
- package/dist/core/runtime/warm-pool.d.ts +4 -0
- package/dist/core/runtime/warm-pool.js +2 -0
- package/dist/core/worktree.d.ts +4 -3
- package/dist/core/worktree.js +73 -27
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
- package/dist/daemon/api/handlers/bash-jobs.js +2 -0
- package/dist/daemon/api/handlers/broker-ops.js +23 -2
- package/dist/daemon/api/handlers/human.js +1 -1
- package/dist/daemon/api/handlers/messages.js +11 -2
- package/dist/daemon/api/handlers/nodes.js +13 -1
- package/dist/daemon/api/handlers/reports.js +14 -6
- package/dist/daemon/api/map.js +1 -0
- package/dist/daemon/api/server.d.ts +9 -12
- package/dist/daemon/api/server.js +30 -19
- package/dist/daemon/companion-retire.js +3 -9
- package/dist/daemon/cron/sinks.js +7 -2
- package/dist/daemon/crtrd.js +4 -1
- package/dist/daemon/fleet.d.ts +2 -16
- package/dist/daemon/fleet.js +63 -150
- package/dist/daemon/human/finish.js +2 -0
- package/dist/daemon/manage.d.ts +11 -0
- package/dist/daemon/manage.js +32 -3
- package/dist/daemon/messaging/node-message.js +7 -1
- package/dist/daemon/reconcilers/bash-deadline.js +2 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
- package/dist/daemon/reconcilers/broker-supervision.js +22 -9
- package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
- package/dist/daemon/reconcilers/live-obligation.js +18 -6
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
- package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
- package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
- package/dist/pi-extensions/canvas-context-intro.js +4 -76
- package/dist/pi-extensions/canvas-inbox-watcher.js +105 -4
- package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
- package/dist/pi-extensions/canvas-review-boundary.js +4 -29
- package/dist/pi-extensions/canvas-stophook.js +1 -6
- package/dist/shared/birth-announcement.d.ts +12 -0
- package/dist/shared/birth-announcement.js +25 -0
- package/dist/shared/generated-context.d.ts +8 -4
- package/dist/shared/generated-context.js +15 -9
- package/package.json +4 -4
- package/runtime.lock.json +2 -2
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
- package/dist/pi-extensions/truncate.d.ts +0 -5
- package/dist/pi-extensions/truncate.js +0 -14
|
@@ -4,6 +4,7 @@ import type { BrokerSdkConfig } from '../launch.js';
|
|
|
4
4
|
import type { BrokerToClient } from '../broker-protocol.js';
|
|
5
5
|
import type { BrokerSession } from './read-ops.js';
|
|
6
6
|
import type { SubscriptionGeneration } from './rebind.js';
|
|
7
|
+
import type { TurnAdmissionGate } from './turn-admission.js';
|
|
7
8
|
type FaultRetryDeps = {
|
|
8
9
|
nodeId: string;
|
|
9
10
|
cfg: BrokerSdkConfig;
|
|
@@ -14,6 +15,7 @@ type FaultRetryDeps = {
|
|
|
14
15
|
registryOf: (services: AgentSessionServices) => ModelRegistry;
|
|
15
16
|
formatModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
|
|
16
17
|
broadcastModelChanged: (persist?: boolean) => Promise<void>;
|
|
18
|
+
turnAdmission: TurnAdmissionGate;
|
|
17
19
|
};
|
|
18
20
|
/** Owns provider fallback and daemon-driven fault retry state for one broker session. */
|
|
19
21
|
export declare class FaultRetry {
|
|
@@ -29,9 +31,16 @@ export declare class FaultRetry {
|
|
|
29
31
|
onEvent(event: BrokerToClient): void;
|
|
30
32
|
isActiveAutoFault(fault: ReturnType<typeof readFault>): fault is NonNullable<ReturnType<typeof readFault>>;
|
|
31
33
|
publishSettledProviderFault(generation: SubscriptionGeneration): void;
|
|
34
|
+
/** Startup re-drive reads only a durable pending episode. Revive deliberately
|
|
35
|
+
* clears the ordinary fault marker before this boundary, so the episode is
|
|
36
|
+
* the recovery authority and admitted/invalidated records remain inert. */
|
|
37
|
+
recoverPendingOnStartup(): void;
|
|
32
38
|
schedule(): void;
|
|
33
39
|
private maybeSwitchProviderForRetry;
|
|
34
40
|
private maybeFallbackOnUnusableModel;
|
|
41
|
+
private recordPendingProviderFault;
|
|
42
|
+
private sessionFile;
|
|
43
|
+
private pendingEpisodeMatches;
|
|
35
44
|
private faultRetryDelayMs;
|
|
36
45
|
private clearAndArmFaultRetry;
|
|
37
46
|
private runFaultRetry;
|
|
@@ -1,7 +1,8 @@
|
|
|
1
|
+
import { realpathSync } from 'node:fs';
|
|
1
2
|
import { envModelIntent, envNodeCwd, envProfileId } from '../../../shared/env.js';
|
|
2
3
|
import { isContextOverflow } from '@earendil-works/pi-ai';
|
|
3
4
|
import { classify } from '../../fault-classifier.js';
|
|
4
|
-
import { clearFault, isModelNotFoundError, isProviderUnconfiguredError, readFault, recordFault } from '../fault.js';
|
|
5
|
+
import { admitProviderRetryEpisode, clearFault, clearProviderRetryEpisode, invalidateProviderRetryEpisode, isModelNotFoundError, isProviderUnconfiguredError, readFault, readProviderRetryEpisode, recordFault, recordPendingProviderRetryFault, } from '../fault.js';
|
|
5
6
|
import { extractCoolingDeadline } from '../managed-provider-cooling.js';
|
|
6
7
|
import { expandModelCandidates, modelRequestFromConfig, probeRouteAvailability } from '../../model-routes.js';
|
|
7
8
|
import { operationIdContext } from '../../events/operation-id.js';
|
|
@@ -77,7 +78,7 @@ export class FaultRetry {
|
|
|
77
78
|
return;
|
|
78
79
|
}
|
|
79
80
|
if (watchdogAbort && last?.stopReason === 'aborted') {
|
|
80
|
-
|
|
81
|
+
this.recordPendingProviderFault(settledSession, {
|
|
81
82
|
link: 'pi→provider', op: 'generation turn', kind: 'connection',
|
|
82
83
|
retry: { disposition: 'auto', by: 'daemon', ...(episode?.attempt !== undefined ? { attempt: episode.attempt } : {}) },
|
|
83
84
|
message: typeof last.errorMessage === 'string' ? last.errorMessage : 'engine error (no errorMessage recorded)',
|
|
@@ -101,7 +102,7 @@ export class FaultRetry {
|
|
|
101
102
|
const coolingDeadline = extractCoolingDeadline(last);
|
|
102
103
|
if (uncompactedOverflow)
|
|
103
104
|
clearFault(this.deps.nodeId, { link: 'pi→provider' });
|
|
104
|
-
|
|
105
|
+
const faultInput = {
|
|
105
106
|
link: 'pi→provider', op: 'generation turn', kind: uncompactedOverflow ? 'context-overflow' : classified.kind,
|
|
106
107
|
retry: uncompactedOverflow
|
|
107
108
|
? { disposition: 'fatal' }
|
|
@@ -117,7 +118,35 @@ export class FaultRetry {
|
|
|
117
118
|
anchorEntryId: !uncompactedOverflow && classified.disposition === 'auto' && episode?.anchorEntryId !== undefined
|
|
118
119
|
? episode.anchorEntryId
|
|
119
120
|
: settledSession.sessionManager.getLeafId?.() ?? undefined,
|
|
120
|
-
}
|
|
121
|
+
};
|
|
122
|
+
if (faultInput.retry.disposition === 'auto')
|
|
123
|
+
this.recordPendingProviderFault(settledSession, faultInput);
|
|
124
|
+
else
|
|
125
|
+
recordFault(this.deps.nodeId, faultInput);
|
|
126
|
+
}
|
|
127
|
+
/** Startup re-drive reads only a durable pending episode. Revive deliberately
|
|
128
|
+
* clears the ordinary fault marker before this boundary, so the episode is
|
|
129
|
+
* the recovery authority and admitted/invalidated records remain inert. */
|
|
130
|
+
recoverPendingOnStartup() {
|
|
131
|
+
const generation = this.deps.installedGeneration();
|
|
132
|
+
if (generation === undefined)
|
|
133
|
+
return;
|
|
134
|
+
const episode = readProviderRetryEpisode(this.deps.nodeId);
|
|
135
|
+
const sessionFile = this.sessionFile(generation.session);
|
|
136
|
+
// A new-cycle launch deliberately roots a new transcript branch even though
|
|
137
|
+
// it reuses the same .jsonl. Its old anchor is no longer a valid recovery
|
|
138
|
+
// coordinate, so mark the episode invalid rather than inferring that from
|
|
139
|
+
// the shared path or letting startup re-drive into the fresh cycle.
|
|
140
|
+
if (this.deps.cfg.newCycle === true) {
|
|
141
|
+
if (episode !== null && sessionFile !== null && episode.sessionFile === sessionFile) {
|
|
142
|
+
invalidateProviderRetryEpisode(this.deps.nodeId, sessionFile);
|
|
143
|
+
}
|
|
144
|
+
return;
|
|
145
|
+
}
|
|
146
|
+
if (episode === null || sessionFile === null || episode.state !== 'pending' || episode.sessionFile !== sessionFile)
|
|
147
|
+
return;
|
|
148
|
+
recordFault(this.deps.nodeId, episode.fault);
|
|
149
|
+
this.schedule();
|
|
121
150
|
}
|
|
122
151
|
schedule() {
|
|
123
152
|
const generation = this.deps.installedGeneration();
|
|
@@ -125,7 +154,11 @@ export class FaultRetry {
|
|
|
125
154
|
if (generation === undefined || session.isStreaming || this.timer !== null)
|
|
126
155
|
return;
|
|
127
156
|
const fault = readFault(this.deps.nodeId);
|
|
128
|
-
|
|
157
|
+
// A live session without a durable transcript path still owns its ordinary
|
|
158
|
+
// in-process timer; it simply cannot survive broker death. Once a path
|
|
159
|
+
// exists, require its pending episode so admitted/invalidated retries stay
|
|
160
|
+
// inert even before a replacement broker starts.
|
|
161
|
+
if (!this.isActiveAutoFault(fault) || (this.sessionFile(session) !== null && !this.pendingEpisodeMatches(session, fault))) {
|
|
129
162
|
this.clearTimer();
|
|
130
163
|
return;
|
|
131
164
|
}
|
|
@@ -215,7 +248,7 @@ export class FaultRetry {
|
|
|
215
248
|
return;
|
|
216
249
|
if (unconfigured)
|
|
217
250
|
clearFault(this.deps.nodeId, { link: 'pi→provider' });
|
|
218
|
-
await operationIdContext.fresh(() => candidateSession.prompt(formatCard('recovery', {}, formatModelFallbackRecovery(currentSpec, resolved, unconfigured ? 'credential' : 'not-found'))));
|
|
251
|
+
await this.deps.turnAdmission.admit((release) => operationIdContext.fresh(() => candidateSession.prompt(formatCard('recovery', {}, formatModelFallbackRecovery(currentSpec, resolved, unconfigured ? 'credential' : 'not-found')), { preflightResult: release })));
|
|
219
252
|
})().catch((err) => {
|
|
220
253
|
emitEvent({ level: 'debug', event: 'broker.runtime.diagnostic', fields: { message: `${unconfigured ? 'unconfigured-provider' : 'not_found'} fallback failed: ${err instanceof Error ? err.message : String(err)}` } });
|
|
221
254
|
}).finally(() => {
|
|
@@ -224,6 +257,36 @@ export class FaultRetry {
|
|
|
224
257
|
});
|
|
225
258
|
this.notFoundFallbackInFlight = fallbackPromise;
|
|
226
259
|
}
|
|
260
|
+
recordPendingProviderFault(session, input) {
|
|
261
|
+
const sessionFile = this.sessionFile(session);
|
|
262
|
+
// A pathless session cannot be crash-recovered, but its live broker still
|
|
263
|
+
// owns the ordinary auto-retry timer.
|
|
264
|
+
if (sessionFile === null) {
|
|
265
|
+
recordFault(this.deps.nodeId, input);
|
|
266
|
+
return;
|
|
267
|
+
}
|
|
268
|
+
recordPendingProviderRetryFault(this.deps.nodeId, sessionFile, input);
|
|
269
|
+
}
|
|
270
|
+
sessionFile(session) {
|
|
271
|
+
if (typeof session.sessionFile !== 'string' || session.sessionFile === '')
|
|
272
|
+
return null;
|
|
273
|
+
try {
|
|
274
|
+
return realpathSync(session.sessionFile);
|
|
275
|
+
}
|
|
276
|
+
catch {
|
|
277
|
+
return session.sessionFile;
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
pendingEpisodeMatches(session, fault) {
|
|
281
|
+
const sessionFile = this.sessionFile(session);
|
|
282
|
+
const episode = readProviderRetryEpisode(this.deps.nodeId);
|
|
283
|
+
return sessionFile !== null &&
|
|
284
|
+
episode !== null &&
|
|
285
|
+
episode.state === 'pending' &&
|
|
286
|
+
episode.sessionFile === sessionFile &&
|
|
287
|
+
episode.fault.since === fault.since &&
|
|
288
|
+
episode.anchorEntryId === (fault.anchorEntryId ?? null);
|
|
289
|
+
}
|
|
227
290
|
faultRetryDelayMs(fault, now) {
|
|
228
291
|
if (fault.retry.nextAt !== undefined) {
|
|
229
292
|
const ms = Date.parse(fault.retry.nextAt) - now;
|
|
@@ -280,11 +343,13 @@ export class FaultRetry {
|
|
|
280
343
|
const policy = policyFor(fault.kind);
|
|
281
344
|
if (attempt >= policy.max) {
|
|
282
345
|
recordFault(this.deps.nodeId, markFatalOnExhaust(fault));
|
|
346
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
283
347
|
return;
|
|
284
348
|
}
|
|
285
349
|
const plan = nextFaultRetry(fault, Date.now());
|
|
286
350
|
if (plan === null) {
|
|
287
351
|
recordFault(this.deps.nodeId, markFatalOnExhaust(fault));
|
|
352
|
+
clearProviderRetryEpisode(this.deps.nodeId);
|
|
288
353
|
return;
|
|
289
354
|
}
|
|
290
355
|
if (fault.anchorEntryId !== undefined) {
|
|
@@ -301,7 +366,28 @@ export class FaultRetry {
|
|
|
301
366
|
}
|
|
302
367
|
if (this.deps.installedGeneration() !== generation || this.deps.currentSession() !== candidateSession)
|
|
303
368
|
return;
|
|
304
|
-
|
|
305
|
-
|
|
369
|
+
await this.deps.turnAdmission.admit((release) => {
|
|
370
|
+
const current = readFault(this.deps.nodeId);
|
|
371
|
+
if (this.deps.installedGeneration() !== generation ||
|
|
372
|
+
this.deps.currentSession() !== candidateSession ||
|
|
373
|
+
candidateSession.isStreaming ||
|
|
374
|
+
!this.isActiveAutoFault(current) ||
|
|
375
|
+
current.since !== fault.since) {
|
|
376
|
+
release();
|
|
377
|
+
return Promise.resolve();
|
|
378
|
+
}
|
|
379
|
+
// This is the admission point: all generation/session/streaming/fault
|
|
380
|
+
// revalidation above has passed, and only now may the durable episode
|
|
381
|
+
// become admitted. Writing it before prompt() fails closed across a broker
|
|
382
|
+
// death in the narrow handoff to the SDK — replaying there could duplicate
|
|
383
|
+
// an already accepted recovery turn.
|
|
384
|
+
const sessionFile = this.sessionFile(candidateSession);
|
|
385
|
+
if (sessionFile !== null && !admitProviderRetryEpisode(this.deps.nodeId, sessionFile, plan.fault)) {
|
|
386
|
+
release();
|
|
387
|
+
return Promise.resolve();
|
|
388
|
+
}
|
|
389
|
+
recordFault(this.deps.nodeId, plan.fault);
|
|
390
|
+
return operationIdContext.fresh(() => candidateSession.prompt(formatCard('recovery', {}, fault.kind === 'connection' ? CONNECTION_FAULT_RECOVERY_BODY : PROVIDER_FAULT_RECOVERY_BODY), { preflightResult: release }));
|
|
391
|
+
});
|
|
306
392
|
}
|
|
307
393
|
}
|
|
@@ -5,6 +5,7 @@ import type { BrokerClient, BrokerClientRegistry } from './client-registry.js';
|
|
|
5
5
|
import type { BrokerSession } from './read-ops.js';
|
|
6
6
|
import { type DriveFrame } from './engine-drive.js';
|
|
7
7
|
import { type PendingDialog } from './frame-client.js';
|
|
8
|
+
import type { TurnAdmissionGate } from './turn-admission.js';
|
|
8
9
|
import type { ToolGroupTracker } from './tool-groups.js';
|
|
9
10
|
import type { EventProjection } from './event-projection.js';
|
|
10
11
|
import type { RebindDriver } from './rebind.js';
|
|
@@ -28,6 +29,7 @@ export type FrameDispatchDeps = {
|
|
|
28
29
|
registryOf: (services: AgentSessionServices) => ModelRegistry;
|
|
29
30
|
formatModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
|
|
30
31
|
formatExactModelSpec: (model: BrokerSession['model'], thinkingLevel?: string) => string | null;
|
|
32
|
+
turnAdmission: TurnAdmissionGate;
|
|
31
33
|
resolveLaunchModel: (services: AgentSessionServices, cfg: BrokerSdkConfig) => Promise<{
|
|
32
34
|
model: NonNullable<ReturnType<ModelRegistry['find']>>;
|
|
33
35
|
thinkingLevel: string | undefined;
|
|
@@ -9,7 +9,7 @@ import { createSessionHandler } from './frame-session.js';
|
|
|
9
9
|
import { createKickoffDriver, createPromptDispatchTracker, createTurnAcceptanceGate, } from './turn-ignition.js';
|
|
10
10
|
/** Owns frame-local input state while dispatching through explicit broker collaborators. */
|
|
11
11
|
export function createFrameDispatchContext(deps) {
|
|
12
|
-
const { nodeId, cfg, registry, toolGroups, projection, rebind, boundaryReviewId, pendingDialogs, sendWelcome, replayExtraPendingDialogsTo, reWelcomeAll, disposeAndExit, persistModelChoice, broadcastModelChanged, registryOf, formatModelSpec, formatExactModelSpec, resolveLaunchModel, } = deps;
|
|
12
|
+
const { nodeId, cfg, registry, toolGroups, projection, rebind, boundaryReviewId, pendingDialogs, sendWelcome, replayExtraPendingDialogsTo, reWelcomeAll, disposeAndExit, persistModelChoice, broadcastModelChanged, registryOf, formatModelSpec, formatExactModelSpec, turnAdmission, resolveLaunchModel, } = deps;
|
|
13
13
|
const currentSession = () => rebind.session();
|
|
14
14
|
const currentServices = () => rebind.services();
|
|
15
15
|
// The reply vocabulary every handler below answers a client with.
|
|
@@ -21,6 +21,7 @@ export function createFrameDispatchContext(deps) {
|
|
|
21
21
|
const kickoffDriver = createKickoffDriver({
|
|
22
22
|
session: currentSession,
|
|
23
23
|
dispatches,
|
|
24
|
+
turnAdmission,
|
|
24
25
|
});
|
|
25
26
|
const authReloadGate = createAuthReloadGate({
|
|
26
27
|
nodeId,
|
|
@@ -33,6 +34,7 @@ export function createFrameDispatchContext(deps) {
|
|
|
33
34
|
formatModelSpec,
|
|
34
35
|
resolveLaunchModel,
|
|
35
36
|
driveKickoff: kickoffDriver.drivePending,
|
|
37
|
+
turnAdmission,
|
|
36
38
|
});
|
|
37
39
|
const driveEngine = createEngineDriver({
|
|
38
40
|
registry,
|
|
@@ -44,6 +46,7 @@ export function createFrameDispatchContext(deps) {
|
|
|
44
46
|
rebind,
|
|
45
47
|
projection,
|
|
46
48
|
toolGroups,
|
|
49
|
+
turnAdmission,
|
|
47
50
|
});
|
|
48
51
|
const handleDriveFrame = (client, frame) => {
|
|
49
52
|
if (notWritable(client, 'drive the engine'))
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
export declare const HELD_DEFERRED_INBOX_PROMPT_JOIN: unique symbol;
|
|
2
|
+
export type HeldDeferredInboxPromptJoin = (session: {
|
|
3
|
+
sendCustomMessage: (message: {
|
|
4
|
+
customType: string;
|
|
5
|
+
content: string;
|
|
6
|
+
display: boolean;
|
|
7
|
+
}) => Promise<void>;
|
|
8
|
+
}) => Promise<void>;
|
|
9
|
+
export declare function heldDeferredInboxPromptJoin(): HeldDeferredInboxPromptJoin | undefined;
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// The inbox watcher is a Pi extension loaded through Jiti, while the broker is
|
|
2
|
+
// native ESM. They therefore cannot share a module singleton. This process-global
|
|
3
|
+
// symbol names the watcher-owned join function across that loader boundary.
|
|
4
|
+
export const HELD_DEFERRED_INBOX_PROMPT_JOIN = Symbol.for('@crouton-kit/crtr:held-deferred-inbox-prompt-join');
|
|
5
|
+
export function heldDeferredInboxPromptJoin() {
|
|
6
|
+
const join = globalThis[HELD_DEFERRED_INBOX_PROMPT_JOIN];
|
|
7
|
+
return typeof join === 'function' ? join : undefined;
|
|
8
|
+
}
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import { existsSync, readFileSync } from 'node:fs';
|
|
4
4
|
import { inboxPath, nodeDir } from '../../canvas/paths.js';
|
|
5
5
|
import { formatInboxCard } from '../../../shared/generated-context.js';
|
|
6
|
+
import { renderBirthAnnouncementBody } from '../../../shared/birth-announcement.js';
|
|
6
7
|
const ENTRY_ID = /^(?!0{32}$)[0-9a-f]{32}$/;
|
|
7
8
|
const BODY_MAX_LINES = 12;
|
|
8
9
|
const BODY_MAX_CHARS = 1000;
|
|
@@ -140,6 +141,9 @@ function inlineReport(entry, reportNodes) {
|
|
|
140
141
|
/** One entry's card body: an inlined short report, a bounded inline message
|
|
141
142
|
* body, or the label when the entry carries neither. */
|
|
142
143
|
function entryBody(entry, reportNodes) {
|
|
144
|
+
const birth = renderBirthAnnouncementBody(entry.data, entry.ts);
|
|
145
|
+
if (birth !== null)
|
|
146
|
+
return birth;
|
|
143
147
|
const report = inlineReport(entry, reportNodes);
|
|
144
148
|
if (report !== null)
|
|
145
149
|
return report.trimEnd();
|
|
@@ -5,6 +5,7 @@ import { emitEvent } from '../../events/emit.js';
|
|
|
5
5
|
import { operationIdContext } from '../../events/operation-id.js';
|
|
6
6
|
import { installPersonaTransitionGate } from '../broker-persona-guidance.js';
|
|
7
7
|
import { abortSessionFromExtension } from './extension-abort.js';
|
|
8
|
+
import { clearFault, invalidateProviderRetryEpisode, readFault } from '../fault.js';
|
|
8
9
|
const MAX_ADMITTED_REPLACEMENTS = 2;
|
|
9
10
|
const REPLACEMENT_DEADLINE_MS = 30_000;
|
|
10
11
|
const REPLACEMENT_DEADLINE_EXCEEDED = Symbol('replacement-deadline-exceeded');
|
|
@@ -95,6 +96,22 @@ export class RebindDriver {
|
|
|
95
96
|
const task = this.rebindTail.then(async () => {
|
|
96
97
|
if (ticket !== this.latestRebindRequest)
|
|
97
98
|
return;
|
|
99
|
+
// A replacement changes the conversation coordinate. The initial bind has
|
|
100
|
+
// no prior generation and must retain a crash-surviving pending episode;
|
|
101
|
+
// every later rebind explicitly marks the old session's episode invalid.
|
|
102
|
+
if (this.installed !== undefined) {
|
|
103
|
+
if (typeof this.sessionValue.sessionFile === 'string' && this.sessionValue.sessionFile !== '') {
|
|
104
|
+
invalidateProviderRetryEpisode(this.deps.nodeId, this.sessionValue.sessionFile);
|
|
105
|
+
}
|
|
106
|
+
// A daemon-owned auto marker belongs to the outgoing session whether
|
|
107
|
+
// that session has a durable path or not. Retaining it lets the
|
|
108
|
+
// replacement inherit an old anchor, attempts, and backoff; fatal and
|
|
109
|
+
// manual markers stay visible.
|
|
110
|
+
const fault = readFault(this.deps.nodeId);
|
|
111
|
+
if (fault?.link === 'pi→provider' && fault.retry.disposition === 'auto' && fault.retry.by === 'daemon') {
|
|
112
|
+
clearFault(this.deps.nodeId, { link: 'pi→provider', preserveProviderRetryEpisode: true });
|
|
113
|
+
}
|
|
114
|
+
}
|
|
98
115
|
this.installed?.executingTools.clear();
|
|
99
116
|
this.removeInstalledSubscription();
|
|
100
117
|
this.deps.setActiveSession(candidateSession);
|
|
@@ -192,6 +209,7 @@ export class RebindDriver {
|
|
|
192
209
|
this.executeSubscriptionCallback(generation, event);
|
|
193
210
|
});
|
|
194
211
|
this.installed = generation;
|
|
212
|
+
this.deps.projection().rearmIfStreaming(generation);
|
|
195
213
|
this.captureSystemPrompt(candidateSession);
|
|
196
214
|
});
|
|
197
215
|
this.rebindTail = task.catch(() => undefined);
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Serializes entry into AgentSession.prompt() until its preflight has decided
|
|
3
|
+
* whether it will start (or join) a run. AgentSession.isStreaming stays false
|
|
4
|
+
* through asynchronous preflight, so checking it before entering prompt() lets
|
|
5
|
+
* two calls reach the SDK's active-run ownership race. Each waiter therefore
|
|
6
|
+
* re-reads live state only after the preceding prompt has reported preflight.
|
|
7
|
+
*/
|
|
8
|
+
export type TurnAdmissionGate = {
|
|
9
|
+
admit<T>(start: (release: () => void) => Promise<T>): Promise<T>;
|
|
10
|
+
};
|
|
11
|
+
export declare function createTurnAdmissionGate(): TurnAdmissionGate;
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export function createTurnAdmissionGate() {
|
|
2
|
+
let tail = Promise.resolve();
|
|
3
|
+
return {
|
|
4
|
+
admit(start) {
|
|
5
|
+
const previous = tail;
|
|
6
|
+
let resolveAdmission;
|
|
7
|
+
const admitted = new Promise((resolve) => {
|
|
8
|
+
resolveAdmission = resolve;
|
|
9
|
+
});
|
|
10
|
+
tail = previous.then(() => admitted);
|
|
11
|
+
return previous.then(() => {
|
|
12
|
+
let released = false;
|
|
13
|
+
const release = () => {
|
|
14
|
+
if (released)
|
|
15
|
+
return;
|
|
16
|
+
released = true;
|
|
17
|
+
resolveAdmission();
|
|
18
|
+
};
|
|
19
|
+
try {
|
|
20
|
+
const attempt = start(release);
|
|
21
|
+
void attempt.then(release, release);
|
|
22
|
+
return attempt;
|
|
23
|
+
}
|
|
24
|
+
catch (error) {
|
|
25
|
+
release();
|
|
26
|
+
throw error;
|
|
27
|
+
}
|
|
28
|
+
});
|
|
29
|
+
},
|
|
30
|
+
};
|
|
31
|
+
}
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { BrokerSession } from './read-ops.js';
|
|
2
|
+
import type { TurnAdmissionGate } from './turn-admission.js';
|
|
2
3
|
/**
|
|
3
4
|
* Prompt dispatches whose promise has not yet settled. `session.prompt()` does
|
|
4
5
|
* async setup (input transforms, model/auth validation, compaction) BEFORE
|
|
@@ -39,4 +40,5 @@ export type KickoffDriver = {
|
|
|
39
40
|
export declare function createKickoffDriver(deps: {
|
|
40
41
|
session: () => BrokerSession;
|
|
41
42
|
dispatches: PromptDispatchTracker;
|
|
43
|
+
turnAdmission: TurnAdmissionGate;
|
|
42
44
|
}): KickoffDriver;
|
|
@@ -41,28 +41,38 @@ export function createTurnAcceptanceGate() {
|
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
43
|
export function createKickoffDriver(deps) {
|
|
44
|
-
const { session, dispatches } = deps;
|
|
44
|
+
const { session, dispatches, turnAdmission } = deps;
|
|
45
45
|
let pendingPrompt;
|
|
46
46
|
let inFlight = false;
|
|
47
47
|
const sessionHasNoModel = () => session().model === undefined || isUnknownModel(session().model);
|
|
48
48
|
const drivePending = () => {
|
|
49
49
|
if (pendingPrompt === undefined || inFlight || sessionHasNoModel())
|
|
50
50
|
return;
|
|
51
|
-
const prompt = pendingPrompt;
|
|
52
|
-
pendingPrompt = undefined;
|
|
53
51
|
inFlight = true;
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
52
|
+
void turnAdmission.admit((release) => {
|
|
53
|
+
const prompt = pendingPrompt;
|
|
54
|
+
if (prompt === undefined || sessionHasNoModel()) {
|
|
55
|
+
release();
|
|
56
|
+
inFlight = false;
|
|
57
|
+
return Promise.resolve();
|
|
58
|
+
}
|
|
59
|
+
pendingPrompt = undefined;
|
|
60
|
+
return dispatches.track(operationIdContext.fresh(() => session()
|
|
61
|
+
.prompt(prompt, {
|
|
62
|
+
preflightResult: release,
|
|
63
|
+
...(session().isStreaming ? { streamingBehavior: 'steer' } : {}),
|
|
64
|
+
})
|
|
65
|
+
.catch((err) => {
|
|
66
|
+
emitEvent({
|
|
67
|
+
level: 'error',
|
|
68
|
+
event: 'broker.prompt.first_failed',
|
|
69
|
+
error: err,
|
|
70
|
+
});
|
|
71
|
+
})
|
|
72
|
+
.finally(() => {
|
|
73
|
+
inFlight = false;
|
|
74
|
+
})));
|
|
75
|
+
});
|
|
66
76
|
};
|
|
67
77
|
return {
|
|
68
78
|
kickoff(prompt) {
|
|
@@ -35,7 +35,7 @@ import { resolveBundledPiPackageDir } from './pi-cli.js';
|
|
|
35
35
|
import { captureSessionChatCapabilities } from './command-surface.js';
|
|
36
36
|
import { PoolCredentialStore } from '../pool-credential-store.js';
|
|
37
37
|
import { clearCleanAbort, markBusy, markCleanAbort } from './busy.js';
|
|
38
|
-
import { recordFaultForCurrentNode } from './fault.js';
|
|
38
|
+
import { clearFault, recordFaultForCurrentNode } from './fault.js';
|
|
39
39
|
import { modelRequestFromConfig, selectInitialModelRoute, unregisteredConcreteModel } from '../model-routes.js';
|
|
40
40
|
import { KNOWN_STREAM_WAIT_UI } from './stream-watchdog.js';
|
|
41
41
|
import { probeOnline as probeOnlineReal } from './connectivity.js';
|
|
@@ -46,6 +46,7 @@ import { BrokerClientRegistry } from './broker/client-registry.js';
|
|
|
46
46
|
import { ToolGroupTracker } from './broker/tool-groups.js';
|
|
47
47
|
import { onNodeNamed } from './broker/node-named.js';
|
|
48
48
|
import { FaultRetry } from './broker/fault-retry.js';
|
|
49
|
+
import { createTurnAdmissionGate } from './broker/turn-admission.js';
|
|
49
50
|
import { EventProjection } from './broker/event-projection.js';
|
|
50
51
|
import { createFrameDispatchContext, handleFrame, } from './broker/frame-dispatch.js';
|
|
51
52
|
import { hydratePersistedAdvertisedCommandMessages, installAdvertisedCommandInvocationContract, } from './advertised-command-invocation.js';
|
|
@@ -325,6 +326,7 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
|
|
|
325
326
|
const spec = persist ? await persistModelChoice(pinnedOverride, userSelected) : liveSpec;
|
|
326
327
|
registry.broadcast({ type: 'model_changed', model: isUnknownModel(liveSession.model) ? undefined : liveSession.model, spec });
|
|
327
328
|
};
|
|
329
|
+
const turnAdmission = createTurnAdmissionGate();
|
|
328
330
|
faultRetry = new FaultRetry({
|
|
329
331
|
nodeId,
|
|
330
332
|
cfg,
|
|
@@ -335,6 +337,7 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
|
|
|
335
337
|
registryOf,
|
|
336
338
|
formatModelSpec,
|
|
337
339
|
broadcastModelChanged,
|
|
340
|
+
turnAdmission,
|
|
338
341
|
});
|
|
339
342
|
projection = new EventProjection({
|
|
340
343
|
registry,
|
|
@@ -527,7 +530,7 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
|
|
|
527
530
|
nodeId, cfg, registry, toolGroups, projection, rebind, boundaryReviewId,
|
|
528
531
|
pendingDialogs, sendWelcome, replayExtraPendingDialogsTo, reWelcomeAll, disposeAndExit,
|
|
529
532
|
persistModelChoice, broadcastModelChanged, registryOf, formatModelSpec,
|
|
530
|
-
formatExactModelSpec, resolveLaunchModel,
|
|
533
|
+
formatExactModelSpec, turnAdmission, resolveLaunchModel,
|
|
531
534
|
});
|
|
532
535
|
notifyTurnAccepted = frameDispatch.notifyTurnAccepted;
|
|
533
536
|
resetFrameState = frameDispatch.resetForSession;
|
|
@@ -591,6 +594,22 @@ export async function runBroker(nodeId, startupAt = process.hrtime.bigint(), sta
|
|
|
591
594
|
disposeAndExit('socket-error', 1);
|
|
592
595
|
});
|
|
593
596
|
server.listen(sockPath, () => {
|
|
597
|
+
// Binding view.sock is the exact inverse of a `connect ENOENT view.sock`
|
|
598
|
+
// fault, so the moment the listener accepts, any socket-absence marker a
|
|
599
|
+
// viewer/relay recorded during the revive→bind gap is provably stale. The
|
|
600
|
+
// broker owns this socket, so it clears those markers here rather than
|
|
601
|
+
// waiting for a client to reconnect — a headless node with nobody attached
|
|
602
|
+
// never gets that reconnect, which is how the stale ⚠ leaked. A genuine
|
|
603
|
+
// post-bind failure is re-recorded by the next failed client op.
|
|
604
|
+
clearFault(nodeId, { link: 'viewer↔broker' });
|
|
605
|
+
clearFault(nodeId, { link: 'relay↔broker' });
|
|
606
|
+
// The listener is also the broker startup boundary: a pending provider
|
|
607
|
+
// episode recovered after an instance death is re-armed only once this
|
|
608
|
+
// replacement can accept its normal viewer/daemon traffic. Independent of
|
|
609
|
+
// the link faults cleared above — those are viewer/relay socket markers,
|
|
610
|
+
// this is the provider retry episode. Admitted and invalidated records are
|
|
611
|
+
// intentionally inert inside FaultRetry.
|
|
612
|
+
faultRetry.recoverPendingOnStartup();
|
|
594
613
|
// A launch prompt delivers only after the listener is accepting, so a
|
|
595
614
|
// root-spawn readiness proof and prompt admission share one boundary. This
|
|
596
615
|
// includes the one continuation carried by a clean runtime-abort resume.
|
|
@@ -1,13 +1,14 @@
|
|
|
1
1
|
import { type SubscriptionRef } from '../canvas/index.js';
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
|
|
2
|
+
import { type InboxTier } from '../feed/inbox.js';
|
|
3
|
+
/** Fan a lifecycle or birth notice to every given subscriber. Active subscribers
|
|
4
|
+
* receive inbox mail; passive subscribers receive passive mail. Shared by
|
|
5
|
+
* terminal-outcome paths and externally created children, so callers preserve
|
|
6
|
+
* the existing subscription's delivery mode rather than inventing a wake path.
|
|
7
|
+
*
|
|
8
|
+
* `tier` defaults to a waking `normal`. A caller whose outcome is not worth
|
|
9
|
+
* starting a manager's cycle passes `deferred`, which rides the manager's next
|
|
10
|
+
* natural cycle instead. */
|
|
11
|
+
export declare function fanDoctrineWake(fromId: string, subscribers: readonly SubscriptionRef[], label: string, data: Record<string, unknown>, tier?: InboxTier): void;
|
|
11
12
|
export interface CloseNodeResult {
|
|
12
13
|
/** The focused node that was closed — the cascade root. */
|
|
13
14
|
root: string;
|
|
@@ -31,4 +32,5 @@ export interface CloseNodeResult {
|
|
|
31
32
|
* status keeps it (finish is only legal from a live status). */
|
|
32
33
|
export declare function closeNode(rootId: string, opts?: {
|
|
33
34
|
rootEvent?: 'cancel' | 'finish';
|
|
35
|
+
callerPid?: number;
|
|
34
36
|
}): CloseNodeResult;
|
|
@@ -30,26 +30,33 @@
|
|
|
30
30
|
// inside the closing set. A node still subscribed to by a manager outside the
|
|
31
31
|
// subtree is left running — "only kill the children if they are only subscribed
|
|
32
32
|
// to by the agent being closed", generalized to any depth via a fixpoint.
|
|
33
|
-
import { getNode, subscriptionsOf, subscribersOf, } from '../canvas/index.js';
|
|
33
|
+
import { cancelCronsOnWake, getNode, subscriptionsOf, subscribersOf, } from '../canvas/index.js';
|
|
34
34
|
import { transition } from './lifecycle.js';
|
|
35
|
-
import {
|
|
35
|
+
import { requestBrokerTeardown } from './host.js';
|
|
36
36
|
import { tearDownNode, reapIfEmpty } from './placement.js';
|
|
37
37
|
import { appendInbox } from '../feed/inbox.js';
|
|
38
38
|
import { appendPassive } from '../feed/passive.js';
|
|
39
|
-
/**
|
|
40
|
-
*
|
|
41
|
-
*
|
|
42
|
-
*
|
|
43
|
-
*
|
|
44
|
-
*
|
|
45
|
-
*
|
|
46
|
-
*
|
|
47
|
-
export function fanDoctrineWake(fromId, subscribers, label, data) {
|
|
39
|
+
/** Fan a lifecycle or birth notice to every given subscriber. Active subscribers
|
|
40
|
+
* receive inbox mail; passive subscribers receive passive mail. Shared by
|
|
41
|
+
* terminal-outcome paths and externally created children, so callers preserve
|
|
42
|
+
* the existing subscription's delivery mode rather than inventing a wake path.
|
|
43
|
+
*
|
|
44
|
+
* `tier` defaults to a waking `normal`. A caller whose outcome is not worth
|
|
45
|
+
* starting a manager's cycle passes `deferred`, which rides the manager's next
|
|
46
|
+
* natural cycle instead. */
|
|
47
|
+
export function fanDoctrineWake(fromId, subscribers, label, data, tier = 'normal') {
|
|
48
48
|
for (const sub of subscribers) {
|
|
49
49
|
try {
|
|
50
|
-
const notice = { from: fromId, tier
|
|
51
|
-
if (sub.active)
|
|
50
|
+
const notice = { from: fromId, tier, kind: 'message', label, data };
|
|
51
|
+
if (sub.active) {
|
|
52
52
|
appendInbox(sub.node_id, notice);
|
|
53
|
+
// A wake-capable notice consumes the subscriber's cancel-on-wake
|
|
54
|
+
// deadline: the thing it was waiting for arrived. `deferred` is the
|
|
55
|
+
// exception — it never wakes a node, riding the next natural cycle
|
|
56
|
+
// instead, so the deadline it was racing must survive.
|
|
57
|
+
if (tier !== 'deferred')
|
|
58
|
+
cancelCronsOnWake(sub.node_id);
|
|
59
|
+
}
|
|
53
60
|
else
|
|
54
61
|
appendPassive(sub.node_id, notice);
|
|
55
62
|
}
|
|
@@ -190,12 +197,9 @@ export function closeNode(rootId, opts = {}) {
|
|
|
190
197
|
else {
|
|
191
198
|
transition(id, 'cancel', { reason: 'closed' });
|
|
192
199
|
}
|
|
193
|
-
// 2)
|
|
194
|
-
//
|
|
195
|
-
|
|
196
|
-
// deliberate close the viewer must be torn down here or it lingers ~30s
|
|
197
|
-
// showing a misleading "reconnecting…" instead of going away at once.
|
|
198
|
-
headlessBrokerHost.teardown(id);
|
|
200
|
+
// 2) Ask the daemon to tear the engine down after this close response
|
|
201
|
+
// settles. The root caller's shell is excluded from its own broker tree.
|
|
202
|
+
requestBrokerTeardown(id, id === rootId ? { excludePid: opts.callerPid } : {});
|
|
199
203
|
tearDownNode(id);
|
|
200
204
|
// 3) Leave the resume notice AFTER the watcher is gone, so it survives.
|
|
201
205
|
appendInbox(id, {
|
|
@@ -11,15 +11,36 @@ export interface FaultRecordInput {
|
|
|
11
11
|
since?: string;
|
|
12
12
|
anchorEntryId?: string;
|
|
13
13
|
}
|
|
14
|
+
/** A provider-retry episode outlives its broker-local timer. Its transcript
|
|
15
|
+
* path and anchor are explicit coordinates, so replacement invalidation never
|
|
16
|
+
* has to guess which conversation a marker belongs to. */
|
|
17
|
+
export interface ProviderRetryEpisode {
|
|
18
|
+
state: 'pending' | 'admitted' | 'invalidated';
|
|
19
|
+
sessionFile: string;
|
|
20
|
+
anchorEntryId: string | null;
|
|
21
|
+
fault: Fault;
|
|
22
|
+
}
|
|
14
23
|
export declare function recordFault(nodeId: string, input: FaultRecordInput): void;
|
|
24
|
+
/** Atomically establishes the crash-recovery authority before publishing its
|
|
25
|
+
* ordinary marker, so a broker death can never leave only a retryable fault. */
|
|
26
|
+
export declare function recordPendingProviderRetryFault(nodeId: string, sessionFile: string, input: FaultRecordInput): Fault | null;
|
|
15
27
|
export declare function recordFaultForCurrentNode(input: FaultRecordInput): boolean;
|
|
16
28
|
export declare function clearFault(nodeId: string, opts?: {
|
|
17
29
|
link?: FaultLink;
|
|
30
|
+
preserveProviderRetryEpisode?: boolean;
|
|
18
31
|
}): boolean;
|
|
19
32
|
export declare function clearFaultForCurrentNode(opts?: {
|
|
20
33
|
link?: FaultLink;
|
|
21
34
|
}): boolean;
|
|
22
35
|
export declare function readFault(nodeId: string): Fault | null;
|
|
36
|
+
export declare function readProviderRetryEpisode(nodeId: string): ProviderRetryEpisode | null;
|
|
37
|
+
/** Persist a newly scheduled provider retry before its broker-local timer is armed. */
|
|
38
|
+
export declare function recordPendingProviderRetryEpisode(nodeId: string, episode: Omit<ProviderRetryEpisode, 'state'>): boolean;
|
|
39
|
+
/** Admission is the irreversible boundary: an admitted retry is never replayed. */
|
|
40
|
+
export declare function admitProviderRetryEpisode(nodeId: string, sessionFile: string, fault: Fault): boolean;
|
|
41
|
+
/** A replacement explicitly invalidates its old transcript's retry episode. */
|
|
42
|
+
export declare function invalidateProviderRetryEpisode(nodeId: string, sessionFile: string): boolean;
|
|
43
|
+
export declare function clearProviderRetryEpisode(nodeId: string): void;
|
|
23
44
|
export declare function readFaultAsync(nodeId: string): Promise<Fault | null>;
|
|
24
45
|
export interface BootFaultAttempt {
|
|
25
46
|
/** The recorded fault's operation + message (e.g. an unresolvable `--model`
|