@north-light/crouter 0.3.282 → 0.3.283
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.d.ts +19 -3
- package/dist/api/client.js +34 -0
- package/dist/api/dto/broker-ops.d.ts +11 -0
- package/dist/api/dto/broker.d.ts +3 -8
- package/dist/api/dto/common.d.ts +1 -1
- package/dist/api/dto/messages.d.ts +2 -2
- package/dist/api/dto/node-outcomes.d.ts +80 -0
- package/dist/api/dto/node-outcomes.js +2 -0
- package/dist/api/dto/nodes.d.ts +9 -0
- package/dist/api/dto/profiles.d.ts +5 -0
- package/dist/api/dto/reports.d.ts +45 -0
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +1 -0
- package/dist/api/routes.d.ts +7 -0
- package/dist/api/routes.js +7 -0
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +24 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +31 -4
- package/dist/clients/attach/overlays/graph.d.ts +5 -0
- package/dist/clients/attach/overlays/graph.js +65 -47
- package/dist/clients/attach/overlays/pickers.js +8 -9
- package/dist/clients/attach/render/card-presentation.js +1 -1
- package/dist/clients/attach/viewer.js +1286 -1260
- package/dist/commands/api-client.js +22 -23
- package/dist/commands/node/create.js +38 -3
- package/dist/commands/node/inspect.js +3 -7
- package/dist/commands/node/message.js +3 -3
- package/dist/commands/node/outcome.d.ts +2 -0
- package/dist/commands/node/outcome.js +134 -0
- package/dist/commands/node.js +2 -1
- package/dist/commands/profile/pause.js +17 -14
- package/dist/commands/push.js +102 -1
- package/dist/commands/sys/__tests__/setup-core.test.js +4 -1
- package/dist/commands/sys/panels/broker-limits-panel.js +7 -1
- package/dist/commands/sys/panels/models-panel.js +2 -2
- package/dist/commands/sys/settings-shell.d.ts +1 -1
- package/dist/commands/sys/settings-shell.js +6 -0
- package/dist/commands/sys/settings.js +3 -3
- package/dist/core/__tests__/fixtures/fake-engine.d.ts +2 -0
- package/dist/core/__tests__/fixtures/fake-engine.js +42 -4
- package/dist/core/__tests__/helpers/harness.d.ts +1 -0
- package/dist/core/__tests__/helpers/harness.js +9 -0
- package/dist/core/__tests__/integration/profile-pause.test.d.ts +1 -0
- package/dist/core/__tests__/integration/profile-pause.test.js +115 -0
- package/dist/core/__tests__/node-outcome.test.d.ts +1 -0
- package/dist/core/__tests__/node-outcome.test.js +184 -0
- package/dist/core/__tests__/revive-parked-fresh.test.js +13 -20
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +6 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +3 -3
- package/dist/core/__tests__/seam/node-outcome-delivery.test.d.ts +1 -0
- package/dist/core/__tests__/seam/node-outcome-delivery.test.js +153 -0
- package/dist/core/bash-jobs.d.ts +6 -0
- package/dist/core/bash-jobs.js +22 -0
- package/dist/core/canvas/canvas.d.ts +41 -3
- package/dist/core/canvas/canvas.js +153 -39
- package/dist/core/canvas/crons.js +2 -1
- package/dist/core/canvas/index.d.ts +1 -0
- package/dist/core/canvas/index.js +1 -0
- package/dist/core/canvas/migrations.js +47 -0
- package/dist/core/canvas/node-outcome-deliveries.d.ts +80 -0
- package/dist/core/canvas/node-outcome-deliveries.js +163 -0
- package/dist/core/canvas/pid.d.ts +7 -0
- package/dist/core/canvas/pid.js +17 -0
- package/dist/core/canvas/render-source.js +2 -2
- package/dist/core/canvas/types.d.ts +42 -1
- package/dist/core/canvas/types.js +2 -1
- package/dist/core/config.d.ts +4 -1
- package/dist/core/config.js +25 -1
- package/dist/core/feed/feed.d.ts +8 -0
- package/dist/core/feed/feed.js +25 -4
- package/dist/core/feed/inbox.d.ts +5 -0
- package/dist/core/feed/inbox.js +13 -0
- package/dist/core/keybindings/catalog.d.ts +1 -1
- package/dist/core/keybindings/catalog.js +3 -0
- package/dist/core/profiles/manifest.d.ts +3 -1
- package/dist/core/profiles/manifest.js +6 -1
- package/dist/core/runtime/broker/daemon-ops.d.ts +3 -1
- package/dist/core/runtime/broker/daemon-ops.js +6 -0
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -3
- package/dist/core/runtime/broker/engine-drive.js +74 -27
- package/dist/core/runtime/broker/frame-dispatch.js +1 -0
- package/dist/core/runtime/broker/read-ops.js +14 -0
- package/dist/core/runtime/broker/rebind.js +0 -7
- package/dist/core/runtime/broker-protocol.d.ts +1 -6
- package/dist/core/runtime/fleet.d.ts +7 -0
- package/dist/core/runtime/headless-pi.d.ts +13 -0
- package/dist/core/runtime/headless-pi.js +10 -2
- package/dist/core/runtime/lifecycle.d.ts +6 -1
- package/dist/core/runtime/lifecycle.js +50 -1
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +2 -1
- package/dist/core/runtime/outcome-document.d.ts +70 -0
- package/dist/core/runtime/outcome-document.js +97 -0
- package/dist/core/runtime/revive.js +16 -21
- package/dist/core/runtime/spawn.d.ts +5 -1
- package/dist/core/runtime/spawn.js +18 -7
- package/dist/core/runtime/stamp/channel.d.ts +3 -5
- package/dist/core/runtime/stamp/channel.js +3 -10
- package/dist/core/runtime/stop-guard.js +2 -2
- package/dist/core/runtime/structured-output.d.ts +24 -3
- package/dist/core/runtime/structured-output.js +42 -3
- package/dist/core/runtime/tmux-bindings.js +4 -0
- package/dist/core/runtime/warm-pool.d.ts +5 -2
- package/dist/core/runtime/warm-pool.js +5 -5
- package/dist/core/subscription-state.js +7 -5
- package/dist/core/substrate/render.js +10 -10
- package/dist/core/user-settings.d.ts +18 -1
- package/dist/core/user-settings.js +37 -0
- package/dist/daemon/__tests__/node-deadline-lane.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-deadline-lane.test.js +76 -0
- package/dist/daemon/__tests__/node-outcome-birth-invariants.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-outcome-birth-invariants.test.js +122 -0
- package/dist/daemon/__tests__/node-outcome-delivery.test.d.ts +1 -0
- package/dist/daemon/__tests__/node-outcome-delivery.test.js +86 -0
- package/dist/daemon/api/__tests__/broker-settle-park.test.js +86 -18
- package/dist/daemon/api/__tests__/profile-launch-gates.test.js +10 -1
- package/dist/daemon/api/__tests__/seam/node-outcome-api.test.d.ts +1 -0
- package/dist/daemon/api/__tests__/seam/node-outcome-api.test.js +159 -0
- package/dist/daemon/api/__tests__/submit-result.test.d.ts +1 -0
- package/dist/daemon/api/__tests__/submit-result.test.js +214 -0
- package/dist/daemon/api/handlers/broker-ops.js +53 -24
- package/dist/daemon/api/handlers/canvas.js +2 -2
- package/dist/daemon/api/handlers/messages.js +12 -2
- package/dist/daemon/api/handlers/node-outcomes.d.ts +2 -0
- package/dist/daemon/api/handlers/node-outcomes.js +136 -0
- package/dist/daemon/api/handlers/nodes.js +67 -1
- package/dist/daemon/api/handlers/profiles.js +53 -1
- package/dist/daemon/api/handlers/reports.js +197 -33
- package/dist/daemon/api/map.d.ts +7 -2
- package/dist/daemon/api/map.js +78 -4
- package/dist/daemon/api/server.js +2 -0
- package/dist/daemon/crtrd.js +12 -0
- package/dist/daemon/fleet.d.ts +4 -8
- package/dist/daemon/fleet.js +48 -9
- package/dist/daemon/messaging/node-message.js +2 -1
- package/dist/daemon/node-outcome/deliver-outcome.d.ts +14 -0
- package/dist/daemon/node-outcome/deliver-outcome.js +155 -0
- package/dist/daemon/park-activity.d.ts +9 -0
- package/dist/daemon/park-activity.js +30 -0
- package/dist/daemon/park-pending.d.ts +10 -5
- package/dist/daemon/park-pending.js +18 -8
- package/dist/daemon/reconcilers/bash-deadline.js +6 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +3 -7
- package/dist/daemon/reconcilers/broker-supervision.js +29 -33
- package/dist/daemon/reconcilers/node-deadline.d.ts +5 -0
- package/dist/daemon/reconcilers/node-deadline.js +82 -0
- package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +9 -1
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +12 -3
- package/dist/daemon/reconcilers/node-outcome-delivery-lane.d.ts +10 -0
- package/dist/daemon/reconcilers/node-outcome-delivery-lane.js +41 -0
- package/dist/pi-extensions/__tests__/canvas-structured-output.test.js +66 -66
- package/dist/pi-extensions/canvas-structured-output.d.ts +9 -5
- package/dist/pi-extensions/canvas-structured-output.js +23 -297
- package/dist/shared/env.d.ts +3 -0
- package/dist/shared/env.js +6 -0
- package/dist/shared/generated-context.d.ts +3 -9
- package/dist/shared/generated-context.js +8 -16
- package/dist/types.d.ts +12 -0
- package/dist/types.js +1 -0
- package/package.json +8 -8
- package/runtime.lock.json +227 -383
package/dist/daemon/crtrd.js
CHANGED
|
@@ -50,6 +50,7 @@ import { listNodes, migrateLegacyPidIdentities, } from '../core/canvas/index.js'
|
|
|
50
50
|
import { createApiServer } from './api/server.js';
|
|
51
51
|
import { reconcileUndeliveredTickets } from './human/sweep.js';
|
|
52
52
|
import { recoverRunningActionDeliveries } from '../core/canvas/human-deliveries.js';
|
|
53
|
+
import { recoverRunningOutcomeDeliveries } from '../core/canvas/node-outcome-deliveries.js';
|
|
53
54
|
import { sweepReviews } from './review/sweep.js';
|
|
54
55
|
import { pidfilePath } from './pidfile.js';
|
|
55
56
|
import { rendererWarning } from '../core/runtime/package-health.js';
|
|
@@ -57,8 +58,10 @@ import { DEFAULT_INTERVAL_MS } from './supervise-cadence.js';
|
|
|
57
58
|
import { BrokerSupervisionReconciler } from './reconcilers/broker-supervision.js';
|
|
58
59
|
import { ControllerDeathReconciler } from './reconcilers/controller-death.js';
|
|
59
60
|
import { NodeLifecycleReconciler } from './reconcilers/node-lifecycle/tick.js';
|
|
61
|
+
import { NodeDeadlineReconciler } from './reconcilers/node-deadline.js';
|
|
60
62
|
import { CronLaneReconciler } from './reconcilers/cron-lane.js';
|
|
61
63
|
import { HumanDeliveryLaneReconciler } from './reconcilers/human-delivery-lane.js';
|
|
64
|
+
import { NodeOutcomeDeliveryLaneReconciler } from './reconcilers/node-outcome-delivery-lane.js';
|
|
62
65
|
import { PendingReviewSubmitReconciler } from './reconcilers/pending-review-submit.js';
|
|
63
66
|
import { StorageMaintenanceReconciler } from './reconcilers/storage-maintenance.js';
|
|
64
67
|
import { BashDeadlineReconciler } from './reconcilers/bash-deadline.js';
|
|
@@ -272,8 +275,10 @@ const directTickLifecycle = {
|
|
|
272
275
|
const brokerSupervision = new BrokerSupervisionReconciler();
|
|
273
276
|
const controllerDeath = new ControllerDeathReconciler();
|
|
274
277
|
const nodeLifecycle = new NodeLifecycleReconciler();
|
|
278
|
+
const nodeDeadline = new NodeDeadlineReconciler();
|
|
275
279
|
const cronLane = new CronLaneReconciler();
|
|
276
280
|
const humanDeliveryLane = new HumanDeliveryLaneReconciler();
|
|
281
|
+
const nodeOutcomeDeliveryLane = new NodeOutcomeDeliveryLaneReconciler();
|
|
277
282
|
const pendingReviewSubmit = new PendingReviewSubmitReconciler();
|
|
278
283
|
const storageMaintenance = new StorageMaintenanceReconciler();
|
|
279
284
|
const bashDeadline = new BashDeadlineReconciler();
|
|
@@ -301,8 +306,12 @@ export async function superviseTick(now = Date.now(), lifecycle = directTickLife
|
|
|
301
306
|
await brokerSupervision.run(now, { rows, fleet, lifecycle });
|
|
302
307
|
controllerDeath.run(now, { rows });
|
|
303
308
|
const frozen = await nodeLifecycle.run(now, { rows: [...rows, ...terminating], fleet });
|
|
309
|
+
// The universal backstop: a node past its deadline with no outcome is
|
|
310
|
+
// canceled and terminalized here, after the lifecycle pass has had its say.
|
|
311
|
+
nodeDeadline.run(now);
|
|
304
312
|
cronLane.run(now, { lifecycle });
|
|
305
313
|
humanDeliveryLane.run(now, { lifecycle });
|
|
314
|
+
nodeOutcomeDeliveryLane.run(now, { lifecycle });
|
|
306
315
|
pendingReviewSubmit.run(now, { lifecycle });
|
|
307
316
|
storageMaintenance.run(now, { lifecycle, fleet });
|
|
308
317
|
await bashDeadline.run(now, { rows });
|
|
@@ -640,6 +649,9 @@ export async function runDaemon(opts = {}) {
|
|
|
640
649
|
await reconcileUndeliveredTickets();
|
|
641
650
|
// Unlike a cron one-shot, a completion is at-least-once: destinations deduplicate request IDs, so a dead daemon's running delivery is re-queued immediately.
|
|
642
651
|
recoverRunningActionDeliveries(Date.now());
|
|
652
|
+
// Same at-least-once contract for node outcome deliveries: a delivery the
|
|
653
|
+
// dead daemon had marked running is re-queued rather than stranded.
|
|
654
|
+
recoverRunningOutcomeDeliveries(Date.now());
|
|
643
655
|
});
|
|
644
656
|
// Boot reconciliation clears stale process identities before any tick can
|
|
645
657
|
// inspect them, then migrates legacy identities only with proven same-boot
|
package/dist/daemon/fleet.d.ts
CHANGED
|
@@ -1,13 +1,7 @@
|
|
|
1
1
|
import { type NodeMeta } from '../core/canvas/index.js';
|
|
2
2
|
import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
|
|
3
|
+
import type { NodeOutcomeDetailV1, TerminalReason } from '../core/canvas/types.js';
|
|
3
4
|
import type { HostHandle } from '../core/runtime/host.js';
|
|
4
|
-
/** Surface a vehicle that never booted.
|
|
5
|
-
*
|
|
6
|
-
* A pi that dies before its first session_start (so `pi_session_id` was never
|
|
7
|
-
* recorded) is invisible to its parent — the spawn returned optimistically.
|
|
8
|
-
* When the daemon observes that death it errors LOUDLY up the spine: an
|
|
9
|
-
* urgent push so the parent learns the child failed to launch instead of just
|
|
10
|
-
* seeing a silent `dead`. */
|
|
11
5
|
export declare function surfaceBootFailure(meta: NodeMeta): Promise<void>;
|
|
12
6
|
/** Whether a dead relaunch should resume the saved session strictly, or retry
|
|
13
7
|
* as a fresh cycle instead. A cycling attempt (cycle_pending) that dies before
|
|
@@ -69,6 +63,8 @@ export declare class DaemonFleet implements FleetRegistry {
|
|
|
69
63
|
reserveSlot(nodeId: string): boolean;
|
|
70
64
|
releaseReservation(nodeId: string): void;
|
|
71
65
|
entries(): Iterable<[string, FleetEntry]>;
|
|
66
|
+
interruptForProfilePause(nodeId: string, pid?: number | undefined): void;
|
|
67
|
+
hasProfilePauseInterruption(nodeId: string): boolean;
|
|
72
68
|
forget(nodeId: string): void;
|
|
73
69
|
onChildExit(nodeId: string, status: FleetExitStatus): void;
|
|
74
70
|
deliverExit(nodeId: string, status: FleetExitStatus): void;
|
|
@@ -78,6 +74,6 @@ export declare class DaemonFleet implements FleetRegistry {
|
|
|
78
74
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
79
75
|
* terminal outcome as a boot failure. The fresh read avoids duplicating
|
|
80
76
|
* reviveNode's launch-failure crash path. */
|
|
81
|
-
terminalizeIfStillNonTerminal(nodeId: string, event: string, error: unknown): Promise<boolean>;
|
|
77
|
+
terminalizeIfStillNonTerminal(nodeId: string, event: string, error: unknown, reason: TerminalReason, detail: NodeOutcomeDetailV1): Promise<boolean>;
|
|
82
78
|
dispose(): void;
|
|
83
79
|
}
|
package/dist/daemon/fleet.js
CHANGED
|
@@ -12,6 +12,7 @@
|
|
|
12
12
|
// before every deliberate exit, and that durable state is robust to every kill
|
|
13
13
|
// path. Codes/signals are logged for diagnostics only.
|
|
14
14
|
import { getNode, getRow, subscribersOf } from '../core/canvas/index.js';
|
|
15
|
+
import { isProfilePaused } from '../core/profiles/manifest.js';
|
|
15
16
|
import { fullName } from '../core/canvas/labels.js';
|
|
16
17
|
import { isSafeNodeId } from '../core/canvas/paths.js';
|
|
17
18
|
import { transition } from '../core/runtime/lifecycle.js';
|
|
@@ -33,14 +34,16 @@ import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
|
|
|
33
34
|
* When the daemon observes that death it errors LOUDLY up the spine: an
|
|
34
35
|
* urgent push so the parent learns the child failed to launch instead of just
|
|
35
36
|
* seeing a silent `dead`. */
|
|
36
|
-
|
|
37
|
-
|
|
37
|
+
function bootFailureBody(meta) {
|
|
38
|
+
return (`⚠ Spawn failed — \`${meta.name}\` (${meta.kind}) never started.\n\n` +
|
|
38
39
|
`Its pi vehicle exited before the session came up (no pi_session_id was ever ` +
|
|
39
40
|
`recorded), so the node produced no output. This is almost always a transient ` +
|
|
40
41
|
`launch failure — e.g. resource pressure when several nodes boot at once — not ` +
|
|
41
42
|
`a fault in the task itself.\n\n` +
|
|
42
|
-
`If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time
|
|
43
|
-
|
|
43
|
+
`If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`);
|
|
44
|
+
}
|
|
45
|
+
export async function surfaceBootFailure(meta) {
|
|
46
|
+
await pushUrgent(meta.node_id, bootFailureBody(meta), { from: meta.node_id });
|
|
44
47
|
}
|
|
45
48
|
async function surfaceCrashLoop(meta) {
|
|
46
49
|
const transportPoison = readTransportPoisonVerdict(meta.node_id);
|
|
@@ -131,6 +134,9 @@ export class DaemonFleet {
|
|
|
131
134
|
#map = new Map();
|
|
132
135
|
/** Slots claimed by launches that have not registered a broker yet. */
|
|
133
136
|
#reservations = new Set();
|
|
137
|
+
/** Brokers deliberately interrupted by profile pause. This belongs to the
|
|
138
|
+
* exact child, so a resume cannot turn its later SIGTERM exit into a crash. */
|
|
139
|
+
#profilePauseInterruptions = new Set();
|
|
134
140
|
#lastCapacityLogAt = Number.NEGATIVE_INFINITY;
|
|
135
141
|
constructor(opts) {
|
|
136
142
|
this.#epoch = opts.epoch;
|
|
@@ -191,6 +197,21 @@ export class DaemonFleet {
|
|
|
191
197
|
entries() {
|
|
192
198
|
return this.#map.entries();
|
|
193
199
|
}
|
|
200
|
+
interruptForProfilePause(nodeId, pid = this.#map.get(nodeId)?.pid) {
|
|
201
|
+
if (pid === undefined)
|
|
202
|
+
return;
|
|
203
|
+
this.#profilePauseInterruptions.add(nodeId);
|
|
204
|
+
try {
|
|
205
|
+
process.kill(pid, 'SIGTERM');
|
|
206
|
+
}
|
|
207
|
+
catch {
|
|
208
|
+
// A live handle or identity-checked PID can exit between its observation
|
|
209
|
+
// and signal. The marker still makes that exact observed exit harmless.
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
hasProfilePauseInterruption(nodeId) {
|
|
213
|
+
return this.#profilePauseInterruptions.has(nodeId);
|
|
214
|
+
}
|
|
194
215
|
forget(nodeId) {
|
|
195
216
|
this.#map.delete(nodeId);
|
|
196
217
|
this.#reservations.delete(nodeId);
|
|
@@ -217,12 +238,14 @@ export class DaemonFleet {
|
|
|
217
238
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
218
239
|
* terminal outcome as a boot failure. The fresh read avoids duplicating
|
|
219
240
|
* reviveNode's launch-failure crash path. */
|
|
220
|
-
async terminalizeIfStillNonTerminal(nodeId, event, error) {
|
|
241
|
+
async terminalizeIfStillNonTerminal(nodeId, event, error, reason, detail) {
|
|
221
242
|
const meta = getNode(nodeId);
|
|
222
243
|
if (meta === null || meta.status === 'done' || meta.status === 'canceled' || meta.status === 'dead')
|
|
223
244
|
return false;
|
|
224
245
|
await this.#terminalize(nodeId, meta, {
|
|
225
246
|
event,
|
|
247
|
+
reason,
|
|
248
|
+
detail,
|
|
226
249
|
wake: `Child crashed — ${fullName(meta)} (${nodeId}) could not be relaunched and is now dead. ` +
|
|
227
250
|
`It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
|
|
228
251
|
surface: () => surfaceRecoveryFailure(meta, error),
|
|
@@ -249,6 +272,17 @@ export class DaemonFleet {
|
|
|
249
272
|
}
|
|
250
273
|
async #decideAndEnact(nodeId) {
|
|
251
274
|
const meta = getNode(nodeId);
|
|
275
|
+
if (meta !== null && isProfilePaused(meta.profile_id)) {
|
|
276
|
+
clearRespawnPolicy(nodeId);
|
|
277
|
+
return 'forget';
|
|
278
|
+
}
|
|
279
|
+
// Consume one unpaused policy pass for the exact broker profile pause
|
|
280
|
+
// interrupted. The following tick owns ordinary recovery regardless of
|
|
281
|
+
// which dead-row verdict the still-recorded PID would otherwise select.
|
|
282
|
+
if (this.#profilePauseInterruptions.delete(nodeId)) {
|
|
283
|
+
clearRespawnPolicy(nodeId);
|
|
284
|
+
return 'forget';
|
|
285
|
+
}
|
|
252
286
|
// Two durable witnesses mean the dead broker had a turn in flight. A dirty
|
|
253
287
|
// death leaves the busy marker behind; a bounded graceful abort clears busy
|
|
254
288
|
// as the run settles, then writes the clean-abort certificate. Both need the
|
|
@@ -284,6 +318,8 @@ export class DaemonFleet {
|
|
|
284
318
|
case 'boot-failure':
|
|
285
319
|
await this.#terminalize(nodeId, meta, {
|
|
286
320
|
event: 'broker.boot.failed',
|
|
321
|
+
reason: 'boot_failed',
|
|
322
|
+
detail: { schema: 'crtr.node-outcome-detail/v1', message: bootFailureBody(meta) },
|
|
287
323
|
wake: `Child crashed — ${fullName(meta)} (${nodeId}) exited before its session ever came up and is now dead. ` +
|
|
288
324
|
`It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
|
|
289
325
|
});
|
|
@@ -328,10 +364,13 @@ export class DaemonFleet {
|
|
|
328
364
|
? recordRespawnPolicy(row, this.#now())
|
|
329
365
|
: { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
|
|
330
366
|
if (attempt.exhausted) {
|
|
367
|
+
const wake = `Child crashed — ${fullName(meta)} (${nodeId}) kept dying without settling a turn (${attempt.failures} consecutive ` +
|
|
368
|
+
`launches) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`;
|
|
331
369
|
await this.#terminalize(nodeId, meta, {
|
|
332
370
|
event: 'broker.respawn.exhausted',
|
|
333
|
-
|
|
334
|
-
|
|
371
|
+
reason: 'crash_looped',
|
|
372
|
+
detail: { schema: 'crtr.node-outcome-detail/v1', message: wake, respawn_failures: attempt.failures },
|
|
373
|
+
wake,
|
|
335
374
|
...(meta.pi_session_id != null ? { surface: () => surfaceCrashLoop(meta) } : {}),
|
|
336
375
|
});
|
|
337
376
|
return;
|
|
@@ -392,7 +431,7 @@ export class DaemonFleet {
|
|
|
392
431
|
// Launch failures terminalize inside reviveNode, but its preflight
|
|
393
432
|
// refusals (for example cwd_missing) happen before that catch. Re-read
|
|
394
433
|
// before terminalizing so the launch path's settled terminal row wins.
|
|
395
|
-
await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err);
|
|
434
|
+
await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err, 'launch_failed', { schema: 'crtr.node-outcome-detail/v1', message: err instanceof Error ? err.message : String(err) });
|
|
396
435
|
}
|
|
397
436
|
}
|
|
398
437
|
/** Terminal outcome shared by table row 8 and respawn exhaustion. */
|
|
@@ -402,7 +441,7 @@ export class DaemonFleet {
|
|
|
402
441
|
event: opts.event,
|
|
403
442
|
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : { fields: { affected_node_id: nodeId } }),
|
|
404
443
|
});
|
|
405
|
-
transition(nodeId, 'crash');
|
|
444
|
+
transition(nodeId, 'crash', { reason: opts.reason, outcome: { detail: opts.detail } });
|
|
406
445
|
try {
|
|
407
446
|
await (opts.surface ?? (() => surfaceBootFailure(meta)))();
|
|
408
447
|
}
|
|
@@ -9,6 +9,7 @@ import { reviveNode } from '../../core/runtime/revive.js';
|
|
|
9
9
|
import { hasNoNaturalCycle } from '../../core/runtime/revive-all.js';
|
|
10
10
|
import { ReviewOperationError } from '../../core/review/types.js';
|
|
11
11
|
import { formatCard } from '../../shared/generated-context.js';
|
|
12
|
+
import { isProfilePaused } from '../../core/profiles/manifest.js';
|
|
12
13
|
function deliveryUnroutable(nodeId, reason) {
|
|
13
14
|
throw new ReviewOperationError('delivery_unroutable', reason === 'origin_missing' ? 'message target was not found' : 'message target has no natural delivery cycle', { node_id: nodeId, reason });
|
|
14
15
|
}
|
|
@@ -33,7 +34,7 @@ export async function deliverNodeMessage(args) {
|
|
|
33
34
|
let target = getNode(args.node_id);
|
|
34
35
|
if (target === null)
|
|
35
36
|
deliveryUnroutable(args.node_id, 'origin_missing');
|
|
36
|
-
if (args.mode === 'interactive' && isBrokerLive(target)) {
|
|
37
|
+
if (args.mode === 'interactive' && !isProfilePaused(target.profile_id) && isBrokerLive(target)) {
|
|
37
38
|
try {
|
|
38
39
|
// Inbox delivery supplies the raw body as an entry in its outer inbox
|
|
39
40
|
// card; only live delivery needs this message's own envelope.
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type OutcomeDeliveryRecord, type OutcomeDeliverySettlement } from '../../core/canvas/node-outcome-deliveries.js';
|
|
2
|
+
export declare const OUTCOME_DELIVERY_TIMEOUT_MS = 120000;
|
|
3
|
+
export declare const OUTCOME_DELIVERY_STDERR_TAIL_BYTES: number;
|
|
4
|
+
export interface OutcomeDeliveryProcessOutcome {
|
|
5
|
+
exitCode: number | null;
|
|
6
|
+
signal: string | null;
|
|
7
|
+
timedOut: boolean;
|
|
8
|
+
spawnError?: Error;
|
|
9
|
+
stderr: string;
|
|
10
|
+
}
|
|
11
|
+
/** The delivery target's exit code is its complete acknowledgment protocol. */
|
|
12
|
+
export declare function outcomeDeliverySettlementForOutcome(outcome: OutcomeDeliveryProcessOutcome, attempt: number, now: number): OutcomeDeliverySettlement;
|
|
13
|
+
/** Spawn and settle one already-claimed outcome delivery. */
|
|
14
|
+
export declare function deliverOutcome(delivery: OutcomeDeliveryRecord, claimOwner: string): Promise<void>;
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { envHomeOverride } from '../../shared/env.js';
|
|
3
|
+
import { outcomeDeliveryBackoffMs, settleOutcomeDelivery, } from '../../core/canvas/node-outcome-deliveries.js';
|
|
4
|
+
import { killProcessGroup } from '../../core/canvas/pid.js';
|
|
5
|
+
import { buildOperationalEnvBase } from '../../core/runtime/spawn-env.js';
|
|
6
|
+
export const OUTCOME_DELIVERY_TIMEOUT_MS = 120_000;
|
|
7
|
+
const OUTCOME_DELIVERY_KILL_GRACE_MS = 2_000;
|
|
8
|
+
export const OUTCOME_DELIVERY_STDERR_TAIL_BYTES = 4 * 1024;
|
|
9
|
+
/** The delivery target's exit code is its complete acknowledgment protocol. */
|
|
10
|
+
export function outcomeDeliverySettlementForOutcome(outcome, attempt, now) {
|
|
11
|
+
if (outcome.spawnError !== undefined) {
|
|
12
|
+
return {
|
|
13
|
+
state: 'pending',
|
|
14
|
+
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
15
|
+
failure: failure('spawn_error', { message: outcome.spawnError.message, stderr: outcome.stderr }),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
if (outcome.timedOut) {
|
|
19
|
+
return {
|
|
20
|
+
state: 'pending',
|
|
21
|
+
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
22
|
+
failure: failure('timeout', { ...(outcome.signal === null ? {} : { signal: outcome.signal }), stderr: outcome.stderr }),
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
if (outcome.signal !== null) {
|
|
26
|
+
return {
|
|
27
|
+
state: 'pending',
|
|
28
|
+
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
29
|
+
failure: failure('signal', { signal: outcome.signal, stderr: outcome.stderr }),
|
|
30
|
+
};
|
|
31
|
+
}
|
|
32
|
+
if (outcome.exitCode === 0)
|
|
33
|
+
return { state: 'accepted' };
|
|
34
|
+
if (outcome.exitCode === 78) {
|
|
35
|
+
return { state: 'permanent_failed', failure: failure('exit', { exitCode: 78, stderr: outcome.stderr }) };
|
|
36
|
+
}
|
|
37
|
+
return {
|
|
38
|
+
state: 'pending',
|
|
39
|
+
nextAttemptAt: now + outcomeDeliveryBackoffMs(attempt),
|
|
40
|
+
failure: failure('exit', { exitCode: outcome.exitCode ?? undefined, stderr: outcome.stderr }),
|
|
41
|
+
};
|
|
42
|
+
}
|
|
43
|
+
function failure(kind, fields) {
|
|
44
|
+
return { kind, ...fields };
|
|
45
|
+
}
|
|
46
|
+
function stderrTailCapture() {
|
|
47
|
+
let tail = Buffer.alloc(0);
|
|
48
|
+
return {
|
|
49
|
+
take(chunk) {
|
|
50
|
+
tail = chunk.length >= OUTCOME_DELIVERY_STDERR_TAIL_BYTES
|
|
51
|
+
? chunk.subarray(chunk.length - OUTCOME_DELIVERY_STDERR_TAIL_BYTES)
|
|
52
|
+
: Buffer.concat([tail, chunk]).subarray(Math.max(0, tail.length + chunk.length - OUTCOME_DELIVERY_STDERR_TAIL_BYTES));
|
|
53
|
+
},
|
|
54
|
+
decode() {
|
|
55
|
+
return tail.toString('utf8');
|
|
56
|
+
},
|
|
57
|
+
};
|
|
58
|
+
}
|
|
59
|
+
/** The stored document is immutable; attempt is its only per-delivery mutation. */
|
|
60
|
+
function documentForAttempt(document, attempt) {
|
|
61
|
+
const outcome = document;
|
|
62
|
+
const delivery = outcome.delivery;
|
|
63
|
+
return JSON.stringify({ ...outcome, delivery: { ...delivery, attempt } });
|
|
64
|
+
}
|
|
65
|
+
/** Spawn and settle one already-claimed outcome delivery. */
|
|
66
|
+
export async function deliverOutcome(delivery, claimOwner) {
|
|
67
|
+
const stderr = stderrTailCapture();
|
|
68
|
+
let document;
|
|
69
|
+
try {
|
|
70
|
+
document = documentForAttempt(delivery.document, delivery.attempt);
|
|
71
|
+
}
|
|
72
|
+
catch (error) {
|
|
73
|
+
settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome({
|
|
74
|
+
exitCode: null,
|
|
75
|
+
signal: null,
|
|
76
|
+
timedOut: false,
|
|
77
|
+
spawnError: error,
|
|
78
|
+
stderr: stderr.decode(),
|
|
79
|
+
}, delivery.attempt, Date.now()));
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
let child;
|
|
83
|
+
try {
|
|
84
|
+
child = spawn(delivery.argv[0], delivery.argv.slice(1), {
|
|
85
|
+
cwd: delivery.cwd,
|
|
86
|
+
env: outcomeDeliveryEnv(delivery.cwd),
|
|
87
|
+
detached: true,
|
|
88
|
+
stdio: ['pipe', 'pipe', 'pipe'],
|
|
89
|
+
});
|
|
90
|
+
}
|
|
91
|
+
catch (error) {
|
|
92
|
+
settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome({
|
|
93
|
+
exitCode: null,
|
|
94
|
+
signal: null,
|
|
95
|
+
timedOut: false,
|
|
96
|
+
spawnError: error,
|
|
97
|
+
stderr: stderr.decode(),
|
|
98
|
+
}, delivery.attempt, Date.now()));
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
await new Promise((resolve) => {
|
|
102
|
+
let settled = false;
|
|
103
|
+
let timedOut = false;
|
|
104
|
+
let killTimer;
|
|
105
|
+
let timeout;
|
|
106
|
+
const finish = (outcome) => {
|
|
107
|
+
if (settled)
|
|
108
|
+
return;
|
|
109
|
+
settled = true;
|
|
110
|
+
if (timeout !== undefined)
|
|
111
|
+
clearTimeout(timeout);
|
|
112
|
+
// A timed-out leader can exit while a SIGTERM-ignoring group member remains.
|
|
113
|
+
if (!outcome.timedOut && killTimer !== undefined)
|
|
114
|
+
clearTimeout(killTimer);
|
|
115
|
+
child.stdin?.destroy();
|
|
116
|
+
child.stdout?.destroy();
|
|
117
|
+
child.stderr?.destroy();
|
|
118
|
+
settle(delivery, claimOwner, outcomeDeliverySettlementForOutcome(outcome, delivery.attempt, Date.now()));
|
|
119
|
+
resolve();
|
|
120
|
+
};
|
|
121
|
+
child.stdout?.on('data', () => { }); // Drain output; stdout is never a control channel.
|
|
122
|
+
child.stderr?.on('data', (chunk) => stderr.take(chunk));
|
|
123
|
+
child.stdin?.on('error', () => { }); // The child may exit before consuming stdin; its exit code is authoritative.
|
|
124
|
+
child.on('error', (error) => {
|
|
125
|
+
finish({ exitCode: null, signal: null, timedOut: false, spawnError: error, stderr: stderr.decode() });
|
|
126
|
+
});
|
|
127
|
+
child.on('exit', (exitCode, signal) => {
|
|
128
|
+
finish({ exitCode, signal, timedOut, stderr: stderr.decode() });
|
|
129
|
+
});
|
|
130
|
+
const killWithEscalation = () => {
|
|
131
|
+
if (child.pid === undefined)
|
|
132
|
+
return;
|
|
133
|
+
killProcessGroup(child.pid);
|
|
134
|
+
killTimer = setTimeout(() => killProcessGroup(child.pid, 'SIGKILL'), OUTCOME_DELIVERY_KILL_GRACE_MS);
|
|
135
|
+
killTimer.unref?.();
|
|
136
|
+
};
|
|
137
|
+
timeout = setTimeout(() => {
|
|
138
|
+
timedOut = true;
|
|
139
|
+
killWithEscalation();
|
|
140
|
+
}, OUTCOME_DELIVERY_TIMEOUT_MS);
|
|
141
|
+
timeout.unref?.();
|
|
142
|
+
child.stdin?.end(document, 'utf8');
|
|
143
|
+
});
|
|
144
|
+
}
|
|
145
|
+
/** Outcome targets cross the same consent boundary as every other crouter spawn. */
|
|
146
|
+
function outcomeDeliveryEnv(cwd) {
|
|
147
|
+
const env = buildOperationalEnvBase({ targetCwd: cwd, targetProfileId: null });
|
|
148
|
+
const canvasHome = envHomeOverride();
|
|
149
|
+
if (canvasHome !== undefined && canvasHome !== '')
|
|
150
|
+
env['CRTR_HOME'] = canvasHome;
|
|
151
|
+
return env;
|
|
152
|
+
}
|
|
153
|
+
function settle(delivery, claimOwner, settlement) {
|
|
154
|
+
settleOutcomeDelivery(delivery.nodeId, delivery.attempt, claimOwner, settlement, Date.now());
|
|
155
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { NodeMeta } from '../core/canvas/types.js';
|
|
2
|
+
import type { PendingPark } from './park-pending.js';
|
|
3
|
+
/** A main-session write after parking was armed is durable evidence of live
|
|
4
|
+
* input even when the turn has since settled and the viewer detached. */
|
|
5
|
+
export declare function mainSessionActivitySince(sessionFile: string | null | undefined, startedAt: number): boolean;
|
|
6
|
+
/** One verdict shared by normal completion and grace expiry. A broker records
|
|
7
|
+
* accepted live input against the process-local marker before it drives Pi, so
|
|
8
|
+
* the verdict cannot race a viewer, live delivery, or durable wake. */
|
|
9
|
+
export declare function shouldParkAfterSummary(nodeId: string, node: Pick<NodeMeta, 'lifecycle' | 'pi_session_file'>, pending: PendingPark): boolean;
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { statSync } from 'node:fs';
|
|
2
|
+
import { emitEvent } from '../core/events/emit.js';
|
|
3
|
+
import { hasWakeMailSince } from '../core/feed/inbox.js';
|
|
4
|
+
import { isBusy } from '../core/runtime/busy.js';
|
|
5
|
+
import { parkEvent } from './reconcilers/live-obligation.js';
|
|
6
|
+
/** A main-session write after parking was armed is durable evidence of live
|
|
7
|
+
* input even when the turn has since settled and the viewer detached. */
|
|
8
|
+
export function mainSessionActivitySince(sessionFile, startedAt) {
|
|
9
|
+
if (sessionFile === null || sessionFile === undefined)
|
|
10
|
+
return false;
|
|
11
|
+
try {
|
|
12
|
+
return statSync(sessionFile).mtimeMs >= startedAt;
|
|
13
|
+
}
|
|
14
|
+
catch (error) {
|
|
15
|
+
if (error.code === 'ENOENT')
|
|
16
|
+
return false;
|
|
17
|
+
emitEvent({ level: 'warn', event: 'broker.park.session_activity_scan_failed', error });
|
|
18
|
+
return true;
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/** One verdict shared by normal completion and grace expiry. A broker records
|
|
22
|
+
* accepted live input against the process-local marker before it drives Pi, so
|
|
23
|
+
* the verdict cannot race a viewer, live delivery, or durable wake. */
|
|
24
|
+
export function shouldParkAfterSummary(nodeId, node, pending) {
|
|
25
|
+
return !pending.activity
|
|
26
|
+
&& !isBusy(nodeId)
|
|
27
|
+
&& !mainSessionActivitySince(node.pi_session_file, pending.startedAt)
|
|
28
|
+
&& !hasWakeMailSince(nodeId, pending.startedAt)
|
|
29
|
+
&& parkEvent(nodeId, node.lifecycle) === 'park';
|
|
30
|
+
}
|
|
@@ -1,13 +1,18 @@
|
|
|
1
|
+
export type PendingPark = {
|
|
2
|
+
startedAt: number;
|
|
3
|
+
activity: boolean;
|
|
4
|
+
};
|
|
1
5
|
/** Arm a pending park for `nodeId`. The caller delivers the parking prompt. */
|
|
2
6
|
export declare function beginPendingPark(nodeId: string, now: number): void;
|
|
7
|
+
/** Record main-engine input accepted while the isolated turn is in flight. */
|
|
8
|
+
export declare function recordPendingParkActivity(nodeId: string): boolean;
|
|
3
9
|
export declare function isPendingPark(nodeId: string): boolean;
|
|
4
10
|
/** When the pending park was armed, or undefined when none is armed. */
|
|
5
11
|
export declare function pendingParkStartedAt(nodeId: string): number | undefined;
|
|
6
|
-
/** Delete-and-report
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
|
|
10
|
-
export declare function consumePendingPark(nodeId: string): boolean;
|
|
12
|
+
/** Delete-and-report the pending marker. Both the supervision tick and the
|
|
13
|
+
* settle handler run on the daemon thread with no await between their check and
|
|
14
|
+
* this call, so exactly one of them ever enacts a given park. */
|
|
15
|
+
export declare function consumePendingPark(nodeId: string): PendingPark | undefined;
|
|
11
16
|
/** Every node with a parking turn in flight — the reconciler's sweep drops the
|
|
12
17
|
* ones whose broker has left the fleet. */
|
|
13
18
|
export declare function pendingParkIds(): string[];
|
|
@@ -15,25 +15,35 @@
|
|
|
15
15
|
// and nothing needs repair. A durable in-flight flag would instead need its own
|
|
16
16
|
// reconciliation on every daemon start to avoid stranding a node that never got
|
|
17
17
|
// its prompt.
|
|
18
|
-
/** node id → the
|
|
18
|
+
/** node id → the pending parking turn and any accepted main-engine input. */
|
|
19
19
|
const pending = new Map();
|
|
20
20
|
/** Arm a pending park for `nodeId`. The caller delivers the parking prompt. */
|
|
21
21
|
export function beginPendingPark(nodeId, now) {
|
|
22
|
-
pending.set(nodeId, now);
|
|
22
|
+
pending.set(nodeId, { startedAt: now, activity: false });
|
|
23
|
+
}
|
|
24
|
+
/** Record main-engine input accepted while the isolated turn is in flight. */
|
|
25
|
+
export function recordPendingParkActivity(nodeId) {
|
|
26
|
+
const marker = pending.get(nodeId);
|
|
27
|
+
if (marker === undefined)
|
|
28
|
+
return false;
|
|
29
|
+
marker.activity = true;
|
|
30
|
+
return true;
|
|
23
31
|
}
|
|
24
32
|
export function isPendingPark(nodeId) {
|
|
25
33
|
return pending.has(nodeId);
|
|
26
34
|
}
|
|
27
35
|
/** When the pending park was armed, or undefined when none is armed. */
|
|
28
36
|
export function pendingParkStartedAt(nodeId) {
|
|
29
|
-
return pending.get(nodeId);
|
|
37
|
+
return pending.get(nodeId)?.startedAt;
|
|
30
38
|
}
|
|
31
|
-
/** Delete-and-report
|
|
32
|
-
*
|
|
33
|
-
*
|
|
34
|
-
* them ever enacts a given park. */
|
|
39
|
+
/** Delete-and-report the pending marker. Both the supervision tick and the
|
|
40
|
+
* settle handler run on the daemon thread with no await between their check and
|
|
41
|
+
* this call, so exactly one of them ever enacts a given park. */
|
|
35
42
|
export function consumePendingPark(nodeId) {
|
|
36
|
-
|
|
43
|
+
const marker = pending.get(nodeId);
|
|
44
|
+
if (marker !== undefined)
|
|
45
|
+
pending.delete(nodeId);
|
|
46
|
+
return marker;
|
|
37
47
|
}
|
|
38
48
|
/** Every node with a parking turn in flight — the reconciler's sweep drops the
|
|
39
49
|
* ones whose broker has left the fleet. */
|
|
@@ -5,6 +5,7 @@ import { cancelCronsOnWake } from '../../core/canvas/crons.js';
|
|
|
5
5
|
import { appendInbox } from '../../core/feed/inbox.js';
|
|
6
6
|
import { emitEvent } from '../../core/events/emit.js';
|
|
7
7
|
import { operationIdContext } from '../../core/events/operation-id.js';
|
|
8
|
+
import { isProfilePaused } from '../../core/profiles/manifest.js';
|
|
8
9
|
// How often the deadline scan runs. The valve arms a 60s default deadline, so a
|
|
9
10
|
// few seconds of enforcement latency is immaterial; scanning every tick would
|
|
10
11
|
// readdir every node's jobs dir needlessly.
|
|
@@ -28,6 +29,11 @@ export class BashDeadlineReconciler {
|
|
|
28
29
|
for (const row of ctx.rows) {
|
|
29
30
|
if (row.status === 'dead')
|
|
30
31
|
continue;
|
|
32
|
+
// A paused profile's node cannot claim anything: pause stopped its broker.
|
|
33
|
+
// Enforcing the window here would cancel the job for a silence the pause
|
|
34
|
+
// imposed, so the deadline waits and resume credits it the same interval.
|
|
35
|
+
if (isProfilePaused(row.profile_id))
|
|
36
|
+
continue;
|
|
31
37
|
const dir = contextDir(row.node_id);
|
|
32
38
|
let expired;
|
|
33
39
|
try {
|
|
@@ -61,16 +61,12 @@ export declare class BrokerSupervisionReconciler {
|
|
|
61
61
|
run(now: number, ctx: BrokerSupervisionContext): Promise<void>;
|
|
62
62
|
/** Every condition that resets the unattended clock EXCEPT `isBusy` — the
|
|
63
63
|
* node is unavailable to park, attended, or has work waiting. Short-circuits
|
|
64
|
-
* in cost order so the common tick does no file reads it can avoid.
|
|
65
|
-
* Evaluated for a pending park too, where a busy engine is precisely what is
|
|
66
|
-
* expected (it is writing the summary) and every other hit aborts. */
|
|
64
|
+
* in cost order so the common tick does no file reads it can avoid. */
|
|
67
65
|
private parkResetCondition;
|
|
68
66
|
private handleUnattendedParking;
|
|
69
67
|
/** Deliver the parking prompt over the live broker's serialized frame loop.
|
|
70
|
-
*
|
|
71
|
-
*
|
|
72
|
-
* `steer` route means a turn started between the idle gate and the frame, so
|
|
73
|
-
* the node is working again and its clock simply restarts. */
|
|
68
|
+
* An unreachable broker falls through to degraded parking; either prompt route
|
|
69
|
+
* starts the same isolated turn, so a concurrent main turn stays passive. */
|
|
74
70
|
private requestParkSummary;
|
|
75
71
|
private warnOverCapInTmux;
|
|
76
72
|
private handleYieldStall;
|