@north-light/crouter 0.3.248 → 0.3.250
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/__tests__/integration/client.test.js +28 -7
- package/dist/api/client.d.ts +1 -1
- package/dist/api/client.js +2 -2
- package/dist/api/dto/lifecycle.d.ts +3 -0
- package/dist/api/dto/messages.d.ts +4 -0
- package/dist/api/dto/nodes.d.ts +4 -0
- package/dist/api/dto/reports.d.ts +7 -2
- package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
- package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
- package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +1 -0
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
- package/dist/clients/inbox/controller.js +32 -5
- package/dist/clients/inbox/tui/ansi.d.ts +3 -1
- package/dist/clients/inbox/tui/ansi.js +14 -10
- package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
- package/dist/commands/api-client.d.ts +2 -2
- package/dist/commands/api-client.js +5 -5
- package/dist/commands/canvas-browse.js +1 -1
- package/dist/commands/node/create.js +3 -1
- package/dist/commands/node/inspect.js +1 -0
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node/message.js +6 -4
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/push.js +36 -37
- package/dist/commands/sys/daemon.js +21 -8
- package/dist/commands/sys/setup-core.js +2 -2
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
- package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
- package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +5 -1
- package/dist/core/__tests__/child-death-wake.test.js +11 -7
- package/dist/core/__tests__/collectible.test.d.ts +1 -0
- package/dist/core/__tests__/collectible.test.js +88 -0
- package/dist/core/__tests__/context-intro.test.js +4 -41
- package/dist/core/__tests__/daemon-boot.test.js +30 -14
- package/dist/core/__tests__/daemon-wedge.test.js +25 -19
- package/dist/core/__tests__/fixtures/fake-engine.js +11 -0
- package/dist/core/__tests__/human-deliver.test.js +5 -5
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
- package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
- package/dist/core/__tests__/integration/revive.test.js +48 -0
- package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
- package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
- package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
- package/dist/core/__tests__/migration.test.js +30 -0
- package/dist/core/__tests__/passive-subscription.test.js +39 -1
- package/dist/core/__tests__/prune.test.js +12 -3
- package/dist/core/__tests__/relaunch-root.test.js +1 -1
- package/dist/core/__tests__/respawn-throttle.test.js +16 -146
- package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
- package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
- package/dist/core/canvas/canvas.d.ts +9 -15
- package/dist/core/canvas/canvas.js +41 -47
- package/dist/core/canvas/crons.d.ts +5 -8
- package/dist/core/canvas/crons.js +5 -8
- package/dist/core/canvas/history.d.ts +5 -0
- package/dist/core/canvas/history.js +12 -1
- package/dist/core/canvas/migrations.js +58 -1
- package/dist/core/canvas/pid.d.ts +1 -1
- package/dist/core/canvas/pid.js +5 -2
- package/dist/core/canvas/types.d.ts +4 -0
- package/dist/core/feed/feed.d.ts +4 -0
- package/dist/core/feed/feed.js +13 -8
- package/dist/core/feed/inbox.d.ts +2 -1
- package/dist/core/feed/inbox.js +22 -3
- package/dist/core/git.d.ts +3 -0
- package/dist/core/git.js +3 -1
- package/dist/core/human/feedback-companion.js +1 -0
- package/dist/core/preview-registry.js +0 -1
- package/dist/core/review/companion.js +1 -0
- package/dist/core/review/realize.js +1 -0
- package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
- package/dist/core/runtime/broker/auth-reload.js +9 -3
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
- package/dist/core/runtime/broker/engine-drive.js +28 -23
- package/dist/core/runtime/broker/event-projection.d.ts +4 -0
- package/dist/core/runtime/broker/event-projection.js +43 -23
- package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
- package/dist/core/runtime/broker/fault-retry.js +94 -8
- package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
- package/dist/core/runtime/broker/frame-dispatch.js +4 -1
- package/dist/core/runtime/broker/inbox.js +4 -0
- package/dist/core/runtime/broker/rebind.js +18 -0
- package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
- package/dist/core/runtime/broker/turn-admission.js +31 -0
- package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
- package/dist/core/runtime/broker/turn-ignition.js +25 -15
- package/dist/core/runtime/broker.js +21 -2
- package/dist/core/runtime/close.d.ts +11 -9
- package/dist/core/runtime/close.js +23 -19
- package/dist/core/runtime/fault.d.ts +21 -0
- package/dist/core/runtime/fault.js +129 -19
- package/dist/core/runtime/fleet.d.ts +0 -6
- package/dist/core/runtime/host.d.ts +7 -1
- package/dist/core/runtime/host.js +20 -2
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +24 -1
- package/dist/core/runtime/placement.d.ts +6 -6
- package/dist/core/runtime/placement.js +15 -8
- package/dist/core/runtime/reset.js +2 -2
- package/dist/core/runtime/revive.js +40 -4
- package/dist/core/runtime/spawn.d.ts +3 -0
- package/dist/core/runtime/spawn.js +1 -0
- package/dist/core/runtime/warm-pool.d.ts +4 -0
- package/dist/core/runtime/warm-pool.js +2 -0
- package/dist/core/worktree.d.ts +4 -3
- package/dist/core/worktree.js +73 -27
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
- package/dist/daemon/api/handlers/bash-jobs.js +2 -0
- package/dist/daemon/api/handlers/broker-ops.js +23 -2
- package/dist/daemon/api/handlers/human.js +1 -1
- package/dist/daemon/api/handlers/messages.js +11 -2
- package/dist/daemon/api/handlers/nodes.js +13 -1
- package/dist/daemon/api/handlers/reports.js +14 -6
- package/dist/daemon/api/map.js +1 -0
- package/dist/daemon/api/server.d.ts +9 -12
- package/dist/daemon/api/server.js +30 -19
- package/dist/daemon/companion-retire.js +3 -9
- package/dist/daemon/cron/sinks.js +7 -2
- package/dist/daemon/crtrd.js +4 -1
- package/dist/daemon/fleet.d.ts +2 -16
- package/dist/daemon/fleet.js +63 -150
- package/dist/daemon/human/finish.js +2 -0
- package/dist/daemon/manage.d.ts +11 -0
- package/dist/daemon/manage.js +32 -3
- package/dist/daemon/messaging/node-message.js +7 -1
- package/dist/daemon/reconcilers/bash-deadline.js +2 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
- package/dist/daemon/reconcilers/broker-supervision.js +22 -9
- package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
- package/dist/daemon/reconcilers/live-obligation.js +18 -6
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
- package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
- package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
- package/dist/pi-extensions/canvas-context-intro.js +4 -76
- package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
- package/dist/pi-extensions/canvas-review-boundary.js +4 -29
- package/dist/pi-extensions/canvas-stophook.js +1 -6
- package/dist/shared/birth-announcement.d.ts +12 -0
- package/dist/shared/birth-announcement.js +25 -0
- package/dist/shared/generated-context.d.ts +8 -4
- package/dist/shared/generated-context.js +15 -9
- package/package.json +4 -4
- package/runtime.lock.json +2 -2
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
- package/dist/clients/inbox/tui/clipboard.d.ts +0 -2
- package/dist/clients/inbox/tui/clipboard.js +0 -26
- package/dist/pi-extensions/truncate.d.ts +0 -5
- package/dist/pi-extensions/truncate.js +0 -14
package/dist/daemon/fleet.js
CHANGED
|
@@ -1,32 +1,31 @@
|
|
|
1
1
|
// src/daemon/fleet.ts — the daemon's broker fleet: the ChildProcess-handle
|
|
2
2
|
// registry (the ONE normal liveness authority), the exit-policy classifier
|
|
3
|
-
// (invariant-5 policy table), and
|
|
3
|
+
// (invariant-5 policy table), and durable respawn recovery.
|
|
4
4
|
//
|
|
5
5
|
// Constructed by runDaemon with the serial work lane's `enqueue` and this
|
|
6
6
|
// daemon's epoch id, then injected into core via `bindFleet` BEFORE the API
|
|
7
|
-
// server or any tick starts. Exit events
|
|
8
|
-
//
|
|
9
|
-
// interleaves with a tick's row reads (design D-3).
|
|
7
|
+
// server or any tick starts. Exit events enqueue observation only; the durable
|
|
8
|
+
// row tick owns recovery policy.
|
|
10
9
|
//
|
|
11
10
|
// Row state (status/intent/busy marker/session identity), never exit codes,
|
|
12
11
|
// is the policy input (design D-4): the stophook writes the exit's explanation
|
|
13
12
|
// before every deliberate exit, and that durable state is robust to every kill
|
|
14
|
-
// path. Codes/signals are logged for diagnostics only.
|
|
15
|
-
|
|
16
|
-
import { getNode, subscribersOf } from '../core/canvas/index.js';
|
|
13
|
+
// path. Codes/signals are logged for diagnostics only.
|
|
14
|
+
import { getNode, getRow, subscribersOf } from '../core/canvas/index.js';
|
|
17
15
|
import { fullName } from '../core/canvas/labels.js';
|
|
18
16
|
import { isSafeNodeId } from '../core/canvas/paths.js';
|
|
19
17
|
import { transition } from '../core/runtime/lifecycle.js';
|
|
20
18
|
import { fanDoctrineWake } from '../core/runtime/close.js';
|
|
21
19
|
import { hasCleanAbort, isBusy } from '../core/runtime/busy.js';
|
|
22
20
|
import { hasPendingWake } from './reconcilers/live-obligation.js';
|
|
21
|
+
import { clearRespawnPolicy, MAX_RESPAWN_ATTEMPTS, recordRespawnPolicy, respawnBackoffMs } from './reconcilers/node-lifecycle/respawn-policy.js';
|
|
23
22
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
24
23
|
import { pushUrgent } from '../core/feed/feed.js';
|
|
25
24
|
import { emitEvent } from '../core/events/emit.js';
|
|
26
25
|
import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
|
|
27
26
|
// Shared crash helpers (moved from crtrd.ts — the fleet's policy paths are now
|
|
28
27
|
// their primary consumers; crtrd imports them back for its surviving passes).
|
|
29
|
-
/** Surface a vehicle that never booted
|
|
28
|
+
/** Surface a vehicle that never booted.
|
|
30
29
|
*
|
|
31
30
|
* A pi that dies before its first session_start (so `pi_session_id` was never
|
|
32
31
|
* recorded) is invisible to its parent — the spawn returned optimistically.
|
|
@@ -42,6 +41,12 @@ export async function surfaceBootFailure(meta) {
|
|
|
42
41
|
`If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`;
|
|
43
42
|
await pushUrgent(meta.node_id, body, { from: meta.node_id });
|
|
44
43
|
}
|
|
44
|
+
async function surfaceCrashLoop(meta) {
|
|
45
|
+
const body = `⚠ Crash loop — \`${meta.name}\` (${meta.kind}) kept dying without settling a turn.\n\n` +
|
|
46
|
+
`Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.\n\n` +
|
|
47
|
+
`Fix the crash cause, then revive the node if the work still needs doing.`;
|
|
48
|
+
await pushUrgent(meta.node_id, body, { from: meta.node_id });
|
|
49
|
+
}
|
|
45
50
|
/** Surface a recovery refusal that happened before a broker could launch. */
|
|
46
51
|
async function surfaceRecoveryFailure(meta, error) {
|
|
47
52
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -110,23 +115,6 @@ export function classifyDeadNode(row, busy, pendingWake = true) {
|
|
|
110
115
|
return 'respawn-fresh'; // row 7: bounded ordinary fork-birth retry
|
|
111
116
|
return 'boot-failure'; // row 8
|
|
112
117
|
}
|
|
113
|
-
// Respawn throttle (replaces the stranded-relaunch counters, event-driven)
|
|
114
|
-
/** An exit with uptime at or above this resets the consecutive-failure
|
|
115
|
-
* counter before classification; a shorter-lived exit increments it. */
|
|
116
|
-
export const HEALTHY_UPTIME_MS = 60_000;
|
|
117
|
-
/** At this many consecutive short-lived exits the respawn loop terminalizes:
|
|
118
|
-
* transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
|
|
119
|
-
export const MAX_RESPAWN_ATTEMPTS = 6;
|
|
120
|
-
const RESPAWN_BACKOFF_BASE_MS = 5_000;
|
|
121
|
-
const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
|
|
122
|
-
/** Backoff before the next respawn given the consecutive-failure count.
|
|
123
|
-
* Zero failures → immediate (the sub-second yield respawn D-3 buys); the
|
|
124
|
-
* n-th delayed attempt waits min(5s·2^n, 10min). */
|
|
125
|
-
export function respawnBackoffMs(consecutiveFailures) {
|
|
126
|
-
if (consecutiveFailures <= 0)
|
|
127
|
-
return 0;
|
|
128
|
-
return Math.min(RESPAWN_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESPAWN_BACKOFF_MAX_MS);
|
|
129
|
-
}
|
|
130
118
|
/** How long a capacity denial stays quiet before it is worth logging again. */
|
|
131
119
|
const CAPACITY_LOG_THROTTLE_MS = 30_000;
|
|
132
120
|
export class DaemonFleet {
|
|
@@ -134,7 +122,6 @@ export class DaemonFleet {
|
|
|
134
122
|
#enqueue;
|
|
135
123
|
#now;
|
|
136
124
|
#map = new Map();
|
|
137
|
-
#throttle = new Map();
|
|
138
125
|
/** Slots claimed but not yet held by a registered broker: a launch in flight,
|
|
139
126
|
* or a broker that exited and whose exit-policy job has not yet decided
|
|
140
127
|
* whether to relaunch it. Counting both against the cap is what keeps a
|
|
@@ -156,21 +143,8 @@ export class DaemonFleet {
|
|
|
156
143
|
if (handle.pid === null) {
|
|
157
144
|
throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
|
|
158
145
|
}
|
|
159
|
-
// A live registration supersedes any scheduled backoff respawn: the node
|
|
160
|
-
// is alive again (a manual revive won the race), so the pending respawn
|
|
161
|
-
// must not fire a second launch.
|
|
162
|
-
const throttle = this.#throttle.get(nodeId);
|
|
163
|
-
if (throttle?.timer !== undefined) {
|
|
164
|
-
clearTimeout(throttle.timer);
|
|
165
|
-
throttle.timer = undefined;
|
|
166
|
-
}
|
|
167
146
|
this.#reservations.delete(nodeId);
|
|
168
|
-
this.#map.set(nodeId, {
|
|
169
|
-
pid: handle.pid,
|
|
170
|
-
launchedAt: this.#now(),
|
|
171
|
-
handle,
|
|
172
|
-
consecutiveFailures: throttle?.failures ?? 0,
|
|
173
|
-
});
|
|
147
|
+
this.#map.set(nodeId, { pid: handle.pid, handle });
|
|
174
148
|
}
|
|
175
149
|
has(nodeId) {
|
|
176
150
|
return this.#map.has(nodeId);
|
|
@@ -212,28 +186,22 @@ export class DaemonFleet {
|
|
|
212
186
|
forget(nodeId) {
|
|
213
187
|
this.#map.delete(nodeId);
|
|
214
188
|
this.#reservations.delete(nodeId);
|
|
215
|
-
this.#clearThrottle(nodeId);
|
|
216
189
|
}
|
|
217
190
|
onChildExit(nodeId, status) {
|
|
218
191
|
// Move the slot from the registry to a reservation in ONE synchronous
|
|
219
192
|
// step: the exit-policy job about to be enqueued may relaunch this node,
|
|
220
|
-
// and holding the slot
|
|
221
|
-
//
|
|
222
|
-
// releases it on every path that does not relaunch.
|
|
223
|
-
const entry = this.#map.get(nodeId);
|
|
193
|
+
// and holding the slot through the exit observation prevents a concurrent
|
|
194
|
+
// launch from entering before the registry reflects the exit.
|
|
224
195
|
this.#map.delete(nodeId);
|
|
225
196
|
this.#reservations.add(nodeId);
|
|
226
|
-
|
|
227
|
-
this.#enqueue(() => this.#runExitPolicy(nodeId, status, launchedAt));
|
|
197
|
+
this.#enqueue(() => this.#runExitPolicy(nodeId, status));
|
|
228
198
|
}
|
|
229
199
|
deliverExit(nodeId, status) {
|
|
230
200
|
this.onChildExit(nodeId, status);
|
|
231
201
|
}
|
|
232
|
-
/** The node-lifecycle tick's enactment entry
|
|
233
|
-
* enact immediately through the SAME policy table exit events use, so tick
|
|
234
|
-
* recovery and exit recovery can never diverge. */
|
|
202
|
+
/** The node-lifecycle tick's enactment entry. */
|
|
235
203
|
async applyDeadRowPolicy(nodeId) {
|
|
236
|
-
await this.#applyPolicy(nodeId
|
|
204
|
+
await this.#applyPolicy(nodeId);
|
|
237
205
|
}
|
|
238
206
|
/** A revive refusal before its launch try/catch leaves the row active but
|
|
239
207
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
@@ -251,49 +219,17 @@ export class DaemonFleet {
|
|
|
251
219
|
});
|
|
252
220
|
return true;
|
|
253
221
|
}
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
#clearThrottle(nodeId) {
|
|
265
|
-
const state = this.#throttle.get(nodeId);
|
|
266
|
-
if (state?.timer !== undefined)
|
|
267
|
-
clearTimeout(state.timer);
|
|
268
|
-
this.#throttle.delete(nodeId);
|
|
269
|
-
}
|
|
270
|
-
async #runExitPolicy(nodeId, status, launchedAt) {
|
|
271
|
-
const uptimeMs = launchedAt === null ? null : Math.max(0, this.#now() - launchedAt);
|
|
272
|
-
// A newer launch may own this node by the time the lane reaches this job
|
|
273
|
-
// (an API revive raced the exit event onto the lane). The exit belonged to
|
|
274
|
-
// the OLD process; the node is alive — nothing to recover, but preserve the
|
|
275
|
-
// observation without charging the fresh launch's throttle budget.
|
|
276
|
-
if (this.#map.has(nodeId)) {
|
|
277
|
-
this.#emitExitObserved(nodeId, status, uptimeMs, this.#throttle.get(nodeId)?.failures ?? 0);
|
|
278
|
-
return;
|
|
279
|
-
}
|
|
280
|
-
const state = this.#throttle.get(nodeId) ?? { failures: 0 };
|
|
281
|
-
if (state.timer !== undefined) {
|
|
282
|
-
clearTimeout(state.timer);
|
|
283
|
-
state.timer = undefined;
|
|
284
|
-
}
|
|
285
|
-
// Healthy uptime resets the counter BEFORE classification; a short-lived
|
|
286
|
-
// exit (or a synthetic exit with no known launch time) increments it.
|
|
287
|
-
if (uptimeMs !== null && uptimeMs >= HEALTHY_UPTIME_MS) {
|
|
288
|
-
state.failures = 0;
|
|
289
|
-
this.#throttle.delete(nodeId);
|
|
222
|
+
dispose() { }
|
|
223
|
+
async #runExitPolicy(nodeId, status) {
|
|
224
|
+
try {
|
|
225
|
+
const launchedAt = getNode(nodeId)?.launched_at;
|
|
226
|
+
const launchedMs = launchedAt == null ? Number.NaN : Date.parse(launchedAt);
|
|
227
|
+
const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
|
|
228
|
+
this.#emitExitObserved(nodeId, status, uptimeMs, getNode(nodeId)?.respawn_failures ?? 0);
|
|
290
229
|
}
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
this.#throttle.set(nodeId, state);
|
|
230
|
+
finally {
|
|
231
|
+
this.releaseReservation(nodeId);
|
|
294
232
|
}
|
|
295
|
-
this.#emitExitObserved(nodeId, status, uptimeMs, state.failures);
|
|
296
|
-
await this.#applyPolicy(nodeId, 'exit');
|
|
297
233
|
}
|
|
298
234
|
#emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures) {
|
|
299
235
|
emitEvent({
|
|
@@ -309,19 +245,10 @@ export class DaemonFleet {
|
|
|
309
245
|
},
|
|
310
246
|
});
|
|
311
247
|
}
|
|
312
|
-
async #applyPolicy(nodeId
|
|
313
|
-
|
|
314
|
-
return await this.#decideAndEnact(nodeId, origin);
|
|
315
|
-
}
|
|
316
|
-
finally {
|
|
317
|
-
// The reservation this node has held since its exit is either consumed by
|
|
318
|
-
// a successful `register` (making this a no-op) or given back here — the
|
|
319
|
-
// one release path covering a declined policy, a deferred backoff, and
|
|
320
|
-
// terminalization alike.
|
|
321
|
-
this.releaseReservation(nodeId);
|
|
322
|
-
}
|
|
248
|
+
async #applyPolicy(nodeId) {
|
|
249
|
+
return this.#decideAndEnact(nodeId);
|
|
323
250
|
}
|
|
324
|
-
async #decideAndEnact(nodeId
|
|
251
|
+
async #decideAndEnact(nodeId) {
|
|
325
252
|
const meta = getNode(nodeId);
|
|
326
253
|
// Two durable witnesses mean the dead broker had a turn in flight. A dirty
|
|
327
254
|
// death leaves the busy marker behind; a bounded graceful abort clears busy
|
|
@@ -332,19 +259,19 @@ export class DaemonFleet {
|
|
|
332
259
|
switch (action) {
|
|
333
260
|
case 'forget':
|
|
334
261
|
// Fanouts for terminal exits already happened at the transition site.
|
|
335
|
-
|
|
262
|
+
clearRespawnPolicy(nodeId);
|
|
336
263
|
break;
|
|
337
264
|
case 'dormant':
|
|
338
265
|
// Chosen dormancy is not a failure — the counter does not survive it.
|
|
339
266
|
// The pathological wake→fault→release→wake loop is bounded by the
|
|
340
|
-
// tick's pass-2 cursor-keyed retry cap (D-11), not this
|
|
341
|
-
|
|
267
|
+
// tick's pass-2 cursor-keyed retry cap (D-11), not this policy.
|
|
268
|
+
clearRespawnPolicy(nodeId);
|
|
342
269
|
break;
|
|
343
270
|
case 'release-idle':
|
|
344
271
|
// Dormancy the daemon chose ON the node's behalf, and the same
|
|
345
272
|
// non-failure: the row lands exactly where a stophook idle-release
|
|
346
273
|
// would have left it, so pass-2 inbox wakes own its next revive.
|
|
347
|
-
|
|
274
|
+
clearRespawnPolicy(nodeId);
|
|
348
275
|
transition(nodeId, 'release');
|
|
349
276
|
emitEvent({
|
|
350
277
|
level: 'info',
|
|
@@ -352,7 +279,6 @@ export class DaemonFleet {
|
|
|
352
279
|
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
|
|
353
280
|
fields: {
|
|
354
281
|
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
355
|
-
origin,
|
|
356
282
|
},
|
|
357
283
|
});
|
|
358
284
|
break;
|
|
@@ -366,51 +292,42 @@ export class DaemonFleet {
|
|
|
366
292
|
case 'respawn-fresh':
|
|
367
293
|
case 'respawn-cycle':
|
|
368
294
|
case 'respawn-resume':
|
|
369
|
-
await this.#respawn(nodeId, meta, action,
|
|
295
|
+
await this.#respawn(nodeId, meta, action, interruptedTurn);
|
|
370
296
|
break;
|
|
371
297
|
}
|
|
372
298
|
return action;
|
|
373
299
|
}
|
|
374
|
-
async #respawn(nodeId, meta, action,
|
|
375
|
-
const
|
|
376
|
-
if (
|
|
300
|
+
async #respawn(nodeId, meta, action, interruptedTurn) {
|
|
301
|
+
const row = getRow(nodeId);
|
|
302
|
+
if (row === null)
|
|
303
|
+
return;
|
|
304
|
+
const assess = row.respawn_not_before == null;
|
|
305
|
+
const priorFailures = row.respawn_failures ?? 0;
|
|
306
|
+
const attempt = assess
|
|
307
|
+
? recordRespawnPolicy(row, this.#now())
|
|
308
|
+
: { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
|
|
309
|
+
if (attempt.exhausted) {
|
|
377
310
|
await this.#terminalize(nodeId, meta, {
|
|
378
311
|
event: 'broker.respawn.exhausted',
|
|
379
|
-
wake: `Child crashed — ${fullName(meta)} (${nodeId}) kept dying
|
|
380
|
-
`
|
|
312
|
+
wake: `Child crashed — ${fullName(meta)} (${nodeId}) kept dying without settling a turn (${attempt.failures} consecutive ` +
|
|
313
|
+
`launches) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
|
|
314
|
+
...(meta.pi_session_id != null ? { surface: () => surfaceCrashLoop(meta) } : {}),
|
|
381
315
|
});
|
|
382
316
|
return;
|
|
383
317
|
}
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
318
|
+
if (assess && attempt.notBefore !== null) {
|
|
319
|
+
emitEvent({
|
|
320
|
+
level: 'info',
|
|
321
|
+
event: 'broker.respawn.deferred',
|
|
322
|
+
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
|
|
323
|
+
fields: {
|
|
324
|
+
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
325
|
+
action,
|
|
326
|
+
backoff_ms: respawnBackoffMs(attempt.failures),
|
|
327
|
+
consecutive_failures: attempt.failures,
|
|
328
|
+
},
|
|
329
|
+
});
|
|
387
330
|
return;
|
|
388
|
-
// Only a fresh exit consults the backoff: a matured timer and a tick
|
|
389
|
-
// enactment relaunch immediately (the timer already WAS the backoff, and a
|
|
390
|
-
// tick candidate reached the lane only after its own gating).
|
|
391
|
-
if (origin === 'exit') {
|
|
392
|
-
const delay = respawnBackoffMs(state.failures);
|
|
393
|
-
if (delay > 0) {
|
|
394
|
-
state.timer = setTimeout(() => {
|
|
395
|
-
state.timer = undefined;
|
|
396
|
-
// A fresh policy job re-reads the row, so a node closed/canceled
|
|
397
|
-
// during the backoff lands in table row 2 and the respawn evaporates.
|
|
398
|
-
this.#enqueue(async () => { await this.#applyPolicy(nodeId, 'timer'); });
|
|
399
|
-
}, delay);
|
|
400
|
-
this.#throttle.set(nodeId, state);
|
|
401
|
-
emitEvent({
|
|
402
|
-
level: 'info',
|
|
403
|
-
event: 'broker.respawn.deferred',
|
|
404
|
-
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
|
|
405
|
-
fields: {
|
|
406
|
-
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
407
|
-
action,
|
|
408
|
-
backoff_ms: delay,
|
|
409
|
-
consecutive_failures: state.failures,
|
|
410
|
-
},
|
|
411
|
-
});
|
|
412
|
-
return;
|
|
413
|
-
}
|
|
414
331
|
}
|
|
415
332
|
emitEvent({
|
|
416
333
|
level: 'info',
|
|
@@ -419,8 +336,7 @@ export class DaemonFleet {
|
|
|
419
336
|
fields: {
|
|
420
337
|
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
421
338
|
action,
|
|
422
|
-
|
|
423
|
-
consecutive_failures: state.failures,
|
|
339
|
+
consecutive_failures: attempt.failures,
|
|
424
340
|
},
|
|
425
341
|
});
|
|
426
342
|
try {
|
|
@@ -450,7 +366,6 @@ export class DaemonFleet {
|
|
|
450
366
|
fields: {
|
|
451
367
|
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
452
368
|
action,
|
|
453
|
-
origin,
|
|
454
369
|
},
|
|
455
370
|
});
|
|
456
371
|
// Launch failures terminalize inside reviveNode, but its preflight
|
|
@@ -459,10 +374,8 @@ export class DaemonFleet {
|
|
|
459
374
|
await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err);
|
|
460
375
|
}
|
|
461
376
|
}
|
|
462
|
-
/** Terminal outcome shared by table row 8 and
|
|
463
|
-
* transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
|
|
377
|
+
/** Terminal outcome shared by table row 8 and respawn exhaustion. */
|
|
464
378
|
async #terminalize(nodeId, meta, opts) {
|
|
465
|
-
this.#clearThrottle(nodeId);
|
|
466
379
|
emitEvent({
|
|
467
380
|
level: 'error',
|
|
468
381
|
event: opts.event,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ApiError } from '../../api/index.js';
|
|
2
2
|
import { getRow, subscribersOf } from '../../core/canvas/canvas.js';
|
|
3
|
+
import { cancelCronsOnWake } from '../../core/canvas/crons.js';
|
|
3
4
|
import { opaqueInboxTicketId, ticketDir } from '../../core/human/root.js';
|
|
4
5
|
import { readTicketActionBinding } from '../../core/human/action-binding.js';
|
|
5
6
|
import { completionEventFor, composeHumanCompletion } from '../../core/human/completion.js';
|
|
@@ -275,6 +276,7 @@ export async function deliverTerminalResult(ticketId) {
|
|
|
275
276
|
data: { body },
|
|
276
277
|
disposition: 'human-canceled',
|
|
277
278
|
});
|
|
279
|
+
cancelCronsOnWake(subscriber.node_id);
|
|
278
280
|
}
|
|
279
281
|
catch {
|
|
280
282
|
failedTo.push(subscriber.node_id);
|
package/dist/daemon/manage.d.ts
CHANGED
|
@@ -46,7 +46,17 @@ export interface SpawnDaemonResult {
|
|
|
46
46
|
pid?: number;
|
|
47
47
|
/** PID of the already-running daemon, if it was already up. */
|
|
48
48
|
existing_pid?: number;
|
|
49
|
+
/** Present only when a live pidfile owner does not answer `/healthz`. */
|
|
50
|
+
running?: true;
|
|
51
|
+
/** Present with `running:true` when the Unix API socket is not serving. */
|
|
52
|
+
serving?: false;
|
|
53
|
+
/** Recovery instruction for a live but non-serving daemon. */
|
|
54
|
+
next?: string;
|
|
49
55
|
}
|
|
56
|
+
/** True only when the mandatory Unix API socket answers `/healthz`. This is
|
|
57
|
+
* deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
|
|
58
|
+
* identifying the process an operator must stop, but it is not readiness. */
|
|
59
|
+
export declare function isDaemonServing(): Promise<boolean>;
|
|
50
60
|
export interface DaemonWaitDeps {
|
|
51
61
|
readPidfile?: () => number | null;
|
|
52
62
|
isPidAlive?: (pid: number) => boolean;
|
|
@@ -58,6 +68,7 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
|
|
|
58
68
|
code: number | null;
|
|
59
69
|
signal: NodeJS.Signals | null;
|
|
60
70
|
} | null;
|
|
71
|
+
isDaemonServing?: () => Promise<boolean>;
|
|
61
72
|
}
|
|
62
73
|
/** Tiny bounded post-spawn guard: wait for the daemon pidfile and live pid to
|
|
63
74
|
* appear before reporting success. This catches the "started:true but not yet
|
package/dist/daemon/manage.js
CHANGED
|
@@ -16,6 +16,7 @@ import { hostExecPath } from '../core/runtime/branded-host.js';
|
|
|
16
16
|
import { isDaemonRunning, readPidfile, isPidAlive } from './pidfile.js';
|
|
17
17
|
import { DAEMON_STARTUP_WINDOW_MS } from './startup-policy.js';
|
|
18
18
|
import { envNoDaemonAutostart } from '../shared/env.js';
|
|
19
|
+
import { CrtrClient } from '../api/index.js';
|
|
19
20
|
// Daemon env sanitization
|
|
20
21
|
/** Env keys that must NEVER reach the daemon process. Restarting crtrd is
|
|
21
22
|
* overwhelmingly done from inside an agent node's own bash tool (`crtr sys
|
|
@@ -201,6 +202,24 @@ export async function execDaemon() {
|
|
|
201
202
|
// Shutdown stays short even though startup may include corpus migration.
|
|
202
203
|
const DAEMON_SHUTDOWN_WINDOW_MS = 20_000;
|
|
203
204
|
const DAEMON_VERIFY_POLL_MS = 10;
|
|
205
|
+
/** A status/start readiness probe must not hang behind a daemon that accepts a
|
|
206
|
+
* Unix connection but never answers it. */
|
|
207
|
+
const DAEMON_HEALTH_PROBE_TIMEOUT_MS = 1_000;
|
|
208
|
+
/** True only when the mandatory Unix API socket answers `/healthz`. This is
|
|
209
|
+
* deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
|
|
210
|
+
* identifying the process an operator must stop, but it is not readiness. */
|
|
211
|
+
export async function isDaemonServing() {
|
|
212
|
+
try {
|
|
213
|
+
const health = await CrtrClient.forLocalSocket({
|
|
214
|
+
autostart: false,
|
|
215
|
+
timeoutMs: DAEMON_HEALTH_PROBE_TIMEOUT_MS,
|
|
216
|
+
}).healthz();
|
|
217
|
+
return health.ok;
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
204
223
|
function sleepMs(ms) {
|
|
205
224
|
return new Promise((resolve) => {
|
|
206
225
|
setTimeout(resolve, ms);
|
|
@@ -223,12 +242,13 @@ export async function verifyDaemonStartup(pid, timeoutMs = DAEMON_STARTUP_WINDOW
|
|
|
223
242
|
const read = deps.readPidfile ?? readPidfile;
|
|
224
243
|
const alive = deps.isPidAlive ?? isPidAlive;
|
|
225
244
|
const exited = deps.childExited;
|
|
245
|
+
const serving = deps.isDaemonServing ?? isDaemonServing;
|
|
226
246
|
const sleep = deps.sleepMs ?? sleepMs;
|
|
227
247
|
const now = deps.now ?? Date.now;
|
|
228
248
|
const deadline = now() + timeoutMs;
|
|
229
249
|
while (now() <= deadline) {
|
|
230
250
|
const owner = read();
|
|
231
|
-
if (owner !== null && alive(owner))
|
|
251
|
+
if (owner !== null && alive(owner) && await serving())
|
|
232
252
|
return owner === pid ? null : owner;
|
|
233
253
|
const exitState = exited?.();
|
|
234
254
|
if (exitState !== null && exitState !== undefined) {
|
|
@@ -314,8 +334,17 @@ export async function stopDaemonProcess(pid, timeoutMs = DAEMON_SHUTDOWN_WINDOW_
|
|
|
314
334
|
* If the daemon is already running, returns {started:false, existing_pid}.
|
|
315
335
|
* If spawning fails (e.g. missing dist — run `npm run build` first), throws. */
|
|
316
336
|
export async function spawnDaemon() {
|
|
317
|
-
|
|
318
|
-
|
|
337
|
+
const recordedPid = readPidfile();
|
|
338
|
+
if (recordedPid !== null && isPidAlive(recordedPid)) {
|
|
339
|
+
if (await isDaemonServing())
|
|
340
|
+
return { started: false, existing_pid: recordedPid };
|
|
341
|
+
return {
|
|
342
|
+
started: false,
|
|
343
|
+
existing_pid: recordedPid,
|
|
344
|
+
running: true,
|
|
345
|
+
serving: false,
|
|
346
|
+
next: 'Run `crtr sys daemon stop` to release the non-serving daemon before starting it again.',
|
|
347
|
+
};
|
|
319
348
|
}
|
|
320
349
|
// Ensure the canvas home directory exists so the daemon can write its pidfile.
|
|
321
350
|
mkdirSync(crtrHome(), { recursive: true });
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { getNode } from '../../core/canvas/canvas.js';
|
|
2
|
+
import { cancelCronsOnWake } from '../../core/canvas/crons.js';
|
|
2
3
|
import { withFreshTerminalGuard } from '../../core/canvas/canvas.js';
|
|
3
4
|
import { appendInbox } from '../../core/feed/inbox.js';
|
|
4
5
|
import { BrokerUnreachableError } from '../../core/runtime/broker-request.js';
|
|
@@ -43,6 +44,7 @@ export async function deliverNodeMessage(args) {
|
|
|
43
44
|
// before reporting an engine receipt so a final that committed during the
|
|
44
45
|
// request is not represented as a routable delivery.
|
|
45
46
|
guardRoutable(args.node_id, () => undefined);
|
|
47
|
+
cancelCronsOnWake(args.node_id);
|
|
46
48
|
return { entry_id: null, via: 'engine', revived: false };
|
|
47
49
|
}
|
|
48
50
|
catch (error) {
|
|
@@ -57,9 +59,13 @@ export async function deliverNodeMessage(args) {
|
|
|
57
59
|
label: args.label,
|
|
58
60
|
data: { ...args.data, body: args.body },
|
|
59
61
|
}));
|
|
62
|
+
if (args.mode !== 'quiet' || target.frozen_at !== null)
|
|
63
|
+
cancelCronsOnWake(args.node_id);
|
|
64
|
+
// The APPENDED entry's tier decides the wake: a quiet send to a terminal
|
|
65
|
+
// target was raised out of `deferred` by the append, and must be delivered.
|
|
60
66
|
let revived = false;
|
|
61
67
|
let reason;
|
|
62
|
-
if (
|
|
68
|
+
if (entry.tier !== 'deferred') {
|
|
63
69
|
target = getNode(args.node_id);
|
|
64
70
|
if (target === null)
|
|
65
71
|
deliveryUnroutable(args.node_id, 'origin_missing');
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { bashJobPaths, expiredAutoBackgroundBashJobs, formatBashElapsed, stopBackgroundBashJob, } from '../../core/bash-jobs.js';
|
|
2
2
|
import { isSafeNodeId } from '../../core/canvas/paths.js';
|
|
3
3
|
import { contextDir } from '../../core/canvas/paths.js';
|
|
4
|
+
import { cancelCronsOnWake } from '../../core/canvas/crons.js';
|
|
4
5
|
import { appendInbox } from '../../core/feed/inbox.js';
|
|
5
6
|
import { emitEvent } from '../../core/events/emit.js';
|
|
6
7
|
import { operationIdContext } from '../../core/events/operation-id.js';
|
|
@@ -64,6 +65,7 @@ export class BashDeadlineReconciler {
|
|
|
64
65
|
`Log: ${paths.jobLog} Command: ${paths.cmdSh}`,
|
|
65
66
|
},
|
|
66
67
|
});
|
|
68
|
+
cancelCronsOnWake(nodeId);
|
|
67
69
|
}
|
|
68
70
|
catch (err) {
|
|
69
71
|
emitEvent({ level: 'error', event: 'bash_deadline.notice_failed', ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}), error: err, fields: { job_id: jobId } });
|
|
@@ -36,6 +36,7 @@ export declare function wedgeVerdict(input: {
|
|
|
36
36
|
busy: boolean;
|
|
37
37
|
quietForMs: number | null;
|
|
38
38
|
cpuPercent: number | null;
|
|
39
|
+
hasDescendants: boolean | null;
|
|
39
40
|
}): WedgeVerdict;
|
|
40
41
|
export declare function sumTreeCpu(psOutput: string, rootPid: number): number | null;
|
|
41
42
|
export declare function treeHasDescendants(psOutput: string, rootPid: number): boolean | null;
|
|
@@ -61,7 +61,9 @@ export function wedgeVerdict(input) {
|
|
|
61
61
|
return 'leave';
|
|
62
62
|
if (input.quietForMs === null || input.quietForMs < WEDGE_QUIET_MS)
|
|
63
63
|
return 'leave';
|
|
64
|
-
if (input.cpuPercent === null
|
|
64
|
+
if (input.cpuPercent === null)
|
|
65
|
+
return 'pending';
|
|
66
|
+
if (input.hasDescendants !== false && input.cpuPercent > WEDGE_CPU_MAX_PERCENT)
|
|
65
67
|
return 'pending';
|
|
66
68
|
return 'wedged';
|
|
67
69
|
}
|
|
@@ -471,18 +473,20 @@ export class BrokerSupervisionReconciler {
|
|
|
471
473
|
const quietForMs = since === null ? null : now - since;
|
|
472
474
|
const sample = quietForMs !== null && quietForMs >= WEDGE_QUIET_MS ? await this.readTreeSample(pid) : null;
|
|
473
475
|
const cpuPercent = sample?.cpuPercent ?? null;
|
|
474
|
-
|
|
476
|
+
const sampleHasDescendants = sample?.hasDescendants ?? null;
|
|
477
|
+
if (wedgeVerdict({ busy: true, quietForMs, cpuPercent, hasDescendants: sampleHasDescendants }) !== 'wedged')
|
|
475
478
|
return;
|
|
476
479
|
if (since !== null && this.wedgeNotifiedAt.get(id) === since)
|
|
477
480
|
return;
|
|
478
|
-
if (sample?.hasDescendants === false && (expectedIdentity == null || recordedPidLiveness(pid, expectedIdentity) !== 'alive'))
|
|
479
|
-
return;
|
|
480
481
|
if (since !== null)
|
|
481
482
|
this.wedgeNotifiedAt.set(id, since);
|
|
482
483
|
const meta = getNode(id);
|
|
483
484
|
const name = meta !== null ? fullName(meta) : id;
|
|
484
485
|
const minutes = Math.round((quietForMs ?? WEDGE_QUIET_MS) / 60_000);
|
|
485
|
-
const hasDescendants =
|
|
486
|
+
const hasDescendants = sampleHasDescendants ?? true;
|
|
487
|
+
const kickSuppressionReason = !hasDescendants && (expectedIdentity == null || recordedPidLiveness(pid, expectedIdentity) !== 'alive')
|
|
488
|
+
? 'pid_identity_unproven'
|
|
489
|
+
: null;
|
|
486
490
|
let label;
|
|
487
491
|
if (hasDescendants) {
|
|
488
492
|
label =
|
|
@@ -491,11 +495,18 @@ export class BrokerSupervisionReconciler {
|
|
|
491
495
|
`\`crtr memory read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this ` +
|
|
492
496
|
`node) so its turn can resume with no lost context.`;
|
|
493
497
|
}
|
|
498
|
+
else if (kickSuppressionReason !== null) {
|
|
499
|
+
label =
|
|
500
|
+
`Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
|
|
501
|
+
`${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did ` +
|
|
502
|
+
`not SIGTERM the broker because its PID identity cannot be ` +
|
|
503
|
+
`confirmed alive.`;
|
|
504
|
+
}
|
|
494
505
|
else {
|
|
495
506
|
label =
|
|
496
507
|
`Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
|
|
497
|
-
`${minutes}+ minutes
|
|
498
|
-
`
|
|
508
|
+
`${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). Kicking it ` +
|
|
509
|
+
`now: SIGTERM to the broker — the daemon will resume a cleanly ` +
|
|
499
510
|
`aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`;
|
|
500
511
|
try {
|
|
501
512
|
process.kill(pid, 'SIGTERM');
|
|
@@ -514,7 +525,8 @@ export class BrokerSupervisionReconciler {
|
|
|
514
525
|
quiet_ms: quietForMs,
|
|
515
526
|
cpu_percent: cpuPercent,
|
|
516
527
|
has_descendants: hasDescendants,
|
|
517
|
-
kicked: !hasDescendants,
|
|
528
|
+
kicked: !hasDescendants && kickSuppressionReason === null,
|
|
529
|
+
...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
|
|
518
530
|
},
|
|
519
531
|
});
|
|
520
532
|
try {
|
|
@@ -539,7 +551,8 @@ export class BrokerSupervisionReconciler {
|
|
|
539
551
|
child: id,
|
|
540
552
|
quiet_ms: quietForMs,
|
|
541
553
|
cpu_percent: cpuPercent,
|
|
542
|
-
kicked: !hasDescendants,
|
|
554
|
+
kicked: !hasDescendants && kickSuppressionReason === null,
|
|
555
|
+
...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
|
|
543
556
|
});
|
|
544
557
|
}
|
|
545
558
|
handleFatalFault(id) {
|
|
@@ -25,16 +25,26 @@ export declare function hasPendingWake(nodeId: string): boolean;
|
|
|
25
25
|
* report is an outstanding deliverable, so such a node keeps its
|
|
26
26
|
* wakeable idle row instead of being written off.
|
|
27
27
|
*
|
|
28
|
-
* Undelivered mail
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
28
|
+
* Undelivered mail holds the row, with one carve-out: a deferred-only unseen
|
|
29
|
+
* set does not hold a RESIDENT. Deferred mail never wakes a node on its own,
|
|
30
|
+
* and parking a resident is reversible — a message sent to it revives it and
|
|
31
|
+
* flushes the held mail on that cycle. A terminalized row takes no such
|
|
32
|
+
* cycle, so unread mail of any tier keeps it from being written off: the
|
|
33
|
+
* invariant is that content a sender already handed over is never dropped.
|
|
32
34
|
*/
|
|
33
35
|
export type ParkEvent = Extract<LifecycleEvent, 'release' | 'park' | 'cancel'>;
|
|
34
36
|
export declare function parkEvent(nodeId: string, lifecycle: Lifecycle): ParkEvent;
|
|
35
37
|
/** Apply an unattended decision and notify subscribers of automatic completion.
|
|
36
38
|
* This is the DEGRADED enactment: no summary turn ran, so subscribers get the
|
|
37
|
-
* doctrine
|
|
39
|
+
* doctrine notice instead of the node's own report. The live clock's good path
|
|
38
40
|
* (broker settlement after a parking turn) enacts `park` directly and fans
|
|
39
|
-
* nothing — the node's report is the notice.
|
|
41
|
+
* nothing — the node's report is the notice.
|
|
42
|
+
*
|
|
43
|
+
* The notice is DEFERRED: a node settling quietly at the end of its unattended
|
|
44
|
+
* grace is not an event worth starting a manager's cycle for, so it rides the
|
|
45
|
+
* manager's next natural cycle and arrives alongside whatever does.
|
|
46
|
+
*
|
|
47
|
+
* Only `park` fans, and only a resident parks, so the notice speaks about a
|
|
48
|
+
* resident throughout: its conversation ended without a final report, its state
|
|
49
|
+
* lives in whatever report it did leave, and a message reopens it. */
|
|
40
50
|
export declare function applyParkEvent(nodeId: string, event: ParkEvent): void;
|