@north-light/crouter 0.3.249 → 0.3.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/__tests__/integration/client.test.js +28 -7
- package/dist/api/client.d.ts +1 -1
- package/dist/api/client.js +2 -2
- package/dist/api/dto/lifecycle.d.ts +3 -0
- package/dist/api/dto/messages.d.ts +4 -0
- package/dist/api/dto/nodes.d.ts +4 -0
- package/dist/api/dto/reports.d.ts +7 -2
- package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
- package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
- package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +1 -0
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
- package/dist/clients/inbox/controller.js +32 -5
- package/dist/clients/inbox/tui/ansi.d.ts +3 -1
- package/dist/clients/inbox/tui/ansi.js +14 -10
- package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
- package/dist/commands/api-client.d.ts +2 -2
- package/dist/commands/api-client.js +5 -5
- package/dist/commands/canvas-browse.js +1 -1
- package/dist/commands/node/create.js +3 -1
- package/dist/commands/node/inspect.js +1 -0
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node/message.js +6 -4
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/push.js +36 -37
- package/dist/commands/sys/daemon.js +21 -8
- package/dist/commands/sys/setup-core.js +2 -2
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
- package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
- package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +100 -1
- package/dist/core/__tests__/child-death-wake.test.js +11 -7
- package/dist/core/__tests__/collectible.test.d.ts +1 -0
- package/dist/core/__tests__/collectible.test.js +88 -0
- package/dist/core/__tests__/context-intro.test.js +4 -41
- package/dist/core/__tests__/daemon-boot.test.js +30 -14
- package/dist/core/__tests__/daemon-wedge.test.js +25 -19
- package/dist/core/__tests__/fixtures/fake-engine.js +20 -2
- package/dist/core/__tests__/human-deliver.test.js +5 -5
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
- package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
- package/dist/core/__tests__/integration/revive.test.js +48 -0
- package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
- package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
- package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
- package/dist/core/__tests__/migration.test.js +30 -0
- package/dist/core/__tests__/passive-subscription.test.js +39 -1
- package/dist/core/__tests__/prune.test.js +12 -3
- package/dist/core/__tests__/relaunch-root.test.js +1 -1
- package/dist/core/__tests__/respawn-throttle.test.js +16 -146
- package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
- package/dist/core/canvas/canvas.d.ts +9 -15
- package/dist/core/canvas/canvas.js +41 -47
- package/dist/core/canvas/crons.d.ts +5 -8
- package/dist/core/canvas/crons.js +5 -8
- package/dist/core/canvas/history.d.ts +5 -0
- package/dist/core/canvas/history.js +12 -1
- package/dist/core/canvas/migrations.js +58 -1
- package/dist/core/canvas/pid.d.ts +1 -1
- package/dist/core/canvas/pid.js +5 -2
- package/dist/core/canvas/types.d.ts +4 -0
- package/dist/core/feed/feed.d.ts +4 -0
- package/dist/core/feed/feed.js +13 -8
- package/dist/core/feed/inbox.d.ts +2 -1
- package/dist/core/feed/inbox.js +22 -3
- package/dist/core/git.d.ts +3 -0
- package/dist/core/git.js +3 -1
- package/dist/core/human/feedback-companion.js +1 -0
- package/dist/core/preview-registry.js +0 -1
- package/dist/core/review/companion.js +1 -0
- package/dist/core/review/realize.js +1 -0
- package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
- package/dist/core/runtime/broker/auth-reload.js +9 -3
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
- package/dist/core/runtime/broker/engine-drive.js +39 -23
- package/dist/core/runtime/broker/event-projection.d.ts +4 -0
- package/dist/core/runtime/broker/event-projection.js +43 -23
- package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
- package/dist/core/runtime/broker/fault-retry.js +94 -8
- package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
- package/dist/core/runtime/broker/frame-dispatch.js +4 -1
- package/dist/core/runtime/broker/held-deferred-inbox.d.ts +9 -0
- package/dist/core/runtime/broker/held-deferred-inbox.js +8 -0
- package/dist/core/runtime/broker/inbox.js +4 -0
- package/dist/core/runtime/broker/rebind.js +18 -0
- package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
- package/dist/core/runtime/broker/turn-admission.js +31 -0
- package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
- package/dist/core/runtime/broker/turn-ignition.js +25 -15
- package/dist/core/runtime/broker.js +21 -2
- package/dist/core/runtime/close.d.ts +11 -9
- package/dist/core/runtime/close.js +23 -19
- package/dist/core/runtime/fault.d.ts +21 -0
- package/dist/core/runtime/fault.js +129 -19
- package/dist/core/runtime/fleet.d.ts +0 -6
- package/dist/core/runtime/host.d.ts +7 -1
- package/dist/core/runtime/host.js +20 -2
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +24 -1
- package/dist/core/runtime/placement.d.ts +6 -6
- package/dist/core/runtime/placement.js +15 -8
- package/dist/core/runtime/reset.js +2 -2
- package/dist/core/runtime/revive.js +40 -4
- package/dist/core/runtime/spawn.d.ts +3 -0
- package/dist/core/runtime/spawn.js +1 -0
- package/dist/core/runtime/warm-pool.d.ts +4 -0
- package/dist/core/runtime/warm-pool.js +2 -0
- package/dist/core/worktree.d.ts +4 -3
- package/dist/core/worktree.js +73 -27
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
- package/dist/daemon/api/handlers/bash-jobs.js +2 -0
- package/dist/daemon/api/handlers/broker-ops.js +23 -2
- package/dist/daemon/api/handlers/human.js +1 -1
- package/dist/daemon/api/handlers/messages.js +11 -2
- package/dist/daemon/api/handlers/nodes.js +13 -1
- package/dist/daemon/api/handlers/reports.js +14 -6
- package/dist/daemon/api/map.js +1 -0
- package/dist/daemon/api/server.d.ts +9 -12
- package/dist/daemon/api/server.js +30 -19
- package/dist/daemon/companion-retire.js +3 -9
- package/dist/daemon/cron/sinks.js +7 -2
- package/dist/daemon/crtrd.js +4 -1
- package/dist/daemon/fleet.d.ts +2 -16
- package/dist/daemon/fleet.js +63 -150
- package/dist/daemon/human/finish.js +2 -0
- package/dist/daemon/manage.d.ts +11 -0
- package/dist/daemon/manage.js +32 -3
- package/dist/daemon/messaging/node-message.js +7 -1
- package/dist/daemon/reconcilers/bash-deadline.js +2 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
- package/dist/daemon/reconcilers/broker-supervision.js +22 -9
- package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
- package/dist/daemon/reconcilers/live-obligation.js +18 -6
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
- package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
- package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
- package/dist/pi-extensions/canvas-context-intro.js +4 -76
- package/dist/pi-extensions/canvas-inbox-watcher.js +105 -4
- package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
- package/dist/pi-extensions/canvas-review-boundary.js +4 -29
- package/dist/pi-extensions/canvas-stophook.js +1 -6
- package/dist/shared/birth-announcement.d.ts +12 -0
- package/dist/shared/birth-announcement.js +25 -0
- package/dist/shared/generated-context.d.ts +8 -4
- package/dist/shared/generated-context.js +15 -9
- package/package.json +4 -4
- package/runtime.lock.json +2 -2
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
- package/dist/pi-extensions/truncate.d.ts +0 -5
- package/dist/pi-extensions/truncate.js +0 -14
package/dist/daemon/manage.js
CHANGED
|
@@ -16,6 +16,7 @@ import { hostExecPath } from '../core/runtime/branded-host.js';
|
|
|
16
16
|
import { isDaemonRunning, readPidfile, isPidAlive } from './pidfile.js';
|
|
17
17
|
import { DAEMON_STARTUP_WINDOW_MS } from './startup-policy.js';
|
|
18
18
|
import { envNoDaemonAutostart } from '../shared/env.js';
|
|
19
|
+
import { CrtrClient } from '../api/index.js';
|
|
19
20
|
// Daemon env sanitization
|
|
20
21
|
/** Env keys that must NEVER reach the daemon process. Restarting crtrd is
|
|
21
22
|
* overwhelmingly done from inside an agent node's own bash tool (`crtr sys
|
|
@@ -201,6 +202,24 @@ export async function execDaemon() {
|
|
|
201
202
|
// Shutdown stays short even though startup may include corpus migration.
|
|
202
203
|
const DAEMON_SHUTDOWN_WINDOW_MS = 20_000;
|
|
203
204
|
const DAEMON_VERIFY_POLL_MS = 10;
|
|
205
|
+
/** A status/start readiness probe must not hang behind a daemon that accepts a
|
|
206
|
+
* Unix connection but never answers it. */
|
|
207
|
+
const DAEMON_HEALTH_PROBE_TIMEOUT_MS = 1_000;
|
|
208
|
+
/** True only when the mandatory Unix API socket answers `/healthz`. This is
|
|
209
|
+
* deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
|
|
210
|
+
* identifying the process an operator must stop, but it is not readiness. */
|
|
211
|
+
export async function isDaemonServing() {
|
|
212
|
+
try {
|
|
213
|
+
const health = await CrtrClient.forLocalSocket({
|
|
214
|
+
autostart: false,
|
|
215
|
+
timeoutMs: DAEMON_HEALTH_PROBE_TIMEOUT_MS,
|
|
216
|
+
}).healthz();
|
|
217
|
+
return health.ok;
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
return false;
|
|
221
|
+
}
|
|
222
|
+
}
|
|
204
223
|
function sleepMs(ms) {
|
|
205
224
|
return new Promise((resolve) => {
|
|
206
225
|
setTimeout(resolve, ms);
|
|
@@ -223,12 +242,13 @@ export async function verifyDaemonStartup(pid, timeoutMs = DAEMON_STARTUP_WINDOW
|
|
|
223
242
|
const read = deps.readPidfile ?? readPidfile;
|
|
224
243
|
const alive = deps.isPidAlive ?? isPidAlive;
|
|
225
244
|
const exited = deps.childExited;
|
|
245
|
+
const serving = deps.isDaemonServing ?? isDaemonServing;
|
|
226
246
|
const sleep = deps.sleepMs ?? sleepMs;
|
|
227
247
|
const now = deps.now ?? Date.now;
|
|
228
248
|
const deadline = now() + timeoutMs;
|
|
229
249
|
while (now() <= deadline) {
|
|
230
250
|
const owner = read();
|
|
231
|
-
if (owner !== null && alive(owner))
|
|
251
|
+
if (owner !== null && alive(owner) && await serving())
|
|
232
252
|
return owner === pid ? null : owner;
|
|
233
253
|
const exitState = exited?.();
|
|
234
254
|
if (exitState !== null && exitState !== undefined) {
|
|
@@ -314,8 +334,17 @@ export async function stopDaemonProcess(pid, timeoutMs = DAEMON_SHUTDOWN_WINDOW_
|
|
|
314
334
|
* If the daemon is already running, returns {started:false, existing_pid}.
|
|
315
335
|
* If spawning fails (e.g. missing dist — run `npm run build` first), throws. */
|
|
316
336
|
export async function spawnDaemon() {
|
|
317
|
-
|
|
318
|
-
|
|
337
|
+
const recordedPid = readPidfile();
|
|
338
|
+
if (recordedPid !== null && isPidAlive(recordedPid)) {
|
|
339
|
+
if (await isDaemonServing())
|
|
340
|
+
return { started: false, existing_pid: recordedPid };
|
|
341
|
+
return {
|
|
342
|
+
started: false,
|
|
343
|
+
existing_pid: recordedPid,
|
|
344
|
+
running: true,
|
|
345
|
+
serving: false,
|
|
346
|
+
next: 'Run `crtr sys daemon stop` to release the non-serving daemon before starting it again.',
|
|
347
|
+
};
|
|
319
348
|
}
|
|
320
349
|
// Ensure the canvas home directory exists so the daemon can write its pidfile.
|
|
321
350
|
mkdirSync(crtrHome(), { recursive: true });
|
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { getNode } from '../../core/canvas/canvas.js';
|
|
2
|
+
import { cancelCronsOnWake } from '../../core/canvas/crons.js';
|
|
2
3
|
import { withFreshTerminalGuard } from '../../core/canvas/canvas.js';
|
|
3
4
|
import { appendInbox } from '../../core/feed/inbox.js';
|
|
4
5
|
import { BrokerUnreachableError } from '../../core/runtime/broker-request.js';
|
|
@@ -43,6 +44,7 @@ export async function deliverNodeMessage(args) {
|
|
|
43
44
|
// before reporting an engine receipt so a final that committed during the
|
|
44
45
|
// request is not represented as a routable delivery.
|
|
45
46
|
guardRoutable(args.node_id, () => undefined);
|
|
47
|
+
cancelCronsOnWake(args.node_id);
|
|
46
48
|
return { entry_id: null, via: 'engine', revived: false };
|
|
47
49
|
}
|
|
48
50
|
catch (error) {
|
|
@@ -57,9 +59,13 @@ export async function deliverNodeMessage(args) {
|
|
|
57
59
|
label: args.label,
|
|
58
60
|
data: { ...args.data, body: args.body },
|
|
59
61
|
}));
|
|
62
|
+
if (args.mode !== 'quiet' || target.frozen_at !== null)
|
|
63
|
+
cancelCronsOnWake(args.node_id);
|
|
64
|
+
// The APPENDED entry's tier decides the wake: a quiet send to a terminal
|
|
65
|
+
// target was raised out of `deferred` by the append, and must be delivered.
|
|
60
66
|
let revived = false;
|
|
61
67
|
let reason;
|
|
62
|
-
if (
|
|
68
|
+
if (entry.tier !== 'deferred') {
|
|
63
69
|
target = getNode(args.node_id);
|
|
64
70
|
if (target === null)
|
|
65
71
|
deliveryUnroutable(args.node_id, 'origin_missing');
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { bashJobPaths, expiredAutoBackgroundBashJobs, formatBashElapsed, stopBackgroundBashJob, } from '../../core/bash-jobs.js';
|
|
2
2
|
import { isSafeNodeId } from '../../core/canvas/paths.js';
|
|
3
3
|
import { contextDir } from '../../core/canvas/paths.js';
|
|
4
|
+
import { cancelCronsOnWake } from '../../core/canvas/crons.js';
|
|
4
5
|
import { appendInbox } from '../../core/feed/inbox.js';
|
|
5
6
|
import { emitEvent } from '../../core/events/emit.js';
|
|
6
7
|
import { operationIdContext } from '../../core/events/operation-id.js';
|
|
@@ -64,6 +65,7 @@ export class BashDeadlineReconciler {
|
|
|
64
65
|
`Log: ${paths.jobLog} Command: ${paths.cmdSh}`,
|
|
65
66
|
},
|
|
66
67
|
});
|
|
68
|
+
cancelCronsOnWake(nodeId);
|
|
67
69
|
}
|
|
68
70
|
catch (err) {
|
|
69
71
|
emitEvent({ level: 'error', event: 'bash_deadline.notice_failed', ...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}), error: err, fields: { job_id: jobId } });
|
|
@@ -36,6 +36,7 @@ export declare function wedgeVerdict(input: {
|
|
|
36
36
|
busy: boolean;
|
|
37
37
|
quietForMs: number | null;
|
|
38
38
|
cpuPercent: number | null;
|
|
39
|
+
hasDescendants: boolean | null;
|
|
39
40
|
}): WedgeVerdict;
|
|
40
41
|
export declare function sumTreeCpu(psOutput: string, rootPid: number): number | null;
|
|
41
42
|
export declare function treeHasDescendants(psOutput: string, rootPid: number): boolean | null;
|
|
@@ -61,7 +61,9 @@ export function wedgeVerdict(input) {
|
|
|
61
61
|
return 'leave';
|
|
62
62
|
if (input.quietForMs === null || input.quietForMs < WEDGE_QUIET_MS)
|
|
63
63
|
return 'leave';
|
|
64
|
-
if (input.cpuPercent === null
|
|
64
|
+
if (input.cpuPercent === null)
|
|
65
|
+
return 'pending';
|
|
66
|
+
if (input.hasDescendants !== false && input.cpuPercent > WEDGE_CPU_MAX_PERCENT)
|
|
65
67
|
return 'pending';
|
|
66
68
|
return 'wedged';
|
|
67
69
|
}
|
|
@@ -471,18 +473,20 @@ export class BrokerSupervisionReconciler {
|
|
|
471
473
|
const quietForMs = since === null ? null : now - since;
|
|
472
474
|
const sample = quietForMs !== null && quietForMs >= WEDGE_QUIET_MS ? await this.readTreeSample(pid) : null;
|
|
473
475
|
const cpuPercent = sample?.cpuPercent ?? null;
|
|
474
|
-
|
|
476
|
+
const sampleHasDescendants = sample?.hasDescendants ?? null;
|
|
477
|
+
if (wedgeVerdict({ busy: true, quietForMs, cpuPercent, hasDescendants: sampleHasDescendants }) !== 'wedged')
|
|
475
478
|
return;
|
|
476
479
|
if (since !== null && this.wedgeNotifiedAt.get(id) === since)
|
|
477
480
|
return;
|
|
478
|
-
if (sample?.hasDescendants === false && (expectedIdentity == null || recordedPidLiveness(pid, expectedIdentity) !== 'alive'))
|
|
479
|
-
return;
|
|
480
481
|
if (since !== null)
|
|
481
482
|
this.wedgeNotifiedAt.set(id, since);
|
|
482
483
|
const meta = getNode(id);
|
|
483
484
|
const name = meta !== null ? fullName(meta) : id;
|
|
484
485
|
const minutes = Math.round((quietForMs ?? WEDGE_QUIET_MS) / 60_000);
|
|
485
|
-
const hasDescendants =
|
|
486
|
+
const hasDescendants = sampleHasDescendants ?? true;
|
|
487
|
+
const kickSuppressionReason = !hasDescendants && (expectedIdentity == null || recordedPidLiveness(pid, expectedIdentity) !== 'alive')
|
|
488
|
+
? 'pid_identity_unproven'
|
|
489
|
+
: null;
|
|
486
490
|
let label;
|
|
487
491
|
if (hasDescendants) {
|
|
488
492
|
label =
|
|
@@ -491,11 +495,18 @@ export class BrokerSupervisionReconciler {
|
|
|
491
495
|
`\`crtr memory read wedged-child-on-runaway-bash\` and kill the runaway SUBPROCESS (not this ` +
|
|
492
496
|
`node) so its turn can resume with no lost context.`;
|
|
493
497
|
}
|
|
498
|
+
else if (kickSuppressionReason !== null) {
|
|
499
|
+
label =
|
|
500
|
+
`Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
|
|
501
|
+
`${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). The daemon did ` +
|
|
502
|
+
`not SIGTERM the broker because its PID identity cannot be ` +
|
|
503
|
+
`confirmed alive.`;
|
|
504
|
+
}
|
|
494
505
|
else {
|
|
495
506
|
label =
|
|
496
507
|
`Child wedged — ${name} (${id}) is alive and mid-turn but has shown NO engine progress for ` +
|
|
497
|
-
`${minutes}+ minutes
|
|
498
|
-
`
|
|
508
|
+
`${minutes}+ minutes and it has no subprocess to kill (the engine itself stalled). Kicking it ` +
|
|
509
|
+
`now: SIGTERM to the broker — the daemon will resume a cleanly ` +
|
|
499
510
|
`aborted session with a continuation, or start a recovery cycle if the abort cannot settle.`;
|
|
500
511
|
try {
|
|
501
512
|
process.kill(pid, 'SIGTERM');
|
|
@@ -514,7 +525,8 @@ export class BrokerSupervisionReconciler {
|
|
|
514
525
|
quiet_ms: quietForMs,
|
|
515
526
|
cpu_percent: cpuPercent,
|
|
516
527
|
has_descendants: hasDescendants,
|
|
517
|
-
kicked: !hasDescendants,
|
|
528
|
+
kicked: !hasDescendants && kickSuppressionReason === null,
|
|
529
|
+
...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
|
|
518
530
|
},
|
|
519
531
|
});
|
|
520
532
|
try {
|
|
@@ -539,7 +551,8 @@ export class BrokerSupervisionReconciler {
|
|
|
539
551
|
child: id,
|
|
540
552
|
quiet_ms: quietForMs,
|
|
541
553
|
cpu_percent: cpuPercent,
|
|
542
|
-
kicked: !hasDescendants,
|
|
554
|
+
kicked: !hasDescendants && kickSuppressionReason === null,
|
|
555
|
+
...(kickSuppressionReason === null ? {} : { kick_suppression_reason: kickSuppressionReason }),
|
|
543
556
|
});
|
|
544
557
|
}
|
|
545
558
|
handleFatalFault(id) {
|
|
@@ -25,16 +25,26 @@ export declare function hasPendingWake(nodeId: string): boolean;
|
|
|
25
25
|
* report is an outstanding deliverable, so such a node keeps its
|
|
26
26
|
* wakeable idle row instead of being written off.
|
|
27
27
|
*
|
|
28
|
-
* Undelivered mail
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
*
|
|
28
|
+
* Undelivered mail holds the row, with one carve-out: a deferred-only unseen
|
|
29
|
+
* set does not hold a RESIDENT. Deferred mail never wakes a node on its own,
|
|
30
|
+
* and parking a resident is reversible — a message sent to it revives it and
|
|
31
|
+
* flushes the held mail on that cycle. A terminalized row takes no such
|
|
32
|
+
* cycle, so unread mail of any tier keeps it from being written off: the
|
|
33
|
+
* invariant is that content a sender already handed over is never dropped.
|
|
32
34
|
*/
|
|
33
35
|
export type ParkEvent = Extract<LifecycleEvent, 'release' | 'park' | 'cancel'>;
|
|
34
36
|
export declare function parkEvent(nodeId: string, lifecycle: Lifecycle): ParkEvent;
|
|
35
37
|
/** Apply an unattended decision and notify subscribers of automatic completion.
|
|
36
38
|
* This is the DEGRADED enactment: no summary turn ran, so subscribers get the
|
|
37
|
-
* doctrine
|
|
39
|
+
* doctrine notice instead of the node's own report. The live clock's good path
|
|
38
40
|
* (broker settlement after a parking turn) enacts `park` directly and fans
|
|
39
|
-
* nothing — the node's report is the notice.
|
|
41
|
+
* nothing — the node's report is the notice.
|
|
42
|
+
*
|
|
43
|
+
* The notice is DEFERRED: a node settling quietly at the end of its unattended
|
|
44
|
+
* grace is not an event worth starting a manager's cycle for, so it rides the
|
|
45
|
+
* manager's next natural cycle and arrives alongside whatever does.
|
|
46
|
+
*
|
|
47
|
+
* Only `park` fans, and only a resident parks, so the notice speaks about a
|
|
48
|
+
* resident throughout: its conversation ended without a final report, its state
|
|
49
|
+
* lives in whatever report it did leave, and a message reopens it. */
|
|
40
50
|
export declare function applyParkEvent(nodeId: string, event: ParkEvent): void;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { hasActiveLiveSubscription, hasLiveMessageWait, hasLiveSubscriber, hasPendingCancelOnWakeCron, subscribersOf, } from '../../core/canvas/index.js';
|
|
1
|
+
import { newestReportRef, hasActiveLiveSubscription, hasLiveMessageWait, hasLiveSubscriber, hasPendingCancelOnWakeCron, subscribersOf, } from '../../core/canvas/index.js';
|
|
2
2
|
import { fullName } from '../../core/canvas/labels.js';
|
|
3
3
|
import { countTickets } from '../../core/canvas/attention.js';
|
|
4
4
|
import { hasUnseenWakeMail, readCursor, readInboxSince } from '../../core/feed/inbox.js';
|
|
@@ -33,22 +33,34 @@ export function hasPendingWake(nodeId) {
|
|
|
33
33
|
export function parkEvent(nodeId, lifecycle) {
|
|
34
34
|
if (hasLiveObligation(nodeId))
|
|
35
35
|
return 'release';
|
|
36
|
+
if (lifecycle === 'resident')
|
|
37
|
+
return hasUnseenWakeMail(nodeId) ? 'release' : 'park';
|
|
36
38
|
if (readInboxSince(nodeId, readCursor(nodeId)).length > 0)
|
|
37
39
|
return 'release';
|
|
38
|
-
if (lifecycle === 'resident')
|
|
39
|
-
return 'park';
|
|
40
40
|
return hasLiveSubscriber(nodeId) ? 'release' : 'cancel';
|
|
41
41
|
}
|
|
42
42
|
/** Apply an unattended decision and notify subscribers of automatic completion.
|
|
43
43
|
* This is the DEGRADED enactment: no summary turn ran, so subscribers get the
|
|
44
|
-
* doctrine
|
|
44
|
+
* doctrine notice instead of the node's own report. The live clock's good path
|
|
45
45
|
* (broker settlement after a parking turn) enacts `park` directly and fans
|
|
46
|
-
* nothing — the node's report is the notice.
|
|
46
|
+
* nothing — the node's report is the notice.
|
|
47
|
+
*
|
|
48
|
+
* The notice is DEFERRED: a node settling quietly at the end of its unattended
|
|
49
|
+
* grace is not an event worth starting a manager's cycle for, so it rides the
|
|
50
|
+
* manager's next natural cycle and arrives alongside whatever does.
|
|
51
|
+
*
|
|
52
|
+
* Only `park` fans, and only a resident parks, so the notice speaks about a
|
|
53
|
+
* resident throughout: its conversation ended without a final report, its state
|
|
54
|
+
* lives in whatever report it did leave, and a message reopens it. */
|
|
47
55
|
export function applyParkEvent(nodeId, event) {
|
|
48
56
|
const node = event === 'cancel'
|
|
49
57
|
? transition(nodeId, event, { reason: 'stranded' })
|
|
50
58
|
: transition(nodeId, event);
|
|
51
59
|
if (event !== 'park')
|
|
52
60
|
return;
|
|
53
|
-
|
|
61
|
+
const ref = newestReportRef(nodeId);
|
|
62
|
+
const pointer = ref === null
|
|
63
|
+
? `It left no report behind; \`crtr node inspect show ${nodeId}\` shows where it stands.`
|
|
64
|
+
: `Its latest report is where it left things: \`crtr canvas history read ${ref}\`.`;
|
|
65
|
+
fanDoctrineWake(nodeId, subscribersOf(nodeId), `Resident concluded — ${fullName(node)} (${nodeId}) had nothing left to wake it through its unattended grace, so its conversation was concluded. It submitted no final report. ${pointer} A message to it reopens it on a fresh cycle.`, { reason: 'child-auto-done', child: nodeId }, 'deferred');
|
|
54
66
|
}
|
|
@@ -1,5 +1,16 @@
|
|
|
1
|
-
import type
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
1
|
+
import { type NodeRow } from '../../../core/canvas/index.js';
|
|
2
|
+
export declare const MAX_RESPAWN_ATTEMPTS = 6;
|
|
3
|
+
export declare function respawnBackoffMs(consecutiveFailures: number): number;
|
|
4
|
+
export interface RespawnAttempt {
|
|
5
|
+
failures: number;
|
|
6
|
+
notBefore: string | null;
|
|
7
|
+
exhausted: boolean;
|
|
8
|
+
}
|
|
9
|
+
/** Whether a durable backoff still holds this row back from a relaunch. */
|
|
5
10
|
export declare function isInBackoff(row: NodeRow, now: number): boolean;
|
|
11
|
+
/** The only respawn-failure policy: progress since launch clears the sequence;
|
|
12
|
+
* every other missing-engine observation advances it and records its backoff. */
|
|
13
|
+
export declare function decideRespawnAttempt(row: Pick<NodeRow, 'last_settled_at' | 'launched_at' | 'respawn_failures'>, now: number): RespawnAttempt;
|
|
14
|
+
/** Persist the policy decision through the daemon-only canvas writer. */
|
|
15
|
+
export declare function recordRespawnPolicy(row: NodeRow, now: number): RespawnAttempt;
|
|
16
|
+
export declare function clearRespawnPolicy(nodeId: string): void;
|
|
@@ -1,6 +1,13 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
1
|
+
import { recordRespawnAttempt } from '../../../core/canvas/index.js';
|
|
2
|
+
export const MAX_RESPAWN_ATTEMPTS = 6;
|
|
3
|
+
const RESPAWN_BACKOFF_BASE_MS = 5_000;
|
|
4
|
+
const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
|
|
5
|
+
export function respawnBackoffMs(consecutiveFailures) {
|
|
6
|
+
if (consecutiveFailures <= 0)
|
|
7
|
+
return 0;
|
|
8
|
+
return Math.min(RESPAWN_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESPAWN_BACKOFF_MAX_MS);
|
|
9
|
+
}
|
|
10
|
+
/** Whether a durable backoff still holds this row back from a relaunch. */
|
|
4
11
|
export function isInBackoff(row, now) {
|
|
5
12
|
const notBefore = row.respawn_not_before;
|
|
6
13
|
if (notBefore == null)
|
|
@@ -8,3 +15,27 @@ export function isInBackoff(row, now) {
|
|
|
8
15
|
const at = Date.parse(notBefore);
|
|
9
16
|
return Number.isFinite(at) && now < at;
|
|
10
17
|
}
|
|
18
|
+
/** The only respawn-failure policy: progress since launch clears the sequence;
|
|
19
|
+
* every other missing-engine observation advances it and records its backoff. */
|
|
20
|
+
export function decideRespawnAttempt(row, now) {
|
|
21
|
+
const settledAt = row.last_settled_at == null ? Number.NaN : Date.parse(row.last_settled_at);
|
|
22
|
+
const launchedAt = row.launched_at == null ? Number.NaN : Date.parse(row.launched_at);
|
|
23
|
+
if (Number.isFinite(settledAt) && Number.isFinite(launchedAt) && settledAt > launchedAt) {
|
|
24
|
+
return { failures: 0, notBefore: null, exhausted: false };
|
|
25
|
+
}
|
|
26
|
+
const failures = (row.respawn_failures ?? 0) + 1;
|
|
27
|
+
return {
|
|
28
|
+
failures,
|
|
29
|
+
notBefore: new Date(now + respawnBackoffMs(failures)).toISOString(),
|
|
30
|
+
exhausted: failures >= MAX_RESPAWN_ATTEMPTS,
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
/** Persist the policy decision through the daemon-only canvas writer. */
|
|
34
|
+
export function recordRespawnPolicy(row, now) {
|
|
35
|
+
const attempt = decideRespawnAttempt(row, now);
|
|
36
|
+
recordRespawnAttempt(row.node_id, { failures: attempt.failures, notBefore: attempt.notBefore });
|
|
37
|
+
return attempt;
|
|
38
|
+
}
|
|
39
|
+
export function clearRespawnPolicy(nodeId) {
|
|
40
|
+
recordRespawnAttempt(nodeId, { failures: 0, notBefore: null });
|
|
41
|
+
}
|
|
@@ -1,2 +1,5 @@
|
|
|
1
1
|
import type { NodeRow } from '../../../core/canvas/index.js';
|
|
2
|
-
|
|
2
|
+
/** Request teardown only while a terminal row still has a daemon handle or a
|
|
3
|
+
* provably live recorded process. An indeterminate pid is left for the next
|
|
4
|
+
* tick: uncertainty never authorizes a signal. */
|
|
5
|
+
export declare function handleTerminatingRow(row: NodeRow, hasFleetHandle: boolean): boolean;
|
|
@@ -1,3 +1,12 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
import { recordedPidLiveness } from '../../../core/canvas/pid.js';
|
|
2
|
+
import { requestBrokerTeardown } from '../../../core/runtime/host.js';
|
|
3
|
+
/** Request teardown only while a terminal row still has a daemon handle or a
|
|
4
|
+
* provably live recorded process. An indeterminate pid is left for the next
|
|
5
|
+
* tick: uncertainty never authorizes a signal. */
|
|
6
|
+
export function handleTerminatingRow(row, hasFleetHandle) {
|
|
7
|
+
if (!hasFleetHandle && (row.pi_pid == null || recordedPidLiveness(row.pi_pid, row.pi_pid_identity) !== 'alive')) {
|
|
8
|
+
return false;
|
|
9
|
+
}
|
|
10
|
+
requestBrokerTeardown(row.node_id);
|
|
11
|
+
return true;
|
|
3
12
|
}
|
|
@@ -59,18 +59,21 @@ export class NodeLifecycleReconciler {
|
|
|
59
59
|
}
|
|
60
60
|
async classifyRow(now, row, ctx, candidates) {
|
|
61
61
|
const id = row.node_id;
|
|
62
|
+
if (isTerminal(row)) {
|
|
63
|
+
if (handleTerminatingRow(row, ctx.fleet.has(id)))
|
|
64
|
+
return;
|
|
65
|
+
if (row.pi_pid != null && recordedPidLiveness(row.pi_pid, row.pi_pid_identity) === 'dead') {
|
|
66
|
+
clearRecordedPid(id);
|
|
67
|
+
}
|
|
68
|
+
if (row.frozen_at != null)
|
|
69
|
+
candidates.push({ row, enact: 'thaw' });
|
|
70
|
+
return;
|
|
71
|
+
}
|
|
62
72
|
// 1. A fleet handle IS execution. Whatever froze this row is over.
|
|
63
73
|
if (ctx.fleet.has(id)) {
|
|
64
74
|
this.thawMark(row);
|
|
65
75
|
return;
|
|
66
76
|
}
|
|
67
|
-
if (isTerminal(row)) {
|
|
68
|
-
if (row.pi_pid != null)
|
|
69
|
-
handleTerminatingRow(row);
|
|
70
|
-
else if (row.frozen_at != null)
|
|
71
|
-
candidates.push({ row, enact: 'thaw' });
|
|
72
|
-
return;
|
|
73
|
-
}
|
|
74
77
|
// A paused profile launches nothing, and its stuck-delivery episode is over
|
|
75
78
|
// rather than merely stalled — the pause is not a delivery failure.
|
|
76
79
|
if (isProfilePaused(row.profile_id)) {
|
|
@@ -101,6 +104,10 @@ export class NodeLifecycleReconciler {
|
|
|
101
104
|
// its way to the launch seam, which owns its own launch and must be left
|
|
102
105
|
// alone rather than raced or written off as a boot failure.
|
|
103
106
|
if (row.pi_pid == null) {
|
|
107
|
+
if (row.respawn_not_before != null) {
|
|
108
|
+
candidates.push({ row, enact: 'policy' });
|
|
109
|
+
return;
|
|
110
|
+
}
|
|
104
111
|
if (row.frozen_at != null)
|
|
105
112
|
candidates.push({ row, enact: 'thaw' });
|
|
106
113
|
return;
|
|
@@ -13,8 +13,8 @@ export declare class StorageMaintenanceReconciler {
|
|
|
13
13
|
/** Milliseconds since meta/telemetry was last written. Inbox activity is
|
|
14
14
|
* deliberately excluded: children can keep writing a dead manager's inbox. */
|
|
15
15
|
private msSinceDiskActivity;
|
|
16
|
-
/** Reap only
|
|
17
|
-
*
|
|
16
|
+
/** Reap only physical placement residue of terminal nodes. The durable row,
|
|
17
|
+
* graph, and on-disk state remain available for a later manual revive. */
|
|
18
18
|
private reapDeadResidue;
|
|
19
19
|
/** Delete focus rows whose recorded tmux pane no longer exists. A failed tmux
|
|
20
20
|
* probe is "can't tell" and never authorizes deleting every viewport. */
|
|
@@ -66,12 +66,12 @@ export class StorageMaintenanceReconciler {
|
|
|
66
66
|
}
|
|
67
67
|
return latest === 0 ? Number.POSITIVE_INFINITY : now - latest;
|
|
68
68
|
}
|
|
69
|
-
/** Reap only
|
|
70
|
-
*
|
|
69
|
+
/** Reap only physical placement residue of terminal nodes. The durable row,
|
|
70
|
+
* graph, and on-disk state remain available for a later manual revive. */
|
|
71
71
|
reapDeadResidue(now) {
|
|
72
72
|
let rows;
|
|
73
73
|
try {
|
|
74
|
-
rows = listNodes({ status: ['dead'] });
|
|
74
|
+
rows = listNodes({ status: ['done', 'canceled', 'dead'] });
|
|
75
75
|
}
|
|
76
76
|
catch (err) {
|
|
77
77
|
operationIdContext.fresh(() => {
|
|
@@ -186,7 +186,7 @@ test('gh-451: finished does NOT shut down while a non-deferred inbox entry sits
|
|
|
186
186
|
assert.equal(getNode('mgr')?.intent ?? null, null, 'no idle-release intent while a non-deferred entry is unseen');
|
|
187
187
|
assert.equal(shutdown, false, 'pi stays alive so the in-process watcher can still deliver/commit');
|
|
188
188
|
});
|
|
189
|
-
test('
|
|
189
|
+
test('a deferred send to a TERMINAL node was raised on append, so it holds the settle instead of riding a later cycle', { timeout: 1_000 }, async () => {
|
|
190
190
|
createNode(node('root', { parent: null, lifecycle: 'resident' }));
|
|
191
191
|
createNode(node('mgr', { parent: 'root', lifecycle: 'terminal', mode: 'orchestrator' }));
|
|
192
192
|
createNode(node('worker', { parent: 'mgr', lifecycle: 'terminal', status: 'active' }));
|
|
@@ -199,9 +199,9 @@ test('gh-451: a DEFERRED-only unseen entry does not block idle-release (matches
|
|
|
199
199
|
let shutdown = false;
|
|
200
200
|
const ctx = { shutdown: () => { shutdown = true; } };
|
|
201
201
|
await settle(pi, stopEvent('still waiting on the worker'), ctx);
|
|
202
|
-
assert.equal(getNode('mgr')?.intent,
|
|
203
|
-
assert.equal(getNode('mgr')?.status, '
|
|
204
|
-
assert.equal(shutdown,
|
|
202
|
+
assert.equal(getNode('mgr')?.intent ?? null, null, 'a terminal node takes no reliable later cycle, so its raised entry blocks the release');
|
|
203
|
+
assert.equal(getNode('mgr')?.status, 'active', 'mgr stays active for the entry the watcher still owes it');
|
|
204
|
+
assert.equal(shutdown, false, 'pi stays alive so the in-process watcher can deliver it');
|
|
205
205
|
});
|
|
206
206
|
test('queued continuation finishes before natural-stop routing commits', { timeout: 1_000 }, async () => {
|
|
207
207
|
createNode(node('leaf', { parent: null, lifecycle: 'terminal' }));
|
|
@@ -23,8 +23,9 @@ interface SessionEntryLike {
|
|
|
23
23
|
}
|
|
24
24
|
interface SessionStartCtxLike {
|
|
25
25
|
sessionManager: {
|
|
26
|
-
|
|
27
|
-
|
|
26
|
+
/** Required by pi's `ReadonlySessionManager`: the active branch, which is
|
|
27
|
+
* the context the next turn actually sees. */
|
|
28
|
+
getBranch: (fromId?: string) => SessionEntryLike[];
|
|
28
29
|
};
|
|
29
30
|
}
|
|
30
31
|
interface CustomMessageLike {
|
|
@@ -37,35 +38,12 @@ interface CustomMessageLike {
|
|
|
37
38
|
nodeId: string;
|
|
38
39
|
};
|
|
39
40
|
}
|
|
40
|
-
/** The message handed to a message renderer. `content` is normally the string we
|
|
41
|
-
* sent, but pi types it as string-or-blocks, so we handle both. */
|
|
42
|
-
interface RenderedMessageLike {
|
|
43
|
-
customType: string;
|
|
44
|
-
content: string | Array<{
|
|
45
|
-
type: string;
|
|
46
|
-
text?: string;
|
|
47
|
-
}>;
|
|
48
|
-
}
|
|
49
|
-
/** Minimal structural match for pi-tui's `Component` (render + invalidate). A
|
|
50
|
-
* plain object of this shape is a valid child for pi's Container. */
|
|
51
|
-
interface ComponentLike {
|
|
52
|
-
render: (width: number) => string[];
|
|
53
|
-
invalidate: () => void;
|
|
54
|
-
}
|
|
55
|
-
/** Subset of pi's `Theme` we touch — `fg(color, text)` wraps text in ANSI. Used
|
|
56
|
-
* defensively (falls back to plain text if absent). */
|
|
57
|
-
interface ThemeLike {
|
|
58
|
-
fg?: (color: string, text: string) => string;
|
|
59
|
-
}
|
|
60
41
|
interface PiLike {
|
|
61
42
|
on: (event: 'session_start', handler: (event: unknown, ctx: SessionStartCtxLike) => void | Promise<void>) => void;
|
|
62
43
|
sendMessage: (message: CustomMessageLike, options?: {
|
|
63
44
|
deliverAs?: string;
|
|
64
45
|
triggerTurn?: boolean;
|
|
65
46
|
}) => void;
|
|
66
|
-
registerMessageRenderer: (customType: string, renderer: (message: RenderedMessageLike, options: {
|
|
67
|
-
expanded?: boolean;
|
|
68
|
-
}, theme: ThemeLike) => ComponentLike | undefined) => void;
|
|
69
47
|
}
|
|
70
48
|
/** Build the full bearings baseline for a conversation without inherited
|
|
71
49
|
* crouter context. Exported for testing. */
|
|
@@ -73,15 +51,6 @@ export declare function buildContextIntro(nodeId: string, target?: ExposureTarge
|
|
|
73
51
|
/** Build the compact node-scoped update for a fork whose copied branch already
|
|
74
52
|
* carries a full crouter baseline. Exported for testing. */
|
|
75
53
|
export declare function buildForkContextIntro(nodeId: string): Promise<string>;
|
|
76
|
-
/**
|
|
77
|
-
* Renderer for `crtr-context` messages. Collapsed (default) shows a one-line
|
|
78
|
-
* stub; expanded (Ctrl+O) shows the label + full body. Returns a plain object
|
|
79
|
-
* matching pi's structural `Component` interface — no pi-tui import. Exported for
|
|
80
|
-
* testing.
|
|
81
|
-
*/
|
|
82
|
-
export declare function renderContextMessage(message: RenderedMessageLike, options: {
|
|
83
|
-
expanded?: boolean;
|
|
84
|
-
}, theme: ThemeLike): ComponentLike;
|
|
85
54
|
/** Register new-context baseline assembly; a strict resume changes nothing. */
|
|
86
55
|
export declare function registerCanvasContextIntro(pi: PiLike): void;
|
|
87
56
|
export default registerCanvasContextIntro;
|