@north-light/crouter 0.3.236 → 0.3.237
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.d.ts +2 -1
- package/dist/api/client.js +3 -0
- package/dist/api/dto/broker-ops.d.ts +1 -1
- package/dist/api/dto/common.d.ts +2 -0
- package/dist/api/dto/lifecycle.d.ts +9 -2
- package/dist/api/dto/messages.d.ts +5 -0
- package/dist/api/dto/nodes.d.ts +10 -2
- package/dist/api/dto/worktree.d.ts +10 -0
- package/dist/api/routes.d.ts +1 -0
- package/dist/api/routes.js +1 -0
- package/dist/builtin-memory/00-runtime-base/00-authoring.md +1 -1
- package/dist/builtin-memory/internal/agent-shaping.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/sysprompt-window.ts +1 -1
- package/dist/clients/attach/render/tool-calls.js +18 -3
- package/dist/clients/attach/viewer.js +426 -426
- package/dist/clients/inbox/review/__tests__/editor-roundtrip.test.js +6 -0
- package/dist/clients/inbox/review/roundtrip.js +1 -1
- package/dist/commands/memory/lint.js +22 -13
- package/dist/commands/memory/shared.js +1 -1
- package/dist/commands/node/create.d.ts +1 -1
- package/dist/commands/node/create.js +1 -1
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node-worktree.js +39 -8
- package/dist/commands/revive.js +11 -2
- package/dist/commands/sys/context/admin/actions.d.ts +48 -0
- package/dist/commands/sys/context/admin/actions.js +246 -0
- package/dist/commands/sys/context/admin/axes-panel.d.ts +21 -0
- package/dist/commands/sys/context/admin/axes-panel.js +117 -0
- package/dist/commands/sys/context/admin/detail-panel.d.ts +53 -0
- package/dist/commands/sys/context/admin/detail-panel.js +178 -0
- package/dist/commands/sys/context/admin/docs-panel.d.ts +45 -0
- package/dist/commands/sys/context/admin/docs-panel.js +186 -0
- package/dist/commands/sys/context/admin/model.d.ts +75 -0
- package/dist/commands/sys/context/admin/model.js +209 -0
- package/dist/commands/sys/context/admin/shell.d.ts +82 -0
- package/dist/commands/sys/context/admin/shell.js +669 -0
- package/dist/commands/sys/context/admin.d.ts +1 -0
- package/dist/commands/sys/context/admin.js +124 -0
- package/dist/commands/sys/context/doc.d.ts +1 -0
- package/dist/commands/sys/context/doc.js +196 -0
- package/dist/commands/sys/context/prompt-review.d.ts +1 -0
- package/dist/commands/sys/{prompt-review.js → context/prompt-review.js} +10 -10
- package/dist/commands/sys/context/resolve.d.ts +55 -0
- package/dist/commands/sys/context/resolve.js +428 -0
- package/dist/commands/sys/context/sysprompt.d.ts +1 -0
- package/dist/commands/sys/{sysprompt.js → context/sysprompt.js} +11 -11
- package/dist/commands/sys/context.d.ts +2 -0
- package/dist/commands/sys/context.js +20 -0
- package/dist/commands/sys.js +3 -4
- package/dist/core/__tests__/child-death-wake.test.js +38 -22
- package/dist/core/__tests__/cron-self-cancel-delivery.test.js +72 -0
- package/dist/core/__tests__/daemon-boot.test.js +40 -28
- package/dist/core/__tests__/human-deliver.test.js +1 -1
- package/dist/core/__tests__/human-node-not-supervised.test.js +10 -15
- package/dist/core/__tests__/integration/revive.test.js +12 -12
- package/dist/core/__tests__/integration/spawn-root.test.js +6 -0
- package/dist/core/__tests__/integration/worktree-reap.test.js +32 -0
- package/dist/core/__tests__/lifecycle-terminal-reasons.test.d.ts +1 -0
- package/dist/core/__tests__/lifecycle-terminal-reasons.test.js +61 -0
- package/dist/core/__tests__/migration.test.js +38 -1
- package/dist/core/__tests__/push-final-guard.test.js +1 -1
- package/dist/core/__tests__/revive-capacity.test.d.ts +1 -0
- package/dist/core/__tests__/revive-capacity.test.js +127 -0
- package/dist/core/__tests__/revive-parked-fresh.test.js +7 -7
- package/dist/core/__tests__/seam/broker-cap-freeze.test.d.ts +1 -0
- package/dist/core/__tests__/seam/broker-cap-freeze.test.js +81 -0
- package/dist/core/__tests__/seam/dormancy-release.test.js +7 -7
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +1 -1
- package/dist/core/canvas/browse/render.js +11 -3
- package/dist/core/canvas/canvas.d.ts +15 -3
- package/dist/core/canvas/canvas.js +52 -16
- package/dist/core/canvas/migrations.js +21 -0
- package/dist/core/canvas/nav-render.js +3 -1
- package/dist/core/canvas/remote-canvas-source.js +2 -0
- package/dist/core/canvas/render-source.d.ts +3 -1
- package/dist/core/canvas/render-source.js +7 -3
- package/dist/core/canvas/status-glyph.d.ts +10 -1
- package/dist/core/canvas/status-glyph.js +19 -3
- package/dist/core/canvas/types.d.ts +20 -1
- package/dist/core/feed/feed.js +1 -1
- package/dist/core/human/feedback-companion.js +5 -2
- package/dist/core/memory/lint.d.ts +19 -0
- package/dist/core/memory/lint.js +97 -0
- package/dist/core/memory-resolver.d.ts +4 -0
- package/dist/core/memory-resolver.js +7 -7
- package/dist/core/preview-registry.js +4 -4
- package/dist/core/review/realize.js +4 -2
- package/dist/core/runtime/close.js +1 -1
- package/dist/core/runtime/fleet.d.ts +29 -1
- package/dist/core/runtime/fleet.js +23 -0
- package/dist/core/runtime/host.js +32 -11
- package/dist/core/runtime/launch-prompt.d.ts +1 -1
- package/dist/core/runtime/launch-prompt.js +2 -2
- package/dist/core/runtime/lifecycle.d.ts +4 -2
- package/dist/core/runtime/lifecycle.js +12 -1
- package/dist/core/runtime/memory.js +1 -1
- package/dist/core/runtime/reopen.js +10 -1
- package/dist/core/runtime/reset.js +1 -1
- package/dist/core/runtime/revive-all.d.ts +3 -0
- package/dist/core/runtime/revive-all.js +10 -4
- package/dist/core/runtime/revive.d.ts +12 -0
- package/dist/core/runtime/revive.js +62 -3
- package/dist/core/runtime/spawn.d.ts +4 -0
- package/dist/core/runtime/spawn.js +36 -7
- package/dist/core/substrate/frontmatter-validation.js +1 -1
- package/dist/core/substrate/index.d.ts +6 -3
- package/dist/core/substrate/index.js +4 -3
- package/dist/core/substrate/on-read.d.ts +0 -8
- package/dist/core/substrate/on-read.js +45 -124
- package/dist/core/substrate/plan.d.ts +94 -0
- package/dist/core/substrate/plan.js +266 -0
- package/dist/core/substrate/render.d.ts +5 -6
- package/dist/core/substrate/render.js +76 -129
- package/dist/core/substrate/schema.d.ts +1 -6
- package/dist/core/substrate/schema.js +1 -15
- package/dist/core/substrate/session-cache.d.ts +2 -10
- package/dist/core/substrate/session-cache.js +9 -47
- package/dist/core/substrate/subject-fields.d.ts +1 -4
- package/dist/core/substrate/subject-fields.js +2 -5
- package/dist/core/substrate/surface-match.d.ts +37 -17
- package/dist/core/substrate/surface-match.js +44 -53
- package/dist/core/tui/page-host.d.ts +6 -0
- package/dist/core/tui/page-host.js +12 -3
- package/dist/core/worktree.d.ts +8 -0
- package/dist/core/worktree.js +65 -0
- package/dist/daemon/__tests__/reconciler-signature.test.js +3 -2
- package/dist/daemon/__tests__/thaw-order.test.d.ts +1 -0
- package/dist/daemon/__tests__/thaw-order.test.js +55 -0
- package/dist/daemon/api/__tests__/canvas-snapshot-fields.test.js +6 -0
- package/dist/daemon/api/__tests__/node-create-description.test.js +6 -0
- package/dist/daemon/api/__tests__/profile-launch-gates.test.js +1 -1
- package/dist/daemon/api/bridge.js +5 -1
- package/dist/daemon/api/handlers/attach.js +5 -1
- package/dist/daemon/api/handlers/bash-jobs.js +1 -1
- package/dist/daemon/api/handlers/messages.js +10 -2
- package/dist/daemon/api/handlers/nodes.js +8 -1
- package/dist/daemon/api/handlers/worktree.js +10 -1
- package/dist/daemon/api/map.js +7 -0
- package/dist/daemon/companion-retire.js +1 -1
- package/dist/daemon/cron/passes.d.ts +0 -5
- package/dist/daemon/cron/passes.js +0 -20
- package/dist/daemon/cron/sinks.js +1 -1
- package/dist/daemon/cron-run.d.ts +5 -6
- package/dist/daemon/cron-run.js +7 -16
- package/dist/daemon/crtrd.d.ts +0 -10
- package/dist/daemon/crtrd.js +23 -61
- package/dist/daemon/fleet.d.ts +10 -7
- package/dist/daemon/fleet.js +71 -12
- package/dist/daemon/messaging/node-message.js +3 -1
- package/dist/daemon/profile-delete.js +1 -1
- package/dist/daemon/reconcilers/broker-supervision.d.ts +26 -9
- package/dist/daemon/reconcilers/broker-supervision.js +39 -52
- package/dist/daemon/reconcilers/controller-death.d.ts +7 -0
- package/dist/daemon/reconcilers/controller-death.js +26 -0
- package/dist/daemon/reconcilers/cron-lane.d.ts +0 -3
- package/dist/daemon/reconcilers/cron-lane.js +0 -3
- package/dist/daemon/reconcilers/live-obligation.js +3 -1
- package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.d.ts +53 -0
- package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +170 -0
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +5 -0
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +16 -0
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -0
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +13 -0
- package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +26 -0
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +140 -0
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
- package/dist/commands/sys/prompt-review.d.ts +0 -1
- package/dist/commands/sys/sysprompt.d.ts +0 -1
- package/dist/core/__tests__/cron-broker-capacity.test.js +0 -119
- package/dist/daemon/reconcilers/dormant-inbox.d.ts +0 -18
- package/dist/daemon/reconcilers/dormant-inbox.js +0 -157
- /package/dist/core/__tests__/{cron-broker-capacity.test.d.ts → cron-self-cancel-delivery.test.d.ts} +0 -0
|
@@ -3,7 +3,7 @@ import { join } from 'node:path';
|
|
|
3
3
|
import { homedir } from 'node:os';
|
|
4
4
|
import { execFile } from 'node:child_process';
|
|
5
5
|
import { promisify } from 'node:util';
|
|
6
|
-
import {
|
|
6
|
+
import { getNode, isWarmSpare, subscribersOf } from '../../core/canvas/index.js';
|
|
7
7
|
import { contextDir, isSafeNodeId } from '../../core/canvas/paths.js';
|
|
8
8
|
import { fullName } from '../../core/canvas/labels.js';
|
|
9
9
|
import { recordedPidLiveness } from '../../core/canvas/pid.js';
|
|
@@ -14,7 +14,6 @@ import { nextFaultRetry } from '../../core/runtime/fault-recovery.js';
|
|
|
14
14
|
import { fanDoctrineWake } from '../../core/runtime/close.js';
|
|
15
15
|
import { headlessBrokerHost } from '../../core/runtime/host.js';
|
|
16
16
|
import { reloadAuthLive } from '../../core/runtime/auth-reload.js';
|
|
17
|
-
import { readConfig } from '../../core/config.js';
|
|
18
17
|
import { emitEvent } from '../../core/events/emit.js';
|
|
19
18
|
import { operationIdContext } from '../../core/events/operation-id.js';
|
|
20
19
|
import { hasUnseenWakeMail } from '../../core/feed/inbox.js';
|
|
@@ -24,22 +23,16 @@ import { hasAttachedBrokerViewers } from '../../core/runtime/broker/client-regis
|
|
|
24
23
|
import { deliverLive } from '../../core/runtime/interactive-deliver.js';
|
|
25
24
|
import { BrokerUnreachableError } from '../../core/runtime/broker-request.js';
|
|
26
25
|
import { formatCard, PARK_SUMMARY_PROMPT } from '../../shared/generated-context.js';
|
|
26
|
+
import { brokerThresholdsForDaemon } from '../../core/runtime/fleet.js';
|
|
27
27
|
import { beginPendingPark, consumePendingPark, pendingParkIds, pendingParkStartedAt } from '../park-pending.js';
|
|
28
28
|
import { applyParkEvent, parkEvent } from './live-obligation.js';
|
|
29
|
-
import {
|
|
29
|
+
import { envNoDaemonAutostart, envTestParkSummaryGraceMs, envTestUnattendedParkMs, } from '../../shared/env.js';
|
|
30
30
|
const execFileAsync = promisify(execFile);
|
|
31
|
-
export function brokerThresholdsForDaemon() {
|
|
32
|
-
const configured = readConfig('user').brokerThresholds;
|
|
33
|
-
const automaticReviveCap = envMaxLiveBrokers() ?? configured.automaticReviveCap;
|
|
34
|
-
const warning = envWarnLiveBrokers() ?? configured.warning;
|
|
35
|
-
return { warning: Math.min(warning, automaticReviveCap), automaticReviveCap };
|
|
36
|
-
}
|
|
37
31
|
export const YIELD_STALL_GRACE_MS = 3 * 60_000;
|
|
38
32
|
const KILL_ESCALATE_MS = 20_000;
|
|
39
33
|
export const WEDGE_QUIET_MS = 20 * 60_000;
|
|
40
34
|
export const WEDGE_CPU_MAX_PERCENT = 2;
|
|
41
35
|
const OVER_CAP_WARN_THROTTLE_MS = 5 * 60_000;
|
|
42
|
-
const CAPACITY_LOG_THROTTLE_MS = 30_000;
|
|
43
36
|
const DEFAULT_UNATTENDED_PARK_MS = 15 * 60_000;
|
|
44
37
|
// Longer than a summary turn and shorter than the unattended clock itself, so a
|
|
45
38
|
// stuck parking turn cannot hold a broker for a whole extra clock period.
|
|
@@ -138,22 +131,31 @@ export class BrokerSupervisionReconciler {
|
|
|
138
131
|
unattendedSince = new Map();
|
|
139
132
|
knownAuthMtimeMs = null;
|
|
140
133
|
lastOverCapWarnAt = Number.NEGATIVE_INFINITY;
|
|
141
|
-
|
|
134
|
+
/** The fleet-wide preamble, run BEFORE the tick's row-list guard: an auth
|
|
135
|
+
* reload does not depend on a successful canvas scan, and the thresholds it
|
|
136
|
+
* returns are read once for the whole tick. */
|
|
137
|
+
async runFleetPreamble(fleet) {
|
|
138
|
+
await this.handleAuthReload(fleet);
|
|
139
|
+
return brokerThresholdsForDaemon();
|
|
140
|
+
}
|
|
141
|
+
/** The saturation notice, run LAST in the tick: the frozen split it reports
|
|
142
|
+
* is only known once the freeze lane has spent this tick's free slots. */
|
|
143
|
+
warnIfSaturated(ctx) {
|
|
144
|
+
if (ctx.fleet.size() < ctx.thresholds.warning || !ctx.lifecycle.acceptsDetachedWork())
|
|
145
|
+
return;
|
|
146
|
+
const warning = operationIdContext.fresh(() => this.warnOverCapInTmux(ctx.fleet.size(), ctx.thresholds, ctx.frozen));
|
|
147
|
+
ctx.lifecycle.registerDetached(warning);
|
|
148
|
+
}
|
|
142
149
|
async run(now, ctx) {
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
+
for (const row of ctx.rows) {
|
|
151
|
+
const id = row.node_id;
|
|
152
|
+
const entry = ctx.fleet.get(id);
|
|
153
|
+
if (entry === undefined || (row.status !== 'active' && row.status !== 'idle')) {
|
|
154
|
+
this.unattendedSince.delete(id);
|
|
155
|
+
continue;
|
|
156
|
+
}
|
|
150
157
|
await operationIdContext.fresh(async () => {
|
|
151
158
|
try {
|
|
152
|
-
const row = getRow(id);
|
|
153
|
-
if (row === null || (row.status !== 'active' && row.status !== 'idle')) {
|
|
154
|
-
this.unattendedSince.delete(id);
|
|
155
|
-
return;
|
|
156
|
-
}
|
|
157
159
|
this.handleYieldStall(row, entry.pid, now);
|
|
158
160
|
await this.handleWedgeDetection(id, entry.pid, row.pi_pid_identity, now);
|
|
159
161
|
this.handleFatalFault(id);
|
|
@@ -180,7 +182,6 @@ export class BrokerSupervisionReconciler {
|
|
|
180
182
|
if (!ctx.fleet.has(id))
|
|
181
183
|
consumePendingPark(id);
|
|
182
184
|
}
|
|
183
|
-
return thresholds;
|
|
184
185
|
}
|
|
185
186
|
/** Every condition that resets the unattended clock EXCEPT `isBusy` — the
|
|
186
187
|
* node is unavailable to park, attended, or has work waiting. Short-circuits
|
|
@@ -325,17 +326,28 @@ export class BrokerSupervisionReconciler {
|
|
|
325
326
|
});
|
|
326
327
|
}
|
|
327
328
|
}
|
|
328
|
-
async warnOverCapInTmux(count,
|
|
329
|
+
async warnOverCapInTmux(count, thresholds, frozen) {
|
|
329
330
|
const now = Date.now();
|
|
330
331
|
if (now - this.lastOverCapWarnAt < OVER_CAP_WARN_THROTTLE_MS)
|
|
331
332
|
return;
|
|
332
333
|
this.lastOverCapWarnAt = now;
|
|
334
|
+
const { warning, automaticReviveCap: max } = thresholds;
|
|
335
|
+
const waiting = frozen.stalled + frozen.waitingOnMail;
|
|
333
336
|
emitEvent({
|
|
334
337
|
level: 'warn',
|
|
335
338
|
event: 'broker.capacity.warning',
|
|
336
|
-
fields: {
|
|
339
|
+
fields: {
|
|
340
|
+
live_count: count,
|
|
341
|
+
warning_threshold: warning,
|
|
342
|
+
hard_cap: max,
|
|
343
|
+
frozen_stalled: frozen.stalled,
|
|
344
|
+
frozen_waiting_on_mail: frozen.waitingOnMail,
|
|
345
|
+
},
|
|
337
346
|
});
|
|
338
|
-
const
|
|
347
|
+
const frozenNote = waiting === 0
|
|
348
|
+
? ''
|
|
349
|
+
: ` — ${waiting} frozen (${frozen.stalled} stalled, ${frozen.waitingOnMail} waiting on mail)`;
|
|
350
|
+
const msg = `⚠ crtrd: ${count} live brokers (warning ${warning}; hard cap ${max})${frozenNote} — possible broker leak. ps -ax | grep broker-cli.js`;
|
|
339
351
|
try {
|
|
340
352
|
const clients = await execFileAsync('tmux', ['list-clients', '-F', '#{client_name}'], {
|
|
341
353
|
encoding: 'utf8',
|
|
@@ -637,29 +649,4 @@ export class BrokerSupervisionReconciler {
|
|
|
637
649
|
}
|
|
638
650
|
});
|
|
639
651
|
}
|
|
640
|
-
hasBrokerCapacity(max, fleet) {
|
|
641
|
-
if (fleet.size() < max)
|
|
642
|
-
return true;
|
|
643
|
-
const now = Date.now();
|
|
644
|
-
if (now - this.lastCapacityLogAt >= CAPACITY_LOG_THROTTLE_MS) {
|
|
645
|
-
this.lastCapacityLogAt = now;
|
|
646
|
-
emitEvent({
|
|
647
|
-
level: 'warn',
|
|
648
|
-
event: 'broker.capacity.revive_deferred',
|
|
649
|
-
fields: { live_count: fleet.size(), hard_cap: max },
|
|
650
|
-
});
|
|
651
|
-
}
|
|
652
|
-
return false;
|
|
653
|
-
}
|
|
654
|
-
hasStartupRecoveryCapacity(fleet) {
|
|
655
|
-
const { automaticReviveCap } = brokerThresholdsForDaemon();
|
|
656
|
-
if (fleet.size() < automaticReviveCap)
|
|
657
|
-
return true;
|
|
658
|
-
emitEvent({
|
|
659
|
-
level: 'warn',
|
|
660
|
-
event: 'daemon.recovery_sweep.capacity_deferred',
|
|
661
|
-
fields: { live_count: fleet.size() },
|
|
662
|
-
});
|
|
663
|
-
return false;
|
|
664
|
-
}
|
|
665
652
|
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { getRow } from '../../core/canvas/index.js';
|
|
2
|
+
import { operationIdContext } from '../../core/events/operation-id.js';
|
|
3
|
+
import { appendControllerDeathNotice } from '../../core/feed/inbox.js';
|
|
4
|
+
export class ControllerDeathReconciler {
|
|
5
|
+
run(now, ctx) {
|
|
6
|
+
void now; // no time-gating here — the death check is already-controller-status-driven; the param is kept for six-way call-site uniformity, matching every other reconciler's run(now, ctx) shape.
|
|
7
|
+
// Settle every wait whose controller has ended, independent of the waiter's
|
|
8
|
+
// current status. Active and focused terminal nodes own a live inbox
|
|
9
|
+
// watcher; an idle-released node is revived by the node-lifecycle tick.
|
|
10
|
+
for (const row of ctx.rows) {
|
|
11
|
+
const current = getRow(row.node_id);
|
|
12
|
+
const controllerId = current?.waiting_for;
|
|
13
|
+
if (controllerId === null || controllerId === undefined)
|
|
14
|
+
continue;
|
|
15
|
+
// Read-gate before the write boundary: only a dead controller warrants a
|
|
16
|
+
// notice. appendControllerDeathNotice re-checks liveness inside its
|
|
17
|
+
// transaction, so this gate cannot misfire.
|
|
18
|
+
const controller = getRow(controllerId);
|
|
19
|
+
if (controller !== null && (controller.status === 'active' || controller.status === 'idle'))
|
|
20
|
+
continue;
|
|
21
|
+
operationIdContext.fresh(() => {
|
|
22
|
+
appendControllerDeathNotice(row.node_id, controllerId);
|
|
23
|
+
});
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
@@ -1,9 +1,6 @@
|
|
|
1
1
|
import type { DetachedWorkLifecycle } from './broker-supervision.js';
|
|
2
2
|
export interface CronLaneContext {
|
|
3
3
|
lifecycle: DetachedWorkLifecycle;
|
|
4
|
-
/** P3.4's capacity authority, bound to the thresholds returned by the live
|
|
5
|
-
* supervision pass for this same tick. */
|
|
6
|
-
hasBrokerCapacity(): boolean;
|
|
7
4
|
}
|
|
8
5
|
export declare class CronLaneReconciler {
|
|
9
6
|
run(now: number, ctx: CronLaneContext): void;
|
|
@@ -20,9 +20,6 @@ export class CronLaneReconciler {
|
|
|
20
20
|
try {
|
|
21
21
|
runDueCrons(now, {
|
|
22
22
|
register: (settlement) => ctx.lifecycle.registerDetached(settlement),
|
|
23
|
-
// A due run that would launch or revive a node stays due until P3.4's
|
|
24
|
-
// capacity authority admits it.
|
|
25
|
-
hasBrokerCapacity: ctx.hasBrokerCapacity,
|
|
26
23
|
});
|
|
27
24
|
}
|
|
28
25
|
catch (err) {
|
|
@@ -45,7 +45,9 @@ export function parkEvent(nodeId, lifecycle) {
|
|
|
45
45
|
* (broker settlement after a parking turn) enacts `park` directly and fans
|
|
46
46
|
* nothing — the node's report is the notice. */
|
|
47
47
|
export function applyParkEvent(nodeId, event) {
|
|
48
|
-
const node =
|
|
48
|
+
const node = event === 'cancel'
|
|
49
|
+
? transition(nodeId, event, { reason: 'stranded' })
|
|
50
|
+
: transition(nodeId, event);
|
|
49
51
|
if (event !== 'park')
|
|
50
52
|
return;
|
|
51
53
|
fanDoctrineWake(nodeId, subscribersOf(nodeId), `Child done — ${fullName(node)} (${nodeId}) was automatically marked done after its unattended grace with nothing left to wake it. It sent no final report.`, { reason: 'child-auto-done', child: nodeId });
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { type NodeRow } from '../../../core/canvas/index.js';
|
|
2
|
+
import type { FleetRegistry } from '../../../core/runtime/fleet.js';
|
|
3
|
+
/** How a candidate spends a slot when its turn comes. `thaw` simply launches a
|
|
4
|
+
* row the cap turned away — its disposition was decided when it froze, and the
|
|
5
|
+
* dead-row table has no jurisdiction over a broker that never existed.
|
|
6
|
+
* `policy` re-enacts the dead-row policy table (which relaunches through
|
|
7
|
+
* `reviveNode`); `idle-release` runs the stuck-delivery episode rules before
|
|
8
|
+
* relaunching. */
|
|
9
|
+
export type ThawEnactment = 'thaw' | 'policy' | 'idle-release';
|
|
10
|
+
export interface ThawCandidate {
|
|
11
|
+
row: NodeRow;
|
|
12
|
+
enact: ThawEnactment;
|
|
13
|
+
}
|
|
14
|
+
/** Tier 1 is a row with nothing waiting for it that simply stopped executing —
|
|
15
|
+
* the stalled set, served first. Tier 2 is a row waiting on mail, plus a row
|
|
16
|
+
* that held a slot last time and settled nothing with it. */
|
|
17
|
+
export type ThawTier = 1 | 2;
|
|
18
|
+
export interface ThawEntry extends ThawCandidate {
|
|
19
|
+
tier: ThawTier;
|
|
20
|
+
}
|
|
21
|
+
export interface FreezeLaneContext {
|
|
22
|
+
fleet: FleetRegistry;
|
|
23
|
+
candidates: readonly ThawCandidate[];
|
|
24
|
+
}
|
|
25
|
+
/** What stayed frozen this tick, split the way the saturation warning reports
|
|
26
|
+
* it. Rows in backoff never reach this lane and are excluded by construction. */
|
|
27
|
+
export interface FrozenCounts {
|
|
28
|
+
stalled: number;
|
|
29
|
+
waitingOnMail: number;
|
|
30
|
+
}
|
|
31
|
+
/** Which tier a candidate competes in. A row that was launched and never
|
|
32
|
+
* settled a turn drops to tier 2: it already spent a slot without producing
|
|
33
|
+
* anything, so it must not keep outranking the mail it stalled behind. A row
|
|
34
|
+
* that never launched at all has spent nothing and stays tier 1. */
|
|
35
|
+
export declare function thawTier(row: NodeRow, pendingWake: boolean): ThawTier;
|
|
36
|
+
/** Thaw order: tier first, then longest-frozen first. A candidate with no
|
|
37
|
+
* `frozen_at` has not waited at all and sorts last within its tier. */
|
|
38
|
+
export declare function compareThaw(a: ThawEntry, b: ThawEntry): number;
|
|
39
|
+
export declare class FreezeLaneReconciler {
|
|
40
|
+
private readonly relaunchState;
|
|
41
|
+
/** End a stuck-delivery episode the tick decided is over — the node's profile
|
|
42
|
+
* paused, or nothing wake-worthy is left in its inbox. */
|
|
43
|
+
endEpisode(nodeId: string): void;
|
|
44
|
+
run(now: number, ctx: FreezeLaneContext): Promise<FrozenCounts>;
|
|
45
|
+
private enact;
|
|
46
|
+
/** A frozen row is owed exactly one thing: the launch the cap refused. Strict
|
|
47
|
+
* resume is the request, and `reviveNode` itself vetoes it for a pending
|
|
48
|
+
* refresh or an unconfirmed cycle — the same modes the dead-row table would
|
|
49
|
+
* have chosen. It is recovery, not a wake: nothing arrived for this node
|
|
50
|
+
* while it waited, so an armed deadline survives the thaw. */
|
|
51
|
+
private enactThaw;
|
|
52
|
+
private enactIdleRelease;
|
|
53
|
+
}
|
|
@@ -0,0 +1,170 @@
|
|
|
1
|
+
// freeze-lane.ts — who gets the free slots, in what order, and what happens to
|
|
2
|
+
// everyone else.
|
|
3
|
+
//
|
|
4
|
+
// The tick hands this lane every row that wants execution and has no engine.
|
|
5
|
+
// Enactment is strictly sequential on the daemon's serial work lane, so the
|
|
6
|
+
// ledger inside `headlessBrokerHost.launch` hands slots out in exactly the
|
|
7
|
+
// order this lane enacts: ordering IS the allocation policy, and no candidate
|
|
8
|
+
// needs to pre-reserve anything. A candidate that finds the ledger full freezes
|
|
9
|
+
// (`reviveNode` with `capacity: 'freeze'` is the single freeze writer) and waits
|
|
10
|
+
// for the next tick.
|
|
11
|
+
//
|
|
12
|
+
// The idle-release episode rules below are the former DormantInboxReconciler's,
|
|
13
|
+
// moved here whole: a relaunch that only ever tried to deliver mail is a launch
|
|
14
|
+
// like any other, and it now competes for slots through the same lane.
|
|
15
|
+
import { getRow, subscribersOf, } from '../../../core/canvas/index.js';
|
|
16
|
+
import { fullName } from '../../../core/canvas/labels.js';
|
|
17
|
+
import { isSafeNodeId } from '../../../core/canvas/paths.js';
|
|
18
|
+
import { emitEvent } from '../../../core/events/emit.js';
|
|
19
|
+
import { operationIdContext } from '../../../core/events/operation-id.js';
|
|
20
|
+
import { readCursor } from '../../../core/feed/inbox.js';
|
|
21
|
+
import { fanDoctrineWake } from '../../../core/runtime/close.js';
|
|
22
|
+
import { transition } from '../../../core/runtime/lifecycle.js';
|
|
23
|
+
import { reviveNode } from '../../../core/runtime/revive.js';
|
|
24
|
+
import { hasPendingWake } from '../live-obligation.js';
|
|
25
|
+
// Idle-release/inbox retry cap (2026-07-10 diagnosis, round 2): the SAME
|
|
26
|
+
// livelock shape as the stranded-relaunch case, but on the pass that resumes a
|
|
27
|
+
// dormant idle-released node when its inbox gains an unseen entry. The counter
|
|
28
|
+
// is keyed on inbox-cursor progress, not row-state residency or pid liveness: a
|
|
29
|
+
// transiently-live broker does not prove that delivery completed.
|
|
30
|
+
const IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS = 6;
|
|
31
|
+
const IDLE_RELEASE_RELAUNCH_MAX_BACKOFF_MS = 10 * 60_000;
|
|
32
|
+
const IDLE_RELEASE_RELAUNCH_BASE_MS = 20_000;
|
|
33
|
+
/** Which tier a candidate competes in. A row that was launched and never
|
|
34
|
+
* settled a turn drops to tier 2: it already spent a slot without producing
|
|
35
|
+
* anything, so it must not keep outranking the mail it stalled behind. A row
|
|
36
|
+
* that never launched at all has spent nothing and stays tier 1. */
|
|
37
|
+
export function thawTier(row, pendingWake) {
|
|
38
|
+
if (pendingWake)
|
|
39
|
+
return 2;
|
|
40
|
+
const launched = row.launched_at;
|
|
41
|
+
if (launched == null)
|
|
42
|
+
return 1;
|
|
43
|
+
const settled = row.last_settled_at;
|
|
44
|
+
if (settled == null)
|
|
45
|
+
return 2;
|
|
46
|
+
return Date.parse(settled) > Date.parse(launched) ? 1 : 2;
|
|
47
|
+
}
|
|
48
|
+
/** Thaw order: tier first, then longest-frozen first. A candidate with no
|
|
49
|
+
* `frozen_at` has not waited at all and sorts last within its tier. */
|
|
50
|
+
export function compareThaw(a, b) {
|
|
51
|
+
if (a.tier !== b.tier)
|
|
52
|
+
return a.tier - b.tier;
|
|
53
|
+
const af = a.row.frozen_at;
|
|
54
|
+
const bf = b.row.frozen_at;
|
|
55
|
+
if (af == null)
|
|
56
|
+
return bf == null ? 0 : 1;
|
|
57
|
+
if (bf == null)
|
|
58
|
+
return -1;
|
|
59
|
+
return Date.parse(af) - Date.parse(bf);
|
|
60
|
+
}
|
|
61
|
+
export class FreezeLaneReconciler {
|
|
62
|
+
relaunchState = new Map();
|
|
63
|
+
/** End a stuck-delivery episode the tick decided is over — the node's profile
|
|
64
|
+
* paused, or nothing wake-worthy is left in its inbox. */
|
|
65
|
+
endEpisode(nodeId) {
|
|
66
|
+
this.relaunchState.delete(nodeId);
|
|
67
|
+
}
|
|
68
|
+
async run(now, ctx) {
|
|
69
|
+
const entries = ctx.candidates.map((candidate) => ({
|
|
70
|
+
...candidate,
|
|
71
|
+
tier: thawTier(candidate.row, hasPendingWake(candidate.row.node_id)),
|
|
72
|
+
}));
|
|
73
|
+
entries.sort(compareThaw);
|
|
74
|
+
const counts = { stalled: 0, waitingOnMail: 0 };
|
|
75
|
+
for (const entry of entries) {
|
|
76
|
+
const outcome = await operationIdContext.fresh(() => this.enact(now, entry, ctx.fleet));
|
|
77
|
+
if (outcome !== 'frozen')
|
|
78
|
+
continue;
|
|
79
|
+
if (entry.tier === 1)
|
|
80
|
+
counts.stalled += 1;
|
|
81
|
+
else
|
|
82
|
+
counts.waitingOnMail += 1;
|
|
83
|
+
}
|
|
84
|
+
return counts;
|
|
85
|
+
}
|
|
86
|
+
async enact(now, entry, fleet) {
|
|
87
|
+
const id = entry.row.node_id;
|
|
88
|
+
try {
|
|
89
|
+
if (entry.enact === 'thaw')
|
|
90
|
+
return this.enactThaw(id);
|
|
91
|
+
if (entry.enact === 'idle-release')
|
|
92
|
+
return this.enactIdleRelease(now, entry.row);
|
|
93
|
+
// The SAME policy table an exit event runs, so tick recovery and exit
|
|
94
|
+
// recovery can never diverge. It relaunches through
|
|
95
|
+
// `reviveNode(capacity: 'freeze')`, which freezes the row when the ledger
|
|
96
|
+
// is full — the outcome this lane reports as saturation.
|
|
97
|
+
await fleet.applyDeadRowPolicy(id);
|
|
98
|
+
if (fleet.has(id))
|
|
99
|
+
return 'launched';
|
|
100
|
+
return getRow(id)?.frozen_at != null ? 'frozen' : 'skipped';
|
|
101
|
+
}
|
|
102
|
+
catch (err) {
|
|
103
|
+
emitEvent({
|
|
104
|
+
level: 'error',
|
|
105
|
+
event: 'node.lifecycle.enact_failed',
|
|
106
|
+
...(isSafeNodeId(id) ? { node_id: id } : {}),
|
|
107
|
+
error: err,
|
|
108
|
+
fields: { ...(isSafeNodeId(id) ? {} : { affected_node_id: id }), enact: entry.enact },
|
|
109
|
+
});
|
|
110
|
+
return 'skipped';
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
/** A frozen row is owed exactly one thing: the launch the cap refused. Strict
|
|
114
|
+
* resume is the request, and `reviveNode` itself vetoes it for a pending
|
|
115
|
+
* refresh or an unconfirmed cycle — the same modes the dead-row table would
|
|
116
|
+
* have chosen. It is recovery, not a wake: nothing arrived for this node
|
|
117
|
+
* while it waited, so an armed deadline survives the thaw. */
|
|
118
|
+
enactThaw(nodeId) {
|
|
119
|
+
return reviveNode(nodeId, { resume: true, capacity: 'freeze', recovery: true }).outcome === 'frozen'
|
|
120
|
+
? 'frozen'
|
|
121
|
+
: 'launched';
|
|
122
|
+
}
|
|
123
|
+
enactIdleRelease(now, row) {
|
|
124
|
+
const id = row.node_id;
|
|
125
|
+
const cursorNow = readCursor(id);
|
|
126
|
+
const prior = this.relaunchState.get(id);
|
|
127
|
+
const cursorChanged = prior === undefined || prior.cursor !== cursorNow;
|
|
128
|
+
const attempts = cursorChanged ? 0 : prior.attempts;
|
|
129
|
+
if (attempts >= IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS) {
|
|
130
|
+
this.relaunchState.delete(id);
|
|
131
|
+
emitEvent({
|
|
132
|
+
level: 'warn',
|
|
133
|
+
event: 'broker.idle_release.exhausted',
|
|
134
|
+
...(isSafeNodeId(id) ? { node_id: id } : {}),
|
|
135
|
+
fields: {
|
|
136
|
+
...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
|
|
137
|
+
attempts,
|
|
138
|
+
max_attempts: IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS,
|
|
139
|
+
},
|
|
140
|
+
});
|
|
141
|
+
transition(id, 'crash');
|
|
142
|
+
fanDoctrineWake(id, subscribersOf(id), `Child crashed — ${fullName(row)} (${id}) failed to relaunch after ${attempts} attempts while trying to ` +
|
|
143
|
+
`deliver a pending inbox message (no delivery progress across any attempt), and is now dead. It stays ` +
|
|
144
|
+
`dead until you revive it — \`crtr node lifecycle revive ${id}\`.`, { reason: 'child-crashed', child: id });
|
|
145
|
+
return 'skipped';
|
|
146
|
+
}
|
|
147
|
+
const lastAttempt = cursorChanged ? undefined : prior?.lastAttempt;
|
|
148
|
+
const backoff = Math.min(IDLE_RELEASE_RELAUNCH_BASE_MS * 2 ** attempts, IDLE_RELEASE_RELAUNCH_MAX_BACKOFF_MS);
|
|
149
|
+
if (lastAttempt !== undefined && now - lastAttempt < backoff)
|
|
150
|
+
return 'skipped';
|
|
151
|
+
// A relaunch the broker cap froze got no slot and therefore proved nothing
|
|
152
|
+
// about this delivery, so it must not spend an attempt: only a launch that
|
|
153
|
+
// actually happened advances the episode toward exhaustion.
|
|
154
|
+
if (reviveNode(id, { resume: true, capacity: 'freeze' }).outcome === 'frozen')
|
|
155
|
+
return 'frozen';
|
|
156
|
+
this.relaunchState.set(id, { cursor: cursorNow, attempts: attempts + 1, lastAttempt: now });
|
|
157
|
+
emitEvent({
|
|
158
|
+
level: 'info',
|
|
159
|
+
event: 'broker.revive.idle_release',
|
|
160
|
+
...(isSafeNodeId(id) ? { node_id: id } : {}),
|
|
161
|
+
attempt: attempts + 1,
|
|
162
|
+
fields: {
|
|
163
|
+
...(isSafeNodeId(id) ? {} : { affected_node_id: id }),
|
|
164
|
+
attempts: attempts + 1,
|
|
165
|
+
max_attempts: IDLE_RELEASE_RELAUNCH_MAX_ATTEMPTS,
|
|
166
|
+
},
|
|
167
|
+
});
|
|
168
|
+
return 'launched';
|
|
169
|
+
}
|
|
170
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
import type { NodeRow } from '../../../core/canvas/index.js';
|
|
2
|
+
/** Whether a durable backoff still holds this row back from a relaunch. A row
|
|
3
|
+
* in backoff holds no slot and is excluded from the saturation figure — it is
|
|
4
|
+
* waiting on a clock, not on capacity. */
|
|
5
|
+
export declare function isInBackoff(row: NodeRow, now: number): boolean;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// respawn-policy.ts — the tick's Backoff gate, and P5's file to own outright.
|
|
2
|
+
//
|
|
3
|
+
// P3 only READS `respawn_not_before`: nothing writes it yet, so the gate is
|
|
4
|
+
// inert in effect. P5 replaces the in-memory throttle in `daemon/fleet.ts` with
|
|
5
|
+
// the durable counter writes (`respawn_failures` + `respawn_not_before`) that
|
|
6
|
+
// make it bite, and adds them here rather than in `tick.ts`.
|
|
7
|
+
/** Whether a durable backoff still holds this row back from a relaunch. A row
|
|
8
|
+
* in backoff holds no slot and is excluded from the saturation figure — it is
|
|
9
|
+
* waiting on a clock, not on capacity. */
|
|
10
|
+
export function isInBackoff(row, now) {
|
|
11
|
+
const notBefore = row.respawn_not_before;
|
|
12
|
+
if (notBefore == null)
|
|
13
|
+
return false;
|
|
14
|
+
const at = Date.parse(notBefore);
|
|
15
|
+
return Number.isFinite(at) && now < at;
|
|
16
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { NodeRow } from '../../../core/canvas/index.js';
|
|
2
|
+
/** Handle a terminal row that still holds a process. Deliberately inert until
|
|
3
|
+
* P4: a row reaching here is already excluded from every other tick duty. */
|
|
4
|
+
export declare function handleTerminatingRow(row: NodeRow): void;
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
// terminating.ts — the Terminating state's owner, and P4's file to own outright.
|
|
2
|
+
//
|
|
3
|
+
// A terminal row that still holds a process is a node whose teardown has been
|
|
4
|
+
// requested but not finished. P4 gives it one bounded shutdown request and an
|
|
5
|
+
// escalation after its grace, moving the inline `headlessBrokerHost.teardown`
|
|
6
|
+
// calls at the close/reset sites into this daemon-side lane. P3 changes no
|
|
7
|
+
// teardown behavior: it only OBSERVES such rows (the tick queries terminal rows
|
|
8
|
+
// carrying a recorded pid) and hands them here, so P4 edits this file alone.
|
|
9
|
+
/** Handle a terminal row that still holds a process. Deliberately inert until
|
|
10
|
+
* P4: a row reaching here is already excluded from every other tick duty. */
|
|
11
|
+
export function handleTerminatingRow(row) {
|
|
12
|
+
void row;
|
|
13
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
import { type NodeRow } from '../../../core/canvas/index.js';
|
|
2
|
+
import type { FleetRegistry } from '../../../core/runtime/fleet.js';
|
|
3
|
+
import { type FrozenCounts } from './freeze-lane.js';
|
|
4
|
+
export interface NodeLifecycleContext {
|
|
5
|
+
/** Every active and idle row (human rows excluded), plus terminal rows that
|
|
6
|
+
* still hold a recorded pid. */
|
|
7
|
+
rows: readonly NodeRow[];
|
|
8
|
+
fleet: FleetRegistry;
|
|
9
|
+
}
|
|
10
|
+
/** Whether this row should have a live engine right now. An active row always
|
|
11
|
+
* should — that status IS the claim that it is executing. An idle row was
|
|
12
|
+
* deliberately released and holds no engine on purpose: only mail it has not
|
|
13
|
+
* seen brings one back, because a live obligation keeps a node WAKEABLE rather
|
|
14
|
+
* than awake, and relaunching on one would re-release it every tick. */
|
|
15
|
+
export declare function wantsExecution(row: NodeRow): boolean;
|
|
16
|
+
export declare class NodeLifecycleReconciler {
|
|
17
|
+
private readonly freezeLane;
|
|
18
|
+
run(now: number, ctx: NodeLifecycleContext): Promise<FrozenCounts>;
|
|
19
|
+
private classifyRow;
|
|
20
|
+
private thawMark;
|
|
21
|
+
/** An idle row with no wake-worthy inbox entry still needs the lifecycle
|
|
22
|
+
* settlement the live park clock applies: once nothing live remains, this is
|
|
23
|
+
* history rather than a permanently protected idle row. The row is already
|
|
24
|
+
* parked, so only a terminalizing verdict is worth writing. */
|
|
25
|
+
private settleParked;
|
|
26
|
+
}
|