@north-light/crouter 0.3.236 → 0.3.238
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/client.d.ts +2 -1
- package/dist/api/client.js +3 -0
- package/dist/api/dto/broker-ops.d.ts +1 -1
- package/dist/api/dto/common.d.ts +2 -0
- package/dist/api/dto/lifecycle.d.ts +9 -2
- package/dist/api/dto/messages.d.ts +5 -0
- package/dist/api/dto/nodes.d.ts +10 -2
- package/dist/api/dto/worktree.d.ts +10 -0
- package/dist/api/routes.d.ts +1 -0
- package/dist/api/routes.js +1 -0
- package/dist/builtin-memory/00-runtime-base/00-authoring.md +1 -1
- package/dist/builtin-memory/internal/agent-shaping.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/sysprompt-window.ts +1 -1
- package/dist/clients/attach/render/tool-calls.js +18 -3
- package/dist/clients/attach/viewer.js +442 -442
- package/dist/clients/inbox/review/__tests__/editor-roundtrip.test.js +6 -0
- package/dist/clients/inbox/review/roundtrip.js +1 -1
- package/dist/commands/memory/lint.js +22 -13
- package/dist/commands/memory/shared.js +1 -1
- package/dist/commands/node/create.d.ts +1 -1
- package/dist/commands/node/create.js +1 -1
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node-worktree.js +39 -8
- package/dist/commands/revive.js +11 -2
- package/dist/commands/sys/context/admin/actions.d.ts +48 -0
- package/dist/commands/sys/context/admin/actions.js +246 -0
- package/dist/commands/sys/context/admin/axes-panel.d.ts +21 -0
- package/dist/commands/sys/context/admin/axes-panel.js +117 -0
- package/dist/commands/sys/context/admin/detail-panel.d.ts +53 -0
- package/dist/commands/sys/context/admin/detail-panel.js +178 -0
- package/dist/commands/sys/context/admin/docs-panel.d.ts +45 -0
- package/dist/commands/sys/context/admin/docs-panel.js +186 -0
- package/dist/commands/sys/context/admin/model.d.ts +75 -0
- package/dist/commands/sys/context/admin/model.js +209 -0
- package/dist/commands/sys/context/admin/shell.d.ts +82 -0
- package/dist/commands/sys/context/admin/shell.js +669 -0
- package/dist/commands/sys/context/admin.d.ts +1 -0
- package/dist/commands/sys/context/admin.js +124 -0
- package/dist/commands/sys/context/doc.d.ts +1 -0
- package/dist/commands/sys/context/doc.js +196 -0
- package/dist/commands/sys/context/prompt-review.d.ts +1 -0
- package/dist/commands/sys/{prompt-review.js → context/prompt-review.js} +10 -10
- package/dist/commands/sys/context/resolve.d.ts +55 -0
- package/dist/commands/sys/context/resolve.js +428 -0
- package/dist/commands/sys/context/sysprompt.d.ts +1 -0
- package/dist/commands/sys/{sysprompt.js → context/sysprompt.js} +11 -11
- package/dist/commands/sys/context.d.ts +2 -0
- package/dist/commands/sys/context.js +20 -0
- package/dist/commands/sys.js +3 -4
- package/dist/core/__tests__/child-death-wake.test.js +38 -22
- package/dist/core/__tests__/cron-self-cancel-delivery.test.js +72 -0
- package/dist/core/__tests__/daemon-boot.test.js +40 -28
- package/dist/core/__tests__/human-deliver.test.js +1 -1
- package/dist/core/__tests__/human-node-not-supervised.test.js +10 -15
- package/dist/core/__tests__/integration/revive.test.js +12 -12
- package/dist/core/__tests__/integration/spawn-root.test.js +6 -0
- package/dist/core/__tests__/integration/worktree-reap.test.js +32 -0
- package/dist/core/__tests__/lifecycle-terminal-reasons.test.d.ts +1 -0
- package/dist/core/__tests__/lifecycle-terminal-reasons.test.js +61 -0
- package/dist/core/__tests__/migration.test.js +38 -1
- package/dist/core/__tests__/push-final-guard.test.js +1 -1
- package/dist/core/__tests__/revive-capacity.test.d.ts +1 -0
- package/dist/core/__tests__/revive-capacity.test.js +127 -0
- package/dist/core/__tests__/revive-parked-fresh.test.js +7 -7
- package/dist/core/__tests__/seam/broker-cap-freeze.test.d.ts +1 -0
- package/dist/core/__tests__/seam/broker-cap-freeze.test.js +81 -0
- package/dist/core/__tests__/seam/dormancy-release.test.js +7 -7
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +1 -1
- package/dist/core/canvas/browse/render.js +11 -3
- package/dist/core/canvas/canvas.d.ts +15 -3
- package/dist/core/canvas/canvas.js +52 -16
- package/dist/core/canvas/migrations.js +21 -0
- package/dist/core/canvas/nav-render.js +3 -1
- package/dist/core/canvas/remote-canvas-source.js +2 -0
- package/dist/core/canvas/render-source.d.ts +3 -1
- package/dist/core/canvas/render-source.js +7 -3
- package/dist/core/canvas/status-glyph.d.ts +10 -1
- package/dist/core/canvas/status-glyph.js +19 -3
- package/dist/core/canvas/types.d.ts +20 -1
- package/dist/core/feed/feed.js +1 -1
- package/dist/core/human/feedback-companion.js +5 -2
- package/dist/core/keybindings/catalog.d.ts +1 -1
- package/dist/core/keybindings/catalog.js +6 -2
- package/dist/core/memory/lint.d.ts +19 -0
- package/dist/core/memory/lint.js +97 -0
- package/dist/core/memory-resolver.d.ts +4 -0
- package/dist/core/memory-resolver.js +7 -7
- package/dist/core/preview-registry.js +4 -4
- package/dist/core/review/realize.js +4 -2
- package/dist/core/runtime/close.js +1 -1
- package/dist/core/runtime/fleet.d.ts +29 -1
- package/dist/core/runtime/fleet.js +23 -0
- package/dist/core/runtime/host.js +32 -11
- package/dist/core/runtime/launch-prompt.d.ts +1 -1
- package/dist/core/runtime/launch-prompt.js +2 -2
- package/dist/core/runtime/lifecycle.d.ts +4 -2
- package/dist/core/runtime/lifecycle.js +12 -1
- package/dist/core/runtime/memory.js +1 -1
- package/dist/core/runtime/reopen.js +10 -1
- package/dist/core/runtime/reset.js +1 -1
- package/dist/core/runtime/revive-all.d.ts +3 -0
- package/dist/core/runtime/revive-all.js +10 -4
- package/dist/core/runtime/revive.d.ts +12 -0
- package/dist/core/runtime/revive.js +62 -3
- package/dist/core/runtime/spawn.d.ts +4 -0
- package/dist/core/runtime/spawn.js +36 -7
- package/dist/core/runtime/tmux-bindings.js +4 -0
- package/dist/core/substrate/frontmatter-validation.js +1 -1
- package/dist/core/substrate/index.d.ts +6 -3
- package/dist/core/substrate/index.js +4 -3
- package/dist/core/substrate/on-read.d.ts +0 -8
- package/dist/core/substrate/on-read.js +45 -124
- package/dist/core/substrate/plan.d.ts +94 -0
- package/dist/core/substrate/plan.js +266 -0
- package/dist/core/substrate/render.d.ts +5 -6
- package/dist/core/substrate/render.js +76 -129
- package/dist/core/substrate/schema.d.ts +1 -6
- package/dist/core/substrate/schema.js +1 -15
- package/dist/core/substrate/session-cache.d.ts +2 -10
- package/dist/core/substrate/session-cache.js +9 -47
- package/dist/core/substrate/subject-fields.d.ts +1 -4
- package/dist/core/substrate/subject-fields.js +2 -5
- package/dist/core/substrate/surface-match.d.ts +37 -17
- package/dist/core/substrate/surface-match.js +44 -53
- package/dist/core/tui/page-host.d.ts +6 -0
- package/dist/core/tui/page-host.js +12 -3
- package/dist/core/worktree.d.ts +8 -0
- package/dist/core/worktree.js +65 -0
- package/dist/daemon/__tests__/reconciler-signature.test.js +3 -2
- package/dist/daemon/__tests__/thaw-order.test.d.ts +1 -0
- package/dist/daemon/__tests__/thaw-order.test.js +55 -0
- package/dist/daemon/api/__tests__/canvas-snapshot-fields.test.js +6 -0
- package/dist/daemon/api/__tests__/node-create-description.test.js +6 -0
- package/dist/daemon/api/__tests__/profile-launch-gates.test.js +1 -1
- package/dist/daemon/api/bridge.js +5 -1
- package/dist/daemon/api/handlers/attach.js +5 -1
- package/dist/daemon/api/handlers/bash-jobs.js +1 -1
- package/dist/daemon/api/handlers/messages.js +10 -2
- package/dist/daemon/api/handlers/nodes.js +8 -1
- package/dist/daemon/api/handlers/worktree.js +10 -1
- package/dist/daemon/api/map.js +7 -0
- package/dist/daemon/companion-retire.js +1 -1
- package/dist/daemon/cron/passes.d.ts +0 -5
- package/dist/daemon/cron/passes.js +0 -20
- package/dist/daemon/cron/sinks.js +1 -1
- package/dist/daemon/cron-run.d.ts +5 -6
- package/dist/daemon/cron-run.js +7 -16
- package/dist/daemon/crtrd.d.ts +0 -10
- package/dist/daemon/crtrd.js +23 -61
- package/dist/daemon/fleet.d.ts +10 -7
- package/dist/daemon/fleet.js +71 -12
- package/dist/daemon/messaging/node-message.js +3 -1
- package/dist/daemon/profile-delete.js +1 -1
- package/dist/daemon/reconcilers/broker-supervision.d.ts +26 -9
- package/dist/daemon/reconcilers/broker-supervision.js +39 -52
- package/dist/daemon/reconcilers/controller-death.d.ts +7 -0
- package/dist/daemon/reconcilers/controller-death.js +26 -0
- package/dist/daemon/reconcilers/cron-lane.d.ts +0 -3
- package/dist/daemon/reconcilers/cron-lane.js +0 -3
- package/dist/daemon/reconcilers/live-obligation.js +3 -1
- package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.d.ts +53 -0
- package/dist/daemon/reconcilers/node-lifecycle/freeze-lane.js +170 -0
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +5 -0
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +16 -0
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -0
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +13 -0
- package/dist/daemon/reconcilers/node-lifecycle/tick.d.ts +26 -0
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +140 -0
- package/package.json +1 -1
- package/runtime.lock.json +2 -2
- package/dist/commands/sys/prompt-review.d.ts +0 -1
- package/dist/commands/sys/sysprompt.d.ts +0 -1
- package/dist/core/__tests__/cron-broker-capacity.test.js +0 -119
- package/dist/daemon/reconcilers/dormant-inbox.d.ts +0 -18
- package/dist/daemon/reconcilers/dormant-inbox.js +0 -157
- /package/dist/core/__tests__/{cron-broker-capacity.test.d.ts → cron-self-cancel-delivery.test.d.ts} +0 -0
|
@@ -2,7 +2,7 @@ import { type RunDueCronOptions } from './cron/passes.js';
|
|
|
2
2
|
import { type Cron, type CronRunRecord } from '../core/canvas/crons.js';
|
|
3
3
|
export { CRON_OUTPUT_HEAD_BYTES } from './cron/capture.js';
|
|
4
4
|
export { cronDaemonInstanceId, markSelfCanceledCronRun } from './cron/live-runs.js';
|
|
5
|
-
export {
|
|
5
|
+
export { type RunDueCronOptions } from './cron/passes.js';
|
|
6
6
|
export { recoverStaleCronLeases } from './cron/lease-recovery.js';
|
|
7
7
|
/** Kick off ONE cron run and return its settlement promise, resolving with
|
|
8
8
|
* the settled run record (null = the idle→running lease was lost, or the row
|
|
@@ -24,9 +24,8 @@ export declare function executeCron(c: Cron, opts: {
|
|
|
24
24
|
* but is NEVER awaited in the tick; returns one settlement promise per
|
|
25
25
|
* admitted run.
|
|
26
26
|
*
|
|
27
|
-
* A run that
|
|
28
|
-
*
|
|
29
|
-
*
|
|
30
|
-
*
|
|
31
|
-
* `fire_at` and is retried on the next poll once capacity frees up. */
|
|
27
|
+
* A run that launches or revives a node is admitted unconditionally: the
|
|
28
|
+
* broker slot ledger is the one admission gate, so such a run's node is born
|
|
29
|
+
* or revived frozen at the cap and the node-lifecycle tick starts it when a
|
|
30
|
+
* slot frees. The cron itself always fires and settles. */
|
|
32
31
|
export declare function runDueCrons(now: number, options?: RunDueCronOptions): Promise<void>[];
|
package/dist/daemon/cron-run.js
CHANGED
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
// surface of the cron lane: subprocess spawn/capture/timeout/lease mechanics
|
|
3
3
|
// for due `crons` rows (the cron-spec scheduler), and the tick pass that fires
|
|
4
4
|
// them. The collaborators it composes live in `cron/`: arm-time context,
|
|
5
|
-
// output capture, the live-run registry, the pre-run passes
|
|
6
|
-
//
|
|
5
|
+
// output capture, the live-run registry, the pre-run passes, stale-lease
|
|
6
|
+
// recovery, the owed-gate decision, delivery sinks,
|
|
7
7
|
// escalation, output disposition, and the one settle tail.
|
|
8
8
|
//
|
|
9
9
|
// A cron RUN is the enactment — there is no separate action to fire —
|
|
@@ -40,7 +40,7 @@ import { clearLiveRun, cronDaemonInstanceId, liveRunFor, registerLiveRun } from
|
|
|
40
40
|
import { buildArmedRunContext } from './cron/armed-context.js';
|
|
41
41
|
import { decideHeldSettle, EXIT_HELD, HELD_DELIVERED } from './cron/held.js';
|
|
42
42
|
import { settleRun } from './cron/settle.js';
|
|
43
|
-
import { applyOverlapPolicies,
|
|
43
|
+
import { applyOverlapPolicies, expireDueCrons, } from './cron/passes.js';
|
|
44
44
|
import { dueClockCrons, acquireCronRunLease, advanceCron, setCronState, } from '../core/canvas/crons.js';
|
|
45
45
|
import { killProcessGroup } from '../core/canvas/pid.js';
|
|
46
46
|
import { emitEvent } from '../core/events/emit.js';
|
|
@@ -51,7 +51,6 @@ import { resolveAdmittedHostCommand } from '../core/runtime/spawn-env.js';
|
|
|
51
51
|
// decomposed into.
|
|
52
52
|
export { CRON_OUTPUT_HEAD_BYTES } from './cron/capture.js';
|
|
53
53
|
export { cronDaemonInstanceId, markSelfCanceledCronRun } from './cron/live-runs.js';
|
|
54
|
-
export { cronLaunchesNode } from './cron/passes.js';
|
|
55
54
|
export { recoverStaleCronLeases } from './cron/lease-recovery.js';
|
|
56
55
|
/** Settle one closed child: choose the outcome (replaced short-circuit, owed
|
|
57
56
|
* gate, or the full disposition), then hand it to the one settle tail. */
|
|
@@ -350,22 +349,16 @@ export function executeCron(c, opts) {
|
|
|
350
349
|
* but is NEVER awaited in the tick; returns one settlement promise per
|
|
351
350
|
* admitted run.
|
|
352
351
|
*
|
|
353
|
-
* A run that
|
|
354
|
-
*
|
|
355
|
-
*
|
|
356
|
-
*
|
|
357
|
-
* `fire_at` and is retried on the next poll once capacity frees up. */
|
|
352
|
+
* A run that launches or revives a node is admitted unconditionally: the
|
|
353
|
+
* broker slot ledger is the one admission gate, so such a run's node is born
|
|
354
|
+
* or revived frozen at the cap and the node-lifecycle tick starts it when a
|
|
355
|
+
* slot frees. The cron itself always fires and settles. */
|
|
358
356
|
export function runDueCrons(now, options = {}) {
|
|
359
357
|
const nowIso = new Date(now).toISOString();
|
|
360
358
|
expireDueCrons(nowIso);
|
|
361
359
|
applyOverlapPolicies(now, nowIso);
|
|
362
360
|
const settlements = [];
|
|
363
|
-
let deferred = 0;
|
|
364
361
|
for (const c of dueClockCrons(nowIso)) {
|
|
365
|
-
if (options.hasBrokerCapacity !== undefined && cronLaunchesNode(c) && !options.hasBrokerCapacity()) {
|
|
366
|
-
deferred += 1;
|
|
367
|
-
continue; // still due — retried next poll, at no cost to the row
|
|
368
|
-
}
|
|
369
362
|
const settlement = executeCron(c, { outOfBand: false }).then(() => undefined);
|
|
370
363
|
settlements.push(settlement);
|
|
371
364
|
options.register?.(settlement);
|
|
@@ -374,7 +367,5 @@ export function runDueCrons(now, options = {}) {
|
|
|
374
367
|
// as a process-level unhandled rejection.
|
|
375
368
|
for (const s of settlements)
|
|
376
369
|
void s.catch(() => { });
|
|
377
|
-
if (deferred > 0)
|
|
378
|
-
emitCapacityDeferral(now, deferred);
|
|
379
370
|
return settlements;
|
|
380
371
|
}
|
package/dist/daemon/crtrd.d.ts
CHANGED
|
@@ -1,18 +1,8 @@
|
|
|
1
1
|
import { type DetachedWorkLifecycle } from './reconcilers/broker-supervision.js';
|
|
2
|
-
import { DaemonFleet } from './fleet.js';
|
|
3
2
|
export declare function brokerCommandDeclaresCanvasHome(command: string, home: string): boolean;
|
|
4
3
|
/** The `--epoch <id>` tag host.ts stamps into every broker's argv, or null
|
|
5
4
|
* for a legacy/untagged broker (which the census treats as prior-epoch). */
|
|
6
5
|
export declare function brokerCommandEpoch(command: string): string | null;
|
|
7
|
-
/** Startup recovery sweep (design flow C step 2): the FIRST job on the serial
|
|
8
|
-
* work lane. The census already reaped every prior-epoch broker, so every
|
|
9
|
-
* active|idle row at boot is known-dead — classify each through the SAME
|
|
10
|
-
* policy table exit events use (fleet.recoverDeadNode → classifyDeadNode,
|
|
11
|
-
* design D-12), so exit recovery and startup recovery can never diverge. A
|
|
12
|
-
* raced API revive is safe: reviveNode's fleet.has guard makes the loser a
|
|
13
|
-
* no-op. Capacity-gated: past the automatic-revive cap the sweep stops — the
|
|
14
|
-
* tick's pass-2 inbox wakes own the remainder as capacity frees up. */
|
|
15
|
-
export declare function runStartupRecoverySweep(fleet: DaemonFleet): Promise<void>;
|
|
16
6
|
interface DaemonTickLifecycle extends DetachedWorkLifecycle {
|
|
17
7
|
}
|
|
18
8
|
/** Supervision tick over the daemon's current live set. */
|
package/dist/daemon/crtrd.js
CHANGED
|
@@ -46,7 +46,7 @@
|
|
|
46
46
|
import { writeFileSync, readFileSync, rmSync, mkdirSync, } from 'node:fs';
|
|
47
47
|
import { dirname } from 'node:path';
|
|
48
48
|
import { randomBytes } from 'node:crypto';
|
|
49
|
-
import { crtrHome
|
|
49
|
+
import { crtrHome } from '../core/canvas/paths.js';
|
|
50
50
|
import { listNodes, migrateLegacyPidIdentities, } from '../core/canvas/index.js';
|
|
51
51
|
import { createApiServer } from './api/server.js';
|
|
52
52
|
import { reconcileUndeliveredTickets } from './human/sweep.js';
|
|
@@ -56,7 +56,8 @@ import { pidfilePath } from './pidfile.js';
|
|
|
56
56
|
import { rendererWarning } from '../core/runtime/package-health.js';
|
|
57
57
|
import { DEFAULT_INTERVAL_MS } from './supervise-cadence.js';
|
|
58
58
|
import { BrokerSupervisionReconciler } from './reconcilers/broker-supervision.js';
|
|
59
|
-
import { ControllerDeathReconciler
|
|
59
|
+
import { ControllerDeathReconciler } from './reconcilers/controller-death.js';
|
|
60
|
+
import { NodeLifecycleReconciler } from './reconcilers/node-lifecycle/tick.js';
|
|
60
61
|
import { CronLaneReconciler } from './reconcilers/cron-lane.js';
|
|
61
62
|
import { HumanDeliveryLaneReconciler } from './reconcilers/human-delivery-lane.js';
|
|
62
63
|
import { PendingReviewSubmitReconciler } from './reconcilers/pending-review-submit.js';
|
|
@@ -229,47 +230,6 @@ async function teardownFleet(fleet) {
|
|
|
229
230
|
fields: { brokers: items.length, sigkilled },
|
|
230
231
|
});
|
|
231
232
|
}
|
|
232
|
-
/** Startup recovery sweep (design flow C step 2): the FIRST job on the serial
|
|
233
|
-
* work lane. The census already reaped every prior-epoch broker, so every
|
|
234
|
-
* active|idle row at boot is known-dead — classify each through the SAME
|
|
235
|
-
* policy table exit events use (fleet.recoverDeadNode → classifyDeadNode,
|
|
236
|
-
* design D-12), so exit recovery and startup recovery can never diverge. A
|
|
237
|
-
* raced API revive is safe: reviveNode's fleet.has guard makes the loser a
|
|
238
|
-
* no-op. Capacity-gated: past the automatic-revive cap the sweep stops — the
|
|
239
|
-
* tick's pass-2 inbox wakes own the remainder as capacity frees up. */
|
|
240
|
-
export async function runStartupRecoverySweep(fleet) {
|
|
241
|
-
let rows;
|
|
242
|
-
try {
|
|
243
|
-
rows = listNodes({ status: ['active', 'idle'] });
|
|
244
|
-
}
|
|
245
|
-
catch (err) {
|
|
246
|
-
emitEvent({ level: 'error', event: 'daemon.recovery_sweep.failed', error: err });
|
|
247
|
-
return;
|
|
248
|
-
}
|
|
249
|
-
for (const row of rows) {
|
|
250
|
-
if (row.kind === 'human')
|
|
251
|
-
continue; // human bridge rows — never broker-hosted
|
|
252
|
-
if (!brokerSupervision.hasStartupRecoveryCapacity(fleet))
|
|
253
|
-
break;
|
|
254
|
-
await operationIdContext.fresh(async () => {
|
|
255
|
-
try {
|
|
256
|
-
await fleet.recoverDeadNode(row.node_id);
|
|
257
|
-
}
|
|
258
|
-
catch (err) {
|
|
259
|
-
emitEvent({
|
|
260
|
-
level: 'error',
|
|
261
|
-
event: 'daemon.recovery_sweep.node_failed',
|
|
262
|
-
...(isSafeNodeId(row.node_id) ? { node_id: row.node_id } : { fields: { affected_node_id: row.node_id } }),
|
|
263
|
-
error: err,
|
|
264
|
-
});
|
|
265
|
-
// reviveNode launch failures terminalize themselves, but preflight
|
|
266
|
-
// refusals happen before its catch. Re-read and surface those rows
|
|
267
|
-
// instead of leaving an active node with no broker to retry it.
|
|
268
|
-
await fleet.terminalizeIfStillNonTerminal(row.node_id, 'daemon.recovery_sweep.node_terminalized', err);
|
|
269
|
-
}
|
|
270
|
-
});
|
|
271
|
-
}
|
|
272
|
-
}
|
|
273
233
|
// Pidfile
|
|
274
234
|
function writePidfile() {
|
|
275
235
|
// Ensure the pidfile parent exists before writing.
|
|
@@ -311,7 +271,7 @@ const directTickLifecycle = {
|
|
|
311
271
|
};
|
|
312
272
|
const brokerSupervision = new BrokerSupervisionReconciler();
|
|
313
273
|
const controllerDeath = new ControllerDeathReconciler();
|
|
314
|
-
const
|
|
274
|
+
const nodeLifecycle = new NodeLifecycleReconciler();
|
|
315
275
|
const cronLane = new CronLaneReconciler();
|
|
316
276
|
const humanDeliveryLane = new HumanDeliveryLaneReconciler();
|
|
317
277
|
const pendingReviewSubmit = new PendingReviewSubmitReconciler();
|
|
@@ -321,26 +281,32 @@ export async function superviseTick(now = Date.now(), lifecycle = directTickLife
|
|
|
321
281
|
const fleet = boundFleet();
|
|
322
282
|
// Auth reload must precede the row-list guard: it is fleet-wide and does not
|
|
323
283
|
// depend on a successful canvas scan.
|
|
324
|
-
const thresholds = await brokerSupervision.
|
|
284
|
+
const thresholds = await brokerSupervision.runFleetPreamble(fleet);
|
|
325
285
|
let rows;
|
|
286
|
+
// A terminal row that still holds a process is the Terminating state, and the
|
|
287
|
+
// lifecycle tick is the only pass that owes it anything.
|
|
288
|
+
let terminating;
|
|
326
289
|
try {
|
|
327
|
-
rows
|
|
290
|
+
// Human bridge rows have no broker engine and participate in none of the
|
|
291
|
+
// recurring row-driven reconciler passes.
|
|
292
|
+
rows = listNodes({ status: ['active', 'idle'] }).filter((row) => row.kind !== 'human');
|
|
293
|
+
terminating = listNodes({ status: ['done', 'canceled', 'dead'], withRecordedPid: true })
|
|
294
|
+
.filter((row) => row.kind !== 'human');
|
|
328
295
|
}
|
|
329
296
|
catch (err) {
|
|
330
297
|
emitEvent({ level: 'error', event: 'canvas.node_list.failed', error: err });
|
|
331
298
|
return;
|
|
332
299
|
}
|
|
333
|
-
|
|
334
|
-
// recurring row-driven reconciler passes. Startup recovery applies the same
|
|
335
|
-
// classification boundary.
|
|
336
|
-
rows = rows.filter((row) => row.kind !== 'human');
|
|
337
|
-
const hasBrokerCapacity = () => brokerSupervision.hasBrokerCapacity(thresholds.automaticReviveCap, fleet);
|
|
300
|
+
await brokerSupervision.run(now, { rows, fleet, lifecycle });
|
|
338
301
|
controllerDeath.run(now, { rows });
|
|
339
|
-
|
|
340
|
-
cronLane.run(now, { lifecycle
|
|
302
|
+
const frozen = await nodeLifecycle.run(now, { rows: [...rows, ...terminating], fleet });
|
|
303
|
+
cronLane.run(now, { lifecycle });
|
|
341
304
|
humanDeliveryLane.run(now, { lifecycle });
|
|
342
305
|
pendingReviewSubmit.run(now, { lifecycle });
|
|
343
306
|
storageMaintenance.run(now, {});
|
|
307
|
+
// Last: the saturation figure is only true once the freeze lane has spent
|
|
308
|
+
// this tick's free slots.
|
|
309
|
+
brokerSupervision.warnIfSaturated({ fleet, lifecycle, thresholds, frozen });
|
|
344
310
|
}
|
|
345
311
|
/** Start the supervisor loop after winning the authoritative canvas claim.
|
|
346
312
|
*
|
|
@@ -704,15 +670,11 @@ export async function runDaemon(opts = {}) {
|
|
|
704
670
|
writePidfile();
|
|
705
671
|
// Start the HTTP+WS API server (§4): unix socket always, TCP when opted in.
|
|
706
672
|
apiServer = createApiServer({ tcp: opts.tcp });
|
|
707
|
-
// The startup recovery sweep is the FIRST job on the lane (flow C step 2):
|
|
708
|
-
// it classifies every boot-time dead row through the same policy table
|
|
709
|
-
// exit events use, before the first tick or any exit event interleaves.
|
|
710
|
-
const bootFleet = fleet;
|
|
711
|
-
enqueue(() => runStartupRecoverySweep(bootFleet));
|
|
712
673
|
// Pre-warm the node pool for the most recent project recipes, so the first
|
|
713
|
-
// create after a boot is warm too.
|
|
714
|
-
//
|
|
715
|
-
//
|
|
674
|
+
// create after a boot is warm too. Fire-and-forget (the mints run off the
|
|
675
|
+
// work lane), so it never delays supervision. Boot-time dead rows are the
|
|
676
|
+
// node-lifecycle tick's business: the first tick classifies every one of
|
|
677
|
+
// them through the same policy table exit events use, under the same cap.
|
|
716
678
|
// Collect the spares this daemon's predecessor left behind BEFORE minting
|
|
717
679
|
// new ones: every broker of the prior epoch was torn down, and no sweep
|
|
718
680
|
// resumes a spare, so each is a dead row holding a node dir.
|
package/dist/daemon/fleet.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type NodeMeta } from '../core/canvas/index.js';
|
|
2
|
-
import type
|
|
2
|
+
import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
|
|
3
3
|
import type { HostHandle } from '../core/runtime/host.js';
|
|
4
4
|
/** Surface a vehicle that never booted (or exhausted its respawn budget).
|
|
5
5
|
*
|
|
@@ -25,8 +25,8 @@ export type DeadNodeAction = 'forget' | 'respawn-fresh' | 'dormant' | 'release-i
|
|
|
25
25
|
* NodeMeta itself) so tests fabricate table rows directly. */
|
|
26
26
|
export type DeadNodeRow = Pick<NodeMeta, 'status' | 'intent' | 'pi_session_id' | 'cycle_pending' | 'fork_from' | 'lifecycle'>;
|
|
27
27
|
/** One pure function producing the action for a node known-dead. Used at
|
|
28
|
-
* exactly two call sites — an exit-policy job and the
|
|
29
|
-
* (design D-12) — so exit recovery and
|
|
28
|
+
* exactly two call sites — an exit-policy job and the node-lifecycle tick
|
|
29
|
+
* (design D-12) — so exit recovery and tick recovery can never diverge.
|
|
30
30
|
* Evaluated in table order; `busy` is the on-disk busy marker.
|
|
31
31
|
*
|
|
32
32
|
* `busy` no longer selects the ACTION. Whether the engine died mid-turn or
|
|
@@ -71,14 +71,17 @@ export declare class DaemonFleet implements FleetRegistry {
|
|
|
71
71
|
has(nodeId: string): boolean;
|
|
72
72
|
get(nodeId: string): FleetEntry | undefined;
|
|
73
73
|
size(): number;
|
|
74
|
+
occupancy(): number;
|
|
75
|
+
reserveSlot(nodeId: string): boolean;
|
|
76
|
+
releaseReservation(nodeId: string): void;
|
|
74
77
|
entries(): Iterable<[string, FleetEntry]>;
|
|
75
78
|
forget(nodeId: string): void;
|
|
76
79
|
onChildExit(nodeId: string, status: FleetExitStatus): void;
|
|
77
80
|
deliverExit(nodeId: string, status: FleetExitStatus): void;
|
|
78
|
-
/**
|
|
79
|
-
*
|
|
80
|
-
*
|
|
81
|
-
|
|
81
|
+
/** The node-lifecycle tick's enactment entry: classify a known-dead row and
|
|
82
|
+
* enact immediately through the SAME policy table exit events use, so tick
|
|
83
|
+
* recovery and exit recovery can never diverge. */
|
|
84
|
+
applyDeadRowPolicy(nodeId: string): Promise<void>;
|
|
82
85
|
/** A revive refusal before its launch try/catch leaves the row active but
|
|
83
86
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
84
87
|
* terminal outcome as a boot failure. The fresh read avoids duplicating
|
package/dist/daemon/fleet.js
CHANGED
|
@@ -23,6 +23,7 @@ import { hasPendingWake } from './reconcilers/live-obligation.js';
|
|
|
23
23
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
24
24
|
import { pushUrgent } from '../core/feed/feed.js';
|
|
25
25
|
import { emitEvent } from '../core/events/emit.js';
|
|
26
|
+
import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
|
|
26
27
|
// Shared crash helpers (moved from crtrd.ts — the fleet's policy paths are now
|
|
27
28
|
// their primary consumers; crtrd imports them back for its surviving passes).
|
|
28
29
|
/** Surface a vehicle that never booted (or exhausted its respawn budget).
|
|
@@ -63,8 +64,8 @@ export function retryResumeMode(meta) {
|
|
|
63
64
|
return meta?.cycle_pending !== true;
|
|
64
65
|
}
|
|
65
66
|
/** One pure function producing the action for a node known-dead. Used at
|
|
66
|
-
* exactly two call sites — an exit-policy job and the
|
|
67
|
-
* (design D-12) — so exit recovery and
|
|
67
|
+
* exactly two call sites — an exit-policy job and the node-lifecycle tick
|
|
68
|
+
* (design D-12) — so exit recovery and tick recovery can never diverge.
|
|
68
69
|
* Evaluated in table order; `busy` is the on-disk busy marker.
|
|
69
70
|
*
|
|
70
71
|
* `busy` no longer selects the ACTION. Whether the engine died mid-turn or
|
|
@@ -126,12 +127,20 @@ export function respawnBackoffMs(consecutiveFailures) {
|
|
|
126
127
|
return 0;
|
|
127
128
|
return Math.min(RESPAWN_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESPAWN_BACKOFF_MAX_MS);
|
|
128
129
|
}
|
|
130
|
+
/** How long a capacity denial stays quiet before it is worth logging again. */
|
|
131
|
+
const CAPACITY_LOG_THROTTLE_MS = 30_000;
|
|
129
132
|
export class DaemonFleet {
|
|
130
133
|
#epoch;
|
|
131
134
|
#enqueue;
|
|
132
135
|
#now;
|
|
133
136
|
#map = new Map();
|
|
134
137
|
#throttle = new Map();
|
|
138
|
+
/** Slots claimed but not yet held by a registered broker: a launch in flight,
|
|
139
|
+
* or a broker that exited and whose exit-policy job has not yet decided
|
|
140
|
+
* whether to relaunch it. Counting both against the cap is what keeps a
|
|
141
|
+
* 1:1 respawn from losing its own slot to a concurrent launcher. */
|
|
142
|
+
#reservations = new Set();
|
|
143
|
+
#lastCapacityLogAt = Number.NEGATIVE_INFINITY;
|
|
135
144
|
constructor(opts) {
|
|
136
145
|
this.#epoch = opts.epoch;
|
|
137
146
|
this.#enqueue = opts.enqueue;
|
|
@@ -155,6 +164,7 @@ export class DaemonFleet {
|
|
|
155
164
|
clearTimeout(throttle.timer);
|
|
156
165
|
throttle.timer = undefined;
|
|
157
166
|
}
|
|
167
|
+
this.#reservations.delete(nodeId);
|
|
158
168
|
this.#map.set(nodeId, {
|
|
159
169
|
pid: handle.pid,
|
|
160
170
|
launchedAt: this.#now(),
|
|
@@ -171,29 +181,59 @@ export class DaemonFleet {
|
|
|
171
181
|
size() {
|
|
172
182
|
return this.#map.size;
|
|
173
183
|
}
|
|
184
|
+
occupancy() {
|
|
185
|
+
return this.#map.size + this.#reservations.size;
|
|
186
|
+
}
|
|
187
|
+
reserveSlot(nodeId) {
|
|
188
|
+
if (this.#reservations.has(nodeId) || this.#map.has(nodeId))
|
|
189
|
+
return true;
|
|
190
|
+
const { automaticReviveCap } = brokerThresholdsForDaemon();
|
|
191
|
+
if (this.occupancy() < automaticReviveCap) {
|
|
192
|
+
this.#reservations.add(nodeId);
|
|
193
|
+
return true;
|
|
194
|
+
}
|
|
195
|
+
const now = this.#now();
|
|
196
|
+
if (now - this.#lastCapacityLogAt >= CAPACITY_LOG_THROTTLE_MS) {
|
|
197
|
+
this.#lastCapacityLogAt = now;
|
|
198
|
+
emitEvent({
|
|
199
|
+
level: 'warn',
|
|
200
|
+
event: 'broker.capacity.revive_deferred',
|
|
201
|
+
fields: { live_count: this.occupancy(), hard_cap: automaticReviveCap },
|
|
202
|
+
});
|
|
203
|
+
}
|
|
204
|
+
return false;
|
|
205
|
+
}
|
|
206
|
+
releaseReservation(nodeId) {
|
|
207
|
+
this.#reservations.delete(nodeId);
|
|
208
|
+
}
|
|
174
209
|
entries() {
|
|
175
210
|
return this.#map.entries();
|
|
176
211
|
}
|
|
177
212
|
forget(nodeId) {
|
|
178
213
|
this.#map.delete(nodeId);
|
|
214
|
+
this.#reservations.delete(nodeId);
|
|
179
215
|
this.#clearThrottle(nodeId);
|
|
180
216
|
}
|
|
181
217
|
onChildExit(nodeId, status) {
|
|
182
|
-
//
|
|
183
|
-
// exit
|
|
218
|
+
// Move the slot from the registry to a reservation in ONE synchronous
|
|
219
|
+
// step: the exit-policy job about to be enqueued may relaunch this node,
|
|
220
|
+
// and holding the slot for it is what keeps a 1:1 respawn from losing its
|
|
221
|
+
// own capacity to a launcher that raced onto the lane first. `#applyPolicy`
|
|
222
|
+
// releases it on every path that does not relaunch.
|
|
184
223
|
const entry = this.#map.get(nodeId);
|
|
185
224
|
this.#map.delete(nodeId);
|
|
225
|
+
this.#reservations.add(nodeId);
|
|
186
226
|
const launchedAt = entry?.launchedAt ?? null;
|
|
187
227
|
this.#enqueue(() => this.#runExitPolicy(nodeId, status, launchedAt));
|
|
188
228
|
}
|
|
189
229
|
deliverExit(nodeId, status) {
|
|
190
230
|
this.onChildExit(nodeId, status);
|
|
191
231
|
}
|
|
192
|
-
/**
|
|
193
|
-
*
|
|
194
|
-
*
|
|
195
|
-
async
|
|
196
|
-
|
|
232
|
+
/** The node-lifecycle tick's enactment entry: classify a known-dead row and
|
|
233
|
+
* enact immediately through the SAME policy table exit events use, so tick
|
|
234
|
+
* recovery and exit recovery can never diverge. */
|
|
235
|
+
async applyDeadRowPolicy(nodeId) {
|
|
236
|
+
await this.#applyPolicy(nodeId, 'tick');
|
|
197
237
|
}
|
|
198
238
|
/** A revive refusal before its launch try/catch leaves the row active but
|
|
199
239
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
@@ -270,6 +310,18 @@ export class DaemonFleet {
|
|
|
270
310
|
});
|
|
271
311
|
}
|
|
272
312
|
async #applyPolicy(nodeId, origin) {
|
|
313
|
+
try {
|
|
314
|
+
return await this.#decideAndEnact(nodeId, origin);
|
|
315
|
+
}
|
|
316
|
+
finally {
|
|
317
|
+
// The reservation this node has held since its exit is either consumed by
|
|
318
|
+
// a successful `register` (making this a no-op) or given back here — the
|
|
319
|
+
// one release path covering a declined policy, a deferred backoff, and
|
|
320
|
+
// terminalization alike.
|
|
321
|
+
this.releaseReservation(nodeId);
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
async #decideAndEnact(nodeId, origin) {
|
|
273
325
|
const meta = getNode(nodeId);
|
|
274
326
|
// Two durable witnesses mean the dead broker had a turn in flight. A dirty
|
|
275
327
|
// death leaves the busy marker behind; a bounded graceful abort clears busy
|
|
@@ -329,9 +381,13 @@ export class DaemonFleet {
|
|
|
329
381
|
});
|
|
330
382
|
return;
|
|
331
383
|
}
|
|
332
|
-
//
|
|
333
|
-
//
|
|
334
|
-
|
|
384
|
+
// A backoff timer already owns this node's next respawn; a tick that
|
|
385
|
+
// enacted the same policy meanwhile would defeat the throttle entirely.
|
|
386
|
+
if (origin === 'tick' && state.timer !== undefined)
|
|
387
|
+
return;
|
|
388
|
+
// Only a fresh exit consults the backoff: a matured timer and a tick
|
|
389
|
+
// enactment relaunch immediately (the timer already WAS the backoff, and a
|
|
390
|
+
// tick candidate reached the lane only after its own gating).
|
|
335
391
|
if (origin === 'exit') {
|
|
336
392
|
const delay = respawnBackoffMs(state.failures);
|
|
337
393
|
if (delay > 0) {
|
|
@@ -369,6 +425,9 @@ export class DaemonFleet {
|
|
|
369
425
|
});
|
|
370
426
|
try {
|
|
371
427
|
reviveNode(nodeId, {
|
|
428
|
+
// A recovery that loses the race for the last slot freezes and waits
|
|
429
|
+
// for one — the tick thaws it — rather than erroring the node away.
|
|
430
|
+
capacity: 'freeze',
|
|
372
431
|
resume: action === 'respawn-resume',
|
|
373
432
|
// A resumed session whose turn was interrupted comes back parked at an
|
|
374
433
|
// idle prompt with the dead turn unfinished. The continuation re-drives
|
|
@@ -64,7 +64,9 @@ export async function deliverNodeMessage(args) {
|
|
|
64
64
|
deliveryUnroutable(args.node_id, 'origin_missing');
|
|
65
65
|
if (!isBrokerLive(target)) {
|
|
66
66
|
try {
|
|
67
|
-
|
|
67
|
+
// Mail is durable, so a wake denied for capacity waits for a slot
|
|
68
|
+
// rather than being lost: the row freezes and the tick relaunches it.
|
|
69
|
+
revived = reviveNode(args.node_id, { resume: true, capacity: 'freeze' }).launch !== undefined;
|
|
68
70
|
}
|
|
69
71
|
catch {
|
|
70
72
|
// The durable entry remains authoritative; one wake attempt is enough.
|
|
@@ -216,7 +216,7 @@ async function detachProfile(plan) {
|
|
|
216
216
|
const detached = new Set(detachedNodeIds);
|
|
217
217
|
for (const nodeId of restartNodeIds) {
|
|
218
218
|
if (detached.has(nodeId))
|
|
219
|
-
reviveNode(nodeId, { resume: true, recovery: true });
|
|
219
|
+
reviveNode(nodeId, { resume: true, recovery: true, capacity: 'freeze' });
|
|
220
220
|
}
|
|
221
221
|
return {
|
|
222
222
|
...resultForPlan(plan),
|
|
@@ -1,17 +1,30 @@
|
|
|
1
1
|
import { type NodeRow } from '../../core/canvas/index.js';
|
|
2
|
-
import type
|
|
2
|
+
import { type FleetRegistry } from '../../core/runtime/fleet.js';
|
|
3
3
|
export interface DetachedWorkLifecycle {
|
|
4
4
|
acceptsDetachedWork(): boolean;
|
|
5
5
|
registerDetached(work: Promise<void>): void;
|
|
6
6
|
}
|
|
7
7
|
export interface BrokerSupervisionContext {
|
|
8
|
+
/** The rows the tick loaded — supervision walks these and looks each one's
|
|
9
|
+
* fleet entry up, rather than re-reading a row per live entry. */
|
|
10
|
+
rows: readonly NodeRow[];
|
|
8
11
|
fleet: FleetRegistry;
|
|
9
12
|
lifecycle: DetachedWorkLifecycle;
|
|
10
13
|
}
|
|
11
|
-
export
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
14
|
+
export interface SaturationContext {
|
|
15
|
+
fleet: FleetRegistry;
|
|
16
|
+
lifecycle: DetachedWorkLifecycle;
|
|
17
|
+
thresholds: {
|
|
18
|
+
warning: number;
|
|
19
|
+
automaticReviveCap: number;
|
|
20
|
+
};
|
|
21
|
+
/** What the freeze lane left frozen this tick, split into rows that stalled
|
|
22
|
+
* and rows waiting on mail. */
|
|
23
|
+
frozen: {
|
|
24
|
+
stalled: number;
|
|
25
|
+
waitingOnMail: number;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
15
28
|
export declare const YIELD_STALL_GRACE_MS: number;
|
|
16
29
|
export declare const WEDGE_QUIET_MS: number;
|
|
17
30
|
export declare const WEDGE_CPU_MAX_PERCENT = 2;
|
|
@@ -34,11 +47,17 @@ export declare class BrokerSupervisionReconciler {
|
|
|
34
47
|
private readonly unattendedSince;
|
|
35
48
|
private knownAuthMtimeMs;
|
|
36
49
|
private lastOverCapWarnAt;
|
|
37
|
-
|
|
38
|
-
|
|
50
|
+
/** The fleet-wide preamble, run BEFORE the tick's row-list guard: an auth
|
|
51
|
+
* reload does not depend on a successful canvas scan, and the thresholds it
|
|
52
|
+
* returns are read once for the whole tick. */
|
|
53
|
+
runFleetPreamble(fleet: FleetRegistry): Promise<{
|
|
39
54
|
warning: number;
|
|
40
55
|
automaticReviveCap: number;
|
|
41
56
|
}>;
|
|
57
|
+
/** The saturation notice, run LAST in the tick: the frozen split it reports
|
|
58
|
+
* is only known once the freeze lane has spent this tick's free slots. */
|
|
59
|
+
warnIfSaturated(ctx: SaturationContext): void;
|
|
60
|
+
run(now: number, ctx: BrokerSupervisionContext): Promise<void>;
|
|
42
61
|
/** Every condition that resets the unattended clock EXCEPT `isBusy` — the
|
|
43
62
|
* node is unavailable to park, attended, or has work waiting. Short-circuits
|
|
44
63
|
* in cost order so the common tick does no file reads it can avoid.
|
|
@@ -58,6 +77,4 @@ export declare class BrokerSupervisionReconciler {
|
|
|
58
77
|
private handleWedgeDetection;
|
|
59
78
|
private handleFatalFault;
|
|
60
79
|
private handleAuthReload;
|
|
61
|
-
hasBrokerCapacity(max: number, fleet: FleetRegistry): boolean;
|
|
62
|
-
hasStartupRecoveryCapacity(fleet: FleetRegistry): boolean;
|
|
63
80
|
}
|