@north-light/crouter 0.3.249 → 0.3.251
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/api/__tests__/integration/client.test.js +28 -7
- package/dist/api/client.d.ts +1 -1
- package/dist/api/client.js +2 -2
- package/dist/api/dto/lifecycle.d.ts +3 -0
- package/dist/api/dto/messages.d.ts +4 -0
- package/dist/api/dto/nodes.d.ts +4 -0
- package/dist/api/dto/reports.d.ts +7 -2
- package/dist/builtin-memory/00-runtime-base/01-escalation.md +1 -1
- package/dist/builtin-memory/05-kinds/review/01-orchestrator.md +3 -0
- package/dist/builtin-memory/internal/nodes-and-canvas.md +1 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/README.md +0 -1
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.js +4 -3
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/provider-rotation.ts +4 -3
- package/dist/clients/attach/session/chrome-refresh.d.ts +3 -0
- package/dist/clients/attach/session/chrome-refresh.js +1 -0
- package/dist/clients/attach/viewer.js +532 -532
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.d.ts +1 -0
- package/dist/clients/inbox/__tests__/integration/stale-row-activation.test.js +53 -0
- package/dist/clients/inbox/controller.js +32 -5
- package/dist/clients/inbox/tui/ansi.d.ts +3 -1
- package/dist/clients/inbox/tui/ansi.js +14 -10
- package/dist/commands/__tests__/api-client-cold-start-diagnostic.test.js +7 -7
- package/dist/commands/api-client.d.ts +2 -2
- package/dist/commands/api-client.js +5 -5
- package/dist/commands/canvas-browse.js +1 -1
- package/dist/commands/node/create.js +3 -1
- package/dist/commands/node/inspect.js +1 -0
- package/dist/commands/node/lifecycle.js +1 -1
- package/dist/commands/node/message.js +6 -4
- package/dist/commands/node/wait.js +1 -1
- package/dist/commands/push.js +36 -37
- package/dist/commands/sys/daemon.js +21 -8
- package/dist/commands/sys/setup-core.js +2 -2
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.d.ts +1 -0
- package/dist/core/__tests__/broker-stream-watchdog-floor.test.js +84 -0
- package/dist/core/__tests__/broker-turn-admission.test.d.ts +1 -0
- package/dist/core/__tests__/broker-turn-admission.test.js +44 -0
- package/dist/core/__tests__/canvas-inbox-watcher-hold.test.js +100 -1
- package/dist/core/__tests__/child-death-wake.test.js +11 -7
- package/dist/core/__tests__/collectible.test.d.ts +1 -0
- package/dist/core/__tests__/collectible.test.js +88 -0
- package/dist/core/__tests__/context-intro.test.js +4 -41
- package/dist/core/__tests__/daemon-boot.test.js +30 -14
- package/dist/core/__tests__/daemon-wedge.test.js +25 -19
- package/dist/core/__tests__/fixtures/fake-engine.js +20 -2
- package/dist/core/__tests__/human-deliver.test.js +5 -5
- package/dist/core/__tests__/integration/deferred-no-wake.test.js +65 -4
- package/dist/core/__tests__/integration/human-deliver-e2e.test.js +3 -3
- package/dist/core/__tests__/integration/revive.test.js +48 -0
- package/dist/core/__tests__/integration/spawn-root.test.js +36 -1
- package/dist/core/__tests__/integration/worktree-land.test.js +82 -3
- package/dist/core/__tests__/integration/worktree-reap.test.js +44 -6
- package/dist/core/__tests__/migration.test.js +30 -0
- package/dist/core/__tests__/passive-subscription.test.js +39 -1
- package/dist/core/__tests__/prune.test.js +12 -3
- package/dist/core/__tests__/relaunch-root.test.js +1 -1
- package/dist/core/__tests__/respawn-throttle.test.js +16 -146
- package/dist/core/__tests__/seam/broker-crash-teardown.test.js +23 -11
- package/dist/core/__tests__/seam/broker-provider-retry.test.js +81 -2
- package/dist/core/__tests__/seam/dormancy-release.test.js +1 -1
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.d.ts +1 -0
- package/dist/core/__tests__/seam/held-deferred-human-prompt.test.js +63 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.d.ts +1 -0
- package/dist/core/__tests__/seam/self-close-teardown.test.js +49 -0
- package/dist/core/__tests__/seam/yield-refresh-transaction.test.js +18 -10
- package/dist/core/canvas/canvas.d.ts +9 -15
- package/dist/core/canvas/canvas.js +41 -47
- package/dist/core/canvas/crons.d.ts +5 -8
- package/dist/core/canvas/crons.js +5 -8
- package/dist/core/canvas/history.d.ts +5 -0
- package/dist/core/canvas/history.js +12 -1
- package/dist/core/canvas/migrations.js +58 -1
- package/dist/core/canvas/pid.d.ts +1 -1
- package/dist/core/canvas/pid.js +5 -2
- package/dist/core/canvas/types.d.ts +4 -0
- package/dist/core/feed/feed.d.ts +4 -0
- package/dist/core/feed/feed.js +13 -8
- package/dist/core/feed/inbox.d.ts +2 -1
- package/dist/core/feed/inbox.js +22 -3
- package/dist/core/git.d.ts +3 -0
- package/dist/core/git.js +3 -1
- package/dist/core/human/feedback-companion.js +1 -0
- package/dist/core/preview-registry.js +0 -1
- package/dist/core/review/companion.js +1 -0
- package/dist/core/review/realize.js +1 -0
- package/dist/core/runtime/broker/auth-reload.d.ts +2 -0
- package/dist/core/runtime/broker/auth-reload.js +9 -3
- package/dist/core/runtime/broker/engine-drive.d.ts +2 -0
- package/dist/core/runtime/broker/engine-drive.js +39 -23
- package/dist/core/runtime/broker/event-projection.d.ts +4 -0
- package/dist/core/runtime/broker/event-projection.js +43 -23
- package/dist/core/runtime/broker/fault-retry.d.ts +9 -0
- package/dist/core/runtime/broker/fault-retry.js +94 -8
- package/dist/core/runtime/broker/frame-dispatch.d.ts +2 -0
- package/dist/core/runtime/broker/frame-dispatch.js +4 -1
- package/dist/core/runtime/broker/held-deferred-inbox.d.ts +9 -0
- package/dist/core/runtime/broker/held-deferred-inbox.js +8 -0
- package/dist/core/runtime/broker/inbox.js +4 -0
- package/dist/core/runtime/broker/rebind.js +18 -0
- package/dist/core/runtime/broker/turn-admission.d.ts +11 -0
- package/dist/core/runtime/broker/turn-admission.js +31 -0
- package/dist/core/runtime/broker/turn-ignition.d.ts +2 -0
- package/dist/core/runtime/broker/turn-ignition.js +25 -15
- package/dist/core/runtime/broker.js +21 -2
- package/dist/core/runtime/close.d.ts +11 -9
- package/dist/core/runtime/close.js +23 -19
- package/dist/core/runtime/fault.d.ts +21 -0
- package/dist/core/runtime/fault.js +129 -19
- package/dist/core/runtime/fleet.d.ts +0 -6
- package/dist/core/runtime/host.d.ts +7 -1
- package/dist/core/runtime/host.js +20 -2
- package/dist/core/runtime/nodes.d.ts +5 -1
- package/dist/core/runtime/nodes.js +24 -1
- package/dist/core/runtime/placement.d.ts +6 -6
- package/dist/core/runtime/placement.js +15 -8
- package/dist/core/runtime/reset.js +2 -2
- package/dist/core/runtime/revive.js +40 -4
- package/dist/core/runtime/spawn.d.ts +3 -0
- package/dist/core/runtime/spawn.js +1 -0
- package/dist/core/runtime/warm-pool.d.ts +4 -0
- package/dist/core/runtime/warm-pool.js +2 -0
- package/dist/core/worktree.d.ts +4 -3
- package/dist/core/worktree.js +73 -27
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.d.ts +1 -0
- package/dist/daemon/__tests__/integration/api-startup-readiness.test.js +81 -0
- package/dist/daemon/api/__tests__/reopen-delivery.test.js +38 -0
- package/dist/daemon/api/__tests__/seam/api-server.test.js +1 -0
- package/dist/daemon/api/__tests__/seam/leaf-api-parity.test.js +13 -2
- package/dist/daemon/api/handlers/bash-jobs.js +2 -0
- package/dist/daemon/api/handlers/broker-ops.js +23 -2
- package/dist/daemon/api/handlers/human.js +1 -1
- package/dist/daemon/api/handlers/messages.js +11 -2
- package/dist/daemon/api/handlers/nodes.js +13 -1
- package/dist/daemon/api/handlers/reports.js +14 -6
- package/dist/daemon/api/map.js +1 -0
- package/dist/daemon/api/server.d.ts +9 -12
- package/dist/daemon/api/server.js +30 -19
- package/dist/daemon/companion-retire.js +3 -9
- package/dist/daemon/cron/sinks.js +7 -2
- package/dist/daemon/crtrd.js +4 -1
- package/dist/daemon/fleet.d.ts +2 -16
- package/dist/daemon/fleet.js +63 -150
- package/dist/daemon/human/finish.js +2 -0
- package/dist/daemon/manage.d.ts +11 -0
- package/dist/daemon/manage.js +32 -3
- package/dist/daemon/messaging/node-message.js +7 -1
- package/dist/daemon/reconcilers/bash-deadline.js +2 -0
- package/dist/daemon/reconcilers/broker-supervision.d.ts +1 -0
- package/dist/daemon/reconcilers/broker-supervision.js +22 -9
- package/dist/daemon/reconcilers/live-obligation.d.ts +16 -6
- package/dist/daemon/reconcilers/live-obligation.js +18 -6
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.d.ts +15 -4
- package/dist/daemon/reconcilers/node-lifecycle/respawn-policy.js +34 -3
- package/dist/daemon/reconcilers/node-lifecycle/terminating.d.ts +4 -1
- package/dist/daemon/reconcilers/node-lifecycle/terminating.js +11 -2
- package/dist/daemon/reconcilers/node-lifecycle/tick.js +14 -7
- package/dist/daemon/reconcilers/storage-maintenance.d.ts +2 -2
- package/dist/daemon/reconcilers/storage-maintenance.js +3 -3
- package/dist/pi-extensions/__tests__/canvas-stophook-agentend.test.js +4 -4
- package/dist/pi-extensions/canvas-context-intro.d.ts +3 -34
- package/dist/pi-extensions/canvas-context-intro.js +4 -76
- package/dist/pi-extensions/canvas-inbox-watcher.js +105 -4
- package/dist/pi-extensions/canvas-review-boundary.d.ts +3 -24
- package/dist/pi-extensions/canvas-review-boundary.js +4 -29
- package/dist/pi-extensions/canvas-stophook.js +1 -6
- package/dist/shared/birth-announcement.d.ts +12 -0
- package/dist/shared/birth-announcement.js +25 -0
- package/dist/shared/generated-context.d.ts +8 -4
- package/dist/shared/generated-context.js +15 -9
- package/package.json +4 -4
- package/runtime.lock.json +2 -2
- package/dist/builtin-pi-packages/pi-crtr-extensions/extensions/statusline.ts +0 -254
- package/dist/pi-extensions/truncate.d.ts +0 -5
- package/dist/pi-extensions/truncate.js +0 -14
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
// server.ts — the crtrd HTTP+WS API server (spec §4, esp. §4.1/§5). A-9.
|
|
2
2
|
//
|
|
3
|
-
//
|
|
3
|
+
// Two `http.Server` listeners share one logical API (a Node server can bind one
|
|
4
|
+
// handle only):
|
|
4
5
|
// - the unix socket at `apiSocketPath()` — ALWAYS bound, owner-only (mode
|
|
5
6
|
// 0600 = the fs-perms-are-auth model);
|
|
6
7
|
// - an opt-in TCP listener, bound only when `--tcp <host:port>` / `CRTRD_TCP`
|
|
@@ -121,19 +122,14 @@ function closeServer(server) {
|
|
|
121
122
|
server.close(() => resolve());
|
|
122
123
|
});
|
|
123
124
|
}
|
|
124
|
-
/** Create and start the crtrd API server
|
|
125
|
-
* `
|
|
126
|
-
*
|
|
127
|
-
* bind failure or a post-listen accept error is emitted and swallowed — it must
|
|
128
|
-
* NEVER become an unhandled 'error' that crashes the daemon.
|
|
125
|
+
/** Create and start the crtrd API server. Returns immediately with a `close()`
|
|
126
|
+
* handle and a `ready` promise for the mandatory Unix socket bind. TCP remains
|
|
127
|
+
* opt-in, and its bind or post-listen accept errors are emitted but non-fatal.
|
|
129
128
|
*
|
|
130
|
-
*
|
|
131
|
-
* `
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
* functions (the standard way to bind one logical API to two transports). The
|
|
135
|
-
* Router and attach bridge are process-wide singletons, so both servers route
|
|
136
|
-
* into identical state. */
|
|
129
|
+
* A Node `net.Server`/`http.Server` binds exactly ONE handle, so the two
|
|
130
|
+
* listeners are separate `http.Server` instances sharing request and upgrade
|
|
131
|
+
* handlers. The Router and attach bridge are process-wide singletons, so both
|
|
132
|
+
* servers route into identical state. */
|
|
137
133
|
export function createApiServer(opts = {}) {
|
|
138
134
|
const router = buildRouter();
|
|
139
135
|
const sockPath = apiSocketPath();
|
|
@@ -199,17 +195,32 @@ export function createApiServer(opts = {}) {
|
|
|
199
195
|
}
|
|
200
196
|
};
|
|
201
197
|
const bindErrorHandler = (server) => {
|
|
202
|
-
//
|
|
203
|
-
//
|
|
198
|
+
// TCP is opt-in, so neither its bind failure nor a later accept error can
|
|
199
|
+
// make the mandatory Unix API unavailable.
|
|
204
200
|
server.on('error', (err) => {
|
|
205
201
|
emitEvent({ event: 'api.server.failed', level: 'error', error: err });
|
|
206
202
|
});
|
|
207
203
|
};
|
|
208
|
-
// Unix socket — ALWAYS bound.
|
|
209
|
-
//
|
|
204
|
+
// Unix socket — ALWAYS bound. Its bind result is startup readiness; after
|
|
205
|
+
// that point, accept errors remain observable but do not tear the daemon down.
|
|
210
206
|
const socketServer = createServer(onRequest);
|
|
211
207
|
socketServer.on('upgrade', onUpgrade);
|
|
212
|
-
|
|
208
|
+
let socketBound = false;
|
|
209
|
+
let resolveReady;
|
|
210
|
+
let rejectReady;
|
|
211
|
+
const ready = new Promise((resolve, reject) => {
|
|
212
|
+
resolveReady = resolve;
|
|
213
|
+
rejectReady = reject;
|
|
214
|
+
});
|
|
215
|
+
socketServer.once('listening', () => {
|
|
216
|
+
socketBound = true;
|
|
217
|
+
resolveReady();
|
|
218
|
+
});
|
|
219
|
+
socketServer.on('error', (err) => {
|
|
220
|
+
emitEvent({ event: 'api.server.failed', level: 'error', error: err });
|
|
221
|
+
if (!socketBound)
|
|
222
|
+
rejectReady(err);
|
|
223
|
+
});
|
|
213
224
|
try {
|
|
214
225
|
if (existsSync(sockPath))
|
|
215
226
|
unlinkSync(sockPath);
|
|
@@ -265,5 +276,5 @@ export function createApiServer(opts = {}) {
|
|
|
265
276
|
/* best-effort cleanup */
|
|
266
277
|
}
|
|
267
278
|
};
|
|
268
|
-
return { close };
|
|
279
|
+
return { ready, close };
|
|
269
280
|
}
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
import { getNode } from '../core/canvas/canvas.js';
|
|
6
6
|
import { FinalizationError, pushFinal } from '../core/feed/feed.js';
|
|
7
7
|
import { transition } from '../core/runtime/lifecycle.js';
|
|
8
|
-
import {
|
|
8
|
+
import { requestBrokerTeardown } from '../core/runtime/host.js';
|
|
9
9
|
/** Retire a companion only through the terminal transition legal for its current status. */
|
|
10
10
|
export async function retireCompanion(nodeId, outcome) {
|
|
11
11
|
const node = getNode(nodeId);
|
|
@@ -23,18 +23,12 @@ export async function retireCompanion(nodeId, outcome) {
|
|
|
23
23
|
throw error;
|
|
24
24
|
result = 'already';
|
|
25
25
|
}
|
|
26
|
-
|
|
27
|
-
headlessBrokerHost.teardown(nodeId);
|
|
28
|
-
}
|
|
29
|
-
catch { /* the canonical final is authoritative */ }
|
|
26
|
+
requestBrokerTeardown(nodeId);
|
|
30
27
|
return result;
|
|
31
28
|
}
|
|
32
29
|
transition(nodeId, 'cancel', { reason: 'retired' });
|
|
33
30
|
if (node.status === 'active' || node.status === 'idle') {
|
|
34
|
-
|
|
35
|
-
headlessBrokerHost.teardown(nodeId);
|
|
36
|
-
}
|
|
37
|
-
catch { /* the terminal transition is authoritative */ }
|
|
31
|
+
requestBrokerTeardown(nodeId);
|
|
38
32
|
}
|
|
39
33
|
return 'retired';
|
|
40
34
|
}
|
|
@@ -5,6 +5,7 @@ import { ticketDir } from '../../core/human/root.js';
|
|
|
5
5
|
import { preparePage, publishPage } from '../../core/human/tickets.js';
|
|
6
6
|
import { resolvePageComponents } from '../../core/config.js';
|
|
7
7
|
import { getNode } from '../../core/canvas/canvas.js';
|
|
8
|
+
import { cancelCronsOnWake } from '../../core/canvas/crons.js';
|
|
8
9
|
import { appendInbox } from '../../core/feed/inbox.js';
|
|
9
10
|
import { hasNoNaturalCycle } from '../../core/runtime/revive-all.js';
|
|
10
11
|
import { isParked } from '../../core/runtime/lifecycle.js';
|
|
@@ -23,15 +24,19 @@ function deliverNodeSink(c, target, stdout) {
|
|
|
23
24
|
throw new Error(`node sink ${target} has no natural cycle ahead (done or finalized)`);
|
|
24
25
|
}
|
|
25
26
|
const from = c.created_by != null && getNode(c.created_by) !== null ? c.created_by : null;
|
|
26
|
-
appendInbox(target, {
|
|
27
|
+
const entry = appendInbox(target, {
|
|
27
28
|
from,
|
|
28
29
|
tier: c.tier,
|
|
29
30
|
kind: 'message',
|
|
30
31
|
label: `⏰ cron ${c.name}`,
|
|
31
32
|
data: { body: stdout },
|
|
32
33
|
});
|
|
34
|
+
if (c.tier !== 'deferred' || meta.frozen_at !== null)
|
|
35
|
+
cancelCronsOnWake(target);
|
|
36
|
+
// The APPENDED tier decides the wake, not the cron's requested one: a deferred
|
|
37
|
+
// cron aimed at a terminal node was raised out of `deferred` by the append.
|
|
33
38
|
// The durable append succeeds even if its best-effort wake fails.
|
|
34
|
-
if (
|
|
39
|
+
if (entry.tier !== 'deferred' && !isBrokerLive(meta)) {
|
|
35
40
|
try {
|
|
36
41
|
reviveNode(target, { resume: true, capacity: 'freeze' });
|
|
37
42
|
}
|
package/dist/daemon/crtrd.js
CHANGED
|
@@ -668,8 +668,11 @@ export async function runDaemon(opts = {}) {
|
|
|
668
668
|
// no adoption across epochs, ever.
|
|
669
669
|
await operationIdContext.fresh(() => reapPriorEpochBrokers(epoch));
|
|
670
670
|
writePidfile();
|
|
671
|
-
//
|
|
671
|
+
// The Unix API socket is the daemon's readiness boundary. A rejected bind
|
|
672
|
+
// enters the existing startup catch, which closes resources and releases
|
|
673
|
+
// this invocation's pidfile mirror and ownership claim before propagating.
|
|
672
674
|
apiServer = createApiServer({ tcp: opts.tcp });
|
|
675
|
+
await apiServer.ready;
|
|
673
676
|
// Pre-warm the node pool for the most recent project recipes, so the first
|
|
674
677
|
// create after a boot is warm too. Fire-and-forget (the mints run off the
|
|
675
678
|
// work lane), so it never delays supervision. Boot-time dead rows are the
|
package/dist/daemon/fleet.d.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type NodeMeta } from '../core/canvas/index.js';
|
|
2
2
|
import { type FleetEntry, type FleetExitStatus, type FleetRegistry } from '../core/runtime/fleet.js';
|
|
3
3
|
import type { HostHandle } from '../core/runtime/host.js';
|
|
4
|
-
/** Surface a vehicle that never booted
|
|
4
|
+
/** Surface a vehicle that never booted.
|
|
5
5
|
*
|
|
6
6
|
* A pi that dies before its first session_start (so `pi_session_id` was never
|
|
7
7
|
* recorded) is invisible to its parent — the spawn returned optimistically.
|
|
@@ -43,16 +43,6 @@ export type DeadNodeRow = Pick<NodeMeta, 'status' | 'intent' | 'pi_session_id' |
|
|
|
43
43
|
* would drive a turn if the engine came back. It selects row 4b and defaults
|
|
44
44
|
* to `true`, so an uninformed caller keeps the conservative respawn. */
|
|
45
45
|
export declare function classifyDeadNode(row: DeadNodeRow | null, busy: boolean, pendingWake?: boolean): DeadNodeAction;
|
|
46
|
-
/** An exit with uptime at or above this resets the consecutive-failure
|
|
47
|
-
* counter before classification; a shorter-lived exit increments it. */
|
|
48
|
-
export declare const HEALTHY_UPTIME_MS = 60000;
|
|
49
|
-
/** At this many consecutive short-lived exits the respawn loop terminalizes:
|
|
50
|
-
* transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
|
|
51
|
-
export declare const MAX_RESPAWN_ATTEMPTS = 6;
|
|
52
|
-
/** Backoff before the next respawn given the consecutive-failure count.
|
|
53
|
-
* Zero failures → immediate (the sub-second yield respawn D-3 buys); the
|
|
54
|
-
* n-th delayed attempt waits min(5s·2^n, 10min). */
|
|
55
|
-
export declare function respawnBackoffMs(consecutiveFailures: number): number;
|
|
56
46
|
export interface DaemonFleetOpts {
|
|
57
47
|
/** This daemon's epoch id (randomBytes(8).toString('hex'), minted by
|
|
58
48
|
* runDaemon after the ownership claim). */
|
|
@@ -78,16 +68,12 @@ export declare class DaemonFleet implements FleetRegistry {
|
|
|
78
68
|
forget(nodeId: string): void;
|
|
79
69
|
onChildExit(nodeId: string, status: FleetExitStatus): void;
|
|
80
70
|
deliverExit(nodeId: string, status: FleetExitStatus): void;
|
|
81
|
-
/** The node-lifecycle tick's enactment entry
|
|
82
|
-
* enact immediately through the SAME policy table exit events use, so tick
|
|
83
|
-
* recovery and exit recovery can never diverge. */
|
|
71
|
+
/** The node-lifecycle tick's enactment entry. */
|
|
84
72
|
applyDeadRowPolicy(nodeId: string): Promise<void>;
|
|
85
73
|
/** A revive refusal before its launch try/catch leaves the row active but
|
|
86
74
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
87
75
|
* terminal outcome as a boot failure. The fresh read avoids duplicating
|
|
88
76
|
* reviveNode's launch-failure crash path. */
|
|
89
77
|
terminalizeIfStillNonTerminal(nodeId: string, event: string, error: unknown): Promise<boolean>;
|
|
90
|
-
/** Daemon shutdown: cancel every pending backoff respawn. Live processes
|
|
91
|
-
* are torn down by the daemon's fleet-teardown stage, not here. */
|
|
92
78
|
dispose(): void;
|
|
93
79
|
}
|
package/dist/daemon/fleet.js
CHANGED
|
@@ -1,32 +1,31 @@
|
|
|
1
1
|
// src/daemon/fleet.ts — the daemon's broker fleet: the ChildProcess-handle
|
|
2
2
|
// registry (the ONE normal liveness authority), the exit-policy classifier
|
|
3
|
-
// (invariant-5 policy table), and
|
|
3
|
+
// (invariant-5 policy table), and durable respawn recovery.
|
|
4
4
|
//
|
|
5
5
|
// Constructed by runDaemon with the serial work lane's `enqueue` and this
|
|
6
6
|
// daemon's epoch id, then injected into core via `bindFleet` BEFORE the API
|
|
7
|
-
// server or any tick starts. Exit events
|
|
8
|
-
//
|
|
9
|
-
// interleaves with a tick's row reads (design D-3).
|
|
7
|
+
// server or any tick starts. Exit events enqueue observation only; the durable
|
|
8
|
+
// row tick owns recovery policy.
|
|
10
9
|
//
|
|
11
10
|
// Row state (status/intent/busy marker/session identity), never exit codes,
|
|
12
11
|
// is the policy input (design D-4): the stophook writes the exit's explanation
|
|
13
12
|
// before every deliberate exit, and that durable state is robust to every kill
|
|
14
|
-
// path. Codes/signals are logged for diagnostics only.
|
|
15
|
-
|
|
16
|
-
import { getNode, subscribersOf } from '../core/canvas/index.js';
|
|
13
|
+
// path. Codes/signals are logged for diagnostics only.
|
|
14
|
+
import { getNode, getRow, subscribersOf } from '../core/canvas/index.js';
|
|
17
15
|
import { fullName } from '../core/canvas/labels.js';
|
|
18
16
|
import { isSafeNodeId } from '../core/canvas/paths.js';
|
|
19
17
|
import { transition } from '../core/runtime/lifecycle.js';
|
|
20
18
|
import { fanDoctrineWake } from '../core/runtime/close.js';
|
|
21
19
|
import { hasCleanAbort, isBusy } from '../core/runtime/busy.js';
|
|
22
20
|
import { hasPendingWake } from './reconcilers/live-obligation.js';
|
|
21
|
+
import { clearRespawnPolicy, MAX_RESPAWN_ATTEMPTS, recordRespawnPolicy, respawnBackoffMs } from './reconcilers/node-lifecycle/respawn-policy.js';
|
|
23
22
|
import { reviveNode } from '../core/runtime/revive.js';
|
|
24
23
|
import { pushUrgent } from '../core/feed/feed.js';
|
|
25
24
|
import { emitEvent } from '../core/events/emit.js';
|
|
26
25
|
import { brokerThresholdsForDaemon } from '../core/runtime/fleet.js';
|
|
27
26
|
// Shared crash helpers (moved from crtrd.ts — the fleet's policy paths are now
|
|
28
27
|
// their primary consumers; crtrd imports them back for its surviving passes).
|
|
29
|
-
/** Surface a vehicle that never booted
|
|
28
|
+
/** Surface a vehicle that never booted.
|
|
30
29
|
*
|
|
31
30
|
* A pi that dies before its first session_start (so `pi_session_id` was never
|
|
32
31
|
* recorded) is invisible to its parent — the spawn returned optimistically.
|
|
@@ -42,6 +41,12 @@ export async function surfaceBootFailure(meta) {
|
|
|
42
41
|
`If the work still needs doing, re-spawn it; if spawns keep dying, spawn fewer at a time.`;
|
|
43
42
|
await pushUrgent(meta.node_id, body, { from: meta.node_id });
|
|
44
43
|
}
|
|
44
|
+
async function surfaceCrashLoop(meta) {
|
|
45
|
+
const body = `⚠ Crash loop — \`${meta.name}\` (${meta.kind}) kept dying without settling a turn.\n\n` +
|
|
46
|
+
`Its session started successfully, but six consecutive relaunches made no progress, so the daemon stopped retrying.\n\n` +
|
|
47
|
+
`Fix the crash cause, then revive the node if the work still needs doing.`;
|
|
48
|
+
await pushUrgent(meta.node_id, body, { from: meta.node_id });
|
|
49
|
+
}
|
|
45
50
|
/** Surface a recovery refusal that happened before a broker could launch. */
|
|
46
51
|
async function surfaceRecoveryFailure(meta, error) {
|
|
47
52
|
const message = error instanceof Error ? error.message : String(error);
|
|
@@ -110,23 +115,6 @@ export function classifyDeadNode(row, busy, pendingWake = true) {
|
|
|
110
115
|
return 'respawn-fresh'; // row 7: bounded ordinary fork-birth retry
|
|
111
116
|
return 'boot-failure'; // row 8
|
|
112
117
|
}
|
|
113
|
-
// Respawn throttle (replaces the stranded-relaunch counters, event-driven)
|
|
114
|
-
/** An exit with uptime at or above this resets the consecutive-failure
|
|
115
|
-
* counter before classification; a shorter-lived exit increments it. */
|
|
116
|
-
export const HEALTHY_UPTIME_MS = 60_000;
|
|
117
|
-
/** At this many consecutive short-lived exits the respawn loop terminalizes:
|
|
118
|
-
* transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
|
|
119
|
-
export const MAX_RESPAWN_ATTEMPTS = 6;
|
|
120
|
-
const RESPAWN_BACKOFF_BASE_MS = 5_000;
|
|
121
|
-
const RESPAWN_BACKOFF_MAX_MS = 10 * 60_000;
|
|
122
|
-
/** Backoff before the next respawn given the consecutive-failure count.
|
|
123
|
-
* Zero failures → immediate (the sub-second yield respawn D-3 buys); the
|
|
124
|
-
* n-th delayed attempt waits min(5s·2^n, 10min). */
|
|
125
|
-
export function respawnBackoffMs(consecutiveFailures) {
|
|
126
|
-
if (consecutiveFailures <= 0)
|
|
127
|
-
return 0;
|
|
128
|
-
return Math.min(RESPAWN_BACKOFF_BASE_MS * 2 ** (consecutiveFailures - 1), RESPAWN_BACKOFF_MAX_MS);
|
|
129
|
-
}
|
|
130
118
|
/** How long a capacity denial stays quiet before it is worth logging again. */
|
|
131
119
|
const CAPACITY_LOG_THROTTLE_MS = 30_000;
|
|
132
120
|
export class DaemonFleet {
|
|
@@ -134,7 +122,6 @@ export class DaemonFleet {
|
|
|
134
122
|
#enqueue;
|
|
135
123
|
#now;
|
|
136
124
|
#map = new Map();
|
|
137
|
-
#throttle = new Map();
|
|
138
125
|
/** Slots claimed but not yet held by a registered broker: a launch in flight,
|
|
139
126
|
* or a broker that exited and whose exit-policy job has not yet decided
|
|
140
127
|
* whether to relaunch it. Counting both against the cap is what keeps a
|
|
@@ -156,21 +143,8 @@ export class DaemonFleet {
|
|
|
156
143
|
if (handle.pid === null) {
|
|
157
144
|
throw new Error(`fleet: refusing to register ${nodeId} with no pid`);
|
|
158
145
|
}
|
|
159
|
-
// A live registration supersedes any scheduled backoff respawn: the node
|
|
160
|
-
// is alive again (a manual revive won the race), so the pending respawn
|
|
161
|
-
// must not fire a second launch.
|
|
162
|
-
const throttle = this.#throttle.get(nodeId);
|
|
163
|
-
if (throttle?.timer !== undefined) {
|
|
164
|
-
clearTimeout(throttle.timer);
|
|
165
|
-
throttle.timer = undefined;
|
|
166
|
-
}
|
|
167
146
|
this.#reservations.delete(nodeId);
|
|
168
|
-
this.#map.set(nodeId, {
|
|
169
|
-
pid: handle.pid,
|
|
170
|
-
launchedAt: this.#now(),
|
|
171
|
-
handle,
|
|
172
|
-
consecutiveFailures: throttle?.failures ?? 0,
|
|
173
|
-
});
|
|
147
|
+
this.#map.set(nodeId, { pid: handle.pid, handle });
|
|
174
148
|
}
|
|
175
149
|
has(nodeId) {
|
|
176
150
|
return this.#map.has(nodeId);
|
|
@@ -212,28 +186,22 @@ export class DaemonFleet {
|
|
|
212
186
|
forget(nodeId) {
|
|
213
187
|
this.#map.delete(nodeId);
|
|
214
188
|
this.#reservations.delete(nodeId);
|
|
215
|
-
this.#clearThrottle(nodeId);
|
|
216
189
|
}
|
|
217
190
|
onChildExit(nodeId, status) {
|
|
218
191
|
// Move the slot from the registry to a reservation in ONE synchronous
|
|
219
192
|
// step: the exit-policy job about to be enqueued may relaunch this node,
|
|
220
|
-
// and holding the slot
|
|
221
|
-
//
|
|
222
|
-
// releases it on every path that does not relaunch.
|
|
223
|
-
const entry = this.#map.get(nodeId);
|
|
193
|
+
// and holding the slot through the exit observation prevents a concurrent
|
|
194
|
+
// launch from entering before the registry reflects the exit.
|
|
224
195
|
this.#map.delete(nodeId);
|
|
225
196
|
this.#reservations.add(nodeId);
|
|
226
|
-
|
|
227
|
-
this.#enqueue(() => this.#runExitPolicy(nodeId, status, launchedAt));
|
|
197
|
+
this.#enqueue(() => this.#runExitPolicy(nodeId, status));
|
|
228
198
|
}
|
|
229
199
|
deliverExit(nodeId, status) {
|
|
230
200
|
this.onChildExit(nodeId, status);
|
|
231
201
|
}
|
|
232
|
-
/** The node-lifecycle tick's enactment entry
|
|
233
|
-
* enact immediately through the SAME policy table exit events use, so tick
|
|
234
|
-
* recovery and exit recovery can never diverge. */
|
|
202
|
+
/** The node-lifecycle tick's enactment entry. */
|
|
235
203
|
async applyDeadRowPolicy(nodeId) {
|
|
236
|
-
await this.#applyPolicy(nodeId
|
|
204
|
+
await this.#applyPolicy(nodeId);
|
|
237
205
|
}
|
|
238
206
|
/** A revive refusal before its launch try/catch leaves the row active but
|
|
239
207
|
* with no fleet entry. Convert that otherwise-silent strand into the same
|
|
@@ -251,49 +219,17 @@ export class DaemonFleet {
|
|
|
251
219
|
});
|
|
252
220
|
return true;
|
|
253
221
|
}
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
#clearThrottle(nodeId) {
|
|
265
|
-
const state = this.#throttle.get(nodeId);
|
|
266
|
-
if (state?.timer !== undefined)
|
|
267
|
-
clearTimeout(state.timer);
|
|
268
|
-
this.#throttle.delete(nodeId);
|
|
269
|
-
}
|
|
270
|
-
async #runExitPolicy(nodeId, status, launchedAt) {
|
|
271
|
-
const uptimeMs = launchedAt === null ? null : Math.max(0, this.#now() - launchedAt);
|
|
272
|
-
// A newer launch may own this node by the time the lane reaches this job
|
|
273
|
-
// (an API revive raced the exit event onto the lane). The exit belonged to
|
|
274
|
-
// the OLD process; the node is alive — nothing to recover, but preserve the
|
|
275
|
-
// observation without charging the fresh launch's throttle budget.
|
|
276
|
-
if (this.#map.has(nodeId)) {
|
|
277
|
-
this.#emitExitObserved(nodeId, status, uptimeMs, this.#throttle.get(nodeId)?.failures ?? 0);
|
|
278
|
-
return;
|
|
279
|
-
}
|
|
280
|
-
const state = this.#throttle.get(nodeId) ?? { failures: 0 };
|
|
281
|
-
if (state.timer !== undefined) {
|
|
282
|
-
clearTimeout(state.timer);
|
|
283
|
-
state.timer = undefined;
|
|
284
|
-
}
|
|
285
|
-
// Healthy uptime resets the counter BEFORE classification; a short-lived
|
|
286
|
-
// exit (or a synthetic exit with no known launch time) increments it.
|
|
287
|
-
if (uptimeMs !== null && uptimeMs >= HEALTHY_UPTIME_MS) {
|
|
288
|
-
state.failures = 0;
|
|
289
|
-
this.#throttle.delete(nodeId);
|
|
222
|
+
dispose() { }
|
|
223
|
+
async #runExitPolicy(nodeId, status) {
|
|
224
|
+
try {
|
|
225
|
+
const launchedAt = getNode(nodeId)?.launched_at;
|
|
226
|
+
const launchedMs = launchedAt == null ? Number.NaN : Date.parse(launchedAt);
|
|
227
|
+
const uptimeMs = Number.isFinite(launchedMs) ? Math.max(0, this.#now() - launchedMs) : null;
|
|
228
|
+
this.#emitExitObserved(nodeId, status, uptimeMs, getNode(nodeId)?.respawn_failures ?? 0);
|
|
290
229
|
}
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
this.#throttle.set(nodeId, state);
|
|
230
|
+
finally {
|
|
231
|
+
this.releaseReservation(nodeId);
|
|
294
232
|
}
|
|
295
|
-
this.#emitExitObserved(nodeId, status, uptimeMs, state.failures);
|
|
296
|
-
await this.#applyPolicy(nodeId, 'exit');
|
|
297
233
|
}
|
|
298
234
|
#emitExitObserved(nodeId, status, uptimeMs, consecutiveFailures) {
|
|
299
235
|
emitEvent({
|
|
@@ -309,19 +245,10 @@ export class DaemonFleet {
|
|
|
309
245
|
},
|
|
310
246
|
});
|
|
311
247
|
}
|
|
312
|
-
async #applyPolicy(nodeId
|
|
313
|
-
|
|
314
|
-
return await this.#decideAndEnact(nodeId, origin);
|
|
315
|
-
}
|
|
316
|
-
finally {
|
|
317
|
-
// The reservation this node has held since its exit is either consumed by
|
|
318
|
-
// a successful `register` (making this a no-op) or given back here — the
|
|
319
|
-
// one release path covering a declined policy, a deferred backoff, and
|
|
320
|
-
// terminalization alike.
|
|
321
|
-
this.releaseReservation(nodeId);
|
|
322
|
-
}
|
|
248
|
+
async #applyPolicy(nodeId) {
|
|
249
|
+
return this.#decideAndEnact(nodeId);
|
|
323
250
|
}
|
|
324
|
-
async #decideAndEnact(nodeId
|
|
251
|
+
async #decideAndEnact(nodeId) {
|
|
325
252
|
const meta = getNode(nodeId);
|
|
326
253
|
// Two durable witnesses mean the dead broker had a turn in flight. A dirty
|
|
327
254
|
// death leaves the busy marker behind; a bounded graceful abort clears busy
|
|
@@ -332,19 +259,19 @@ export class DaemonFleet {
|
|
|
332
259
|
switch (action) {
|
|
333
260
|
case 'forget':
|
|
334
261
|
// Fanouts for terminal exits already happened at the transition site.
|
|
335
|
-
|
|
262
|
+
clearRespawnPolicy(nodeId);
|
|
336
263
|
break;
|
|
337
264
|
case 'dormant':
|
|
338
265
|
// Chosen dormancy is not a failure — the counter does not survive it.
|
|
339
266
|
// The pathological wake→fault→release→wake loop is bounded by the
|
|
340
|
-
// tick's pass-2 cursor-keyed retry cap (D-11), not this
|
|
341
|
-
|
|
267
|
+
// tick's pass-2 cursor-keyed retry cap (D-11), not this policy.
|
|
268
|
+
clearRespawnPolicy(nodeId);
|
|
342
269
|
break;
|
|
343
270
|
case 'release-idle':
|
|
344
271
|
// Dormancy the daemon chose ON the node's behalf, and the same
|
|
345
272
|
// non-failure: the row lands exactly where a stophook idle-release
|
|
346
273
|
// would have left it, so pass-2 inbox wakes own its next revive.
|
|
347
|
-
|
|
274
|
+
clearRespawnPolicy(nodeId);
|
|
348
275
|
transition(nodeId, 'release');
|
|
349
276
|
emitEvent({
|
|
350
277
|
level: 'info',
|
|
@@ -352,7 +279,6 @@ export class DaemonFleet {
|
|
|
352
279
|
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
|
|
353
280
|
fields: {
|
|
354
281
|
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
355
|
-
origin,
|
|
356
282
|
},
|
|
357
283
|
});
|
|
358
284
|
break;
|
|
@@ -366,51 +292,42 @@ export class DaemonFleet {
|
|
|
366
292
|
case 'respawn-fresh':
|
|
367
293
|
case 'respawn-cycle':
|
|
368
294
|
case 'respawn-resume':
|
|
369
|
-
await this.#respawn(nodeId, meta, action,
|
|
295
|
+
await this.#respawn(nodeId, meta, action, interruptedTurn);
|
|
370
296
|
break;
|
|
371
297
|
}
|
|
372
298
|
return action;
|
|
373
299
|
}
|
|
374
|
-
async #respawn(nodeId, meta, action,
|
|
375
|
-
const
|
|
376
|
-
if (
|
|
300
|
+
async #respawn(nodeId, meta, action, interruptedTurn) {
|
|
301
|
+
const row = getRow(nodeId);
|
|
302
|
+
if (row === null)
|
|
303
|
+
return;
|
|
304
|
+
const assess = row.respawn_not_before == null;
|
|
305
|
+
const priorFailures = row.respawn_failures ?? 0;
|
|
306
|
+
const attempt = assess
|
|
307
|
+
? recordRespawnPolicy(row, this.#now())
|
|
308
|
+
: { failures: priorFailures, notBefore: row.respawn_not_before, exhausted: priorFailures >= MAX_RESPAWN_ATTEMPTS };
|
|
309
|
+
if (attempt.exhausted) {
|
|
377
310
|
await this.#terminalize(nodeId, meta, {
|
|
378
311
|
event: 'broker.respawn.exhausted',
|
|
379
|
-
wake: `Child crashed — ${fullName(meta)} (${nodeId}) kept dying
|
|
380
|
-
`
|
|
312
|
+
wake: `Child crashed — ${fullName(meta)} (${nodeId}) kept dying without settling a turn (${attempt.failures} consecutive ` +
|
|
313
|
+
`launches) and is now dead. It stays dead until you revive it — \`crtr node lifecycle revive ${nodeId}\`.`,
|
|
314
|
+
...(meta.pi_session_id != null ? { surface: () => surfaceCrashLoop(meta) } : {}),
|
|
381
315
|
});
|
|
382
316
|
return;
|
|
383
317
|
}
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
318
|
+
if (assess && attempt.notBefore !== null) {
|
|
319
|
+
emitEvent({
|
|
320
|
+
level: 'info',
|
|
321
|
+
event: 'broker.respawn.deferred',
|
|
322
|
+
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
|
|
323
|
+
fields: {
|
|
324
|
+
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
325
|
+
action,
|
|
326
|
+
backoff_ms: respawnBackoffMs(attempt.failures),
|
|
327
|
+
consecutive_failures: attempt.failures,
|
|
328
|
+
},
|
|
329
|
+
});
|
|
387
330
|
return;
|
|
388
|
-
// Only a fresh exit consults the backoff: a matured timer and a tick
|
|
389
|
-
// enactment relaunch immediately (the timer already WAS the backoff, and a
|
|
390
|
-
// tick candidate reached the lane only after its own gating).
|
|
391
|
-
if (origin === 'exit') {
|
|
392
|
-
const delay = respawnBackoffMs(state.failures);
|
|
393
|
-
if (delay > 0) {
|
|
394
|
-
state.timer = setTimeout(() => {
|
|
395
|
-
state.timer = undefined;
|
|
396
|
-
// A fresh policy job re-reads the row, so a node closed/canceled
|
|
397
|
-
// during the backoff lands in table row 2 and the respawn evaporates.
|
|
398
|
-
this.#enqueue(async () => { await this.#applyPolicy(nodeId, 'timer'); });
|
|
399
|
-
}, delay);
|
|
400
|
-
this.#throttle.set(nodeId, state);
|
|
401
|
-
emitEvent({
|
|
402
|
-
level: 'info',
|
|
403
|
-
event: 'broker.respawn.deferred',
|
|
404
|
-
...(isSafeNodeId(nodeId) ? { node_id: nodeId } : {}),
|
|
405
|
-
fields: {
|
|
406
|
-
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
407
|
-
action,
|
|
408
|
-
backoff_ms: delay,
|
|
409
|
-
consecutive_failures: state.failures,
|
|
410
|
-
},
|
|
411
|
-
});
|
|
412
|
-
return;
|
|
413
|
-
}
|
|
414
331
|
}
|
|
415
332
|
emitEvent({
|
|
416
333
|
level: 'info',
|
|
@@ -419,8 +336,7 @@ export class DaemonFleet {
|
|
|
419
336
|
fields: {
|
|
420
337
|
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
421
338
|
action,
|
|
422
|
-
|
|
423
|
-
consecutive_failures: state.failures,
|
|
339
|
+
consecutive_failures: attempt.failures,
|
|
424
340
|
},
|
|
425
341
|
});
|
|
426
342
|
try {
|
|
@@ -450,7 +366,6 @@ export class DaemonFleet {
|
|
|
450
366
|
fields: {
|
|
451
367
|
...(isSafeNodeId(nodeId) ? {} : { affected_node_id: nodeId }),
|
|
452
368
|
action,
|
|
453
|
-
origin,
|
|
454
369
|
},
|
|
455
370
|
});
|
|
456
371
|
// Launch failures terminalize inside reviveNode, but its preflight
|
|
@@ -459,10 +374,8 @@ export class DaemonFleet {
|
|
|
459
374
|
await this.terminalizeIfStillNonTerminal(nodeId, 'broker.respawn.refused', err);
|
|
460
375
|
}
|
|
461
376
|
}
|
|
462
|
-
/** Terminal outcome shared by table row 8 and
|
|
463
|
-
* transition('crash') + surfaceBootFailure + doctrine wake, counter cleared. */
|
|
377
|
+
/** Terminal outcome shared by table row 8 and respawn exhaustion. */
|
|
464
378
|
async #terminalize(nodeId, meta, opts) {
|
|
465
|
-
this.#clearThrottle(nodeId);
|
|
466
379
|
emitEvent({
|
|
467
380
|
level: 'error',
|
|
468
381
|
event: opts.event,
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { ApiError } from '../../api/index.js';
|
|
2
2
|
import { getRow, subscribersOf } from '../../core/canvas/canvas.js';
|
|
3
|
+
import { cancelCronsOnWake } from '../../core/canvas/crons.js';
|
|
3
4
|
import { opaqueInboxTicketId, ticketDir } from '../../core/human/root.js';
|
|
4
5
|
import { readTicketActionBinding } from '../../core/human/action-binding.js';
|
|
5
6
|
import { completionEventFor, composeHumanCompletion } from '../../core/human/completion.js';
|
|
@@ -275,6 +276,7 @@ export async function deliverTerminalResult(ticketId) {
|
|
|
275
276
|
data: { body },
|
|
276
277
|
disposition: 'human-canceled',
|
|
277
278
|
});
|
|
279
|
+
cancelCronsOnWake(subscriber.node_id);
|
|
278
280
|
}
|
|
279
281
|
catch {
|
|
280
282
|
failedTo.push(subscriber.node_id);
|
package/dist/daemon/manage.d.ts
CHANGED
|
@@ -46,7 +46,17 @@ export interface SpawnDaemonResult {
|
|
|
46
46
|
pid?: number;
|
|
47
47
|
/** PID of the already-running daemon, if it was already up. */
|
|
48
48
|
existing_pid?: number;
|
|
49
|
+
/** Present only when a live pidfile owner does not answer `/healthz`. */
|
|
50
|
+
running?: true;
|
|
51
|
+
/** Present with `running:true` when the Unix API socket is not serving. */
|
|
52
|
+
serving?: false;
|
|
53
|
+
/** Recovery instruction for a live but non-serving daemon. */
|
|
54
|
+
next?: string;
|
|
49
55
|
}
|
|
56
|
+
/** True only when the mandatory Unix API socket answers `/healthz`. This is
|
|
57
|
+
* deliberately separate from `isDaemonRunning()`: a pidfile remains useful for
|
|
58
|
+
* identifying the process an operator must stop, but it is not readiness. */
|
|
59
|
+
export declare function isDaemonServing(): Promise<boolean>;
|
|
50
60
|
export interface DaemonWaitDeps {
|
|
51
61
|
readPidfile?: () => number | null;
|
|
52
62
|
isPidAlive?: (pid: number) => boolean;
|
|
@@ -58,6 +68,7 @@ export interface DaemonStartupDeps extends DaemonWaitDeps {
|
|
|
58
68
|
code: number | null;
|
|
59
69
|
signal: NodeJS.Signals | null;
|
|
60
70
|
} | null;
|
|
71
|
+
isDaemonServing?: () => Promise<boolean>;
|
|
61
72
|
}
|
|
62
73
|
/** Tiny bounded post-spawn guard: wait for the daemon pidfile and live pid to
|
|
63
74
|
* appear before reporting success. This catches the "started:true but not yet
|