@vellumai/cli 0.11.4-dev.202608201710.fcf0423 → 0.11.4-dev.202608201915.5ca6ffa
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/wake.test.ts +120 -0
- package/src/commands/wake.ts +13 -1
- package/src/lib/__tests__/local-daemon-readiness.test.ts +128 -0
- package/src/lib/local.ts +161 -38
package/package.json
CHANGED
|
@@ -15,6 +15,7 @@ import { join } from "node:path";
|
|
|
15
15
|
import * as assistantConfig from "../lib/assistant-config.js";
|
|
16
16
|
import * as docker from "../lib/docker.js";
|
|
17
17
|
import * as guardianToken from "../lib/guardian-token.js";
|
|
18
|
+
import * as httpClient from "../lib/http-client.js";
|
|
18
19
|
import * as ingressConfig from "../lib/ingress-config.js";
|
|
19
20
|
import * as local from "../lib/local.js";
|
|
20
21
|
import * as nginxIngress from "../lib/nginx-ingress.js";
|
|
@@ -167,12 +168,28 @@ mock.module("../lib/nginx-ingress.js", () => ({
|
|
|
167
168
|
readIngressState: readIngressStateMock,
|
|
168
169
|
}));
|
|
169
170
|
|
|
171
|
+
const realHttpClient = { ...httpClient };
|
|
172
|
+
|
|
173
|
+
const probeDaemonReadinessWithRetryMock = mock<
|
|
174
|
+
typeof httpClient.probeDaemonReadinessWithRetry
|
|
175
|
+
>(async () => "ready");
|
|
176
|
+
const waitForDaemonMigrationsReadyMock = mock<
|
|
177
|
+
typeof httpClient.waitForDaemonMigrationsReady
|
|
178
|
+
>(async () => "ready");
|
|
179
|
+
|
|
180
|
+
mock.module("../lib/http-client.js", () => ({
|
|
181
|
+
...realHttpClient,
|
|
182
|
+
probeDaemonReadinessWithRetry: probeDaemonReadinessWithRetryMock,
|
|
183
|
+
waitForDaemonMigrationsReady: waitForDaemonMigrationsReadyMock,
|
|
184
|
+
}));
|
|
185
|
+
|
|
170
186
|
const { wake } = await import("../commands/wake.js");
|
|
171
187
|
|
|
172
188
|
let tempDir: string;
|
|
173
189
|
let originalArgv: string[];
|
|
174
190
|
let logSpy: ReturnType<typeof spyOn>;
|
|
175
191
|
let warnSpy: ReturnType<typeof spyOn>;
|
|
192
|
+
let errorSpy: ReturnType<typeof spyOn>;
|
|
176
193
|
let localEntry: AssistantEntry;
|
|
177
194
|
|
|
178
195
|
function makeLocalEntry(): AssistantEntry {
|
|
@@ -199,6 +216,7 @@ beforeEach(() => {
|
|
|
199
216
|
process.argv = ["bun", "vellum", "wake", "--watch", "local-assistant"];
|
|
200
217
|
logSpy = spyOn(console, "log").mockImplementation(() => {});
|
|
201
218
|
warnSpy = spyOn(console, "warn").mockImplementation(() => {});
|
|
219
|
+
errorSpy = spyOn(console, "error").mockImplementation(() => {});
|
|
202
220
|
|
|
203
221
|
localEntry = makeLocalEntry();
|
|
204
222
|
resolveTargetAssistantMock.mockReset();
|
|
@@ -231,6 +249,10 @@ beforeEach(() => {
|
|
|
231
249
|
startCesMock.mockResolvedValue(undefined);
|
|
232
250
|
isProcessAliveMock.mockReset();
|
|
233
251
|
isProcessAliveMock.mockReturnValue({ alive: false, pid: null });
|
|
252
|
+
probeDaemonReadinessWithRetryMock.mockReset();
|
|
253
|
+
probeDaemonReadinessWithRetryMock.mockResolvedValue("ready");
|
|
254
|
+
waitForDaemonMigrationsReadyMock.mockReset();
|
|
255
|
+
waitForDaemonMigrationsReadyMock.mockResolvedValue("ready");
|
|
234
256
|
seedGuardianTokenFromSiblingEnvMock.mockReset();
|
|
235
257
|
seedGuardianTokenFromSiblingEnvMock.mockReturnValue(false);
|
|
236
258
|
loadGuardianTokenMock.mockReset();
|
|
@@ -263,6 +285,7 @@ afterEach(() => {
|
|
|
263
285
|
process.argv = originalArgv;
|
|
264
286
|
logSpy.mockRestore();
|
|
265
287
|
warnSpy.mockRestore();
|
|
288
|
+
errorSpy.mockRestore();
|
|
266
289
|
if (tempDir) {
|
|
267
290
|
rmSync(tempDir, { recursive: true, force: true });
|
|
268
291
|
}
|
|
@@ -277,6 +300,7 @@ afterAll(() => {
|
|
|
277
300
|
mock.module("../lib/ngrok", () => realNgrok);
|
|
278
301
|
mock.module("../lib/ingress-config.js", () => realIngressConfig);
|
|
279
302
|
mock.module("../lib/nginx-ingress.js", () => realNginxIngress);
|
|
303
|
+
mock.module("../lib/http-client.js", () => realHttpClient);
|
|
280
304
|
});
|
|
281
305
|
|
|
282
306
|
describe("vellum wake", () => {
|
|
@@ -456,6 +480,102 @@ describe("vellum wake", () => {
|
|
|
456
480
|
});
|
|
457
481
|
});
|
|
458
482
|
|
|
483
|
+
describe("vellum wake: daemon startup failure", () => {
|
|
484
|
+
const daemonPidOf = (dir: string) => join(dir, ".vellum", "daemon.pid");
|
|
485
|
+
|
|
486
|
+
beforeEach(() => {
|
|
487
|
+
process.argv = ["bun", "vellum", "wake", "local-assistant"];
|
|
488
|
+
// The daemon needs starting; the gateway is already serving.
|
|
489
|
+
resolveProcessStateMock.mockImplementation(
|
|
490
|
+
async (_pidFile, _port, label) =>
|
|
491
|
+
label === "Gateway"
|
|
492
|
+
? { status: "healthy", pid: 456 }
|
|
493
|
+
: { status: "needs_start", pid: null },
|
|
494
|
+
);
|
|
495
|
+
});
|
|
496
|
+
|
|
497
|
+
test("fails when the freshly spawned daemon is not running", async () => {
|
|
498
|
+
/**
|
|
499
|
+
* Tests that a daemon which aborts during startup (an occupied runtime
|
|
500
|
+
* HTTP port) fails the command instead of reporting a successful wake.
|
|
501
|
+
*/
|
|
502
|
+
|
|
503
|
+
// GIVEN the spawned daemon left no live process behind
|
|
504
|
+
isProcessAliveMock.mockReturnValue({ alive: false, pid: null });
|
|
505
|
+
|
|
506
|
+
// AND a foreign listener on the runtime HTTP port answers readiness in
|
|
507
|
+
// the dead daemon's place
|
|
508
|
+
probeDaemonReadinessWithRetryMock.mockResolvedValue("ready");
|
|
509
|
+
const exitSpy = spyOn(process, "exit").mockImplementation(((
|
|
510
|
+
code?: number,
|
|
511
|
+
) => {
|
|
512
|
+
throw new Error(`process.exit:${code}`);
|
|
513
|
+
}) as never);
|
|
514
|
+
|
|
515
|
+
// WHEN wake runs
|
|
516
|
+
const result = wake();
|
|
517
|
+
|
|
518
|
+
// THEN it exits nonzero, names the port on stderr, and never claims
|
|
519
|
+
// the wake completed
|
|
520
|
+
await expect(result).rejects.toThrow("process.exit:1");
|
|
521
|
+
expect(errorSpy).toHaveBeenCalledWith(
|
|
522
|
+
expect.stringContaining("exited during startup"),
|
|
523
|
+
);
|
|
524
|
+
expect(errorSpy).toHaveBeenCalledWith(expect.stringContaining("7821"));
|
|
525
|
+
expect(logSpy).not.toHaveBeenCalledWith("Wake complete.");
|
|
526
|
+
expect(startGatewayMock).not.toHaveBeenCalled();
|
|
527
|
+
exitSpy.mockRestore();
|
|
528
|
+
});
|
|
529
|
+
|
|
530
|
+
test("completes when the freshly spawned daemon is running", async () => {
|
|
531
|
+
/**
|
|
532
|
+
* Tests that a healthy fresh spawn still reports a completed wake.
|
|
533
|
+
*/
|
|
534
|
+
|
|
535
|
+
// GIVEN the spawned daemon is alive and ready
|
|
536
|
+
isProcessAliveMock.mockImplementation((pidFile) =>
|
|
537
|
+
pidFile === daemonPidOf(tempDir)
|
|
538
|
+
? { alive: true, pid: 123 }
|
|
539
|
+
: { alive: false, pid: null },
|
|
540
|
+
);
|
|
541
|
+
probeDaemonReadinessWithRetryMock.mockResolvedValue("ready");
|
|
542
|
+
|
|
543
|
+
// WHEN wake runs
|
|
544
|
+
await wake();
|
|
545
|
+
|
|
546
|
+
// THEN it reports success and starts the gateway
|
|
547
|
+
expect(logSpy).toHaveBeenCalledWith("Wake complete.");
|
|
548
|
+
expect(errorSpy).not.toHaveBeenCalled();
|
|
549
|
+
expect(startLocalDaemonMock).toHaveBeenCalledTimes(1);
|
|
550
|
+
});
|
|
551
|
+
|
|
552
|
+
test("completes when the freshly spawned daemon is still migrating", async () => {
|
|
553
|
+
/**
|
|
554
|
+
* Tests that a live daemon running database migrations is reported as a
|
|
555
|
+
* degraded success, not a failure.
|
|
556
|
+
*/
|
|
557
|
+
|
|
558
|
+
// GIVEN the spawned daemon is alive but still migrating past the
|
|
559
|
+
// gateway-coordination wait
|
|
560
|
+
isProcessAliveMock.mockImplementation((pidFile) =>
|
|
561
|
+
pidFile === daemonPidOf(tempDir)
|
|
562
|
+
? { alive: true, pid: 123 }
|
|
563
|
+
: { alive: false, pid: null },
|
|
564
|
+
);
|
|
565
|
+
probeDaemonReadinessWithRetryMock.mockResolvedValue("migrating");
|
|
566
|
+
waitForDaemonMigrationsReadyMock.mockResolvedValue("migrating");
|
|
567
|
+
|
|
568
|
+
// WHEN wake runs
|
|
569
|
+
await wake();
|
|
570
|
+
|
|
571
|
+
// THEN it reports the migration state and completes
|
|
572
|
+
expect(logSpy).toHaveBeenCalledWith(
|
|
573
|
+
"Assistant is still running database migrations; DB-backed routes return 503 until they finish.",
|
|
574
|
+
);
|
|
575
|
+
expect(logSpy).toHaveBeenCalledWith("Wake complete.");
|
|
576
|
+
});
|
|
577
|
+
});
|
|
578
|
+
|
|
459
579
|
describe("vellum wake — tunnel edge restore", () => {
|
|
460
580
|
const webhookConfig = { telegram: { botUsername: "bot" } };
|
|
461
581
|
const enabledConfig = {
|
package/src/commands/wake.ts
CHANGED
|
@@ -229,7 +229,19 @@ export async function wake(): Promise<void> {
|
|
|
229
229
|
startCes(watch, resources),
|
|
230
230
|
startLocalDaemon(watch, resources, { foreground, signingKey }),
|
|
231
231
|
]);
|
|
232
|
-
//
|
|
232
|
+
// A daemon that aborts during startup (an occupied runtime HTTP port, a
|
|
233
|
+
// fatal subsystem failure) leaves no process behind, and readiness alone
|
|
234
|
+
// cannot see that: whatever foreign listener holds the port answers the
|
|
235
|
+
// probe in its place. Liveness is the authoritative signal, so check it
|
|
236
|
+
// first and fail the command instead of reporting a successful wake for
|
|
237
|
+
// an assistant that is not running.
|
|
238
|
+
if (!isProcessAlive(pidFile).alive) {
|
|
239
|
+
console.error(
|
|
240
|
+
`Error: the assistant exited during startup and is not running. Its runtime HTTP port ${resources.daemonPort} may be held by another process (check \`lsof -i :${resources.daemonPort}\`); the daemon log records the startup error it reported.`,
|
|
241
|
+
);
|
|
242
|
+
process.exit(1);
|
|
243
|
+
}
|
|
244
|
+
// startLocalDaemon's post-spawn wait is bounded (60s), and a longer
|
|
233
245
|
// migration outlives it. Classify the fresh spawn the same way the
|
|
234
246
|
// attach path does, so the gateway-coordination wait below applies to
|
|
235
247
|
// both paths and wake's closing summary stays honest.
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { tmpdir } from "node:os";
|
|
3
|
+
import { join } from "node:path";
|
|
4
|
+
|
|
5
|
+
import { afterEach, beforeEach, describe, expect, spyOn, test } from "bun:test";
|
|
6
|
+
|
|
7
|
+
import { type DaemonSpawn, reportFreshSpawnReadiness } from "../local.js";
|
|
8
|
+
|
|
9
|
+
// Above every platform default pid ceiling, so no live process owns it.
|
|
10
|
+
const DEAD_PID = 999999;
|
|
11
|
+
// No daemon listens here, so port ownership can never be confirmed and the
|
|
12
|
+
// settle window decides the outcome.
|
|
13
|
+
const UNSERVED_PORT = 7999;
|
|
14
|
+
|
|
15
|
+
describe("reportFreshSpawnReadiness", () => {
|
|
16
|
+
let tempDir: string;
|
|
17
|
+
let pidFile: string;
|
|
18
|
+
let logSpy: ReturnType<typeof spyOn>;
|
|
19
|
+
|
|
20
|
+
const spawnHandle = (exited: boolean): DaemonSpawn => ({
|
|
21
|
+
hasExited: () => exited,
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
beforeEach(() => {
|
|
25
|
+
tempDir = mkdtempSync(join(tmpdir(), "vellum-readiness-"));
|
|
26
|
+
pidFile = join(tempDir, "daemon.pid");
|
|
27
|
+
logSpy = spyOn(console, "log").mockImplementation(() => {});
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
afterEach(() => {
|
|
31
|
+
logSpy.mockRestore();
|
|
32
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
33
|
+
});
|
|
34
|
+
|
|
35
|
+
test("reports the readiness of a daemon that survived its spawn", async () => {
|
|
36
|
+
/**
|
|
37
|
+
* Tests that a daemon still running after the settle window has its
|
|
38
|
+
* probed state reported.
|
|
39
|
+
*/
|
|
40
|
+
|
|
41
|
+
// GIVEN the spawned daemon is running
|
|
42
|
+
writeFileSync(pidFile, String(process.pid), "utf-8");
|
|
43
|
+
|
|
44
|
+
// WHEN its readiness is reported
|
|
45
|
+
await reportFreshSpawnReadiness(
|
|
46
|
+
spawnHandle(false),
|
|
47
|
+
pidFile,
|
|
48
|
+
UNSERVED_PORT,
|
|
49
|
+
"ready",
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
// THEN the probed state reaches the operator
|
|
53
|
+
expect(logSpy).toHaveBeenCalledWith(" Assistant ready\n");
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("reports a startup failure instead of a probe answered by a foreign listener", async () => {
|
|
57
|
+
/**
|
|
58
|
+
* Tests that a daemon which aborted during startup is reported as not
|
|
59
|
+
* running, even when the probe reports it as up.
|
|
60
|
+
*/
|
|
61
|
+
|
|
62
|
+
// GIVEN the spawned daemon exited, while a foreign listener holding its
|
|
63
|
+
// runtime HTTP port answered the readiness probe in its place
|
|
64
|
+
writeFileSync(pidFile, String(DEAD_PID), "utf-8");
|
|
65
|
+
|
|
66
|
+
// WHEN its readiness is reported
|
|
67
|
+
await reportFreshSpawnReadiness(
|
|
68
|
+
spawnHandle(true),
|
|
69
|
+
pidFile,
|
|
70
|
+
UNSERVED_PORT,
|
|
71
|
+
"migrating",
|
|
72
|
+
);
|
|
73
|
+
|
|
74
|
+
// THEN the failure is reported rather than the probe's optimistic state
|
|
75
|
+
expect(logSpy).toHaveBeenCalledWith(
|
|
76
|
+
" ⚠️ Assistant exited during startup and is not running\n",
|
|
77
|
+
);
|
|
78
|
+
expect(logSpy).not.toHaveBeenCalledWith(
|
|
79
|
+
expect.stringContaining("database migrations still running"),
|
|
80
|
+
);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("reports a startup failure when the spawn dies during the settle window", async () => {
|
|
84
|
+
/**
|
|
85
|
+
* Tests that a daemon which is still alive when its readiness is probed,
|
|
86
|
+
* and aborts moments later, is reported as not running.
|
|
87
|
+
*/
|
|
88
|
+
|
|
89
|
+
// GIVEN a spawn that is alive when reported and exits shortly after
|
|
90
|
+
writeFileSync(pidFile, String(process.pid), "utf-8");
|
|
91
|
+
let exited = false;
|
|
92
|
+
const spawn: DaemonSpawn = { hasExited: () => exited };
|
|
93
|
+
setTimeout(() => {
|
|
94
|
+
exited = true;
|
|
95
|
+
}, 150);
|
|
96
|
+
|
|
97
|
+
// WHEN its readiness is reported
|
|
98
|
+
await reportFreshSpawnReadiness(spawn, pidFile, UNSERVED_PORT, "ready");
|
|
99
|
+
|
|
100
|
+
// THEN the failure is reported rather than the probe's optimistic state
|
|
101
|
+
expect(logSpy).toHaveBeenCalledWith(
|
|
102
|
+
" ⚠️ Assistant exited during startup and is not running\n",
|
|
103
|
+
);
|
|
104
|
+
expect(logSpy).not.toHaveBeenCalledWith(" Assistant ready\n");
|
|
105
|
+
});
|
|
106
|
+
|
|
107
|
+
test("throws for a failed spawn when the caller requires readiness", async () => {
|
|
108
|
+
/**
|
|
109
|
+
* Tests that callers which cannot continue without a running assistant
|
|
110
|
+
* (hatch) fail rather than proceeding on a foreign listener's probe.
|
|
111
|
+
*/
|
|
112
|
+
|
|
113
|
+
// GIVEN the spawned daemon exited and the caller requires readiness
|
|
114
|
+
writeFileSync(pidFile, String(DEAD_PID), "utf-8");
|
|
115
|
+
|
|
116
|
+
// WHEN its readiness is reported
|
|
117
|
+
const report = reportFreshSpawnReadiness(
|
|
118
|
+
spawnHandle(true),
|
|
119
|
+
pidFile,
|
|
120
|
+
UNSERVED_PORT,
|
|
121
|
+
"ready",
|
|
122
|
+
true,
|
|
123
|
+
);
|
|
124
|
+
|
|
125
|
+
// THEN startup fails loudly
|
|
126
|
+
await expect(report).rejects.toThrow("exited during startup");
|
|
127
|
+
});
|
|
128
|
+
});
|
package/src/lib/local.ts
CHANGED
|
@@ -1,4 +1,10 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
type ChildProcess,
|
|
3
|
+
execFileSync,
|
|
4
|
+
execSync,
|
|
5
|
+
spawn,
|
|
6
|
+
spawnSync,
|
|
7
|
+
} from "child_process";
|
|
2
8
|
import { createHash, randomBytes } from "crypto";
|
|
3
9
|
import {
|
|
4
10
|
existsSync,
|
|
@@ -40,6 +46,7 @@ import { stopIngressNginx } from "./nginx-ingress.js";
|
|
|
40
46
|
import {
|
|
41
47
|
type ProcessState,
|
|
42
48
|
executableName,
|
|
49
|
+
isProcessAlive,
|
|
43
50
|
pathListDelimiter,
|
|
44
51
|
resolveProcessState,
|
|
45
52
|
stopProcess,
|
|
@@ -637,12 +644,12 @@ function logDaemonReadiness(
|
|
|
637
644
|
break;
|
|
638
645
|
case "migrating":
|
|
639
646
|
console.log(
|
|
640
|
-
" Assistant is up
|
|
647
|
+
" Assistant is up. Database migrations still running; DB-backed commands return 503 until they finish\n",
|
|
641
648
|
);
|
|
642
649
|
break;
|
|
643
650
|
case "failed":
|
|
644
651
|
console.log(
|
|
645
|
-
" ⚠️ Assistant database migrations FAILED
|
|
652
|
+
" ⚠️ Assistant database migrations FAILED. DB-backed commands return 503 until the assistant is restarted\n",
|
|
646
653
|
);
|
|
647
654
|
break;
|
|
648
655
|
default:
|
|
@@ -652,11 +659,100 @@ function logDaemonReadiness(
|
|
|
652
659
|
);
|
|
653
660
|
}
|
|
654
661
|
console.log(
|
|
655
|
-
" ⚠️ Assistant did not become ready within 60s
|
|
662
|
+
" ⚠️ Assistant did not become ready within 60s, continuing anyway\n",
|
|
656
663
|
);
|
|
657
664
|
}
|
|
658
665
|
}
|
|
659
666
|
|
|
667
|
+
/**
|
|
668
|
+
* Handle to a daemon this process spawned, reporting whether that child has
|
|
669
|
+
* since exited. Attach paths have no handle, since the daemon they found is
|
|
670
|
+
* not a child of this process.
|
|
671
|
+
*/
|
|
672
|
+
export type DaemonSpawn = { hasExited: () => boolean };
|
|
673
|
+
|
|
674
|
+
function trackDaemonSpawn(child: ChildProcess): DaemonSpawn {
|
|
675
|
+
let exited = false;
|
|
676
|
+
child.once("exit", () => {
|
|
677
|
+
exited = true;
|
|
678
|
+
});
|
|
679
|
+
return { hasExited: () => exited };
|
|
680
|
+
}
|
|
681
|
+
|
|
682
|
+
/** How long a fresh spawn gets to abort before its readiness is trusted. */
|
|
683
|
+
const FRESH_SPAWN_SETTLE_MS = 2000;
|
|
684
|
+
const FRESH_SPAWN_POLL_MS = 100;
|
|
685
|
+
|
|
686
|
+
function sleep(ms: number): Promise<void> {
|
|
687
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
688
|
+
}
|
|
689
|
+
|
|
690
|
+
/**
|
|
691
|
+
* True when a freshly spawned daemon is the process serving its runtime HTTP
|
|
692
|
+
* port.
|
|
693
|
+
*
|
|
694
|
+
* A readiness answer proves only that something listens on the port: a foreign
|
|
695
|
+
* listener holding it answers in the daemon's place, so a daemon aborting over
|
|
696
|
+
* the collision still reads as "up" or "migrating". Ownership is confirmed
|
|
697
|
+
* from the listening PID where the platform can report it. Otherwise the spawn
|
|
698
|
+
* gets a short settle window, which is long enough because transports bind
|
|
699
|
+
* before migrations run, so an address collision aborts the daemon early.
|
|
700
|
+
*/
|
|
701
|
+
async function freshSpawnServesRuntimePort(
|
|
702
|
+
spawn: DaemonSpawn,
|
|
703
|
+
pidFile: string,
|
|
704
|
+
daemonPort: number,
|
|
705
|
+
): Promise<boolean> {
|
|
706
|
+
const spawnPid = (): number | null => {
|
|
707
|
+
if (spawn.hasExited()) {
|
|
708
|
+
return null;
|
|
709
|
+
}
|
|
710
|
+
return isProcessAlive(pidFile).pid;
|
|
711
|
+
};
|
|
712
|
+
|
|
713
|
+
const pid = spawnPid();
|
|
714
|
+
if (pid === null) {
|
|
715
|
+
return false;
|
|
716
|
+
}
|
|
717
|
+
if (findPidListeningOnPort(daemonPort) === pid) {
|
|
718
|
+
return true;
|
|
719
|
+
}
|
|
720
|
+
|
|
721
|
+
const deadline = Date.now() + FRESH_SPAWN_SETTLE_MS;
|
|
722
|
+
while (Date.now() < deadline) {
|
|
723
|
+
await sleep(FRESH_SPAWN_POLL_MS);
|
|
724
|
+
if (spawnPid() === null) {
|
|
725
|
+
return false;
|
|
726
|
+
}
|
|
727
|
+
}
|
|
728
|
+
return true;
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
/**
|
|
732
|
+
* Report a freshly spawned daemon's readiness, gated on that daemon serving
|
|
733
|
+
* the runtime HTTP port the readiness was probed on.
|
|
734
|
+
*
|
|
735
|
+
* A daemon that aborts during startup (an occupied runtime HTTP port, a fatal
|
|
736
|
+
* subsystem failure) is reported as the startup failure it is instead of an
|
|
737
|
+
* optimistic line about an assistant that is not running.
|
|
738
|
+
*/
|
|
739
|
+
export async function reportFreshSpawnReadiness(
|
|
740
|
+
spawn: DaemonSpawn,
|
|
741
|
+
pidFile: string,
|
|
742
|
+
daemonPort: number,
|
|
743
|
+
readiness: DaemonReadiness,
|
|
744
|
+
requireReady = false,
|
|
745
|
+
): Promise<void> {
|
|
746
|
+
if (!(await freshSpawnServesRuntimePort(spawn, pidFile, daemonPort))) {
|
|
747
|
+
if (requireReady) {
|
|
748
|
+
throw new Error("Assistant exited during startup and is not running.");
|
|
749
|
+
}
|
|
750
|
+
console.log(" ⚠️ Assistant exited during startup and is not running\n");
|
|
751
|
+
return;
|
|
752
|
+
}
|
|
753
|
+
logDaemonReadiness(readiness, requireReady);
|
|
754
|
+
}
|
|
755
|
+
|
|
660
756
|
function logAssistantAlreadyRunning(
|
|
661
757
|
pid: number,
|
|
662
758
|
status: ProcessState["status"],
|
|
@@ -676,7 +772,7 @@ async function startDaemonFromSource(
|
|
|
676
772
|
assistantIndex: string,
|
|
677
773
|
resources: LocalInstanceResources,
|
|
678
774
|
options?: DaemonStartOptions,
|
|
679
|
-
): Promise<
|
|
775
|
+
): Promise<DaemonSpawn | null> {
|
|
680
776
|
const foreground = options?.foreground ?? false;
|
|
681
777
|
const daemonMainPath = resolveDaemonMainPath(assistantIndex);
|
|
682
778
|
|
|
@@ -686,7 +782,9 @@ async function startDaemonFromSource(
|
|
|
686
782
|
mkdirSync(dirname(pidFile), { recursive: true });
|
|
687
783
|
|
|
688
784
|
// --- Lifecycle guard: prevent split-brain daemon state ---
|
|
689
|
-
if (await awaitStartingSentinel(pidFile, resources.daemonPort))
|
|
785
|
+
if (await awaitStartingSentinel(pidFile, resources.daemonPort)) {
|
|
786
|
+
return null;
|
|
787
|
+
}
|
|
690
788
|
|
|
691
789
|
const daemonState = await resolveProcessState(
|
|
692
790
|
pidFile,
|
|
@@ -697,10 +795,12 @@ async function startDaemonFromSource(
|
|
|
697
795
|
);
|
|
698
796
|
if (daemonState.status !== "needs_start") {
|
|
699
797
|
logAssistantAlreadyRunning(daemonState.pid, daemonState.status);
|
|
700
|
-
return
|
|
798
|
+
return null;
|
|
701
799
|
}
|
|
702
800
|
|
|
703
|
-
if (await checkOrphanedDaemon(pidFile, resources.daemonPort))
|
|
801
|
+
if (await checkOrphanedDaemon(pidFile, resources.daemonPort)) {
|
|
802
|
+
return null;
|
|
803
|
+
}
|
|
704
804
|
|
|
705
805
|
const env: Record<string, string | undefined> = {
|
|
706
806
|
...process.env,
|
|
@@ -744,14 +844,14 @@ async function startDaemonFromSource(
|
|
|
744
844
|
return c;
|
|
745
845
|
})();
|
|
746
846
|
|
|
747
|
-
if (child.pid) {
|
|
748
|
-
writeFileSync(pidFile, String(child.pid), "utf-8");
|
|
749
|
-
} else {
|
|
847
|
+
if (!child.pid) {
|
|
750
848
|
try {
|
|
751
849
|
unlinkSync(pidFile);
|
|
752
850
|
} catch {}
|
|
851
|
+
return null;
|
|
753
852
|
}
|
|
754
|
-
|
|
853
|
+
writeFileSync(pidFile, String(child.pid), "utf-8");
|
|
854
|
+
return trackDaemonSpawn(child);
|
|
755
855
|
}
|
|
756
856
|
|
|
757
857
|
// NOTE: startDaemonWatchFromSource() is the CLI-side watch-mode daemon
|
|
@@ -762,7 +862,7 @@ async function startDaemonWatchFromSource(
|
|
|
762
862
|
assistantIndex: string,
|
|
763
863
|
resources: LocalInstanceResources,
|
|
764
864
|
options?: DaemonStartOptions,
|
|
765
|
-
): Promise<
|
|
865
|
+
): Promise<DaemonSpawn | null> {
|
|
766
866
|
const mainPath = resolveDaemonMainPath(assistantIndex);
|
|
767
867
|
if (!existsSync(mainPath)) {
|
|
768
868
|
throw new Error(`Daemon main.ts not found at ${mainPath}`);
|
|
@@ -772,7 +872,9 @@ async function startDaemonWatchFromSource(
|
|
|
772
872
|
mkdirSync(dirname(pidFile), { recursive: true });
|
|
773
873
|
|
|
774
874
|
// --- Lifecycle guard: prevent split-brain daemon state ---
|
|
775
|
-
if (await awaitStartingSentinel(pidFile, resources.daemonPort))
|
|
875
|
+
if (await awaitStartingSentinel(pidFile, resources.daemonPort)) {
|
|
876
|
+
return null;
|
|
877
|
+
}
|
|
776
878
|
|
|
777
879
|
const daemonState = await resolveProcessState(
|
|
778
880
|
pidFile,
|
|
@@ -783,10 +885,12 @@ async function startDaemonWatchFromSource(
|
|
|
783
885
|
);
|
|
784
886
|
if (daemonState.status !== "needs_start") {
|
|
785
887
|
logAssistantAlreadyRunning(daemonState.pid, daemonState.status);
|
|
786
|
-
return
|
|
888
|
+
return null;
|
|
787
889
|
}
|
|
788
890
|
|
|
789
|
-
if (await checkOrphanedDaemon(pidFile, resources.daemonPort))
|
|
891
|
+
if (await checkOrphanedDaemon(pidFile, resources.daemonPort)) {
|
|
892
|
+
return null;
|
|
893
|
+
}
|
|
790
894
|
|
|
791
895
|
const env: Record<string, string | undefined> = {
|
|
792
896
|
...process.env,
|
|
@@ -811,16 +915,16 @@ async function startDaemonWatchFromSource(
|
|
|
811
915
|
const daemonPid = child.pid;
|
|
812
916
|
|
|
813
917
|
// Overwrite sentinel with real PID, or clean up on spawn failure.
|
|
814
|
-
if (daemonPid) {
|
|
815
|
-
writeFileSync(pidFile, String(daemonPid), "utf-8");
|
|
816
|
-
} else {
|
|
918
|
+
if (!daemonPid) {
|
|
817
919
|
try {
|
|
818
920
|
unlinkSync(pidFile);
|
|
819
921
|
} catch {}
|
|
922
|
+
return null;
|
|
820
923
|
}
|
|
924
|
+
writeFileSync(pidFile, String(daemonPid), "utf-8");
|
|
821
925
|
|
|
822
926
|
console.log(" Assistant started in watch mode (bun --watch)");
|
|
823
|
-
return
|
|
927
|
+
return trackDaemonSpawn(child);
|
|
824
928
|
}
|
|
825
929
|
|
|
826
930
|
function resolveGatewayDir(resources?: LocalInstanceResources): string {
|
|
@@ -1394,10 +1498,16 @@ export async function startLocalDaemon(
|
|
|
1394
1498
|
// already-running daemon was classified and logged inside
|
|
1395
1499
|
// startDaemonFromSource, and re-waiting would just block on a migration
|
|
1396
1500
|
// the user was already told about.
|
|
1397
|
-
|
|
1398
|
-
|
|
1399
|
-
|
|
1400
|
-
|
|
1501
|
+
const runtimeSpawn = await startDaemonFromSource(
|
|
1502
|
+
runtimeAssistantIndex,
|
|
1503
|
+
resources,
|
|
1504
|
+
options,
|
|
1505
|
+
);
|
|
1506
|
+
if (runtimeSpawn) {
|
|
1507
|
+
await reportFreshSpawnReadiness(
|
|
1508
|
+
runtimeSpawn,
|
|
1509
|
+
getDaemonPidPath(resources),
|
|
1510
|
+
resources.daemonPort,
|
|
1401
1511
|
await waitForDaemonMigrationsReady(
|
|
1402
1512
|
resources.daemonPort,
|
|
1403
1513
|
Date.now() + 60000,
|
|
@@ -1437,6 +1547,7 @@ export async function startLocalDaemon(
|
|
|
1437
1547
|
if (daemonAlive) {
|
|
1438
1548
|
logAssistantAlreadyRunning(daemonState.pid, daemonState.status);
|
|
1439
1549
|
}
|
|
1550
|
+
let daemonSpawn: DaemonSpawn | null = null;
|
|
1440
1551
|
|
|
1441
1552
|
if (!daemonAlive) {
|
|
1442
1553
|
if (await checkOrphanedDaemon(pidFile, resources.daemonPort)) {
|
|
@@ -1558,6 +1669,7 @@ export async function startLocalDaemon(
|
|
|
1558
1669
|
// Overwrite sentinel with real PID, or clean up on spawn failure.
|
|
1559
1670
|
if (daemonPid) {
|
|
1560
1671
|
writeFileSync(pidFile, String(daemonPid), "utf-8");
|
|
1672
|
+
daemonSpawn = trackDaemonSpawn(child);
|
|
1561
1673
|
} else {
|
|
1562
1674
|
try {
|
|
1563
1675
|
unlinkSync(pidFile);
|
|
@@ -1596,19 +1708,17 @@ export async function startLocalDaemon(
|
|
|
1596
1708
|
const assistantIndex = resolveAssistantIndexPath(resources);
|
|
1597
1709
|
if (assistantIndex) {
|
|
1598
1710
|
console.log(
|
|
1599
|
-
" Bundled assistant not healthy after 60s
|
|
1711
|
+
" Bundled assistant not healthy after 60s, falling back to source assistant...",
|
|
1600
1712
|
);
|
|
1601
1713
|
// Kill the bundled daemon to avoid two processes competing for the same port
|
|
1602
1714
|
await stopProcessByPidFile(pidFile, "bundled daemon");
|
|
1603
|
-
|
|
1604
|
-
await startDaemonWatchFromSource(
|
|
1605
|
-
|
|
1606
|
-
|
|
1607
|
-
|
|
1608
|
-
|
|
1609
|
-
|
|
1610
|
-
await startDaemonFromSource(assistantIndex, resources, options);
|
|
1611
|
-
}
|
|
1715
|
+
daemonSpawn = watch
|
|
1716
|
+
? await startDaemonWatchFromSource(
|
|
1717
|
+
assistantIndex,
|
|
1718
|
+
resources,
|
|
1719
|
+
options,
|
|
1720
|
+
)
|
|
1721
|
+
: await startDaemonFromSource(assistantIndex, resources, options);
|
|
1612
1722
|
readiness = await waitForDaemonMigrationsReady(
|
|
1613
1723
|
resources.daemonPort,
|
|
1614
1724
|
Date.now() + 60000,
|
|
@@ -1621,7 +1731,17 @@ export async function startLocalDaemon(
|
|
|
1621
1731
|
readiness = await probeDaemonReadiness(resources.daemonPort);
|
|
1622
1732
|
}
|
|
1623
1733
|
|
|
1624
|
-
|
|
1734
|
+
if (daemonSpawn) {
|
|
1735
|
+
await reportFreshSpawnReadiness(
|
|
1736
|
+
daemonSpawn,
|
|
1737
|
+
pidFile,
|
|
1738
|
+
resources.daemonPort,
|
|
1739
|
+
readiness,
|
|
1740
|
+
options?.requireReady,
|
|
1741
|
+
);
|
|
1742
|
+
} else {
|
|
1743
|
+
logDaemonReadiness(readiness, options?.requireReady);
|
|
1744
|
+
}
|
|
1625
1745
|
}
|
|
1626
1746
|
} else {
|
|
1627
1747
|
console.log("🔨 Starting local assistant...");
|
|
@@ -1633,12 +1753,15 @@ export async function startLocalDaemon(
|
|
|
1633
1753
|
" Ensure the daemon binary is bundled alongside the CLI, or run from the source tree.",
|
|
1634
1754
|
);
|
|
1635
1755
|
}
|
|
1636
|
-
const
|
|
1756
|
+
const sourceSpawn = watch
|
|
1637
1757
|
? await startDaemonWatchFromSource(assistantIndex, resources, options)
|
|
1638
1758
|
: await startDaemonFromSource(assistantIndex, resources, options);
|
|
1639
1759
|
// Attach case was classified and logged inside the start function.
|
|
1640
|
-
if (
|
|
1641
|
-
|
|
1760
|
+
if (sourceSpawn) {
|
|
1761
|
+
await reportFreshSpawnReadiness(
|
|
1762
|
+
sourceSpawn,
|
|
1763
|
+
getDaemonPidPath(resources),
|
|
1764
|
+
resources.daemonPort,
|
|
1642
1765
|
await waitForDaemonMigrationsReady(
|
|
1643
1766
|
resources.daemonPort,
|
|
1644
1767
|
Date.now() + 60000,
|