@vellumai/cli 0.10.12-dev.202607270141.42f7d3c → 0.10.12-dev.202607270523.ec1a36e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/drain.test.ts +337 -0
- package/src/__tests__/sleep.test.ts +171 -2
- package/src/commands/sleep.ts +151 -2
- package/src/lib/drain.ts +346 -0
package/package.json
CHANGED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { drainAssistant, parseWaitDuration } from "../lib/drain.js";
|
|
4
|
+
|
|
5
|
+
const BASE = "http://127.0.0.1:7823";
|
|
6
|
+
|
|
7
|
+
interface RecordedCall {
|
|
8
|
+
url: string;
|
|
9
|
+
method: string;
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* Fetch stub: routes requests through `handler` and records every call.
|
|
14
|
+
* Throwing from the handler simulates a network failure.
|
|
15
|
+
*/
|
|
16
|
+
function fetchStub(
|
|
17
|
+
handler: (url: string, method: string) => Response | Promise<Response>,
|
|
18
|
+
): {
|
|
19
|
+
impl: (url: string, init?: RequestInit) => Promise<Response>;
|
|
20
|
+
calls: RecordedCall[];
|
|
21
|
+
} {
|
|
22
|
+
const calls: RecordedCall[] = [];
|
|
23
|
+
const impl = async (url: string, init?: RequestInit): Promise<Response> => {
|
|
24
|
+
const method = init?.method ?? "GET";
|
|
25
|
+
calls.push({ url, method });
|
|
26
|
+
return handler(url, method);
|
|
27
|
+
};
|
|
28
|
+
return { impl, calls };
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function jsonResponse(body: unknown, status = 200): Response {
|
|
32
|
+
return new Response(JSON.stringify(body), {
|
|
33
|
+
status,
|
|
34
|
+
headers: { "Content-Type": "application/json" },
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
function busyStatus(): unknown {
|
|
39
|
+
return {
|
|
40
|
+
quiescedUntil: Date.now() + 60_000,
|
|
41
|
+
idle: false,
|
|
42
|
+
activeConversations: [],
|
|
43
|
+
memoryJobs: [
|
|
44
|
+
{ id: "job-1", type: "memory_retrospective", startedAt: Date.now() },
|
|
45
|
+
],
|
|
46
|
+
scheduleRuns: [],
|
|
47
|
+
workflowRuns: [],
|
|
48
|
+
heartbeatRuns: [],
|
|
49
|
+
};
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function workflowBusyStatus(): unknown {
|
|
53
|
+
return {
|
|
54
|
+
quiescedUntil: Date.now() + 60_000,
|
|
55
|
+
idle: false,
|
|
56
|
+
activeConversations: [],
|
|
57
|
+
memoryJobs: [],
|
|
58
|
+
scheduleRuns: [],
|
|
59
|
+
workflowRuns: [
|
|
60
|
+
{ runId: "wf-1", name: "nightly-report", startedAt: Date.now() },
|
|
61
|
+
],
|
|
62
|
+
heartbeatRuns: [],
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function idleStatus(): unknown {
|
|
67
|
+
return {
|
|
68
|
+
quiescedUntil: Date.now() + 60_000,
|
|
69
|
+
idle: true,
|
|
70
|
+
activeConversations: [],
|
|
71
|
+
memoryJobs: [],
|
|
72
|
+
scheduleRuns: [],
|
|
73
|
+
workflowRuns: [],
|
|
74
|
+
heartbeatRuns: [],
|
|
75
|
+
};
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
function idleExpiredLeaseStatus(): unknown {
|
|
79
|
+
return { ...(idleStatus() as Record<string, unknown>), quiescedUntil: null };
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
describe("parseWaitDuration", () => {
|
|
83
|
+
test("parses bare seconds, seconds, and minutes", () => {
|
|
84
|
+
expect(parseWaitDuration("90")).toBe(90_000);
|
|
85
|
+
expect(parseWaitDuration("90s")).toBe(90_000);
|
|
86
|
+
expect(parseWaitDuration("10m")).toBe(600_000);
|
|
87
|
+
expect(parseWaitDuration(" 5s ")).toBe(5_000);
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
test("rejects invalid durations", () => {
|
|
91
|
+
expect(parseWaitDuration("0")).toBeNull();
|
|
92
|
+
expect(parseWaitDuration("abc")).toBeNull();
|
|
93
|
+
expect(parseWaitDuration("5x")).toBeNull();
|
|
94
|
+
expect(parseWaitDuration("-5")).toBeNull();
|
|
95
|
+
expect(parseWaitDuration("")).toBeNull();
|
|
96
|
+
});
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
describe("drainAssistant", () => {
|
|
100
|
+
test("arms the lease, narrates, and returns drained when work finishes", async () => {
|
|
101
|
+
let statusCalls = 0;
|
|
102
|
+
const { impl, calls } = fetchStub((url, method) => {
|
|
103
|
+
if (url.endsWith("/v1/lifecycle/quiesce") && method === "POST") {
|
|
104
|
+
return jsonResponse({ quiescedUntil: Date.now() + 60_000 });
|
|
105
|
+
}
|
|
106
|
+
statusCalls += 1;
|
|
107
|
+
return jsonResponse(statusCalls < 3 ? busyStatus() : idleStatus());
|
|
108
|
+
});
|
|
109
|
+
const lines: string[] = [];
|
|
110
|
+
|
|
111
|
+
const outcome = await drainAssistant({
|
|
112
|
+
baseUrl: BASE,
|
|
113
|
+
token: "tok",
|
|
114
|
+
deadlineAt: null,
|
|
115
|
+
pollIntervalMs: 1,
|
|
116
|
+
fetchImpl: impl,
|
|
117
|
+
log: (line) => lines.push(line),
|
|
118
|
+
});
|
|
119
|
+
|
|
120
|
+
expect(outcome).toBe("drained");
|
|
121
|
+
expect(calls[0]).toEqual({
|
|
122
|
+
url: `${BASE}/v1/lifecycle/quiesce`,
|
|
123
|
+
method: "POST",
|
|
124
|
+
});
|
|
125
|
+
expect(lines.join("\n")).toContain("memory memory_retrospective");
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test("a running workflow keeps the drain busy and is narrated", async () => {
|
|
129
|
+
let statusCalls = 0;
|
|
130
|
+
const { impl } = fetchStub((_url, method) => {
|
|
131
|
+
if (method === "POST") {
|
|
132
|
+
return jsonResponse({ quiescedUntil: Date.now() + 60_000 });
|
|
133
|
+
}
|
|
134
|
+
statusCalls += 1;
|
|
135
|
+
return jsonResponse(
|
|
136
|
+
statusCalls < 2 ? workflowBusyStatus() : idleStatus(),
|
|
137
|
+
);
|
|
138
|
+
});
|
|
139
|
+
const lines: string[] = [];
|
|
140
|
+
|
|
141
|
+
const outcome = await drainAssistant({
|
|
142
|
+
baseUrl: BASE,
|
|
143
|
+
token: "tok",
|
|
144
|
+
deadlineAt: null,
|
|
145
|
+
pollIntervalMs: 1,
|
|
146
|
+
fetchImpl: impl,
|
|
147
|
+
log: (line) => lines.push(line),
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
expect(outcome).toBe("drained");
|
|
151
|
+
expect(lines.join("\n")).toContain('workflow "nightly-report"');
|
|
152
|
+
});
|
|
153
|
+
|
|
154
|
+
test("a single idle snapshot is not enough — drained needs two in a row", async () => {
|
|
155
|
+
let statusCalls = 0;
|
|
156
|
+
const sequence = [idleStatus(), busyStatus(), idleStatus(), idleStatus()];
|
|
157
|
+
const { impl } = fetchStub((_url, method) => {
|
|
158
|
+
if (method === "POST") {
|
|
159
|
+
return jsonResponse({ quiescedUntil: Date.now() + 60_000 });
|
|
160
|
+
}
|
|
161
|
+
const status = sequence[Math.min(statusCalls, sequence.length - 1)];
|
|
162
|
+
statusCalls += 1;
|
|
163
|
+
return jsonResponse(status);
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
const outcome = await drainAssistant({
|
|
167
|
+
baseUrl: BASE,
|
|
168
|
+
token: "tok",
|
|
169
|
+
deadlineAt: null,
|
|
170
|
+
pollIntervalMs: 1,
|
|
171
|
+
fetchImpl: impl,
|
|
172
|
+
log: () => {},
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
expect(outcome).toBe("drained");
|
|
176
|
+
// idle, busy (streak reset), idle, idle — four reads, not one.
|
|
177
|
+
expect(statusCalls).toBe(4);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("idle without a live lease re-arms instead of counting toward drained", async () => {
|
|
181
|
+
let statusCalls = 0;
|
|
182
|
+
let armCalls = 0;
|
|
183
|
+
const { impl } = fetchStub((_url, method) => {
|
|
184
|
+
if (method === "POST") {
|
|
185
|
+
armCalls += 1;
|
|
186
|
+
return jsonResponse({ quiescedUntil: Date.now() + 60_000 });
|
|
187
|
+
}
|
|
188
|
+
statusCalls += 1;
|
|
189
|
+
// The lease has lapsed on the first idle read; once re-armed, the
|
|
190
|
+
// subsequent idle reads carry a live lease.
|
|
191
|
+
return jsonResponse(
|
|
192
|
+
statusCalls === 1 ? idleExpiredLeaseStatus() : idleStatus(),
|
|
193
|
+
);
|
|
194
|
+
});
|
|
195
|
+
|
|
196
|
+
const outcome = await drainAssistant({
|
|
197
|
+
baseUrl: BASE,
|
|
198
|
+
token: "tok",
|
|
199
|
+
deadlineAt: null,
|
|
200
|
+
pollIntervalMs: 1,
|
|
201
|
+
fetchImpl: impl,
|
|
202
|
+
log: () => {},
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
expect(outcome).toBe("drained");
|
|
206
|
+
// Initial arm + the re-arm triggered by the lapsed lease.
|
|
207
|
+
expect(armCalls).toBeGreaterThanOrEqual(2);
|
|
208
|
+
// The expired-lease idle read did not count: two live-lease reads follow.
|
|
209
|
+
expect(statusCalls).toBeGreaterThanOrEqual(3);
|
|
210
|
+
});
|
|
211
|
+
|
|
212
|
+
test("an unmaintainable lease fails the drain instead of trusting idle", async () => {
|
|
213
|
+
let armCalls = 0;
|
|
214
|
+
const { impl } = fetchStub((_url, method) => {
|
|
215
|
+
if (method === "POST") {
|
|
216
|
+
armCalls += 1;
|
|
217
|
+
if (armCalls === 1) {
|
|
218
|
+
return jsonResponse({ quiescedUntil: Date.now() + 60_000 });
|
|
219
|
+
}
|
|
220
|
+
return jsonResponse({ error: "boom" }, 500);
|
|
221
|
+
}
|
|
222
|
+
return jsonResponse(idleExpiredLeaseStatus());
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
const outcome = await drainAssistant({
|
|
226
|
+
baseUrl: BASE,
|
|
227
|
+
token: "tok",
|
|
228
|
+
deadlineAt: null,
|
|
229
|
+
pollIntervalMs: 1,
|
|
230
|
+
fetchImpl: impl,
|
|
231
|
+
log: () => {},
|
|
232
|
+
});
|
|
233
|
+
|
|
234
|
+
expect(outcome).toBe("unreachable");
|
|
235
|
+
});
|
|
236
|
+
|
|
237
|
+
test("returns unsupported when the assistant lacks the quiesce route", async () => {
|
|
238
|
+
const { impl } = fetchStub(() => jsonResponse({ error: "nope" }, 404));
|
|
239
|
+
|
|
240
|
+
const outcome = await drainAssistant({
|
|
241
|
+
baseUrl: BASE,
|
|
242
|
+
token: "tok",
|
|
243
|
+
deadlineAt: null,
|
|
244
|
+
fetchImpl: impl,
|
|
245
|
+
log: () => {},
|
|
246
|
+
});
|
|
247
|
+
|
|
248
|
+
expect(outcome).toBe("unsupported");
|
|
249
|
+
});
|
|
250
|
+
|
|
251
|
+
test("returns unreachable when the initial lease request fails", async () => {
|
|
252
|
+
const { impl } = fetchStub(() => {
|
|
253
|
+
throw new Error("connection refused");
|
|
254
|
+
});
|
|
255
|
+
|
|
256
|
+
const outcome = await drainAssistant({
|
|
257
|
+
baseUrl: BASE,
|
|
258
|
+
token: "tok",
|
|
259
|
+
deadlineAt: null,
|
|
260
|
+
fetchImpl: impl,
|
|
261
|
+
log: () => {},
|
|
262
|
+
});
|
|
263
|
+
|
|
264
|
+
expect(outcome).toBe("unreachable");
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
test("returns timeout when the deadline passes while still busy", async () => {
|
|
268
|
+
const { impl } = fetchStub((_url, method) =>
|
|
269
|
+
method === "POST"
|
|
270
|
+
? jsonResponse({ quiescedUntil: Date.now() + 60_000 })
|
|
271
|
+
: jsonResponse(busyStatus()),
|
|
272
|
+
);
|
|
273
|
+
|
|
274
|
+
const outcome = await drainAssistant({
|
|
275
|
+
baseUrl: BASE,
|
|
276
|
+
token: "tok",
|
|
277
|
+
deadlineAt: Date.now() + 40,
|
|
278
|
+
pollIntervalMs: 1,
|
|
279
|
+
fetchImpl: impl,
|
|
280
|
+
log: () => {},
|
|
281
|
+
});
|
|
282
|
+
|
|
283
|
+
expect(outcome).toBe("timeout");
|
|
284
|
+
});
|
|
285
|
+
|
|
286
|
+
test("returns unreachable after sustained status failures", async () => {
|
|
287
|
+
const { impl } = fetchStub((_url, method) => {
|
|
288
|
+
if (method === "POST") {
|
|
289
|
+
return jsonResponse({ quiescedUntil: Date.now() + 60_000 });
|
|
290
|
+
}
|
|
291
|
+
throw new Error("boom");
|
|
292
|
+
});
|
|
293
|
+
|
|
294
|
+
const outcome = await drainAssistant({
|
|
295
|
+
baseUrl: BASE,
|
|
296
|
+
token: "tok",
|
|
297
|
+
deadlineAt: null,
|
|
298
|
+
pollIntervalMs: 1,
|
|
299
|
+
fetchImpl: impl,
|
|
300
|
+
log: () => {},
|
|
301
|
+
});
|
|
302
|
+
|
|
303
|
+
expect(outcome).toBe("unreachable");
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
test("cancel releases the lease and returns cancelled", async () => {
|
|
307
|
+
const controller = new AbortController();
|
|
308
|
+
const { impl, calls } = fetchStub((_url, method) => {
|
|
309
|
+
if (method === "POST") {
|
|
310
|
+
return jsonResponse({ quiescedUntil: Date.now() + 60_000 });
|
|
311
|
+
}
|
|
312
|
+
if (method === "DELETE") {
|
|
313
|
+
return jsonResponse({ released: true });
|
|
314
|
+
}
|
|
315
|
+
// First busy status arrives, then the user hits Ctrl-C.
|
|
316
|
+
controller.abort();
|
|
317
|
+
return jsonResponse(busyStatus());
|
|
318
|
+
});
|
|
319
|
+
|
|
320
|
+
const outcome = await drainAssistant({
|
|
321
|
+
baseUrl: BASE,
|
|
322
|
+
token: "tok",
|
|
323
|
+
deadlineAt: null,
|
|
324
|
+
pollIntervalMs: 1,
|
|
325
|
+
fetchImpl: impl,
|
|
326
|
+
signal: controller.signal,
|
|
327
|
+
log: () => {},
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
expect(outcome).toBe("cancelled");
|
|
331
|
+
expect(
|
|
332
|
+
calls.some(
|
|
333
|
+
(c) => c.method === "DELETE" && c.url.endsWith("/v1/lifecycle/quiesce"),
|
|
334
|
+
),
|
|
335
|
+
).toBe(true);
|
|
336
|
+
});
|
|
337
|
+
});
|
|
@@ -30,10 +30,53 @@ mock.module("../lib/process.js", () => ({
|
|
|
30
30
|
stopProcessByPidFile: stopProcessByPidFileMock,
|
|
31
31
|
}));
|
|
32
32
|
|
|
33
|
-
//
|
|
33
|
+
// Mock the drain phase and the guardian-token loader so `--wait` tests can
|
|
34
|
+
// steer outcomes without a live daemon. Both spread the real module so
|
|
35
|
+
// untouched exports keep working (parseWaitDuration stays real).
|
|
36
|
+
import type { DrainOptions, DrainOutcome } from "../lib/drain.js";
|
|
37
|
+
|
|
38
|
+
const drainAssistantMock = mock(
|
|
39
|
+
async (_opts: DrainOptions): Promise<DrainOutcome> => "drained",
|
|
40
|
+
);
|
|
41
|
+
const realDrain = { ...(await import("../lib/drain.js")) };
|
|
42
|
+
mock.module("../lib/drain.js", () => ({
|
|
43
|
+
...realDrain,
|
|
44
|
+
drainAssistant: drainAssistantMock,
|
|
45
|
+
}));
|
|
46
|
+
|
|
47
|
+
const realGuardianToken = { ...(await import("../lib/guardian-token.js")) };
|
|
48
|
+
const loadGuardianTokenMock = mock(
|
|
49
|
+
(_assistantId: string) =>
|
|
50
|
+
({ accessToken: "test-guardian-token" }) as unknown as ReturnType<
|
|
51
|
+
typeof realGuardianToken.loadGuardianToken
|
|
52
|
+
>,
|
|
53
|
+
);
|
|
54
|
+
mock.module("../lib/guardian-token.js", () => ({
|
|
55
|
+
...realGuardianToken,
|
|
56
|
+
loadGuardianToken: loadGuardianTokenMock,
|
|
57
|
+
}));
|
|
58
|
+
|
|
59
|
+
// Stub the token-refresh helper without importing the real client module
|
|
60
|
+
// (it drags in the interactive chat client's dependency graph). Pass-through
|
|
61
|
+
// by default: the stored token is returned as-is.
|
|
62
|
+
const resolveFreshBearerTokenMock = mock(
|
|
63
|
+
async (
|
|
64
|
+
_runtimeUrl: string,
|
|
65
|
+
_assistantId: string,
|
|
66
|
+
bearerToken: string | undefined,
|
|
67
|
+
_cloud: string | undefined,
|
|
68
|
+
) => bearerToken,
|
|
69
|
+
);
|
|
70
|
+
mock.module("../commands/client.js", () => ({
|
|
71
|
+
resolveFreshBearerToken: resolveFreshBearerTokenMock,
|
|
72
|
+
}));
|
|
73
|
+
|
|
74
|
+
// Restore the real modules once this file finishes so the mocks do not leak
|
|
34
75
|
// into other test files in the same `bun test` run.
|
|
35
76
|
afterAll(() => {
|
|
36
77
|
mock.module("../lib/process.js", () => realProcess);
|
|
78
|
+
mock.module("../lib/drain.js", () => realDrain);
|
|
79
|
+
mock.module("../lib/guardian-token.js", () => realGuardianToken);
|
|
37
80
|
});
|
|
38
81
|
|
|
39
82
|
import { sleep } from "../commands/sleep.js";
|
|
@@ -45,7 +88,7 @@ import {
|
|
|
45
88
|
|
|
46
89
|
// Write a lockfile entry so the real resolveTargetAssistant() finds our test
|
|
47
90
|
// assistant without needing to mock the entire assistant-config module.
|
|
48
|
-
function writeLockfile(): void {
|
|
91
|
+
function writeLockfile(extraEntryFields: Record<string, unknown> = {}): void {
|
|
49
92
|
writeFileSync(
|
|
50
93
|
join(testDir, ".vellum.lock.json"),
|
|
51
94
|
JSON.stringify(
|
|
@@ -61,6 +104,7 @@ function writeLockfile(): void {
|
|
|
61
104
|
gatewayPort: DEFAULT_GATEWAY_PORT,
|
|
62
105
|
qdrantPort: DEFAULT_QDRANT_PORT,
|
|
63
106
|
},
|
|
107
|
+
...extraEntryFields,
|
|
64
108
|
},
|
|
65
109
|
],
|
|
66
110
|
activeAssistant: "sleep-test",
|
|
@@ -99,6 +143,16 @@ describe("sleep command", () => {
|
|
|
99
143
|
isProcessAliveMock.mockReturnValue({ alive: false, pid: null });
|
|
100
144
|
stopProcessByPidFileMock.mockReset();
|
|
101
145
|
stopProcessByPidFileMock.mockResolvedValue(true);
|
|
146
|
+
drainAssistantMock.mockReset();
|
|
147
|
+
drainAssistantMock.mockResolvedValue("drained");
|
|
148
|
+
loadGuardianTokenMock.mockReset();
|
|
149
|
+
loadGuardianTokenMock.mockReturnValue({
|
|
150
|
+
accessToken: "test-guardian-token",
|
|
151
|
+
} as unknown as ReturnType<typeof realGuardianToken.loadGuardianToken>);
|
|
152
|
+
resolveFreshBearerTokenMock.mockReset();
|
|
153
|
+
resolveFreshBearerTokenMock.mockImplementation(
|
|
154
|
+
async (_url, _id, bearerToken, _cloud) => bearerToken,
|
|
155
|
+
);
|
|
102
156
|
rmSync(assistantRootDir, { recursive: true, force: true });
|
|
103
157
|
writeLockfile();
|
|
104
158
|
});
|
|
@@ -187,4 +241,119 @@ describe("sleep command", () => {
|
|
|
187
241
|
"credential-executor",
|
|
188
242
|
);
|
|
189
243
|
});
|
|
244
|
+
|
|
245
|
+
describe("--wait", () => {
|
|
246
|
+
test("runs the drain phase against localUrl before stopping", async () => {
|
|
247
|
+
writeLockfile({ localUrl: "http://127.0.0.1:9999" });
|
|
248
|
+
isProcessAliveMock.mockReturnValue({ alive: true, pid: 4321 });
|
|
249
|
+
process.argv = ["bun", "vellum", "sleep", "sleep-test", "--wait"];
|
|
250
|
+
|
|
251
|
+
const consoleLog = spyOn(console, "log").mockImplementation(() => {});
|
|
252
|
+
try {
|
|
253
|
+
await sleep();
|
|
254
|
+
} finally {
|
|
255
|
+
consoleLog.mockRestore();
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
expect(drainAssistantMock).toHaveBeenCalledTimes(1);
|
|
259
|
+
const opts = drainAssistantMock.mock.calls[0]![0];
|
|
260
|
+
expect(opts.baseUrl).toBe("http://127.0.0.1:9999");
|
|
261
|
+
expect(opts.token).toBe("test-guardian-token");
|
|
262
|
+
// Bare --wait waits as long as it takes.
|
|
263
|
+
expect(opts.deadlineAt).toBeNull();
|
|
264
|
+
expect(stopProcessByPidFileMock).toHaveBeenCalledTimes(3);
|
|
265
|
+
});
|
|
266
|
+
|
|
267
|
+
test("a duration bounds the wait with a deadline", async () => {
|
|
268
|
+
isProcessAliveMock.mockReturnValue({ alive: true, pid: 4321 });
|
|
269
|
+
process.argv = ["bun", "vellum", "sleep", "sleep-test", "--wait", "90s"];
|
|
270
|
+
|
|
271
|
+
const consoleLog = spyOn(console, "log").mockImplementation(() => {});
|
|
272
|
+
const before = Date.now();
|
|
273
|
+
try {
|
|
274
|
+
await sleep();
|
|
275
|
+
} finally {
|
|
276
|
+
consoleLog.mockRestore();
|
|
277
|
+
}
|
|
278
|
+
|
|
279
|
+
const opts = drainAssistantMock.mock.calls[0]![0];
|
|
280
|
+
expect(opts.deadlineAt).toBeGreaterThanOrEqual(before + 89_000);
|
|
281
|
+
expect(opts.deadlineAt).toBeLessThanOrEqual(Date.now() + 91_000);
|
|
282
|
+
expect(stopProcessByPidFileMock).toHaveBeenCalledTimes(3);
|
|
283
|
+
});
|
|
284
|
+
|
|
285
|
+
test("cancelled drain exits 130 without stopping anything", async () => {
|
|
286
|
+
isProcessAliveMock.mockReturnValue({ alive: true, pid: 4321 });
|
|
287
|
+
drainAssistantMock.mockResolvedValue("cancelled");
|
|
288
|
+
process.argv = ["bun", "vellum", "sleep", "sleep-test", "--wait"];
|
|
289
|
+
|
|
290
|
+
const consoleLog = spyOn(console, "log").mockImplementation(() => {});
|
|
291
|
+
const exitMock = mock((code?: number) => {
|
|
292
|
+
throw new Error(`process.exit:${code}`);
|
|
293
|
+
});
|
|
294
|
+
const originalExit = process.exit;
|
|
295
|
+
process.exit = exitMock as unknown as typeof process.exit;
|
|
296
|
+
try {
|
|
297
|
+
await expect(sleep()).rejects.toThrow("process.exit:130");
|
|
298
|
+
} finally {
|
|
299
|
+
process.exit = originalExit;
|
|
300
|
+
consoleLog.mockRestore();
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
expect(stopProcessByPidFileMock).not.toHaveBeenCalled();
|
|
304
|
+
});
|
|
305
|
+
|
|
306
|
+
test("skips the drain when the assistant is not running", async () => {
|
|
307
|
+
isProcessAliveMock.mockReturnValue({ alive: false, pid: null });
|
|
308
|
+
process.argv = ["bun", "vellum", "sleep", "sleep-test", "--wait"];
|
|
309
|
+
|
|
310
|
+
const consoleLog = spyOn(console, "log").mockImplementation(() => {});
|
|
311
|
+
try {
|
|
312
|
+
await sleep();
|
|
313
|
+
} finally {
|
|
314
|
+
consoleLog.mockRestore();
|
|
315
|
+
}
|
|
316
|
+
|
|
317
|
+
expect(drainAssistantMock).not.toHaveBeenCalled();
|
|
318
|
+
expect(stopProcessByPidFileMock).toHaveBeenCalledTimes(3);
|
|
319
|
+
});
|
|
320
|
+
|
|
321
|
+
test("stops without waiting when no guardian token is available", async () => {
|
|
322
|
+
isProcessAliveMock.mockReturnValue({ alive: true, pid: 4321 });
|
|
323
|
+
loadGuardianTokenMock.mockReturnValue(null);
|
|
324
|
+
process.argv = ["bun", "vellum", "sleep", "sleep-test", "--wait"];
|
|
325
|
+
|
|
326
|
+
const consoleLog = spyOn(console, "log").mockImplementation(() => {});
|
|
327
|
+
try {
|
|
328
|
+
await sleep();
|
|
329
|
+
} finally {
|
|
330
|
+
consoleLog.mockRestore();
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
expect(drainAssistantMock).not.toHaveBeenCalled();
|
|
334
|
+
expect(stopProcessByPidFileMock).toHaveBeenCalledTimes(3);
|
|
335
|
+
});
|
|
336
|
+
|
|
337
|
+
test("rejects an invalid --wait duration", async () => {
|
|
338
|
+
process.argv = ["bun", "vellum", "sleep", "sleep-test", "--wait=abc"];
|
|
339
|
+
|
|
340
|
+
const consoleError = spyOn(console, "error").mockImplementation(() => {});
|
|
341
|
+
const exitMock = mock((code?: number) => {
|
|
342
|
+
throw new Error(`process.exit:${code}`);
|
|
343
|
+
});
|
|
344
|
+
const originalExit = process.exit;
|
|
345
|
+
process.exit = exitMock as unknown as typeof process.exit;
|
|
346
|
+
try {
|
|
347
|
+
await expect(sleep()).rejects.toThrow("process.exit:1");
|
|
348
|
+
expect(consoleError).toHaveBeenCalledWith(
|
|
349
|
+
expect.stringContaining("invalid --wait duration"),
|
|
350
|
+
);
|
|
351
|
+
} finally {
|
|
352
|
+
process.exit = originalExit;
|
|
353
|
+
consoleError.mockRestore();
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
expect(stopProcessByPidFileMock).not.toHaveBeenCalled();
|
|
357
|
+
});
|
|
358
|
+
});
|
|
190
359
|
});
|
package/src/commands/sleep.ts
CHANGED
|
@@ -7,8 +7,15 @@ import {
|
|
|
7
7
|
} from "../lib/assistant-config.js";
|
|
8
8
|
import type { AssistantEntry } from "../lib/assistant-config.js";
|
|
9
9
|
import { dockerResourceNames, sleepContainers } from "../lib/docker.js";
|
|
10
|
+
import {
|
|
11
|
+
drainAssistant,
|
|
12
|
+
type DrainOutcome,
|
|
13
|
+
parseWaitDuration,
|
|
14
|
+
} from "../lib/drain.js";
|
|
15
|
+
import { loadGuardianToken } from "../lib/guardian-token.js";
|
|
10
16
|
import { stopIngressNginx } from "../lib/nginx-ingress.js";
|
|
11
17
|
import { isProcessAlive, stopProcessByPidFile } from "../lib/process";
|
|
18
|
+
import { resolveFreshBearerToken } from "./client.js";
|
|
12
19
|
|
|
13
20
|
const ACTIVE_CALL_LEASES_FILE = "active-call-leases.json";
|
|
14
21
|
|
|
@@ -49,7 +56,7 @@ function readActiveCallLeases(vellumDir: string): ActiveCallLease[] {
|
|
|
49
56
|
export async function sleep(): Promise<void> {
|
|
50
57
|
const args = process.argv.slice(3);
|
|
51
58
|
if (args.includes("--help") || args.includes("-h")) {
|
|
52
|
-
console.log("Usage: vellum sleep [<name>] [--force]");
|
|
59
|
+
console.log("Usage: vellum sleep [<name>] [--force] [--wait [<duration>]]");
|
|
53
60
|
console.log("");
|
|
54
61
|
console.log("Stop the assistant and gateway processes.");
|
|
55
62
|
console.log("");
|
|
@@ -62,14 +69,66 @@ export async function sleep(): Promise<void> {
|
|
|
62
69
|
console.log(
|
|
63
70
|
" --force Stop the assistant even if a phone call keepalive lease is active",
|
|
64
71
|
);
|
|
72
|
+
console.log(
|
|
73
|
+
" --wait Wait for in-flight background work (heartbeats, memory jobs,",
|
|
74
|
+
);
|
|
75
|
+
console.log(
|
|
76
|
+
" schedules) to finish before stopping. Waits as long as it",
|
|
77
|
+
);
|
|
78
|
+
console.log(
|
|
79
|
+
" takes by default; pass a duration (e.g. --wait 90s, --wait 10m)",
|
|
80
|
+
);
|
|
81
|
+
console.log(
|
|
82
|
+
" to bound the wait. Ctrl-C cancels and leaves the assistant running.",
|
|
83
|
+
);
|
|
65
84
|
process.exit(0);
|
|
66
85
|
}
|
|
67
86
|
|
|
68
87
|
const force = args.includes("--force");
|
|
69
|
-
|
|
88
|
+
let wait = false;
|
|
89
|
+
let waitMs: number | null = null;
|
|
90
|
+
let waitDurationToken: string | null = null;
|
|
91
|
+
const waitEqArg = args.find((a) => a.startsWith("--wait="));
|
|
92
|
+
if (waitEqArg) {
|
|
93
|
+
wait = true;
|
|
94
|
+
const raw = waitEqArg.slice("--wait=".length);
|
|
95
|
+
waitMs = parseWaitDuration(raw);
|
|
96
|
+
if (waitMs == null) {
|
|
97
|
+
console.error(
|
|
98
|
+
`Error: invalid --wait duration '${raw}'. Use seconds or a value like 90s or 10m.`,
|
|
99
|
+
);
|
|
100
|
+
process.exit(1);
|
|
101
|
+
}
|
|
102
|
+
} else {
|
|
103
|
+
const waitIndex = args.indexOf("--wait");
|
|
104
|
+
if (waitIndex !== -1) {
|
|
105
|
+
wait = true;
|
|
106
|
+
// A bare `--wait` waits indefinitely; a following duration token
|
|
107
|
+
// (e.g. `90s`, `10m`, `120`) bounds it. Anything else is the name arg.
|
|
108
|
+
const next = args[waitIndex + 1];
|
|
109
|
+
if (next !== undefined && /^\d+(s|m)?$/.test(next)) {
|
|
110
|
+
waitDurationToken = next;
|
|
111
|
+
waitMs = parseWaitDuration(next);
|
|
112
|
+
if (waitMs == null) {
|
|
113
|
+
console.error(
|
|
114
|
+
`Error: invalid --wait duration '${next}'. Use seconds or a value like 90s or 10m.`,
|
|
115
|
+
);
|
|
116
|
+
process.exit(1);
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
const nameArg = args.find(
|
|
122
|
+
(a) => !a.startsWith("-") && a !== waitDurationToken,
|
|
123
|
+
);
|
|
70
124
|
const entry = resolveTargetAssistant(nameArg);
|
|
71
125
|
|
|
72
126
|
if (entry.cloud === "docker") {
|
|
127
|
+
if (wait) {
|
|
128
|
+
console.log(
|
|
129
|
+
"--wait is not supported for Docker assistants yet — stopping containers now.",
|
|
130
|
+
);
|
|
131
|
+
}
|
|
73
132
|
const res = dockerResourceNames(entry.assistantId);
|
|
74
133
|
await sleepContainers(res);
|
|
75
134
|
console.log("Docker containers stopped.");
|
|
@@ -135,6 +194,14 @@ export async function sleep(): Promise<void> {
|
|
|
135
194
|
}
|
|
136
195
|
}
|
|
137
196
|
|
|
197
|
+
if (wait) {
|
|
198
|
+
const outcome = await runDrainPhase(entry, assistantPidFile, waitMs);
|
|
199
|
+
if (outcome === "cancelled") {
|
|
200
|
+
console.log("Sleep cancelled — the assistant is still running.");
|
|
201
|
+
process.exit(130);
|
|
202
|
+
}
|
|
203
|
+
}
|
|
204
|
+
|
|
138
205
|
// Stop assistant — use a generous timeout. On SIGTERM the daemon runs a
|
|
139
206
|
// WAL checkpoint before exiting, which can take several seconds on a
|
|
140
207
|
// multi-GB database. The default 2s grace in stopProcess() would SIGKILL a
|
|
@@ -186,3 +253,85 @@ export async function sleep(): Promise<void> {
|
|
|
186
253
|
console.log("nginx ingress stopped.");
|
|
187
254
|
}
|
|
188
255
|
}
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Wait for the assistant's in-flight background work (heartbeats, memory
|
|
259
|
+
* jobs, schedule runs, active conversation turns) to finish before the stop.
|
|
260
|
+
* Every failure mode falls back to a normal stop with a note — a drain must
|
|
261
|
+
* never make `vellum sleep` less reliable than it is without `--wait`.
|
|
262
|
+
*/
|
|
263
|
+
async function runDrainPhase(
|
|
264
|
+
entry: AssistantEntry,
|
|
265
|
+
assistantPidFile: string,
|
|
266
|
+
waitMs: number | null,
|
|
267
|
+
): Promise<DrainOutcome | "skipped"> {
|
|
268
|
+
if (!isProcessAlive(assistantPidFile).alive) {
|
|
269
|
+
return "skipped";
|
|
270
|
+
}
|
|
271
|
+
const baseUrl = entry.localUrl ?? entry.runtimeUrl;
|
|
272
|
+
const stored = loadGuardianToken(entry.assistantId)?.accessToken;
|
|
273
|
+
if (!baseUrl || !stored) {
|
|
274
|
+
console.log(
|
|
275
|
+
"Cannot reach the assistant API to wait for background work (missing URL or guardian token) — stopping without waiting.",
|
|
276
|
+
);
|
|
277
|
+
return "skipped";
|
|
278
|
+
}
|
|
279
|
+
// A token past its renewal point would 401 and read as "unreachable",
|
|
280
|
+
// silently skipping the wait — refresh-and-fall-back like other CLI
|
|
281
|
+
// request paths.
|
|
282
|
+
const token = await resolveFreshBearerToken(
|
|
283
|
+
baseUrl,
|
|
284
|
+
entry.assistantId,
|
|
285
|
+
stored,
|
|
286
|
+
entry.cloud,
|
|
287
|
+
);
|
|
288
|
+
if (!token) {
|
|
289
|
+
console.log(
|
|
290
|
+
"Cannot reach the assistant API to wait for background work (missing URL or guardian token) — stopping without waiting.",
|
|
291
|
+
);
|
|
292
|
+
return "skipped";
|
|
293
|
+
}
|
|
294
|
+
|
|
295
|
+
console.log(
|
|
296
|
+
waitMs == null
|
|
297
|
+
? "Waiting for the assistant to finish background work (Ctrl-C cancels)…"
|
|
298
|
+
: `Waiting up to ${Math.round(waitMs / 1000)}s for the assistant to finish background work (Ctrl-C cancels)…`,
|
|
299
|
+
);
|
|
300
|
+
|
|
301
|
+
const controller = new AbortController();
|
|
302
|
+
const onSigint = (): void => controller.abort();
|
|
303
|
+
process.once("SIGINT", onSigint);
|
|
304
|
+
try {
|
|
305
|
+
const outcome = await drainAssistant({
|
|
306
|
+
baseUrl,
|
|
307
|
+
token,
|
|
308
|
+
deadlineAt: waitMs == null ? null : Date.now() + waitMs,
|
|
309
|
+
signal: controller.signal,
|
|
310
|
+
});
|
|
311
|
+
switch (outcome) {
|
|
312
|
+
case "drained":
|
|
313
|
+
console.log("Background work finished — stopping now.");
|
|
314
|
+
break;
|
|
315
|
+
case "timeout":
|
|
316
|
+
console.log(
|
|
317
|
+
"Wait limit reached — stopping now. Interrupted background jobs resume after the next start.",
|
|
318
|
+
);
|
|
319
|
+
break;
|
|
320
|
+
case "unsupported":
|
|
321
|
+
console.log(
|
|
322
|
+
"This assistant version does not support --wait — stopping without waiting.",
|
|
323
|
+
);
|
|
324
|
+
break;
|
|
325
|
+
case "unreachable":
|
|
326
|
+
console.log(
|
|
327
|
+
"Could not reach the assistant to check background work — stopping without waiting.",
|
|
328
|
+
);
|
|
329
|
+
break;
|
|
330
|
+
case "cancelled":
|
|
331
|
+
break;
|
|
332
|
+
}
|
|
333
|
+
return outcome;
|
|
334
|
+
} finally {
|
|
335
|
+
process.removeListener("SIGINT", onSigint);
|
|
336
|
+
}
|
|
337
|
+
}
|
package/src/lib/drain.ts
ADDED
|
@@ -0,0 +1,346 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Drain phase for `vellum sleep --wait`.
|
|
3
|
+
*
|
|
4
|
+
* Arms the assistant's quiesce lease (pausing the START of new background
|
|
5
|
+
* work — heartbeats, schedule claims, memory-job claims), then polls the
|
|
6
|
+
* drain-status endpoint until nothing is in flight. The lease is TTL-bound
|
|
7
|
+
* and refreshed while polling, so a killed CLI can never leave background
|
|
8
|
+
* work paused. The wait narrates what it is waiting on — background jobs can
|
|
9
|
+
* legitimately run for many minutes, and a silent wait is indistinguishable
|
|
10
|
+
* from a hang.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { loopbackSafeFetch } from "./loopback-fetch.js";
|
|
14
|
+
|
|
15
|
+
const QUIESCE_TTL_MS = 60_000;
|
|
16
|
+
const QUIESCE_REFRESH_INTERVAL_MS = 20_000;
|
|
17
|
+
const REQUEST_TIMEOUT_MS = 2_500;
|
|
18
|
+
const DEFAULT_POLL_INTERVAL_MS = 1_000;
|
|
19
|
+
const NARRATE_HEARTBEAT_MS = 15_000;
|
|
20
|
+
/** Consecutive failed polls before giving up on the drain. Transient
|
|
21
|
+
* failures (a brief daemon event-loop stall) recover; a wall of failures
|
|
22
|
+
* means the assistant cannot report status and waiting is pointless. */
|
|
23
|
+
const MAX_CONSECUTIVE_FAILURES = 10;
|
|
24
|
+
/** Consecutive failed lease arms/refreshes before giving up. Without a live
|
|
25
|
+
* lease the claim gates are open and an idle snapshot proves nothing, so an
|
|
26
|
+
* unmaintainable lease makes the drain unsound. */
|
|
27
|
+
const MAX_LEASE_FAILURES = 3;
|
|
28
|
+
|
|
29
|
+
export type DrainOutcome =
|
|
30
|
+
| "drained"
|
|
31
|
+
| "timeout"
|
|
32
|
+
| "unsupported"
|
|
33
|
+
| "unreachable"
|
|
34
|
+
| "cancelled";
|
|
35
|
+
|
|
36
|
+
interface DrainStatusResponse {
|
|
37
|
+
quiescedUntil: number | null;
|
|
38
|
+
idle: boolean;
|
|
39
|
+
activeConversations: Array<{
|
|
40
|
+
conversationId: string;
|
|
41
|
+
title: string | null;
|
|
42
|
+
originChannel: string | null;
|
|
43
|
+
originInterface: string | null;
|
|
44
|
+
processingStartedAt: number;
|
|
45
|
+
}>;
|
|
46
|
+
memoryJobs: Array<{
|
|
47
|
+
id: string;
|
|
48
|
+
type: string;
|
|
49
|
+
startedAt: number | null;
|
|
50
|
+
}>;
|
|
51
|
+
scheduleRuns: Array<{
|
|
52
|
+
runId: string;
|
|
53
|
+
scheduleName: string | null;
|
|
54
|
+
startedAt: number;
|
|
55
|
+
}>;
|
|
56
|
+
workflowRuns?: Array<{
|
|
57
|
+
runId: string;
|
|
58
|
+
name: string | null;
|
|
59
|
+
startedAt: number | null;
|
|
60
|
+
}>;
|
|
61
|
+
heartbeatRuns?: Array<{
|
|
62
|
+
runId: string;
|
|
63
|
+
startedAt: number;
|
|
64
|
+
}>;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
export interface DrainOptions {
|
|
68
|
+
baseUrl: string;
|
|
69
|
+
token: string;
|
|
70
|
+
/** Epoch ms to stop waiting at, or null to wait as long as it takes. */
|
|
71
|
+
deadlineAt: number | null;
|
|
72
|
+
/** Aborted on Ctrl-C — releases the lease and returns "cancelled". */
|
|
73
|
+
signal?: AbortSignal;
|
|
74
|
+
pollIntervalMs?: number;
|
|
75
|
+
fetchImpl?: (url: string, init?: RequestInit) => Promise<Response>;
|
|
76
|
+
log?: (line: string) => void;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Parse a `--wait` duration: bare seconds (`90`), seconds (`90s`), or
|
|
81
|
+
* minutes (`10m`). Returns milliseconds, or null when not a valid duration.
|
|
82
|
+
*/
|
|
83
|
+
export function parseWaitDuration(raw: string): number | null {
|
|
84
|
+
const match = /^(\d+)(s|m)?$/.exec(raw.trim());
|
|
85
|
+
if (!match) {
|
|
86
|
+
return null;
|
|
87
|
+
}
|
|
88
|
+
const value = Number.parseInt(match[1]!, 10);
|
|
89
|
+
if (!Number.isFinite(value) || value < 1) {
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
return match[2] === "m" ? value * 60_000 : value * 1_000;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
function formatElapsed(sinceMs: number | null, now: number): string {
|
|
96
|
+
if (sinceMs == null) {
|
|
97
|
+
return "just started";
|
|
98
|
+
}
|
|
99
|
+
const totalSeconds = Math.max(0, Math.floor((now - sinceMs) / 1000));
|
|
100
|
+
const minutes = Math.floor(totalSeconds / 60);
|
|
101
|
+
const seconds = totalSeconds % 60;
|
|
102
|
+
return minutes > 0
|
|
103
|
+
? `${minutes}m${seconds.toString().padStart(2, "0")}s`
|
|
104
|
+
: `${seconds}s`;
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
function describeBusyWork(status: DrainStatusResponse, now: number): string[] {
|
|
108
|
+
const items: string[] = [];
|
|
109
|
+
for (const conv of status.activeConversations) {
|
|
110
|
+
const label =
|
|
111
|
+
conv.title != null && conv.title.length > 0
|
|
112
|
+
? `conversation "${conv.title}"`
|
|
113
|
+
: `background turn (${conv.originInterface ?? conv.originChannel ?? "unknown origin"})`;
|
|
114
|
+
items.push(`${label} (${formatElapsed(conv.processingStartedAt, now)})`);
|
|
115
|
+
}
|
|
116
|
+
for (const job of status.memoryJobs) {
|
|
117
|
+
items.push(`memory ${job.type} (${formatElapsed(job.startedAt, now)})`);
|
|
118
|
+
}
|
|
119
|
+
for (const run of status.scheduleRuns) {
|
|
120
|
+
const label = run.scheduleName ?? run.runId.slice(0, 8);
|
|
121
|
+
items.push(`schedule "${label}" (${formatElapsed(run.startedAt, now)})`);
|
|
122
|
+
}
|
|
123
|
+
for (const run of status.workflowRuns ?? []) {
|
|
124
|
+
const label = run.name ?? run.runId.slice(0, 8);
|
|
125
|
+
items.push(`workflow "${label}" (${formatElapsed(run.startedAt, now)})`);
|
|
126
|
+
}
|
|
127
|
+
for (const run of status.heartbeatRuns ?? []) {
|
|
128
|
+
items.push(`heartbeat (${formatElapsed(run.startedAt, now)})`);
|
|
129
|
+
}
|
|
130
|
+
return items;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function busySignature(status: DrainStatusResponse): string {
|
|
134
|
+
return [
|
|
135
|
+
...status.activeConversations.map((c) => c.conversationId),
|
|
136
|
+
...status.memoryJobs.map((j) => j.id),
|
|
137
|
+
...status.scheduleRuns.map((r) => r.runId),
|
|
138
|
+
...(status.workflowRuns ?? []).map((r) => r.runId),
|
|
139
|
+
...(status.heartbeatRuns ?? []).map((r) => r.runId),
|
|
140
|
+
].join("|");
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function sleepMs(ms: number, signal?: AbortSignal): Promise<void> {
|
|
144
|
+
return new Promise((resolve) => {
|
|
145
|
+
const timer = setTimeout(done, ms);
|
|
146
|
+
function done(): void {
|
|
147
|
+
clearTimeout(timer);
|
|
148
|
+
signal?.removeEventListener("abort", done);
|
|
149
|
+
resolve();
|
|
150
|
+
}
|
|
151
|
+
signal?.addEventListener("abort", done, { once: true });
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
/**
|
|
156
|
+
* Wait for the assistant's in-flight background work to finish.
|
|
157
|
+
*
|
|
158
|
+
* Never throws: every failure mode maps onto a {@link DrainOutcome} so the
|
|
159
|
+
* caller can decide how to proceed with the stop.
|
|
160
|
+
*/
|
|
161
|
+
export async function drainAssistant(
|
|
162
|
+
opts: DrainOptions,
|
|
163
|
+
): Promise<DrainOutcome> {
|
|
164
|
+
const fetchImpl = opts.fetchImpl ?? loopbackSafeFetch;
|
|
165
|
+
const log = opts.log ?? console.log;
|
|
166
|
+
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
|
167
|
+
const base = opts.baseUrl.replace(/\/+$/, "");
|
|
168
|
+
|
|
169
|
+
const request = async (
|
|
170
|
+
path: string,
|
|
171
|
+
init: { method: string; body?: string },
|
|
172
|
+
): Promise<Response> => {
|
|
173
|
+
// Per-request controller aborted by either the request timeout or the
|
|
174
|
+
// caller's cancel signal; listeners are removed per request so a long
|
|
175
|
+
// wait never accumulates them on the shared cancel signal.
|
|
176
|
+
const controller = new AbortController();
|
|
177
|
+
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
|
|
178
|
+
const onOuterAbort = (): void => controller.abort();
|
|
179
|
+
opts.signal?.addEventListener("abort", onOuterAbort, { once: true });
|
|
180
|
+
try {
|
|
181
|
+
return await fetchImpl(`${base}${path}`, {
|
|
182
|
+
method: init.method,
|
|
183
|
+
headers: {
|
|
184
|
+
Authorization: `Bearer ${opts.token}`,
|
|
185
|
+
"Content-Type": "application/json",
|
|
186
|
+
},
|
|
187
|
+
...(init.body !== undefined && { body: init.body }),
|
|
188
|
+
signal: controller.signal,
|
|
189
|
+
});
|
|
190
|
+
} finally {
|
|
191
|
+
clearTimeout(timer);
|
|
192
|
+
opts.signal?.removeEventListener("abort", onOuterAbort);
|
|
193
|
+
}
|
|
194
|
+
};
|
|
195
|
+
|
|
196
|
+
const armLease = () =>
|
|
197
|
+
request("/v1/lifecycle/quiesce", {
|
|
198
|
+
method: "POST",
|
|
199
|
+
body: JSON.stringify({ ttlMs: QUIESCE_TTL_MS }),
|
|
200
|
+
});
|
|
201
|
+
|
|
202
|
+
const releaseLease = async (): Promise<void> => {
|
|
203
|
+
try {
|
|
204
|
+
await request("/v1/lifecycle/quiesce", { method: "DELETE" });
|
|
205
|
+
} catch {
|
|
206
|
+
// Best-effort — the lease self-expires.
|
|
207
|
+
}
|
|
208
|
+
};
|
|
209
|
+
|
|
210
|
+
// Arm the lease before anything else: without it a busy queue keeps
|
|
211
|
+
// claiming new work and the wait never converges.
|
|
212
|
+
let response: Response;
|
|
213
|
+
try {
|
|
214
|
+
response = await armLease();
|
|
215
|
+
} catch {
|
|
216
|
+
if (opts.signal?.aborted) {
|
|
217
|
+
// The abort may have landed after the server persisted the lease, so
|
|
218
|
+
// its outcome is ambiguous — release best-effort rather than leaving
|
|
219
|
+
// background work paused for a TTL on a cancelled sleep.
|
|
220
|
+
await releaseLease();
|
|
221
|
+
return "cancelled";
|
|
222
|
+
}
|
|
223
|
+
return "unreachable";
|
|
224
|
+
}
|
|
225
|
+
if (response.status === 404 || response.status === 405) {
|
|
226
|
+
return "unsupported";
|
|
227
|
+
}
|
|
228
|
+
if (!response.ok) {
|
|
229
|
+
return "unreachable";
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
let lastLeaseAt = Date.now();
|
|
233
|
+
let consecutiveFailures = 0;
|
|
234
|
+
let leaseFailureStreak = 0;
|
|
235
|
+
let lastSignature: string | null = null;
|
|
236
|
+
let lastNarratedAt = 0;
|
|
237
|
+
let idleStreak = 0;
|
|
238
|
+
|
|
239
|
+
for (;;) {
|
|
240
|
+
if (opts.signal?.aborted) {
|
|
241
|
+
await releaseLease();
|
|
242
|
+
return "cancelled";
|
|
243
|
+
}
|
|
244
|
+
if (opts.deadlineAt != null && Date.now() >= opts.deadlineAt) {
|
|
245
|
+
return "timeout";
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
try {
|
|
249
|
+
const statusResponse = await request("/v1/lifecycle/drain-status", {
|
|
250
|
+
method: "GET",
|
|
251
|
+
});
|
|
252
|
+
if (!statusResponse.ok) {
|
|
253
|
+
throw new Error(`drain-status returned ${statusResponse.status}`);
|
|
254
|
+
}
|
|
255
|
+
const status = (await statusResponse.json()) as DrainStatusResponse;
|
|
256
|
+
consecutiveFailures = 0;
|
|
257
|
+
|
|
258
|
+
if (status.idle) {
|
|
259
|
+
// Idle only counts under a LIVE lease: with the lease expired the
|
|
260
|
+
// claim gates are open again, so new work invisible to this snapshot
|
|
261
|
+
// could be starting. Re-arm and retry rather than trusting the read.
|
|
262
|
+
const leaseActive =
|
|
263
|
+
status.quiescedUntil != null && status.quiescedUntil > Date.now();
|
|
264
|
+
if (!leaseActive) {
|
|
265
|
+
idleStreak = 0;
|
|
266
|
+
try {
|
|
267
|
+
const rearm = await armLease();
|
|
268
|
+
if (rearm.ok) {
|
|
269
|
+
lastLeaseAt = Date.now();
|
|
270
|
+
leaseFailureStreak = 0;
|
|
271
|
+
} else {
|
|
272
|
+
leaseFailureStreak += 1;
|
|
273
|
+
}
|
|
274
|
+
} catch {
|
|
275
|
+
if (opts.signal?.aborted) {
|
|
276
|
+
await releaseLease();
|
|
277
|
+
return "cancelled";
|
|
278
|
+
}
|
|
279
|
+
leaseFailureStreak += 1;
|
|
280
|
+
}
|
|
281
|
+
if (leaseFailureStreak >= MAX_LEASE_FAILURES) {
|
|
282
|
+
return "unreachable";
|
|
283
|
+
}
|
|
284
|
+
await sleepMs(pollIntervalMs, opts.signal);
|
|
285
|
+
continue;
|
|
286
|
+
}
|
|
287
|
+
// Require two consecutive idle snapshots: work that raced past a
|
|
288
|
+
// pre-lease gate check lands its run row within a tick, so a second
|
|
289
|
+
// read one poll later rules out a stop landing on that window.
|
|
290
|
+
idleStreak += 1;
|
|
291
|
+
if (idleStreak >= 2) {
|
|
292
|
+
return "drained";
|
|
293
|
+
}
|
|
294
|
+
await sleepMs(pollIntervalMs, opts.signal);
|
|
295
|
+
continue;
|
|
296
|
+
}
|
|
297
|
+
idleStreak = 0;
|
|
298
|
+
|
|
299
|
+
const now = Date.now();
|
|
300
|
+
const signature = busySignature(status);
|
|
301
|
+
if (
|
|
302
|
+
signature !== lastSignature ||
|
|
303
|
+
now - lastNarratedAt >= NARRATE_HEARTBEAT_MS
|
|
304
|
+
) {
|
|
305
|
+
const items = describeBusyWork(status, now);
|
|
306
|
+
const shown = items.slice(0, 4);
|
|
307
|
+
const more = items.length - shown.length;
|
|
308
|
+
log(
|
|
309
|
+
`Waiting for the assistant to finish: ${shown.join(", ")}${
|
|
310
|
+
more > 0 ? ` and ${more} more` : ""
|
|
311
|
+
}…`,
|
|
312
|
+
);
|
|
313
|
+
lastSignature = signature;
|
|
314
|
+
lastNarratedAt = now;
|
|
315
|
+
}
|
|
316
|
+
} catch {
|
|
317
|
+
if (opts.signal?.aborted) {
|
|
318
|
+
await releaseLease();
|
|
319
|
+
return "cancelled";
|
|
320
|
+
}
|
|
321
|
+
consecutiveFailures += 1;
|
|
322
|
+
if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
|
|
323
|
+
return "unreachable";
|
|
324
|
+
}
|
|
325
|
+
}
|
|
326
|
+
|
|
327
|
+
if (Date.now() - lastLeaseAt >= QUIESCE_REFRESH_INTERVAL_MS) {
|
|
328
|
+
try {
|
|
329
|
+
const refresh = await armLease();
|
|
330
|
+
if (refresh.ok) {
|
|
331
|
+
lastLeaseAt = Date.now();
|
|
332
|
+
leaseFailureStreak = 0;
|
|
333
|
+
} else {
|
|
334
|
+
leaseFailureStreak += 1;
|
|
335
|
+
}
|
|
336
|
+
} catch {
|
|
337
|
+
leaseFailureStreak += 1;
|
|
338
|
+
}
|
|
339
|
+
if (leaseFailureStreak >= MAX_LEASE_FAILURES) {
|
|
340
|
+
return "unreachable";
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
await sleepMs(pollIntervalMs, opts.signal);
|
|
345
|
+
}
|
|
346
|
+
}
|