@vellumai/cli 0.10.11-dev.202607241533.d936a4e → 0.10.11-dev.202607241637.6d2be3e
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/local-hatch-lock.test.ts +163 -0
- package/src/lib/hatch-local.ts +98 -86
- package/src/lib/local-hatch-lock.ts +322 -0
- package/src/lib/local.ts +20 -2
package/package.json
CHANGED
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
import { afterEach, describe, expect, test } from "bun:test";
|
|
2
|
+
import { existsSync, mkdtempSync, rmSync, utimesSync, writeFileSync } from "fs";
|
|
3
|
+
import { tmpdir } from "os";
|
|
4
|
+
import { join } from "path";
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
resolveLocalHatchLockPath,
|
|
8
|
+
withLocalHatchLock,
|
|
9
|
+
} from "../lib/local-hatch-lock.js";
|
|
10
|
+
|
|
11
|
+
const tempDirs: string[] = [];
|
|
12
|
+
|
|
13
|
+
afterEach(() => {
|
|
14
|
+
for (const tempDir of tempDirs.splice(0)) {
|
|
15
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
16
|
+
}
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
function makeLockPath(): string {
|
|
20
|
+
const tempDir = mkdtempSync(join(tmpdir(), "vellum-hatch-lock-test-"));
|
|
21
|
+
tempDirs.push(tempDir);
|
|
22
|
+
return join(tempDir, "local-hatch.lock");
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
describe("withLocalHatchLock", () => {
|
|
26
|
+
test("uses the same machine lock when callers have different temp directories", () => {
|
|
27
|
+
const originalTmpDir = process.env.TMPDIR;
|
|
28
|
+
const firstTmpDir = mkdtempSync(join(tmpdir(), "vellum-tmp-a-"));
|
|
29
|
+
const secondTmpDir = mkdtempSync(join(tmpdir(), "vellum-tmp-b-"));
|
|
30
|
+
tempDirs.push(firstTmpDir, secondTmpDir);
|
|
31
|
+
|
|
32
|
+
try {
|
|
33
|
+
process.env.TMPDIR = firstTmpDir;
|
|
34
|
+
const firstPath = resolveLocalHatchLockPath();
|
|
35
|
+
process.env.TMPDIR = secondTmpDir;
|
|
36
|
+
const secondPath = resolveLocalHatchLockPath();
|
|
37
|
+
|
|
38
|
+
expect(secondPath).toBe(firstPath);
|
|
39
|
+
expect(firstPath.startsWith(firstTmpDir)).toBe(false);
|
|
40
|
+
expect(firstPath.startsWith(secondTmpDir)).toBe(false);
|
|
41
|
+
} finally {
|
|
42
|
+
if (originalTmpDir === undefined) {
|
|
43
|
+
delete process.env.TMPDIR;
|
|
44
|
+
} else {
|
|
45
|
+
process.env.TMPDIR = originalTmpDir;
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
test("serializes startup work that shares the machine-wide lock", async () => {
|
|
51
|
+
const lockPath = makeLockPath();
|
|
52
|
+
let releaseFirst!: () => void;
|
|
53
|
+
let firstStarted!: () => void;
|
|
54
|
+
const firstStartedPromise = new Promise<void>((resolve) => {
|
|
55
|
+
firstStarted = resolve;
|
|
56
|
+
});
|
|
57
|
+
const firstReleasePromise = new Promise<void>((resolve) => {
|
|
58
|
+
releaseFirst = resolve;
|
|
59
|
+
});
|
|
60
|
+
let secondEntered = false;
|
|
61
|
+
|
|
62
|
+
const first = withLocalHatchLock(
|
|
63
|
+
async () => {
|
|
64
|
+
firstStarted();
|
|
65
|
+
await firstReleasePromise;
|
|
66
|
+
},
|
|
67
|
+
{ lockPath, timeoutMs: 1_000, retryMs: 5 },
|
|
68
|
+
);
|
|
69
|
+
await firstStartedPromise;
|
|
70
|
+
|
|
71
|
+
const second = withLocalHatchLock(
|
|
72
|
+
async () => {
|
|
73
|
+
secondEntered = true;
|
|
74
|
+
},
|
|
75
|
+
{ lockPath, timeoutMs: 1_000, retryMs: 5 },
|
|
76
|
+
);
|
|
77
|
+
await Bun.sleep(20);
|
|
78
|
+
expect(secondEntered).toBe(false);
|
|
79
|
+
|
|
80
|
+
releaseFirst();
|
|
81
|
+
await Promise.all([first, second]);
|
|
82
|
+
|
|
83
|
+
expect(secondEntered).toBe(true);
|
|
84
|
+
expect(existsSync(lockPath)).toBe(false);
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("recovers a lock owned by a dead process", async () => {
|
|
88
|
+
const lockPath = makeLockPath();
|
|
89
|
+
writeFileSync(
|
|
90
|
+
lockPath,
|
|
91
|
+
JSON.stringify({ pid: 2_147_483_647, token: "stale-owner" }) + "\n",
|
|
92
|
+
);
|
|
93
|
+
|
|
94
|
+
const result = await withLocalHatchLock(async () => "started", {
|
|
95
|
+
lockPath,
|
|
96
|
+
timeoutMs: 1_000,
|
|
97
|
+
retryMs: 5,
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
expect(result).toBe("started");
|
|
101
|
+
expect(existsSync(lockPath)).toBe(false);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("recovers an expired lock even when its PID has been reused", async () => {
|
|
105
|
+
const lockPath = makeLockPath();
|
|
106
|
+
writeFileSync(
|
|
107
|
+
lockPath,
|
|
108
|
+
JSON.stringify({
|
|
109
|
+
pid: process.pid,
|
|
110
|
+
processStartedAt: "not-the-current-process",
|
|
111
|
+
token: "reused-pid-owner",
|
|
112
|
+
}) + "\n",
|
|
113
|
+
);
|
|
114
|
+
const expired = new Date(Date.now() - 300_000);
|
|
115
|
+
utimesSync(lockPath, expired, expired);
|
|
116
|
+
|
|
117
|
+
const result = await withLocalHatchLock(async () => "started", {
|
|
118
|
+
lockPath,
|
|
119
|
+
liveOwnerRecoveryGraceMs: 0,
|
|
120
|
+
timeoutMs: 1_000,
|
|
121
|
+
retryMs: 5,
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
expect(result).toBe("started");
|
|
125
|
+
expect(existsSync(lockPath)).toBe(false);
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test("recovers a fresh lock when its PID has been reused", async () => {
|
|
129
|
+
const lockPath = makeLockPath();
|
|
130
|
+
writeFileSync(
|
|
131
|
+
lockPath,
|
|
132
|
+
JSON.stringify({
|
|
133
|
+
pid: process.pid,
|
|
134
|
+
processStartedAt: "not-the-current-process",
|
|
135
|
+
token: "fresh-reused-pid-owner",
|
|
136
|
+
}) + "\n",
|
|
137
|
+
);
|
|
138
|
+
|
|
139
|
+
const result = await withLocalHatchLock(async () => "started", {
|
|
140
|
+
lockPath,
|
|
141
|
+
timeoutMs: 1_000,
|
|
142
|
+
retryMs: 5,
|
|
143
|
+
});
|
|
144
|
+
|
|
145
|
+
expect(result).toBe("started");
|
|
146
|
+
expect(existsSync(lockPath)).toBe(false);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("releases the lock when startup fails", async () => {
|
|
150
|
+
const lockPath = makeLockPath();
|
|
151
|
+
|
|
152
|
+
await expect(
|
|
153
|
+
withLocalHatchLock(
|
|
154
|
+
async () => {
|
|
155
|
+
throw new Error("startup failed");
|
|
156
|
+
},
|
|
157
|
+
{ lockPath },
|
|
158
|
+
),
|
|
159
|
+
).rejects.toThrow("startup failed");
|
|
160
|
+
|
|
161
|
+
expect(existsSync(lockPath)).toBe(false);
|
|
162
|
+
});
|
|
163
|
+
});
|
package/src/lib/hatch-local.ts
CHANGED
|
@@ -46,6 +46,7 @@ import {
|
|
|
46
46
|
import { logHatchNextSteps } from "./hatch-next-steps.js";
|
|
47
47
|
import { checkProviderApiKey } from "./api-key-check.js";
|
|
48
48
|
import { loopbackSafeFetch } from "./loopback-fetch.js";
|
|
49
|
+
import { withLocalHatchLock } from "./local-hatch-lock.js";
|
|
49
50
|
|
|
50
51
|
/**
|
|
51
52
|
* Attempts to place a symlink at the given path pointing to cliBinary.
|
|
@@ -179,89 +180,116 @@ export async function hatchLocal(
|
|
|
179
180
|
name ?? process.env.VELLUM_ASSISTANT_NAME,
|
|
180
181
|
);
|
|
181
182
|
|
|
182
|
-
|
|
183
|
+
const { resources, runtimeUrl, loopbackUrl, bootstrapSecret } =
|
|
184
|
+
await withLocalHatchLock(async () => {
|
|
185
|
+
reporter.progress(1, 6, "Allocating resources...");
|
|
183
186
|
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
187
|
+
const existing = findAssistantByName(instanceName);
|
|
188
|
+
if (existing && (!existing.cloud || existing.cloud === "local")) {
|
|
189
|
+
throw new Error(
|
|
190
|
+
`An assistant named "${instanceName}" is already hatched.\n` +
|
|
191
|
+
`Run \`vellum wake\` to restart it, or \`vellum retire ${instanceName}\` to remove it first.`,
|
|
192
|
+
);
|
|
193
|
+
}
|
|
191
194
|
|
|
192
|
-
|
|
195
|
+
const resources = await allocateLocalResources(instanceName);
|
|
193
196
|
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
197
|
+
const logsDir = join(
|
|
198
|
+
resources.instanceDir,
|
|
199
|
+
".vellum",
|
|
200
|
+
"workspace",
|
|
201
|
+
"data",
|
|
202
|
+
"logs",
|
|
203
|
+
);
|
|
204
|
+
archiveLogFile("hatch.log", logsDir);
|
|
205
|
+
resetLogFile("hatch.log");
|
|
203
206
|
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
+
reporter.log(`🥚 Hatching local assistant: ${instanceName}`);
|
|
208
|
+
reporter.log(` Species: ${species}`);
|
|
209
|
+
reporter.log("");
|
|
207
210
|
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
211
|
+
const apiKeyCheck = checkProviderApiKey();
|
|
212
|
+
if (!apiKeyCheck.hasKey) {
|
|
213
|
+
reporter.warn(
|
|
214
|
+
"Warning: No LLM provider API key is configured. The assistant will fail when you try to send a message.",
|
|
215
|
+
);
|
|
216
|
+
reporter.warn(" To fix, export your key before running vellum hatch:");
|
|
217
|
+
reporter.warn(" export ANTHROPIC_API_KEY=<your-key>");
|
|
218
|
+
reporter.warn("");
|
|
219
|
+
}
|
|
217
220
|
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
+
if (!process.env.APP_VERSION) {
|
|
222
|
+
process.env.APP_VERSION = cliPkg.version;
|
|
223
|
+
}
|
|
221
224
|
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
225
|
+
reporter.progress(2, 6, "Writing configuration...");
|
|
226
|
+
const hatchConfigValues = buildHatchConfigValues(configValues, provider);
|
|
227
|
+
const defaultWorkspaceConfigPath = writeInitialConfig(hatchConfigValues);
|
|
228
|
+
|
|
229
|
+
reporter.progress(3, 6, "Starting assistant...");
|
|
230
|
+
const signingKey = generateLocalSigningKey();
|
|
231
|
+
const bootstrapSecret = generateLocalSigningKey();
|
|
232
|
+
let runtimeUrl = `http://127.0.0.1:${resources.gatewayPort}`;
|
|
233
|
+
try {
|
|
234
|
+
// Launch the CES sibling alongside the daemon, in parallel — matching the
|
|
235
|
+
// Docker topology. The assistant does not spawn its own CES, so a freshly
|
|
236
|
+
// hatched instance would otherwise come up with CES unavailable.
|
|
237
|
+
// startCes always launches the CES sibling.
|
|
238
|
+
const startupResults = await Promise.allSettled([
|
|
239
|
+
startCes(watch, resources),
|
|
240
|
+
startLocalDaemon(watch, resources, {
|
|
241
|
+
defaultWorkspaceConfigPath,
|
|
242
|
+
requireReady: true,
|
|
243
|
+
signingKey,
|
|
244
|
+
}),
|
|
245
|
+
]);
|
|
246
|
+
const startupFailure = startupResults.find(
|
|
247
|
+
(result): result is PromiseRejectedResult =>
|
|
248
|
+
result.status === "rejected",
|
|
249
|
+
);
|
|
250
|
+
if (startupFailure) {
|
|
251
|
+
throw startupFailure.reason;
|
|
252
|
+
}
|
|
253
|
+
|
|
254
|
+
reporter.progress(4, 6, "Starting gateway...");
|
|
255
|
+
runtimeUrl = await startGateway(watch, resources, {
|
|
256
|
+
signingKey,
|
|
257
|
+
bootstrapSecret,
|
|
258
|
+
envOverrides: flagEnvVars,
|
|
259
|
+
requireReady: true,
|
|
260
|
+
});
|
|
261
|
+
} catch (error) {
|
|
262
|
+
reporter.error(
|
|
263
|
+
`\n❌ Local assistant startup failed — stopping partial processes.`,
|
|
264
|
+
);
|
|
265
|
+
await stopLocalProcesses(resources);
|
|
266
|
+
throw error;
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
const loopbackUrl = `http://127.0.0.1:${resources.gatewayPort}`;
|
|
270
|
+
const localEntry: AssistantEntry = {
|
|
271
|
+
assistantId: instanceName,
|
|
272
|
+
runtimeUrl,
|
|
273
|
+
localUrl: `http://127.0.0.1:${resources.gatewayPort}`,
|
|
274
|
+
cloud: "local",
|
|
275
|
+
species,
|
|
276
|
+
hatchedAt: new Date().toISOString(),
|
|
277
|
+
resources: { ...resources, signingKey },
|
|
278
|
+
guardianBootstrapSecret: bootstrapSecret,
|
|
279
|
+
};
|
|
280
|
+
|
|
281
|
+
reporter.progress(5, 6, "Saving configuration...");
|
|
282
|
+
saveAssistantEntry(localEntry);
|
|
283
|
+
setActiveAssistant(instanceName);
|
|
284
|
+
|
|
285
|
+
return { resources, runtimeUrl, loopbackUrl, bootstrapSecret };
|
|
248
286
|
});
|
|
249
|
-
} catch (error) {
|
|
250
|
-
// Gateway failed — stop the daemon we just started so we don't leave
|
|
251
|
-
// orphaned processes with no lock file entry.
|
|
252
|
-
reporter.error(
|
|
253
|
-
`\n❌ Gateway startup failed — stopping assistant to avoid orphaned processes.`,
|
|
254
|
-
);
|
|
255
|
-
await stopLocalProcesses(resources);
|
|
256
|
-
throw error;
|
|
257
|
-
}
|
|
258
287
|
|
|
259
288
|
// Lease a guardian token so the desktop app can import it on first launch
|
|
260
289
|
// instead of hitting /v1/guardian/init itself. Use loopback to satisfy
|
|
261
290
|
// the daemon's local-only check — the mDNS runtimeUrl resolves to a LAN
|
|
262
291
|
// IP which the daemon rejects as non-loopback.
|
|
263
|
-
reporter.progress(
|
|
264
|
-
const loopbackUrl = `http://127.0.0.1:${resources.gatewayPort}`;
|
|
292
|
+
reporter.progress(6, 6, "Securing connection...");
|
|
265
293
|
const maxLeaseAttempts = 3;
|
|
266
294
|
let guardianAccessToken: string | undefined;
|
|
267
295
|
for (let attempt = 1; attempt <= maxLeaseAttempts; attempt++) {
|
|
@@ -290,22 +318,6 @@ export async function hatchLocal(
|
|
|
290
318
|
}
|
|
291
319
|
}
|
|
292
320
|
|
|
293
|
-
// Auto-start ngrok if webhook integrations (e.g. Telegram, Twilio) are configured.
|
|
294
|
-
const localEntry: AssistantEntry = {
|
|
295
|
-
assistantId: instanceName,
|
|
296
|
-
runtimeUrl,
|
|
297
|
-
localUrl: `http://127.0.0.1:${resources.gatewayPort}`,
|
|
298
|
-
cloud: "local",
|
|
299
|
-
species,
|
|
300
|
-
hatchedAt: new Date().toISOString(),
|
|
301
|
-
resources: { ...resources, signingKey },
|
|
302
|
-
guardianBootstrapSecret: bootstrapSecret,
|
|
303
|
-
};
|
|
304
|
-
|
|
305
|
-
reporter.progress(6, 6, "Saving configuration...");
|
|
306
|
-
saveAssistantEntry(localEntry);
|
|
307
|
-
setActiveAssistant(instanceName);
|
|
308
|
-
|
|
309
321
|
if (process.env.VELLUM_DESKTOP_APP) {
|
|
310
322
|
installCLISymlink();
|
|
311
323
|
}
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
import { execFileSync } from "child_process";
|
|
2
|
+
import { createHash, randomUUID } from "crypto";
|
|
3
|
+
import {
|
|
4
|
+
closeSync,
|
|
5
|
+
mkdirSync,
|
|
6
|
+
openSync,
|
|
7
|
+
readFileSync,
|
|
8
|
+
statSync,
|
|
9
|
+
unlinkSync,
|
|
10
|
+
utimesSync,
|
|
11
|
+
writeFileSync,
|
|
12
|
+
} from "fs";
|
|
13
|
+
import { hostname, userInfo } from "os";
|
|
14
|
+
import { dirname, join } from "path";
|
|
15
|
+
|
|
16
|
+
const DEFAULT_TIMEOUT_MS = 240_000;
|
|
17
|
+
const DEFAULT_RETRY_MS = 100;
|
|
18
|
+
const MAX_LOCK_AGE_MS = DEFAULT_TIMEOUT_MS - 60_000;
|
|
19
|
+
const HEARTBEAT_INTERVAL_MS = 30_000;
|
|
20
|
+
const LIVE_OWNER_RECOVERY_GRACE_MS = 10_000;
|
|
21
|
+
|
|
22
|
+
interface LocalHatchLockOptions {
|
|
23
|
+
lockPath?: string;
|
|
24
|
+
liveOwnerRecoveryGraceMs?: number;
|
|
25
|
+
timeoutMs?: number;
|
|
26
|
+
retryMs?: number;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
interface LockOwner {
|
|
30
|
+
pid: number;
|
|
31
|
+
processStartedAt?: string;
|
|
32
|
+
token: string;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
interface LockSnapshot {
|
|
36
|
+
owner?: LockOwner;
|
|
37
|
+
signature: string;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
export function resolveLocalHatchLockPath(): string {
|
|
41
|
+
const machineKey = createHash("sha256")
|
|
42
|
+
.update(hostname())
|
|
43
|
+
.digest("hex")
|
|
44
|
+
.slice(0, 12);
|
|
45
|
+
return join(
|
|
46
|
+
userInfo().homedir,
|
|
47
|
+
".cache",
|
|
48
|
+
"vellum",
|
|
49
|
+
"runtime",
|
|
50
|
+
machineKey,
|
|
51
|
+
"local-hatch.lock",
|
|
52
|
+
);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function processIsAlive(pid: number): boolean {
|
|
56
|
+
try {
|
|
57
|
+
process.kill(pid, 0);
|
|
58
|
+
return true;
|
|
59
|
+
} catch (error) {
|
|
60
|
+
return (error as NodeJS.ErrnoException).code === "EPERM";
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
function processStartedAt(pid: number): string | undefined {
|
|
65
|
+
try {
|
|
66
|
+
const value = execFileSync("ps", ["-o", "lstart=", "-p", String(pid)], {
|
|
67
|
+
encoding: "utf-8",
|
|
68
|
+
stdio: ["ignore", "pipe", "ignore"],
|
|
69
|
+
timeout: 1_000,
|
|
70
|
+
}).trim();
|
|
71
|
+
return value || undefined;
|
|
72
|
+
} catch {
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function parseOwner(raw: string): LockOwner | undefined {
|
|
78
|
+
try {
|
|
79
|
+
const parsed = JSON.parse(raw) as {
|
|
80
|
+
pid?: unknown;
|
|
81
|
+
processStartedAt?: unknown;
|
|
82
|
+
token?: unknown;
|
|
83
|
+
};
|
|
84
|
+
if (
|
|
85
|
+
typeof parsed.pid === "number" &&
|
|
86
|
+
Number.isInteger(parsed.pid) &&
|
|
87
|
+
parsed.pid > 0 &&
|
|
88
|
+
typeof parsed.token === "string" &&
|
|
89
|
+
parsed.token.length > 0
|
|
90
|
+
) {
|
|
91
|
+
return {
|
|
92
|
+
pid: parsed.pid,
|
|
93
|
+
processStartedAt:
|
|
94
|
+
typeof parsed.processStartedAt === "string" &&
|
|
95
|
+
parsed.processStartedAt.length > 0
|
|
96
|
+
? parsed.processStartedAt
|
|
97
|
+
: undefined,
|
|
98
|
+
token: parsed.token,
|
|
99
|
+
};
|
|
100
|
+
}
|
|
101
|
+
} catch {}
|
|
102
|
+
return undefined;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
function readSnapshot(path: string): LockSnapshot | undefined {
|
|
106
|
+
try {
|
|
107
|
+
const raw = readFileSync(path, "utf-8");
|
|
108
|
+
return {
|
|
109
|
+
owner: parseOwner(raw),
|
|
110
|
+
signature: createHash("sha256").update(raw).digest("hex"),
|
|
111
|
+
};
|
|
112
|
+
} catch {
|
|
113
|
+
return undefined;
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function readOwner(path: string): LockOwner | undefined {
|
|
118
|
+
return readSnapshot(path)?.owner;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
function fileAgeMs(path: string): number | undefined {
|
|
122
|
+
try {
|
|
123
|
+
return Date.now() - statSync(path).mtimeMs;
|
|
124
|
+
} catch {
|
|
125
|
+
return undefined;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
const expiredLiveOwners = new Map<string, number>();
|
|
130
|
+
|
|
131
|
+
function ownerIsActive(
|
|
132
|
+
path: string,
|
|
133
|
+
owner: LockOwner | undefined,
|
|
134
|
+
liveOwnerRecoveryGraceMs: number,
|
|
135
|
+
): boolean {
|
|
136
|
+
const ageMs = fileAgeMs(path);
|
|
137
|
+
if (!owner) {
|
|
138
|
+
return ageMs !== undefined && ageMs < MAX_LOCK_AGE_MS;
|
|
139
|
+
}
|
|
140
|
+
if (!processIsAlive(owner.pid)) {
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
if (owner.processStartedAt) {
|
|
144
|
+
const currentStartedAt = processStartedAt(owner.pid);
|
|
145
|
+
if (currentStartedAt) {
|
|
146
|
+
expiredLiveOwners.delete(owner.token);
|
|
147
|
+
return currentStartedAt === owner.processStartedAt;
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
if (ageMs !== undefined && ageMs < MAX_LOCK_AGE_MS) {
|
|
151
|
+
expiredLiveOwners.delete(owner.token);
|
|
152
|
+
return true;
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
const firstExpiredAt = expiredLiveOwners.get(owner.token) ?? Date.now();
|
|
156
|
+
expiredLiveOwners.set(owner.token, firstExpiredAt);
|
|
157
|
+
if (Date.now() - firstExpiredAt < liveOwnerRecoveryGraceMs) {
|
|
158
|
+
return true;
|
|
159
|
+
}
|
|
160
|
+
return false;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
function writeOwnerFile(path: string, owner: LockOwner): void {
|
|
164
|
+
const fd = openSync(path, "wx", 0o600);
|
|
165
|
+
try {
|
|
166
|
+
writeFileSync(fd, JSON.stringify(owner) + "\n");
|
|
167
|
+
} finally {
|
|
168
|
+
closeSync(fd);
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
function acquireRecoveryClaim(
|
|
173
|
+
lockPath: string,
|
|
174
|
+
staleSignature: string,
|
|
175
|
+
owner: LockOwner,
|
|
176
|
+
liveOwnerRecoveryGraceMs: number,
|
|
177
|
+
): (() => void) | undefined {
|
|
178
|
+
const recoveryKey = staleSignature.slice(0, 16);
|
|
179
|
+
for (let generation = 0; generation < 100; generation++) {
|
|
180
|
+
const claimPath = `${lockPath}.recovery-${recoveryKey}-${generation}`;
|
|
181
|
+
try {
|
|
182
|
+
writeOwnerFile(claimPath, owner);
|
|
183
|
+
return () => releaseLock(claimPath, owner.token);
|
|
184
|
+
} catch (error) {
|
|
185
|
+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
|
|
186
|
+
throw error;
|
|
187
|
+
}
|
|
188
|
+
if (
|
|
189
|
+
ownerIsActive(claimPath, readOwner(claimPath), liveOwnerRecoveryGraceMs)
|
|
190
|
+
) {
|
|
191
|
+
return undefined;
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
return undefined;
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
function removeStaleLock(
|
|
199
|
+
lockPath: string,
|
|
200
|
+
recoveryOwner: LockOwner,
|
|
201
|
+
liveOwnerRecoveryGraceMs: number,
|
|
202
|
+
): boolean {
|
|
203
|
+
const staleSnapshot = readSnapshot(lockPath);
|
|
204
|
+
if (!staleSnapshot) {
|
|
205
|
+
return true;
|
|
206
|
+
}
|
|
207
|
+
if (ownerIsActive(lockPath, staleSnapshot.owner, liveOwnerRecoveryGraceMs)) {
|
|
208
|
+
return false;
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
const releaseRecoveryClaim = acquireRecoveryClaim(
|
|
212
|
+
lockPath,
|
|
213
|
+
staleSnapshot.signature,
|
|
214
|
+
recoveryOwner,
|
|
215
|
+
liveOwnerRecoveryGraceMs,
|
|
216
|
+
);
|
|
217
|
+
if (!releaseRecoveryClaim) {
|
|
218
|
+
return false;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
try {
|
|
222
|
+
const currentSnapshot = readSnapshot(lockPath);
|
|
223
|
+
if (!currentSnapshot) {
|
|
224
|
+
return true;
|
|
225
|
+
}
|
|
226
|
+
if (currentSnapshot.signature !== staleSnapshot.signature) {
|
|
227
|
+
return false;
|
|
228
|
+
}
|
|
229
|
+
if (
|
|
230
|
+
ownerIsActive(lockPath, currentSnapshot.owner, liveOwnerRecoveryGraceMs)
|
|
231
|
+
) {
|
|
232
|
+
return false;
|
|
233
|
+
}
|
|
234
|
+
try {
|
|
235
|
+
unlinkSync(lockPath);
|
|
236
|
+
return true;
|
|
237
|
+
} catch (error) {
|
|
238
|
+
return (error as NodeJS.ErrnoException).code === "ENOENT";
|
|
239
|
+
}
|
|
240
|
+
} finally {
|
|
241
|
+
releaseRecoveryClaim();
|
|
242
|
+
}
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
function releaseLock(lockPath: string, token: string): void {
|
|
246
|
+
if (readOwner(lockPath)?.token !== token) {
|
|
247
|
+
return;
|
|
248
|
+
}
|
|
249
|
+
try {
|
|
250
|
+
unlinkSync(lockPath);
|
|
251
|
+
} catch (error) {
|
|
252
|
+
if ((error as NodeJS.ErrnoException).code !== "ENOENT") {
|
|
253
|
+
throw error;
|
|
254
|
+
}
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
|
|
258
|
+
function refreshLock(lockPath: string, token: string): void {
|
|
259
|
+
if (readOwner(lockPath)?.token !== token) {
|
|
260
|
+
return;
|
|
261
|
+
}
|
|
262
|
+
try {
|
|
263
|
+
const now = new Date();
|
|
264
|
+
utimesSync(lockPath, now, now);
|
|
265
|
+
} catch {}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Serialize local assistant startup across CLI processes. Port availability is
|
|
270
|
+
* machine-wide even when callers use different lockfile or XDG directories,
|
|
271
|
+
* so allocation must remain exclusive until the new services have bound their
|
|
272
|
+
* ports and the assistant entry has been persisted.
|
|
273
|
+
*/
|
|
274
|
+
export async function withLocalHatchLock<T>(
|
|
275
|
+
action: () => Promise<T>,
|
|
276
|
+
options: LocalHatchLockOptions = {},
|
|
277
|
+
): Promise<T> {
|
|
278
|
+
const lockPath = options.lockPath ?? resolveLocalHatchLockPath();
|
|
279
|
+
const timeoutMs = options.timeoutMs ?? DEFAULT_TIMEOUT_MS;
|
|
280
|
+
const retryMs = options.retryMs ?? DEFAULT_RETRY_MS;
|
|
281
|
+
const liveOwnerRecoveryGraceMs =
|
|
282
|
+
options.liveOwnerRecoveryGraceMs ?? LIVE_OWNER_RECOVERY_GRACE_MS;
|
|
283
|
+
const deadline = Date.now() + timeoutMs;
|
|
284
|
+
const owner: LockOwner = {
|
|
285
|
+
pid: process.pid,
|
|
286
|
+
processStartedAt: processStartedAt(process.pid),
|
|
287
|
+
token: `${process.pid}-${randomUUID()}`,
|
|
288
|
+
};
|
|
289
|
+
|
|
290
|
+
mkdirSync(dirname(lockPath), { recursive: true, mode: 0o700 });
|
|
291
|
+
|
|
292
|
+
while (true) {
|
|
293
|
+
try {
|
|
294
|
+
writeOwnerFile(lockPath, owner);
|
|
295
|
+
} catch (error) {
|
|
296
|
+
if ((error as NodeJS.ErrnoException).code !== "EEXIST") {
|
|
297
|
+
throw error;
|
|
298
|
+
}
|
|
299
|
+
if (removeStaleLock(lockPath, owner, liveOwnerRecoveryGraceMs)) {
|
|
300
|
+
continue;
|
|
301
|
+
}
|
|
302
|
+
if (Date.now() >= deadline) {
|
|
303
|
+
throw new Error(
|
|
304
|
+
`Timed out waiting for another local assistant to finish starting (${lockPath})`,
|
|
305
|
+
);
|
|
306
|
+
}
|
|
307
|
+
await new Promise((resolve) => setTimeout(resolve, retryMs));
|
|
308
|
+
continue;
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
const heartbeat = setInterval(
|
|
312
|
+
() => refreshLock(lockPath, owner.token),
|
|
313
|
+
HEARTBEAT_INTERVAL_MS,
|
|
314
|
+
);
|
|
315
|
+
try {
|
|
316
|
+
return await action();
|
|
317
|
+
} finally {
|
|
318
|
+
clearInterval(heartbeat);
|
|
319
|
+
releaseLock(lockPath, owner.token);
|
|
320
|
+
}
|
|
321
|
+
}
|
|
322
|
+
}
|
package/src/lib/local.ts
CHANGED
|
@@ -548,6 +548,7 @@ export function generateLocalSigningKey(): string {
|
|
|
548
548
|
type DaemonStartOptions = {
|
|
549
549
|
foreground?: boolean;
|
|
550
550
|
defaultWorkspaceConfigPath?: string;
|
|
551
|
+
requireReady?: boolean;
|
|
551
552
|
signingKey?: string;
|
|
552
553
|
};
|
|
553
554
|
|
|
@@ -597,7 +598,10 @@ function applyDaemonEnvOverrides(
|
|
|
597
598
|
applyIpcSocketDirOverride(env);
|
|
598
599
|
}
|
|
599
600
|
|
|
600
|
-
function logDaemonReadiness(
|
|
601
|
+
function logDaemonReadiness(
|
|
602
|
+
readiness: DaemonReadiness,
|
|
603
|
+
requireReady = false,
|
|
604
|
+
): void {
|
|
601
605
|
switch (readiness) {
|
|
602
606
|
case "ready":
|
|
603
607
|
console.log(" Assistant ready\n");
|
|
@@ -613,6 +617,11 @@ function logDaemonReadiness(readiness: DaemonReadiness): void {
|
|
|
613
617
|
);
|
|
614
618
|
break;
|
|
615
619
|
default:
|
|
620
|
+
if (requireReady) {
|
|
621
|
+
throw new Error(
|
|
622
|
+
"Assistant did not bind its local port within 60 seconds.",
|
|
623
|
+
);
|
|
624
|
+
}
|
|
616
625
|
console.log(
|
|
617
626
|
" ⚠️ Assistant did not become ready within 60s — continuing anyway\n",
|
|
618
627
|
);
|
|
@@ -1320,6 +1329,7 @@ export async function startLocalDaemon(
|
|
|
1320
1329
|
resources.daemonPort,
|
|
1321
1330
|
Date.now() + 60000,
|
|
1322
1331
|
),
|
|
1332
|
+
options?.requireReady,
|
|
1323
1333
|
);
|
|
1324
1334
|
}
|
|
1325
1335
|
return;
|
|
@@ -1362,6 +1372,7 @@ export async function startLocalDaemon(
|
|
|
1362
1372
|
// classifies it without blocking on an in-flight migration.
|
|
1363
1373
|
logDaemonReadiness(
|
|
1364
1374
|
await probeDaemonReadinessWithRetry(resources.daemonPort),
|
|
1375
|
+
options?.requireReady,
|
|
1365
1376
|
);
|
|
1366
1377
|
return;
|
|
1367
1378
|
}
|
|
@@ -1513,7 +1524,7 @@ export async function startLocalDaemon(
|
|
|
1513
1524
|
readiness = await probeDaemonReadiness(resources.daemonPort);
|
|
1514
1525
|
}
|
|
1515
1526
|
|
|
1516
|
-
logDaemonReadiness(readiness);
|
|
1527
|
+
logDaemonReadiness(readiness, options?.requireReady);
|
|
1517
1528
|
}
|
|
1518
1529
|
} else {
|
|
1519
1530
|
console.log("🔨 Starting local assistant...");
|
|
@@ -1535,6 +1546,7 @@ export async function startLocalDaemon(
|
|
|
1535
1546
|
resources.daemonPort,
|
|
1536
1547
|
Date.now() + 60000,
|
|
1537
1548
|
),
|
|
1549
|
+
options?.requireReady,
|
|
1538
1550
|
);
|
|
1539
1551
|
}
|
|
1540
1552
|
}
|
|
@@ -1547,6 +1559,7 @@ export async function startGateway(
|
|
|
1547
1559
|
signingKey?: string;
|
|
1548
1560
|
bootstrapSecret?: string;
|
|
1549
1561
|
envOverrides?: Record<string, string>;
|
|
1562
|
+
requireReady?: boolean;
|
|
1550
1563
|
},
|
|
1551
1564
|
): Promise<string> {
|
|
1552
1565
|
const effectiveGatewayPort = resources?.gatewayPort ?? GATEWAY_PORT;
|
|
@@ -1676,6 +1689,11 @@ export async function startGateway(
|
|
|
1676
1689
|
// connection-refused errors.
|
|
1677
1690
|
const ready = await waitForDaemonReady(effectiveGatewayPort, 30000);
|
|
1678
1691
|
if (!ready) {
|
|
1692
|
+
if (options?.requireReady) {
|
|
1693
|
+
throw new Error(
|
|
1694
|
+
"Assistant gateway did not bind its local port within 30 seconds.",
|
|
1695
|
+
);
|
|
1696
|
+
}
|
|
1679
1697
|
console.warn(
|
|
1680
1698
|
"⚠ Gateway started but health check did not respond within 30s",
|
|
1681
1699
|
);
|