@vellumai/assistant 0.12.2-dev.202609171418.b9a2adf → 0.12.2-dev.202609171516.2acdfd4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/config-watcher.test.ts +24 -0
- package/src/__tests__/credential-execution-client.test.ts +5 -5
- package/src/config/schemas/__tests__/tools.test.ts +31 -0
- package/src/config/schemas/mcp.ts +2 -1
- package/src/config/schemas/tools.ts +10 -0
- package/src/config/skills.ts +2 -6
- package/src/credential-execution/executable-discovery.ts +1 -11
- package/src/daemon/config-watcher.ts +10 -0
- package/src/mcp/__tests__/manager-tool-caps.test.ts +33 -0
- package/src/mcp/__tests__/tool-caps.test.ts +28 -1
- package/src/mcp/manager.ts +3 -0
- package/src/mcp/tool-caps.ts +12 -4
- package/src/monitoring/__tests__/control.test.ts +17 -5
- package/src/monitoring/control.ts +15 -5
- package/src/plugins/defaults/memory/__tests__/worker-control.test.ts +19 -6
- package/src/plugins/defaults/memory/worker-control.ts +15 -5
- package/src/routes/control.ts +7 -5
- package/src/schedule/__tests__/worker-control.test.ts +36 -5
- package/src/schedule/worker-control.ts +15 -5
- package/src/skills/clawhub.ts +3 -8
- package/src/skills/managed-store.ts +1 -5
- package/src/util/__tests__/worker-process-command.test.ts +5 -4
- package/src/util/worker-process.ts +77 -35
package/package.json
CHANGED
|
@@ -315,6 +315,30 @@ describe("ConfigWatcher workspace file handlers", () => {
|
|
|
315
315
|
expect(evictCallCount).toBe(1);
|
|
316
316
|
});
|
|
317
317
|
|
|
318
|
+
test("config.json change reloads MCP when tools.mcpGlobalMaxTools changes", async () => {
|
|
319
|
+
watcher.initFingerprint({
|
|
320
|
+
tools: { exclude: [], mcpGlobalMaxTools: 50 },
|
|
321
|
+
} as never);
|
|
322
|
+
watcher.refreshConfigFromSources = async () => true;
|
|
323
|
+
watcher.start();
|
|
324
|
+
simulateFileChange(WORKSPACE_DIR, "config.json");
|
|
325
|
+
await new Promise((r) => setTimeout(r, WAIT_MS));
|
|
326
|
+
expect(mcpReloadCallCount).toBe(1);
|
|
327
|
+
expect(evictCallCount).toBe(1);
|
|
328
|
+
});
|
|
329
|
+
|
|
330
|
+
test("config.json change does not reload MCP when the global max is unchanged", async () => {
|
|
331
|
+
watcher.initFingerprint({
|
|
332
|
+
tools: { exclude: [] },
|
|
333
|
+
} as never);
|
|
334
|
+
watcher.refreshConfigFromSources = async () => true;
|
|
335
|
+
watcher.start();
|
|
336
|
+
simulateFileChange(WORKSPACE_DIR, "config.json");
|
|
337
|
+
await new Promise((r) => setTimeout(r, WAIT_MS));
|
|
338
|
+
expect(mcpReloadCallCount).toBe(0);
|
|
339
|
+
expect(evictCallCount).toBe(1);
|
|
340
|
+
});
|
|
341
|
+
|
|
318
342
|
test("config.json change is suppressed when suppressConfigReload is true", async () => {
|
|
319
343
|
let refreshCalled = false;
|
|
320
344
|
watcher.refreshConfigFromSources = async () => {
|
|
@@ -31,8 +31,8 @@ import {
|
|
|
31
31
|
createCesClient,
|
|
32
32
|
} from "../credential-execution/client.js";
|
|
33
33
|
import {
|
|
34
|
+
discoverCes,
|
|
34
35
|
discoverCesWithRetry,
|
|
35
|
-
discoverManagedCes,
|
|
36
36
|
} from "../credential-execution/executable-discovery.js";
|
|
37
37
|
|
|
38
38
|
// ---------------------------------------------------------------------------
|
|
@@ -100,7 +100,7 @@ describe("managed CES discovery", () => {
|
|
|
100
100
|
const bootstrapDir = mkdtempSync(join(tmpdir(), "ces-missing-"));
|
|
101
101
|
const restore = withBootstrapDir(bootstrapDir);
|
|
102
102
|
try {
|
|
103
|
-
const result =
|
|
103
|
+
const result = discoverCes();
|
|
104
104
|
expect(result.mode).toBe("unavailable");
|
|
105
105
|
expect((result as { reason: string }).reason).toContain(
|
|
106
106
|
"CES bootstrap socket not found",
|
|
@@ -115,7 +115,7 @@ describe("managed CES discovery", () => {
|
|
|
115
115
|
const bootstrapDir = mkdtempSync(join(tmpdir(), "ces-missing-"));
|
|
116
116
|
const restore = withBootstrapDir(bootstrapDir);
|
|
117
117
|
try {
|
|
118
|
-
const result =
|
|
118
|
+
const result = discoverCes();
|
|
119
119
|
expect(["managed", "unavailable"]).toContain(result.mode);
|
|
120
120
|
} finally {
|
|
121
121
|
restore();
|
|
@@ -132,7 +132,7 @@ describe("CES bootstrap socket discovery", () => {
|
|
|
132
132
|
const socketPath = resolveIpcEndpoint("ces", {
|
|
133
133
|
workspaceDir: bootstrapDir,
|
|
134
134
|
}).path;
|
|
135
|
-
const result =
|
|
135
|
+
const result = discoverCes();
|
|
136
136
|
expect(result.mode).toBe("unavailable");
|
|
137
137
|
expect((result as { reason: string }).reason).toContain(socketPath);
|
|
138
138
|
} finally {
|
|
@@ -145,7 +145,7 @@ describe("CES bootstrap socket discovery", () => {
|
|
|
145
145
|
const bootstrapDir = mkdtempSync(join(tmpdir(), "ces-bootstrap-"));
|
|
146
146
|
const restore = withBootstrapDir(bootstrapDir);
|
|
147
147
|
try {
|
|
148
|
-
const result =
|
|
148
|
+
const result = discoverCes();
|
|
149
149
|
const workspaceDir = process.env.VELLUM_WORKSPACE_DIR;
|
|
150
150
|
expect(workspaceDir).toBeDefined();
|
|
151
151
|
const workspaceSocket = resolveIpcEndpoint("ces", {
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import { describe, expect, test } from "bun:test";
|
|
2
|
+
|
|
3
|
+
import { MCP_GLOBAL_MAX_TOOLS } from "../mcp.js";
|
|
4
|
+
import { ToolsConfigSchema } from "../tools.js";
|
|
5
|
+
|
|
6
|
+
describe("ToolsConfigSchema.mcpGlobalMaxTools", () => {
|
|
7
|
+
test("omits the override so the shipped MCP cap remains in force", () => {
|
|
8
|
+
const parsed = ToolsConfigSchema.parse({});
|
|
9
|
+
expect(parsed.mcpGlobalMaxTools).toBeUndefined();
|
|
10
|
+
expect(parsed.exclude).toEqual([]);
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
test("accepts a workspace override", () => {
|
|
14
|
+
expect(
|
|
15
|
+
ToolsConfigSchema.parse({ mcpGlobalMaxTools: 80 }).mcpGlobalMaxTools,
|
|
16
|
+
).toBe(80);
|
|
17
|
+
expect(
|
|
18
|
+
ToolsConfigSchema.parse({
|
|
19
|
+
mcpGlobalMaxTools: MCP_GLOBAL_MAX_TOOLS,
|
|
20
|
+
}).mcpGlobalMaxTools,
|
|
21
|
+
).toBe(MCP_GLOBAL_MAX_TOOLS);
|
|
22
|
+
expect(
|
|
23
|
+
ToolsConfigSchema.parse({ mcpGlobalMaxTools: 1000 }).mcpGlobalMaxTools,
|
|
24
|
+
).toBe(1000);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("rejects zero and fractions", () => {
|
|
28
|
+
expect(() => ToolsConfigSchema.parse({ mcpGlobalMaxTools: 0 })).toThrow();
|
|
29
|
+
expect(() => ToolsConfigSchema.parse({ mcpGlobalMaxTools: 1.5 })).toThrow();
|
|
30
|
+
});
|
|
31
|
+
});
|
|
@@ -18,7 +18,8 @@ export const PLUGIN_MCP_RISK_LEVEL = "low" as const;
|
|
|
18
18
|
export const MCP_MAX_TOOLS_PER_SERVER = 20;
|
|
19
19
|
|
|
20
20
|
/**
|
|
21
|
-
* Cap on tools registered across every MCP server
|
|
21
|
+
* Cap on tools registered across every MCP server when the workspace
|
|
22
|
+
* does not set `tools.mcpGlobalMaxTools` in config.json. Selection is a
|
|
22
23
|
* deterministic round-robin by server id (see `mcp/tool-caps.ts`), so a
|
|
23
24
|
* later server is not emptied just because earlier ones filled the budget.
|
|
24
25
|
*/
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
2
|
|
|
3
|
+
import { MCP_GLOBAL_MAX_TOOLS } from "./mcp.js";
|
|
4
|
+
|
|
3
5
|
export const ToolsConfigSchema = z
|
|
4
6
|
.object({
|
|
5
7
|
exclude: z
|
|
@@ -8,6 +10,14 @@ export const ToolsConfigSchema = z
|
|
|
8
10
|
.describe(
|
|
9
11
|
"Tool names to suppress. Excluded tools are not sent to the LLM. Names match `ToolDefinition.name` exactly (e.g. `bash`, `mcp__server__tool`).",
|
|
10
12
|
),
|
|
13
|
+
mcpGlobalMaxTools: z
|
|
14
|
+
.number({ error: "tools.mcpGlobalMaxTools must be a number" })
|
|
15
|
+
.int("tools.mcpGlobalMaxTools must be an integer")
|
|
16
|
+
.min(1, "tools.mcpGlobalMaxTools must be >= 1")
|
|
17
|
+
.optional()
|
|
18
|
+
.describe(
|
|
19
|
+
`Override the code-owned cap on MCP tools registered across every server. Omit to use the shipped default of ${MCP_GLOBAL_MAX_TOOLS}. Selection is a fair round-robin by server id.`,
|
|
20
|
+
),
|
|
11
21
|
})
|
|
12
22
|
.describe("Tool visibility configuration");
|
|
13
23
|
|
package/src/config/skills.ts
CHANGED
|
@@ -225,10 +225,6 @@ export interface SkillToolManifestMeta {
|
|
|
225
225
|
|
|
226
226
|
// ─── Helpers ─────────────────────────────────────────────────────────────────
|
|
227
227
|
|
|
228
|
-
function getSkillsDir(): string {
|
|
229
|
-
return getWorkspaceSkillsDir();
|
|
230
|
-
}
|
|
231
|
-
|
|
232
228
|
export function getBundledSkillsDir(): string {
|
|
233
229
|
const dir = import.meta.dir;
|
|
234
230
|
|
|
@@ -1114,7 +1110,7 @@ export function loadSkillCatalog(
|
|
|
1114
1110
|
}
|
|
1115
1111
|
|
|
1116
1112
|
// Load managed (user) skills, which take precedence over bundled skills with the same ID
|
|
1117
|
-
const skillsDir =
|
|
1113
|
+
const skillsDir = getWorkspaceSkillsDir();
|
|
1118
1114
|
const directories = discoverSkillDirectories(skillsDir);
|
|
1119
1115
|
|
|
1120
1116
|
for (const directory of directories) {
|
|
@@ -1354,7 +1350,7 @@ function loadSkillDefinition(skill: SkillSummary): SkillLookupResult {
|
|
|
1354
1350
|
} else {
|
|
1355
1351
|
loaded = readSkillFromDirectory(
|
|
1356
1352
|
skill.directoryPath,
|
|
1357
|
-
|
|
1353
|
+
getWorkspaceSkillsDir(),
|
|
1358
1354
|
skill.source,
|
|
1359
1355
|
);
|
|
1360
1356
|
}
|
|
@@ -53,9 +53,7 @@ export type DiscoveryResult = ManagedDiscoverySuccess | DiscoveryFailure;
|
|
|
53
53
|
* processes can each connect. The actual connection is made later by
|
|
54
54
|
* `CesProcessManager.start()`.
|
|
55
55
|
*/
|
|
56
|
-
export function
|
|
57
|
-
| ManagedDiscoverySuccess
|
|
58
|
-
| DiscoveryFailure {
|
|
56
|
+
export function discoverCes(): DiscoveryResult {
|
|
59
57
|
const socketPath = getCesSocketPath();
|
|
60
58
|
|
|
61
59
|
if (!isNamedPipePath(socketPath) && !existsSync(socketPath)) {
|
|
@@ -68,14 +66,6 @@ export function discoverManagedCes():
|
|
|
68
66
|
return { mode: "managed", socketPath };
|
|
69
67
|
}
|
|
70
68
|
|
|
71
|
-
/**
|
|
72
|
-
* Discover CES for the current process. Local and managed topologies use
|
|
73
|
-
* the same bootstrap socket path.
|
|
74
|
-
*/
|
|
75
|
-
export function discoverCes(): DiscoveryResult {
|
|
76
|
-
return discoverManagedCes();
|
|
77
|
-
}
|
|
78
|
-
|
|
79
69
|
/** How long to poll for the CES socket before giving up. */
|
|
80
70
|
const MANAGED_DISCOVERY_TIMEOUT_MS = 3_000;
|
|
81
71
|
|
|
@@ -190,12 +190,22 @@ export class ConfigWatcher {
|
|
|
190
190
|
if (this.suppressReload) {
|
|
191
191
|
return;
|
|
192
192
|
}
|
|
193
|
+
const prevMcpGlobalMax = this.lastConfig?.tools.mcpGlobalMaxTools;
|
|
193
194
|
try {
|
|
194
195
|
const changed = await this.refreshConfigFromSources();
|
|
195
196
|
if (changed) {
|
|
196
197
|
evictConversationsForReload();
|
|
197
198
|
refreshAuthenticatedApiRateLimit();
|
|
198
199
|
publishConfigChanged();
|
|
200
|
+
const nextMcpGlobalMax = getConfig().tools.mcpGlobalMaxTools;
|
|
201
|
+
if (prevMcpGlobalMax !== nextMcpGlobalMax) {
|
|
202
|
+
reloadMcpServers().catch((err: unknown) => {
|
|
203
|
+
log.error(
|
|
204
|
+
{ err },
|
|
205
|
+
"MCP reload after tools.mcpGlobalMaxTools change failed",
|
|
206
|
+
);
|
|
207
|
+
});
|
|
208
|
+
}
|
|
199
209
|
}
|
|
200
210
|
} catch (err) {
|
|
201
211
|
log.error(
|
|
@@ -11,6 +11,13 @@ import type { ResolvedMcpConfig } from "../../config/schemas/mcp.js";
|
|
|
11
11
|
|
|
12
12
|
const toolsByServer = new Map<string, Array<{ name: string }>>();
|
|
13
13
|
const connectDelays = new Map<string, number>();
|
|
14
|
+
let mcpGlobalMaxTools: number | undefined;
|
|
15
|
+
|
|
16
|
+
mock.module("../../config/loader.js", () => ({
|
|
17
|
+
getConfig: () => ({
|
|
18
|
+
tools: { exclude: [], mcpGlobalMaxTools },
|
|
19
|
+
}),
|
|
20
|
+
}));
|
|
14
21
|
|
|
15
22
|
mock.module("../client.js", () => ({
|
|
16
23
|
McpClient: class {
|
|
@@ -60,6 +67,7 @@ describe("McpServerManager tool selection", () => {
|
|
|
60
67
|
beforeEach(() => {
|
|
61
68
|
toolsByServer.clear();
|
|
62
69
|
connectDelays.clear();
|
|
70
|
+
mcpGlobalMaxTools = undefined;
|
|
63
71
|
});
|
|
64
72
|
|
|
65
73
|
test("later servers keep tools when eight servers exceed the global cap", async () => {
|
|
@@ -108,4 +116,29 @@ describe("McpServerManager tool selection", () => {
|
|
|
108
116
|
).toBe(true);
|
|
109
117
|
await manager.stop();
|
|
110
118
|
});
|
|
119
|
+
|
|
120
|
+
test("a workspace global-max override raises how many tools are kept", async () => {
|
|
121
|
+
const ids = Array.from(
|
|
122
|
+
{ length: 8 },
|
|
123
|
+
(_, i) => `server-${String(i + 1).padStart(2, "0")}`,
|
|
124
|
+
);
|
|
125
|
+
for (const id of ids) {
|
|
126
|
+
toolsByServer.set(
|
|
127
|
+
id,
|
|
128
|
+
Array.from({ length: 10 }, (_, i) => ({ name: `tool_${i}` })),
|
|
129
|
+
);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
mcpGlobalMaxTools = 80;
|
|
133
|
+
const manager = new McpServerManager();
|
|
134
|
+
const started = await manager.start(configWith(ids));
|
|
135
|
+
|
|
136
|
+
expect(started.discoveredToolCount).toBe(80);
|
|
137
|
+
expect(started.keptToolCount).toBe(80);
|
|
138
|
+
expect(started.droppedToolCount).toBe(0);
|
|
139
|
+
expect(started.servers.every((server) => server.tools.length === 10)).toBe(
|
|
140
|
+
true,
|
|
141
|
+
);
|
|
142
|
+
await manager.stop();
|
|
143
|
+
});
|
|
111
144
|
});
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
MCP_GLOBAL_MAX_TOOLS,
|
|
11
11
|
MCP_MAX_TOOLS_PER_SERVER,
|
|
12
12
|
} from "../../config/schemas/mcp.js";
|
|
13
|
-
import { applyMcpToolCaps } from "../tool-caps.js";
|
|
13
|
+
import { applyMcpToolCaps, resolveMcpGlobalMaxTools } from "../tool-caps.js";
|
|
14
14
|
|
|
15
15
|
function toolsNamed(prefix: string, count: number): string[] {
|
|
16
16
|
return Array.from({ length: count }, (_, i) => `${prefix}-tool-${i}`);
|
|
@@ -104,4 +104,31 @@ describe("applyMcpToolCaps", () => {
|
|
|
104
104
|
4,
|
|
105
105
|
);
|
|
106
106
|
});
|
|
107
|
+
|
|
108
|
+
test("honors a workspace global-max override", () => {
|
|
109
|
+
const servers = Array.from({ length: 8 }, (_, i) => ({
|
|
110
|
+
serverId: `server-${String(i + 1).padStart(2, "0")}`,
|
|
111
|
+
tools: toolsNamed(`s${i + 1}`, 10),
|
|
112
|
+
}));
|
|
113
|
+
|
|
114
|
+
const result = applyMcpToolCaps(servers, { globalMax: 80, perServerMax: 20 });
|
|
115
|
+
|
|
116
|
+
expect(result.keptToolCount).toBe(80);
|
|
117
|
+
expect(result.droppedToolCount).toBe(0);
|
|
118
|
+
expect(result.servers.every((server) => server.tools.length === 10)).toBe(
|
|
119
|
+
true,
|
|
120
|
+
);
|
|
121
|
+
});
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
describe("resolveMcpGlobalMaxTools", () => {
|
|
125
|
+
test("uses the shipped default when the workspace omits the override", () => {
|
|
126
|
+
expect(resolveMcpGlobalMaxTools()).toBe(MCP_GLOBAL_MAX_TOOLS);
|
|
127
|
+
expect(resolveMcpGlobalMaxTools({})).toBe(MCP_GLOBAL_MAX_TOOLS);
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
test("uses tools.mcpGlobalMaxTools from config.json when set", () => {
|
|
131
|
+
expect(resolveMcpGlobalMaxTools({ mcpGlobalMaxTools: 80 })).toBe(80);
|
|
132
|
+
expect(resolveMcpGlobalMaxTools({ mcpGlobalMaxTools: 1 })).toBe(1);
|
|
133
|
+
});
|
|
107
134
|
});
|
package/src/mcp/manager.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { getConfig } from "../config/loader.js";
|
|
1
2
|
import type {
|
|
2
3
|
ResolvedMcpConfig,
|
|
3
4
|
ResolvedMcpServerConfig,
|
|
@@ -6,6 +7,7 @@ import { getLogger } from "../util/logger.js";
|
|
|
6
7
|
import { McpClient, type McpToolInfo } from "./client.js";
|
|
7
8
|
import {
|
|
8
9
|
applyMcpToolCaps,
|
|
10
|
+
resolveMcpGlobalMaxTools,
|
|
9
11
|
truncatedServerIdsFromCaps,
|
|
10
12
|
} from "./tool-caps.js";
|
|
11
13
|
|
|
@@ -78,6 +80,7 @@ export class McpServerManager {
|
|
|
78
80
|
serverId: result.serverId,
|
|
79
81
|
tools: result.tools,
|
|
80
82
|
})),
|
|
83
|
+
{ globalMax: resolveMcpGlobalMaxTools(getConfig().tools) },
|
|
81
84
|
);
|
|
82
85
|
const keptByServer = new Map(
|
|
83
86
|
capped.servers.map((server) => [server.serverId, server.tools]),
|
package/src/mcp/tool-caps.ts
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* Code-owned MCP tool-count caps and the selection used when they bind.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
4
|
+
* The shipped global cap is {@link MCP_GLOBAL_MAX_TOOLS}. A workspace may
|
|
5
|
+
* raise or lower it with `tools.mcpGlobalMaxTools` in config.json. The
|
|
6
|
+
* per-server cap stays code-owned. These numbers are a context-window
|
|
7
|
+
* budget, not a permission gate. Allowlists and risk still decide what
|
|
8
|
+
* a turn may call.
|
|
8
9
|
*
|
|
9
10
|
* The global cap is applied by a deterministic round-robin across
|
|
10
11
|
* servers sorted by id. Insertion order must not empty a later server
|
|
@@ -15,6 +16,7 @@ import {
|
|
|
15
16
|
MCP_GLOBAL_MAX_TOOLS,
|
|
16
17
|
MCP_MAX_TOOLS_PER_SERVER,
|
|
17
18
|
} from "../config/schemas/mcp.js";
|
|
19
|
+
import type { ToolsConfig } from "../config/schemas/tools.js";
|
|
18
20
|
|
|
19
21
|
export interface McpServerToolSet<T> {
|
|
20
22
|
serverId: string;
|
|
@@ -39,6 +41,12 @@ export interface McpToolCapResult<T> {
|
|
|
39
41
|
decisions: McpToolCapDecision[];
|
|
40
42
|
}
|
|
41
43
|
|
|
44
|
+
export function resolveMcpGlobalMaxTools(
|
|
45
|
+
tools?: Pick<ToolsConfig, "mcpGlobalMaxTools">,
|
|
46
|
+
): number {
|
|
47
|
+
return tools?.mcpGlobalMaxTools ?? MCP_GLOBAL_MAX_TOOLS;
|
|
48
|
+
}
|
|
49
|
+
|
|
42
50
|
export function applyMcpToolCaps<T>(
|
|
43
51
|
servers: ReadonlyArray<McpServerToolSet<T>>,
|
|
44
52
|
options?: { globalMax?: number; perServerMax?: number },
|
|
@@ -167,19 +167,26 @@ describe("spawnMonitoringWorkerProcess", () => {
|
|
|
167
167
|
}
|
|
168
168
|
});
|
|
169
169
|
|
|
170
|
-
test("
|
|
170
|
+
test("spawns when the PID file names an unrelated live process", async () => {
|
|
171
171
|
writeFileSync(pidPath, String(process.pid));
|
|
172
172
|
let spawned = false;
|
|
173
173
|
const restore = stubBunSpawn(() => {
|
|
174
174
|
spawned = true;
|
|
175
|
-
|
|
175
|
+
writeFileSync(pidPath, "4242");
|
|
176
|
+
return {
|
|
177
|
+
unref: () => {},
|
|
178
|
+
kill: () => {},
|
|
179
|
+
pid: 4242,
|
|
180
|
+
exited: neverExits(),
|
|
181
|
+
};
|
|
176
182
|
});
|
|
177
183
|
try {
|
|
178
184
|
const result = await spawnMonitoringWorkerProcess({
|
|
179
|
-
pidWaitTimeoutMs:
|
|
185
|
+
pidWaitTimeoutMs: 1_000,
|
|
186
|
+
pidPollIntervalMs: 10,
|
|
180
187
|
});
|
|
181
|
-
expect(result).toEqual({ pid:
|
|
182
|
-
expect(spawned).toBe(
|
|
188
|
+
expect(result).toEqual({ pid: 4242, alreadyRunning: false });
|
|
189
|
+
expect(spawned).toBe(true);
|
|
183
190
|
} finally {
|
|
184
191
|
restore();
|
|
185
192
|
}
|
|
@@ -201,6 +208,11 @@ describe("probeMonitoringWorker", () => {
|
|
|
201
208
|
}
|
|
202
209
|
});
|
|
203
210
|
|
|
211
|
+
test("reports not_running when the PID file names a live process that is not this worker", () => {
|
|
212
|
+
writeFileSync(pidPath, String(process.pid));
|
|
213
|
+
expect(probeMonitoringWorker()).toEqual({ status: "not_running" });
|
|
214
|
+
});
|
|
215
|
+
|
|
204
216
|
test("reports running (not throws) when the process exists but is not signalable (EPERM)", () => {
|
|
205
217
|
writeFileSync(pidPath, "4321");
|
|
206
218
|
const restore = stubProcessKill(new Set(), new Set([4321]));
|
|
@@ -19,15 +19,21 @@ import {
|
|
|
19
19
|
type WorkerProcessStatus,
|
|
20
20
|
} from "../util/worker-process.js";
|
|
21
21
|
|
|
22
|
+
const MONITORING_WORKER_ENTRY = new URL("./worker.ts", import.meta.url);
|
|
23
|
+
|
|
22
24
|
const log = getLogger("monitoring-control");
|
|
23
25
|
|
|
24
26
|
/**
|
|
25
27
|
* Read the PID file and report liveness. A missing or malformed file reports
|
|
26
|
-
* not_running; a file pointing at a dead process
|
|
27
|
-
* not_running.
|
|
28
|
+
* not_running; a file pointing at a dead process, or at a live process that
|
|
29
|
+
* is not this worker, is cleaned up and reported as not_running.
|
|
28
30
|
*/
|
|
29
31
|
export function probeMonitoringWorker(): WorkerProcessStatus {
|
|
30
|
-
return probeWorkerPidFile(
|
|
32
|
+
return probeWorkerPidFile(
|
|
33
|
+
getMonitoringPidPath(),
|
|
34
|
+
MONITORING_WORKER_ENTRY,
|
|
35
|
+
"monitoring",
|
|
36
|
+
);
|
|
31
37
|
}
|
|
32
38
|
|
|
33
39
|
export class MonitoringWorkerSpawnError extends WorkerProcessSpawnError {}
|
|
@@ -44,7 +50,7 @@ export async function spawnMonitoringWorkerProcess(
|
|
|
44
50
|
try {
|
|
45
51
|
return await spawnWorkerProcess({
|
|
46
52
|
pidPath: getMonitoringPidPath(),
|
|
47
|
-
entry:
|
|
53
|
+
entry: MONITORING_WORKER_ENTRY,
|
|
48
54
|
packagedEntry: "monitoring",
|
|
49
55
|
workerLabel: "Resource monitor",
|
|
50
56
|
options: opts,
|
|
@@ -63,7 +69,11 @@ export async function spawnMonitoringWorkerProcess(
|
|
|
63
69
|
* (e.g. EPERM) — a not-running monitor is a no-op.
|
|
64
70
|
*/
|
|
65
71
|
export function stopMonitoringWorkerProcess(): WorkerProcessStatus {
|
|
66
|
-
return stopWorkerProcess(
|
|
72
|
+
return stopWorkerProcess(
|
|
73
|
+
getMonitoringPidPath(),
|
|
74
|
+
MONITORING_WORKER_ENTRY,
|
|
75
|
+
"monitoring",
|
|
76
|
+
);
|
|
67
77
|
}
|
|
68
78
|
|
|
69
79
|
/**
|
|
@@ -231,18 +231,26 @@ describe("spawnMemoryWorkerProcess", () => {
|
|
|
231
231
|
}
|
|
232
232
|
});
|
|
233
233
|
|
|
234
|
-
test("
|
|
235
|
-
// A live PID file (this test process) makes the probe report running.
|
|
234
|
+
test("spawns when the PID file names an unrelated live process", async () => {
|
|
236
235
|
writeFileSync(pidPath, String(process.pid));
|
|
237
236
|
let spawned = false;
|
|
238
237
|
const restore = stubBunSpawn(() => {
|
|
239
238
|
spawned = true;
|
|
240
|
-
|
|
239
|
+
writeFileSync(pidPath, "4242");
|
|
240
|
+
return {
|
|
241
|
+
unref: () => {},
|
|
242
|
+
kill: () => {},
|
|
243
|
+
pid: 4242,
|
|
244
|
+
exited: neverExits(),
|
|
245
|
+
};
|
|
241
246
|
});
|
|
242
247
|
try {
|
|
243
|
-
const result = await spawnMemoryWorkerProcess({
|
|
244
|
-
|
|
245
|
-
|
|
248
|
+
const result = await spawnMemoryWorkerProcess({
|
|
249
|
+
pidWaitTimeoutMs: 1_000,
|
|
250
|
+
pidPollIntervalMs: 10,
|
|
251
|
+
});
|
|
252
|
+
expect(result).toEqual({ pid: 4242, alreadyRunning: false });
|
|
253
|
+
expect(spawned).toBe(true);
|
|
246
254
|
} finally {
|
|
247
255
|
restore();
|
|
248
256
|
}
|
|
@@ -250,6 +258,11 @@ describe("spawnMemoryWorkerProcess", () => {
|
|
|
250
258
|
});
|
|
251
259
|
|
|
252
260
|
describe("probeMemoryWorker", () => {
|
|
261
|
+
test("reports not_running when the PID file names a live process that is not this worker", () => {
|
|
262
|
+
writeFileSync(pidPath, String(process.pid));
|
|
263
|
+
expect(probeMemoryWorker()).toEqual({ status: "not_running" });
|
|
264
|
+
});
|
|
265
|
+
|
|
253
266
|
test("reports running (not throws) when the process exists but is not signalable (EPERM)", () => {
|
|
254
267
|
writeFileSync(pidPath, "4321");
|
|
255
268
|
const restore = stubProcessKill(new Set(), new Set([4321]));
|
|
@@ -18,13 +18,19 @@ import {
|
|
|
18
18
|
type WorkerProcessStatus,
|
|
19
19
|
} from "../../../util/worker-process.js";
|
|
20
20
|
|
|
21
|
+
const MEMORY_WORKER_ENTRY = new URL("./worker.ts", import.meta.url);
|
|
22
|
+
|
|
21
23
|
/**
|
|
22
24
|
* Inspect the PID file to determine whether the worker process is alive.
|
|
23
|
-
* A stale PID file (pointing at a dead process
|
|
24
|
-
* as not_running.
|
|
25
|
+
* A stale PID file (pointing at a dead process, or at a live process that
|
|
26
|
+
* is not this worker) is cleaned up and reported as not_running.
|
|
25
27
|
*/
|
|
26
28
|
export function probeMemoryWorker(): WorkerProcessStatus {
|
|
27
|
-
return probeWorkerPidFile(
|
|
29
|
+
return probeWorkerPidFile(
|
|
30
|
+
getMemoryWorkerPidPath(),
|
|
31
|
+
MEMORY_WORKER_ENTRY,
|
|
32
|
+
"memory",
|
|
33
|
+
);
|
|
28
34
|
}
|
|
29
35
|
|
|
30
36
|
export class MemoryWorkerSpawnError extends WorkerProcessSpawnError {}
|
|
@@ -50,7 +56,7 @@ export async function spawnMemoryWorkerProcess(
|
|
|
50
56
|
try {
|
|
51
57
|
return await spawnWorkerProcess({
|
|
52
58
|
pidPath: getMemoryWorkerPidPath(),
|
|
53
|
-
entry:
|
|
59
|
+
entry: MEMORY_WORKER_ENTRY,
|
|
54
60
|
packagedEntry: "memory",
|
|
55
61
|
workerLabel: "Memory worker",
|
|
56
62
|
options: opts,
|
|
@@ -71,5 +77,9 @@ export async function spawnMemoryWorkerProcess(
|
|
|
71
77
|
* (e.g. EPERM) — a not-running worker is a no-op.
|
|
72
78
|
*/
|
|
73
79
|
export function stopMemoryWorkerProcess(): WorkerProcessStatus {
|
|
74
|
-
return stopWorkerProcess(
|
|
80
|
+
return stopWorkerProcess(
|
|
81
|
+
getMemoryWorkerPidPath(),
|
|
82
|
+
MEMORY_WORKER_ENTRY,
|
|
83
|
+
"memory",
|
|
84
|
+
);
|
|
75
85
|
}
|
package/src/routes/control.ts
CHANGED
|
@@ -44,13 +44,15 @@ function routeHostPidPath(): string {
|
|
|
44
44
|
return getProcPidPath(ROUTE_HOST_PROC_NAME);
|
|
45
45
|
}
|
|
46
46
|
|
|
47
|
+
const ROUTE_HOST_ENTRY = new URL("./worker.ts", import.meta.url);
|
|
48
|
+
|
|
47
49
|
/**
|
|
48
50
|
* Read the PID file and report liveness. A missing or malformed file reports
|
|
49
|
-
* not_running; a file pointing at a dead process
|
|
50
|
-
* not_running.
|
|
51
|
+
* not_running; a file pointing at a dead process, or at a live process that
|
|
52
|
+
* is not this worker, is cleaned up and reported as not_running.
|
|
51
53
|
*/
|
|
52
54
|
export function probeRouteHostWorker(): WorkerProcessStatus {
|
|
53
|
-
return probeWorkerPidFile(routeHostPidPath());
|
|
55
|
+
return probeWorkerPidFile(routeHostPidPath(), ROUTE_HOST_ENTRY, "routes");
|
|
54
56
|
}
|
|
55
57
|
|
|
56
58
|
export class RouteHostSpawnError extends WorkerProcessSpawnError {}
|
|
@@ -67,7 +69,7 @@ export async function spawnRouteHostWorkerProcess(
|
|
|
67
69
|
try {
|
|
68
70
|
return await spawnWorkerProcess({
|
|
69
71
|
pidPath: routeHostPidPath(),
|
|
70
|
-
entry:
|
|
72
|
+
entry: ROUTE_HOST_ENTRY,
|
|
71
73
|
packagedEntry: "routes",
|
|
72
74
|
workerLabel: "Route host",
|
|
73
75
|
options: opts,
|
|
@@ -86,7 +88,7 @@ export async function spawnRouteHostWorkerProcess(
|
|
|
86
88
|
* (e.g. EPERM) — a not-running host is a no-op.
|
|
87
89
|
*/
|
|
88
90
|
export function stopRouteHostWorkerProcess(): WorkerProcessStatus {
|
|
89
|
-
return stopWorkerProcess(routeHostPidPath());
|
|
91
|
+
return stopWorkerProcess(routeHostPidPath(), ROUTE_HOST_ENTRY, "routes");
|
|
90
92
|
}
|
|
91
93
|
|
|
92
94
|
/**
|
|
@@ -206,19 +206,26 @@ describe("spawnScheduleWorkerProcess", () => {
|
|
|
206
206
|
}
|
|
207
207
|
});
|
|
208
208
|
|
|
209
|
-
test("
|
|
209
|
+
test("spawns when the PID file names an unrelated live process", async () => {
|
|
210
210
|
writeFileSync(pidPath, String(process.pid));
|
|
211
211
|
let spawned = false;
|
|
212
212
|
const restore = stubBunSpawn(() => {
|
|
213
213
|
spawned = true;
|
|
214
|
-
|
|
214
|
+
writeFileSync(pidPath, "4242");
|
|
215
|
+
return {
|
|
216
|
+
unref: () => {},
|
|
217
|
+
kill: () => {},
|
|
218
|
+
pid: 4242,
|
|
219
|
+
exited: neverExits(),
|
|
220
|
+
};
|
|
215
221
|
});
|
|
216
222
|
try {
|
|
217
223
|
const result = await spawnScheduleWorkerProcess({
|
|
218
|
-
pidWaitTimeoutMs:
|
|
224
|
+
pidWaitTimeoutMs: 1_000,
|
|
225
|
+
pidPollIntervalMs: 10,
|
|
219
226
|
});
|
|
220
|
-
expect(result).toEqual({ pid:
|
|
221
|
-
expect(spawned).toBe(
|
|
227
|
+
expect(result).toEqual({ pid: 4242, alreadyRunning: false });
|
|
228
|
+
expect(spawned).toBe(true);
|
|
222
229
|
} finally {
|
|
223
230
|
restore();
|
|
224
231
|
}
|
|
@@ -240,6 +247,11 @@ describe("probeScheduleWorker", () => {
|
|
|
240
247
|
}
|
|
241
248
|
});
|
|
242
249
|
|
|
250
|
+
test("reports not_running when the PID file names a live process that is not this worker", () => {
|
|
251
|
+
writeFileSync(pidPath, String(process.pid));
|
|
252
|
+
expect(probeScheduleWorker()).toEqual({ status: "not_running" });
|
|
253
|
+
});
|
|
254
|
+
|
|
243
255
|
test("reports running (not throws) when the process exists but is not signalable (EPERM)", () => {
|
|
244
256
|
writeFileSync(pidPath, "4321");
|
|
245
257
|
const restore = stubProcessKill(new Set(), new Set([4321]));
|
|
@@ -256,6 +268,25 @@ describe("stopScheduleWorkerProcess", () => {
|
|
|
256
268
|
expect(stopScheduleWorkerProcess()).toEqual({ status: "not_running" });
|
|
257
269
|
});
|
|
258
270
|
|
|
271
|
+
test("does not signal a recycled PID that is not this worker", () => {
|
|
272
|
+
writeFileSync(pidPath, String(process.pid));
|
|
273
|
+
const signalled: Array<[number, string | number | undefined]> = [];
|
|
274
|
+
const original = process.kill.bind(process);
|
|
275
|
+
process.kill = ((pid: number, signal?: string | number) => {
|
|
276
|
+
if ((signal ?? 0) === 0) {
|
|
277
|
+
return original(pid, signal);
|
|
278
|
+
}
|
|
279
|
+
signalled.push([pid, signal]);
|
|
280
|
+
return true;
|
|
281
|
+
}) as typeof process.kill;
|
|
282
|
+
try {
|
|
283
|
+
expect(stopScheduleWorkerProcess()).toEqual({ status: "not_running" });
|
|
284
|
+
expect(signalled).toEqual([]);
|
|
285
|
+
} finally {
|
|
286
|
+
process.kill = original;
|
|
287
|
+
}
|
|
288
|
+
});
|
|
289
|
+
|
|
259
290
|
test("signals a running worker and reports its prior state", () => {
|
|
260
291
|
writeFileSync(pidPath, "4321");
|
|
261
292
|
const signalled: Array<[number, string | number | undefined]> = [];
|
|
@@ -19,6 +19,8 @@ import {
|
|
|
19
19
|
type WorkerProcessStatus,
|
|
20
20
|
} from "../util/worker-process.js";
|
|
21
21
|
|
|
22
|
+
const SCHEDULE_WORKER_ENTRY = new URL("./worker.ts", import.meta.url);
|
|
23
|
+
|
|
22
24
|
const log = getLogger("schedule-worker-control");
|
|
23
25
|
|
|
24
26
|
/**
|
|
@@ -42,11 +44,15 @@ export function setScheduleWorkerAdministrativelyStopped(value: boolean): void {
|
|
|
42
44
|
|
|
43
45
|
/**
|
|
44
46
|
* Inspect the PID file to determine whether the schedule worker process is
|
|
45
|
-
* alive. A stale PID file (pointing at a dead process
|
|
46
|
-
* reported as not_running.
|
|
47
|
+
* alive. A stale PID file (pointing at a dead process, or at a live process
|
|
48
|
+
* that is not this worker) is cleaned up and reported as not_running.
|
|
47
49
|
*/
|
|
48
50
|
export function probeScheduleWorker(): WorkerProcessStatus {
|
|
49
|
-
return probeWorkerPidFile(
|
|
51
|
+
return probeWorkerPidFile(
|
|
52
|
+
getScheduleWorkerPidPath(),
|
|
53
|
+
SCHEDULE_WORKER_ENTRY,
|
|
54
|
+
"schedule",
|
|
55
|
+
);
|
|
50
56
|
}
|
|
51
57
|
|
|
52
58
|
export class ScheduleWorkerSpawnError extends WorkerProcessSpawnError {}
|
|
@@ -95,7 +101,7 @@ async function spawnScheduleWorkerProcessUncoalesced(
|
|
|
95
101
|
try {
|
|
96
102
|
return await spawnWorkerProcess({
|
|
97
103
|
pidPath: getScheduleWorkerPidPath(),
|
|
98
|
-
entry:
|
|
104
|
+
entry: SCHEDULE_WORKER_ENTRY,
|
|
99
105
|
packagedEntry: "schedule",
|
|
100
106
|
workerLabel: "Schedule worker",
|
|
101
107
|
options: opts,
|
|
@@ -114,7 +120,11 @@ async function spawnScheduleWorkerProcessUncoalesced(
|
|
|
114
120
|
* `process.kill` itself fails (e.g. EPERM) — a not-running worker is a no-op.
|
|
115
121
|
*/
|
|
116
122
|
export function stopScheduleWorkerProcess(): WorkerProcessStatus {
|
|
117
|
-
return stopWorkerProcess(
|
|
123
|
+
return stopWorkerProcess(
|
|
124
|
+
getScheduleWorkerPidPath(),
|
|
125
|
+
SCHEDULE_WORKER_ENTRY,
|
|
126
|
+
"schedule",
|
|
127
|
+
);
|
|
118
128
|
}
|
|
119
129
|
|
|
120
130
|
/**
|
package/src/skills/clawhub.ts
CHANGED
|
@@ -11,11 +11,6 @@ import {
|
|
|
11
11
|
|
|
12
12
|
const log = getLogger("clawhub");
|
|
13
13
|
|
|
14
|
-
// Managed skills directory — where installed skill folders live
|
|
15
|
-
function getManagedSkillsDir(): string {
|
|
16
|
-
return getWorkspaceSkillsDir();
|
|
17
|
-
}
|
|
18
|
-
|
|
19
14
|
// ClaWHub project root — clawhub creates a `skills/` subdir inside its cwd,
|
|
20
15
|
// so we use the parent of the managed skills dir as the project root.
|
|
21
16
|
function getClawhubProjectRoot(): string {
|
|
@@ -39,7 +34,7 @@ interface IntegrityRecord {
|
|
|
39
34
|
type IntegrityManifest = Record<string, IntegrityRecord>;
|
|
40
35
|
|
|
41
36
|
function getIntegrityPath(): string {
|
|
42
|
-
return join(
|
|
37
|
+
return join(getWorkspaceSkillsDir(), ".integrity.json");
|
|
43
38
|
}
|
|
44
39
|
|
|
45
40
|
function loadIntegrityManifest(): IntegrityManifest {
|
|
@@ -66,7 +61,7 @@ function loadIntegrityManifest(): IntegrityManifest {
|
|
|
66
61
|
* from skills that haven't been migrated yet.
|
|
67
62
|
*/
|
|
68
63
|
export function verifyAndRecordSkillHash(slug: string): void {
|
|
69
|
-
const skillDir = join(
|
|
64
|
+
const skillDir = join(getWorkspaceSkillsDir(), slug);
|
|
70
65
|
const hash = computeSkillHash(skillDir);
|
|
71
66
|
if (!hash) {
|
|
72
67
|
log.warn({ slug }, "Could not compute content hash for installed skill");
|
|
@@ -232,7 +227,7 @@ function getSkillNameFromSlug(slug: string): string {
|
|
|
232
227
|
}
|
|
233
228
|
|
|
234
229
|
function getClawhubSkillsDir(projectRoot?: string): string {
|
|
235
|
-
return projectRoot ? join(projectRoot, "skills") :
|
|
230
|
+
return projectRoot ? join(projectRoot, "skills") : getWorkspaceSkillsDir();
|
|
236
231
|
}
|
|
237
232
|
|
|
238
233
|
function hasRootSkillFile(skillDir: string): boolean {
|
|
@@ -44,13 +44,9 @@ export function validateManagedSkillId(id: string): string | null {
|
|
|
44
44
|
|
|
45
45
|
// ─── Path helpers ────────────────────────────────────────────────────────────
|
|
46
46
|
|
|
47
|
-
function getManagedSkillsDir(): string {
|
|
48
|
-
return getWorkspaceSkillsDir();
|
|
49
|
-
}
|
|
50
|
-
|
|
51
47
|
/** Absolute path of a managed skill's directory (whether or not it exists). */
|
|
52
48
|
export function getManagedSkillDir(id: string): string {
|
|
53
|
-
return join(
|
|
49
|
+
return join(getWorkspaceSkillsDir(), id);
|
|
54
50
|
}
|
|
55
51
|
|
|
56
52
|
interface ResolvedCompanionPath {
|
|
@@ -205,11 +205,12 @@ describe("decideWorkerSlot", () => {
|
|
|
205
205
|
).toEqual({ action: "adopt", pid: WORKER });
|
|
206
206
|
});
|
|
207
207
|
|
|
208
|
-
// The cases below
|
|
208
|
+
// The cases below must never reach a kill. Unmatched command lines spawn
|
|
209
|
+
// a new worker without signalling the stranger holding the recycled PID.
|
|
209
210
|
test("never signals an unrelated process on a recycled PID", () => {
|
|
210
211
|
expect(
|
|
211
212
|
decide({ pid: WORKER, ppid: 1, command: "/usr/bin/postgres -D /data" }),
|
|
212
|
-
).toEqual({ action: "
|
|
213
|
+
).toEqual({ action: "spawn" });
|
|
213
214
|
});
|
|
214
215
|
|
|
215
216
|
test("never signals another project's worker.ts on a recycled PID", () => {
|
|
@@ -219,7 +220,7 @@ describe("decideWorkerSlot", () => {
|
|
|
219
220
|
ppid: 1,
|
|
220
221
|
command: "bun run /home/dev/side-project/worker.ts",
|
|
221
222
|
}),
|
|
222
|
-
).toEqual({ action: "
|
|
223
|
+
).toEqual({ action: "spawn" });
|
|
223
224
|
});
|
|
224
225
|
|
|
225
226
|
test("never signals a different worker kind holding this slot", () => {
|
|
@@ -229,7 +230,7 @@ describe("decideWorkerSlot", () => {
|
|
|
229
230
|
ppid: 1,
|
|
230
231
|
command: "bun --smol run /app/runtime/0.10.11/src/monitoring/worker.ts",
|
|
231
232
|
}),
|
|
232
|
-
).toEqual({ action: "
|
|
233
|
+
).toEqual({ action: "spawn" });
|
|
233
234
|
});
|
|
234
235
|
|
|
235
236
|
test("never signals when the process table could not be read", () => {
|
|
@@ -50,13 +50,11 @@ function isEsrchError(err: unknown): boolean {
|
|
|
50
50
|
);
|
|
51
51
|
}
|
|
52
52
|
|
|
53
|
-
/**
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
*/
|
|
59
|
-
export function probeWorkerPidFile(path: string): WorkerProcessStatus {
|
|
53
|
+
/** Kill-0 plus command-line identity for a PID file. */
|
|
54
|
+
function probePidFile(
|
|
55
|
+
path: string,
|
|
56
|
+
signature: readonly string[],
|
|
57
|
+
): WorkerProcessStatus {
|
|
60
58
|
if (!existsSync(path)) {
|
|
61
59
|
return { status: "not_running" };
|
|
62
60
|
}
|
|
@@ -69,22 +67,49 @@ export function probeWorkerPidFile(path: string): WorkerProcessStatus {
|
|
|
69
67
|
|
|
70
68
|
try {
|
|
71
69
|
process.kill(pid, 0);
|
|
72
|
-
return { status: "running", pid };
|
|
73
70
|
} catch (err: unknown) {
|
|
74
71
|
if (isEsrchError(err)) {
|
|
75
|
-
|
|
76
|
-
try {
|
|
77
|
-
unlinkSync(path);
|
|
78
|
-
} catch {
|
|
79
|
-
// best-effort
|
|
80
|
-
}
|
|
72
|
+
unlinkPidFileIfNames(path, pid);
|
|
81
73
|
return { status: "not_running" };
|
|
82
74
|
}
|
|
83
75
|
// Any other error (e.g. EPERM: the process exists but this caller may not
|
|
84
|
-
// signal it) means the process is alive.
|
|
85
|
-
// letting the error escape a status probe.
|
|
86
|
-
|
|
76
|
+
// signal it) means the process is alive. Fall through to the identity
|
|
77
|
+
// check rather than letting the error escape a status probe.
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
const fate = classifyOrphanAfterWait(
|
|
81
|
+
true,
|
|
82
|
+
readRawProcessCommand(pid),
|
|
83
|
+
signature,
|
|
84
|
+
);
|
|
85
|
+
if (fate === "gone") {
|
|
86
|
+
log.info(
|
|
87
|
+
{ pid, pidPath: path },
|
|
88
|
+
"Worker PID file names a live process this runtime does not recognise as its worker; releasing the slot rather than treating it as running",
|
|
89
|
+
);
|
|
90
|
+
unlinkPidFileIfNames(path, pid);
|
|
91
|
+
return { status: "not_running" };
|
|
87
92
|
}
|
|
93
|
+
return { status: "running", pid };
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Read a PID file and report whether this worker is actually running there.
|
|
98
|
+
*
|
|
99
|
+
* Identity comes from the same `entry` / `packagedEntry` spawn uses. A missing
|
|
100
|
+
* or malformed file reports not_running. A file pointing at a dead process, or
|
|
101
|
+
* at a live process whose command line is not this worker (a recycled PID
|
|
102
|
+
* after a container restart), is cleaned up and reported as not_running. A
|
|
103
|
+
* live PID whose command line cannot be read is reported running: uncertainty
|
|
104
|
+
* must not start a second worker next to a maybe-live one, and must not be
|
|
105
|
+
* treated as a license to signal a stranger.
|
|
106
|
+
*/
|
|
107
|
+
export function probeWorkerPidFile(
|
|
108
|
+
pidPath: string,
|
|
109
|
+
entry: URL,
|
|
110
|
+
packagedEntry?: PackagedWorkerEntry,
|
|
111
|
+
): WorkerProcessStatus {
|
|
112
|
+
return probePidFile(pidPath, workerKindSignature(entry, packagedEntry));
|
|
88
113
|
}
|
|
89
114
|
|
|
90
115
|
/** Thrown when a worker process fails to come up within the wait window. */
|
|
@@ -394,11 +419,13 @@ async function stopOrphanedWorker(
|
|
|
394
419
|
/**
|
|
395
420
|
* What to do about the process a worker's PID file currently names.
|
|
396
421
|
*
|
|
397
|
-
* - `adopt`: reuse it.
|
|
398
|
-
*
|
|
399
|
-
*
|
|
422
|
+
* - `adopt`: reuse it. It is this process's own worker, it belongs to
|
|
423
|
+
* another live owner, or its identity could not be read so a second
|
|
424
|
+
* worker must not be started next to a maybe-live one.
|
|
400
425
|
* - `reclaim`: an orphan left by an owner that is gone. Stop it, then spawn.
|
|
401
|
-
* - `spawn`: nothing is holding the slot
|
|
426
|
+
* - `spawn`: nothing is holding the slot, including a live PID whose
|
|
427
|
+
* command line is not this worker (a recycled PID). Never signal that
|
|
428
|
+
* process.
|
|
402
429
|
*/
|
|
403
430
|
export type WorkerSlotDecision =
|
|
404
431
|
| { action: "adopt"; pid: number }
|
|
@@ -410,9 +437,12 @@ export type WorkerSlotDecision =
|
|
|
410
437
|
*
|
|
411
438
|
* `reclaim` is the only outcome that signals a process, and it requires two
|
|
412
439
|
* independent things to agree: the command line marks the process as this
|
|
413
|
-
* worker, and parentage says no daemon owns it. A missing row
|
|
414
|
-
* command line
|
|
415
|
-
*
|
|
440
|
+
* worker, and parentage says no daemon owns it. A missing row and an
|
|
441
|
+
* unreadable command line fall back to `adopt`, so an uncertain answer never
|
|
442
|
+
* costs a process its life and never starts a duplicate. A command line that
|
|
443
|
+
* does not match is `spawn` without a signal: the PID file is stale, and
|
|
444
|
+
* treating that PID as the worker would leave the slot occupied by a
|
|
445
|
+
* stranger forever.
|
|
416
446
|
*
|
|
417
447
|
* `isOwnerAlive` is the caller's definition of a legitimate owner. These
|
|
418
448
|
* workers have exactly one, the daemon, so passing a plain liveness probe
|
|
@@ -435,7 +465,7 @@ export function decideWorkerSlot(
|
|
|
435
465
|
return { action: "adopt", pid: status.pid };
|
|
436
466
|
}
|
|
437
467
|
if (!matchesSignature(row.command, signature)) {
|
|
438
|
-
return { action: "
|
|
468
|
+
return { action: "spawn" };
|
|
439
469
|
}
|
|
440
470
|
const ownership = classifyWorkerOwnership(
|
|
441
471
|
row,
|
|
@@ -458,7 +488,7 @@ function inspectWorkerSlot(
|
|
|
458
488
|
pidPath: string,
|
|
459
489
|
signature: readonly string[],
|
|
460
490
|
): WorkerSlotDecision {
|
|
461
|
-
const status =
|
|
491
|
+
const status = probePidFile(pidPath, signature);
|
|
462
492
|
if (status.status !== "running" || status.pid == null) {
|
|
463
493
|
return { action: "spawn" };
|
|
464
494
|
}
|
|
@@ -479,11 +509,11 @@ function inspectWorkerSlot(
|
|
|
479
509
|
if (row && !matchesSignature(row.command, signature)) {
|
|
480
510
|
log.info(
|
|
481
511
|
{ pid: row.pid, pidPath },
|
|
482
|
-
"Worker PID file names a live process this runtime does not recognise as its worker;
|
|
512
|
+
"Worker PID file names a live process this runtime does not recognise as its worker; releasing the slot rather than treating it as running",
|
|
483
513
|
);
|
|
484
514
|
}
|
|
485
515
|
|
|
486
|
-
|
|
516
|
+
const decision = decideWorkerSlot(
|
|
487
517
|
status,
|
|
488
518
|
row,
|
|
489
519
|
signature,
|
|
@@ -491,6 +521,12 @@ function inspectWorkerSlot(
|
|
|
491
521
|
(pid) => isDaemonCommand(commandOf(pid)),
|
|
492
522
|
pid1OwnsWorkers(commandOf(1)),
|
|
493
523
|
);
|
|
524
|
+
// A spawn decision while the PID file still names a live stranger would
|
|
525
|
+
// make waitForWorkerPidFile treat that stale file as readiness.
|
|
526
|
+
if (decision.action === "spawn") {
|
|
527
|
+
unlinkPidFileIfNames(pidPath, status.pid);
|
|
528
|
+
}
|
|
529
|
+
return decision;
|
|
494
530
|
}
|
|
495
531
|
|
|
496
532
|
/**
|
|
@@ -508,7 +544,7 @@ async function reclaimWorkerSlot(
|
|
|
508
544
|
// Still running and beyond our reach. One stale worker beats two live ones.
|
|
509
545
|
return pid;
|
|
510
546
|
}
|
|
511
|
-
const replacement =
|
|
547
|
+
const replacement = probePidFile(pidPath, signature);
|
|
512
548
|
return replacement.status === "running" && replacement.pid != null
|
|
513
549
|
? replacement.pid
|
|
514
550
|
: null;
|
|
@@ -519,7 +555,8 @@ async function reclaimWorkerSlot(
|
|
|
519
555
|
* process, and wait for it to report readiness by writing its PID file. The
|
|
520
556
|
* child is `unref`'d, so the spawning process never blocks on it.
|
|
521
557
|
*
|
|
522
|
-
* If a worker is already running (
|
|
558
|
+
* If a worker is already running (the PID file names a live process whose
|
|
559
|
+
* command line matches this worker), returns its PID with
|
|
523
560
|
* `alreadyRunning: true` rather than spawning a second one. Throws
|
|
524
561
|
* {@link WorkerProcessSpawnError} if the child crashes during startup or
|
|
525
562
|
* never writes its PID file within the wait window.
|
|
@@ -633,14 +670,19 @@ export async function spawnWorkerProcess(args: {
|
|
|
633
670
|
|
|
634
671
|
/**
|
|
635
672
|
* Send SIGTERM to the worker process behind `pidPath` if it is actually
|
|
636
|
-
*
|
|
673
|
+
* this worker.
|
|
637
674
|
*
|
|
638
675
|
* Returns the status observed before signalling, so callers can report
|
|
639
|
-
* whether anything was stopped.
|
|
640
|
-
*
|
|
676
|
+
* whether anything was stopped. A PID file that names a dead process or a
|
|
677
|
+
* live process that is not this worker is a no-op. Only throws if
|
|
678
|
+
* `process.kill` itself fails (e.g. EPERM).
|
|
641
679
|
*/
|
|
642
|
-
export function stopWorkerProcess(
|
|
643
|
-
|
|
680
|
+
export function stopWorkerProcess(
|
|
681
|
+
pidPath: string,
|
|
682
|
+
entry: URL,
|
|
683
|
+
packagedEntry?: PackagedWorkerEntry,
|
|
684
|
+
): WorkerProcessStatus {
|
|
685
|
+
const current = probeWorkerPidFile(pidPath, entry, packagedEntry);
|
|
644
686
|
if (current.status === "running" && current.pid != null) {
|
|
645
687
|
process.kill(current.pid, "SIGTERM");
|
|
646
688
|
}
|