@koda-sl/baker-bridge 0.124.0 → 0.125.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/broker/broker-process.d.ts +10 -0
- package/dist/broker/broker-process.d.ts.map +1 -0
- package/dist/broker/broker-process.js +20 -0
- package/dist/broker/broker-process.js.map +1 -0
- package/dist/broker/broker-process.test.d.ts +2 -0
- package/dist/broker/broker-process.test.d.ts.map +1 -0
- package/dist/broker/broker-process.test.js +13 -0
- package/dist/broker/broker-process.test.js.map +1 -0
- package/dist/broker/env.d.ts +2 -0
- package/dist/broker/env.d.ts.map +1 -1
- package/dist/broker/env.js +4 -0
- package/dist/broker/env.js.map +1 -1
- package/dist/broker/main.js +8 -2
- package/dist/broker/main.js.map +1 -1
- package/dist/broker/socket-service.d.ts.map +1 -1
- package/dist/broker/socket-service.js +12 -2
- package/dist/broker/socket-service.js.map +1 -1
- package/dist/broker/socket-service.test.js +7 -0
- package/dist/broker/socket-service.test.js.map +1 -1
- package/dist/broker/supervisor-main.d.ts +3 -0
- package/dist/broker/supervisor-main.d.ts.map +1 -0
- package/dist/broker/supervisor-main.js +70 -0
- package/dist/broker/supervisor-main.js.map +1 -0
- package/dist/broker/supervisor.d.ts +25 -0
- package/dist/broker/supervisor.d.ts.map +1 -0
- package/dist/broker/supervisor.js +88 -0
- package/dist/broker/supervisor.js.map +1 -0
- package/dist/broker/supervisor.test.d.ts +2 -0
- package/dist/broker/supervisor.test.d.ts.map +1 -0
- package/dist/broker/supervisor.test.js +161 -0
- package/dist/broker/supervisor.test.js.map +1 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +2 -0
- package/dist/cli.js.map +1 -1
- package/dist/hono/bridge-job.d.ts.map +1 -1
- package/dist/hono/bridge-job.js +4 -0
- package/dist/hono/bridge-job.js.map +1 -1
- package/dist/hono/bridge-job.test.js +13 -0
- package/dist/hono/bridge-job.test.js.map +1 -1
- package/dist/hono/oom-priority.agent.test.d.ts +2 -0
- package/dist/hono/oom-priority.agent.test.d.ts.map +1 -0
- package/dist/hono/oom-priority.agent.test.js +138 -0
- package/dist/hono/oom-priority.agent.test.js.map +1 -0
- package/dist/hono/oom-priority.d.ts +39 -0
- package/dist/hono/oom-priority.d.ts.map +1 -0
- package/dist/hono/oom-priority.js +138 -0
- package/dist/hono/oom-priority.js.map +1 -0
- package/dist/hono/oom-priority.test.d.ts +2 -0
- package/dist/hono/oom-priority.test.d.ts.map +1 -0
- package/dist/hono/oom-priority.test.js +52 -0
- package/dist/hono/oom-priority.test.js.map +1 -0
- package/package.json +3 -2
package/README.md
CHANGED
|
@@ -58,7 +58,7 @@ All variables are validated at startup via `@t3-oss/env-core` + `zod`.
|
|
|
58
58
|
|
|
59
59
|
## Credential broker (`baker-credential-broker`)
|
|
60
60
|
|
|
61
|
-
A second bin, run as root in the sandbox before the agent starts, so the agent can use the user's stored credentials without ever seeing them. The agent writes `{{credential:NAME}}` in a request; the broker is an HTTP(S) proxy on `127.0.0.1` that swaps in the real value and swaps it back out of the response.
|
|
61
|
+
A second bin, run as root in the sandbox before the agent starts, so the agent can use the user's stored credentials without ever seeing them. A third, `baker-credential-broker-supervisor`, takes the same environment, runs the broker and starts it again whenever it exits, stops answering `GET /health`, or fails to start at all; on `SIGTERM` it stops the broker and kills it if it is still running 5 s later; it is what a Runtime starts. It runs out of the OOM killer's reach with the broker's heap capped, and the bridge marks every process its agent starts as the OOM killer's first choice — through a command wrapper it passes as `CLAUDE_CODE_SHELL_PREFIX` (unless the Runtime set one), backed by a 2-second scan — so memory pressure ends one command, never the turn. `src/hono/oom-priority.agent.test.ts` runs the SDK's own agent binary through the wrapper, so an upgrade that changes how it calls `CLAUDE_CODE_SHELL_PREFIX` fails the suite. The agent writes `{{credential:NAME}}` in a request; the broker is an HTTP(S) proxy on `127.0.0.1` that swaps in the real value and swaps it back out of the response.
|
|
62
62
|
|
|
63
63
|
| Variable | Description |
|
|
64
64
|
|---|---|
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { BrokerProcess } from "./supervisor.ts";
|
|
2
|
+
/**
|
|
3
|
+
* Starts the broker as a child of the supervisor. A spawn that fails (ENOENT,
|
|
4
|
+
* EAGAIN, EMFILE — likeliest under the memory pressure the supervisor exists
|
|
5
|
+
* for) emits `error` and possibly no `exit`; it settles `exited` all the same,
|
|
6
|
+
* so the supervisor's back-off retries it. A synchronous throw is the
|
|
7
|
+
* supervisor's to catch.
|
|
8
|
+
*/
|
|
9
|
+
export declare function spawnBroker(command: string, args: readonly string[], env: NodeJS.ProcessEnv, log: (message: string) => void): BrokerProcess;
|
|
10
|
+
//# sourceMappingURL=broker-process.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"broker-process.d.ts","sourceRoot":"","sources":["../../src/broker/broker-process.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAc,aAAa,EAAE,MAAM,iBAAiB,CAAC;AAEjE;;;;;;GAMG;AACH,wBAAgB,WAAW,CACzB,OAAO,EAAE,MAAM,EACf,IAAI,EAAE,SAAS,MAAM,EAAE,EACvB,GAAG,EAAE,MAAM,CAAC,UAAU,EACtB,GAAG,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,GAC7B,aAAa,CAUf"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { spawn } from "node:child_process";
|
|
2
|
+
/**
|
|
3
|
+
* Starts the broker as a child of the supervisor. A spawn that fails (ENOENT,
|
|
4
|
+
* EAGAIN, EMFILE — likeliest under the memory pressure the supervisor exists
|
|
5
|
+
* for) emits `error` and possibly no `exit`; it settles `exited` all the same,
|
|
6
|
+
* so the supervisor's back-off retries it. A synchronous throw is the
|
|
7
|
+
* supervisor's to catch.
|
|
8
|
+
*/
|
|
9
|
+
export function spawnBroker(command, args, env, log) {
|
|
10
|
+
const child = spawn(command, args, { stdio: "inherit", env });
|
|
11
|
+
const exited = new Promise((resolve) => {
|
|
12
|
+
child.once("exit", (code, signal) => resolve({ code, signal }));
|
|
13
|
+
child.once("error", (error) => {
|
|
14
|
+
log(`could not start broker: ${error.message}`);
|
|
15
|
+
resolve({ code: null, signal: null });
|
|
16
|
+
});
|
|
17
|
+
});
|
|
18
|
+
return { exited, kill: (signal) => void child.kill(signal) };
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=broker-process.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"broker-process.js","sourceRoot":"","sources":["../../src/broker/broker-process.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAG3C;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CACzB,OAAe,EACf,IAAuB,EACvB,GAAsB,EACtB,GAA8B;IAE9B,MAAM,KAAK,GAAG,KAAK,CAAC,OAAO,EAAE,IAAI,EAAE,EAAE,KAAK,EAAE,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC;IAC9D,MAAM,MAAM,GAAG,IAAI,OAAO,CAAa,CAAC,OAAO,EAAE,EAAE;QACjD,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;QAChE,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC,KAAK,EAAE,EAAE;YAC5B,GAAG,CAAC,2BAA2B,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;YAChD,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IACH,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,MAAM,EAAE,EAAE,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;AAC/D,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"broker-process.test.d.ts","sourceRoot":"","sources":["../../src/broker/broker-process.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import { describe, expect, it } from "vitest";
|
|
2
|
+
import { spawnBroker } from "./broker-process.js";
|
|
3
|
+
describe("spawnBroker", () => {
|
|
4
|
+
// Node reports ENOENT, EAGAIN and EMFILE as an `error` event, with no `exit`
|
|
5
|
+
// after it: unheard, it crashed the supervisor; heard, it is a broker that exited.
|
|
6
|
+
it("reports a broker that could not be spawned as one that exited, with the reason", async () => {
|
|
7
|
+
const reasons = [];
|
|
8
|
+
const broker = spawnBroker("/nonexistent/node", [], {}, (line) => reasons.push(line));
|
|
9
|
+
await expect(broker.exited).resolves.toEqual({ code: null, signal: null });
|
|
10
|
+
expect(reasons.join("")).toContain("ENOENT");
|
|
11
|
+
});
|
|
12
|
+
});
|
|
13
|
+
//# sourceMappingURL=broker-process.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"broker-process.test.js","sourceRoot":"","sources":["../../src/broker/broker-process.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAElD,QAAQ,CAAC,aAAa,EAAE,GAAG,EAAE;IAC3B,6EAA6E;IAC7E,mFAAmF;IACnF,EAAE,CAAC,gFAAgF,EAAE,KAAK,IAAI,EAAE;QAC9F,MAAM,OAAO,GAAa,EAAE,CAAC;QAE7B,MAAM,MAAM,GAAG,WAAW,CAAC,mBAAmB,EAAE,EAAE,EAAE,EAAE,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;QAEtF,MAAM,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3E,MAAM,CAAC,OAAO,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
package/dist/broker/env.d.ts
CHANGED
|
@@ -15,5 +15,7 @@ export declare function parseBrokerEnv(env?: NodeJS.ProcessEnv): z.ZodSafeParseR
|
|
|
15
15
|
BAKER_CA_DIR: string;
|
|
16
16
|
BAKER_BROKER_SOCKET: string;
|
|
17
17
|
}>;
|
|
18
|
+
/** The supervisor's own environment for the broker it starts, with `NODE_OPTIONS` set. */
|
|
19
|
+
export declare function brokerChildEnv(nodeOptions: string, env?: NodeJS.ProcessEnv): NodeJS.ProcessEnv;
|
|
18
20
|
export {};
|
|
19
21
|
//# sourceMappingURL=env.d.ts.map
|
package/dist/broker/env.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"env.d.ts","sourceRoot":"","sources":["../../src/broker/env.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,QAAA,MAAM,SAAS;;;;;;iBAMb,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,CAAC;AAElD,wBAAgB,cAAc,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB;;;;;;GAElE"}
|
|
1
|
+
{"version":3,"file":"env.d.ts","sourceRoot":"","sources":["../../src/broker/env.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,QAAA,MAAM,SAAS;;;;;;iBAMb,CAAC;AAEH,MAAM,MAAM,SAAS,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,SAAS,CAAC,CAAC;AAElD,wBAAgB,cAAc,CAAC,GAAG,GAAE,MAAM,CAAC,UAAwB;;;;;;GAElE;AAED,0FAA0F;AAC1F,wBAAgB,cAAc,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,GAAE,MAAM,CAAC,UAAwB,GAAG,MAAM,CAAC,UAAU,CAE3G"}
|
package/dist/broker/env.js
CHANGED
|
@@ -13,4 +13,8 @@ const envSchema = z.object({
|
|
|
13
13
|
export function parseBrokerEnv(env = process.env) {
|
|
14
14
|
return envSchema.safeParse(env);
|
|
15
15
|
}
|
|
16
|
+
/** The supervisor's own environment for the broker it starts, with `NODE_OPTIONS` set. */
|
|
17
|
+
export function brokerChildEnv(nodeOptions, env = process.env) {
|
|
18
|
+
return { ...env, NODE_OPTIONS: nodeOptions };
|
|
19
|
+
}
|
|
16
20
|
//# sourceMappingURL=env.js.map
|
package/dist/broker/env.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../src/broker/env.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhD,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;IACzB,uBAAuB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,qBAAqB,EAAE,CAAC,CAAC,GAAG,EAAE;IAC9B,2BAA2B,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC;IACnG,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;IACvD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;CACtE,CAAC,CAAC;AAIH,MAAM,UAAU,cAAc,CAAC,GAAG,GAAsB,OAAO,CAAC,GAAG;IACjE,OAAO,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;AAClC,CAAC"}
|
|
1
|
+
{"version":3,"file":"env.js","sourceRoot":"","sources":["../../src/broker/env.ts"],"names":[],"mappings":"AAAA,wDAAwD;AACxD,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,qBAAqB,EAAE,MAAM,oBAAoB,CAAC;AAC3D,OAAO,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,kBAAkB,EAAE,MAAM,YAAY,CAAC;AAEhD,MAAM,SAAS,GAAG,CAAC,CAAC,MAAM,CAAC;IACzB,uBAAuB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,qBAAqB,EAAE,CAAC,CAAC,GAAG,EAAE;IAC9B,2BAA2B,EAAE,CAAC,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,CAAC,kBAAkB,CAAC;IACnG,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC;IACvD,mBAAmB,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,qBAAqB,CAAC;CACtE,CAAC,CAAC;AAIH,MAAM,UAAU,cAAc,CAAC,GAAG,GAAsB,OAAO,CAAC,GAAG;IACjE,OAAO,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;AAClC,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,cAAc,CAAC,WAAmB,EAAE,GAAG,GAAsB,OAAO,CAAC,GAAG;IACtF,OAAO,EAAE,GAAG,GAAG,EAAE,YAAY,EAAE,WAAW,EAAE,CAAC;AAC/C,CAAC"}
|
package/dist/broker/main.js
CHANGED
|
@@ -37,12 +37,18 @@ async function main() {
|
|
|
37
37
|
const port = await broker.start();
|
|
38
38
|
await socket.listen();
|
|
39
39
|
log(`listening on 127.0.0.1:${port}, socket ${env.BAKER_BROKER_SOCKET}`);
|
|
40
|
+
// A stop signals the broker twice — once directly, once from its supervisor —
|
|
41
|
+
// and a second signal with no handler would kill it before it removes its socket.
|
|
42
|
+
let stopping = false;
|
|
40
43
|
const shutdown = async () => {
|
|
44
|
+
if (stopping)
|
|
45
|
+
return;
|
|
46
|
+
stopping = true;
|
|
41
47
|
await Promise.allSettled([socket.close(), broker.stop()]);
|
|
42
48
|
process.exit(0);
|
|
43
49
|
};
|
|
44
|
-
process.
|
|
45
|
-
process.
|
|
50
|
+
process.on("SIGTERM", () => void shutdown());
|
|
51
|
+
process.on("SIGINT", () => void shutdown());
|
|
46
52
|
}
|
|
47
53
|
main().catch((error) => {
|
|
48
54
|
log(`failed to start: ${error instanceof Error ? error.message : String(error)}`);
|
package/dist/broker/main.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/broker/main.ts"],"names":[],"mappings":";AACA;;;;;GAKG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,0BAA0B,EAAE,MAAM,SAAS,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,SAAS,GAAG,CAAC,OAAe;IAC1B,OAAO,CAAC,KAAK,CAAC,uBAAuB,OAAO,EAAE,CAAC,CAAC;AAClD,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,GAAG,CAAC,wBAAwB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC;IACxB,MAAM,GAAG,GAAG,oBAAoB,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,qBAAqB,EAAE,KAAK,EAAE,GAAG,CAAC,uBAAuB,EAAE,CAAC,CAAC;IAC7G,MAAM,EAAE,GAAG,MAAM,0BAA0B,CAAC;QAC1C,GAAG,EAAE,GAAG,CAAC,YAAY;QACrB,kBAAkB,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,KAAK,CAAC;QAC5C,GAAG;KACJ,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,YAAY,CAAC;QAC1B,EAAE;QACF,IAAI,EAAE,GAAG,CAAC,2BAA2B;QACrC,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,GAAG;KACJ,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,mBAAmB,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,CAAC;IACtG,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IAClC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;IACtB,GAAG,CAAC,0BAA0B,IAAI,YAAY,GAAG,CAAC,mBAAmB,EAAE,CAAC,CAAC;IAEzE,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,CAAC,
|
|
1
|
+
{"version":3,"file":"main.js","sourceRoot":"","sources":["../../src/broker/main.ts"],"names":[],"mappings":";AACA;;;;;GAKG;AACH,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AACtD,OAAO,EAAE,0BAA0B,EAAE,MAAM,SAAS,CAAC;AACrD,OAAO,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAC1C,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,SAAS,GAAG,CAAC,OAAe;IAC1B,OAAO,CAAC,KAAK,CAAC,uBAAuB,OAAO,EAAE,CAAC,CAAC;AAClD,CAAC;AAED,KAAK,UAAU,IAAI;IACjB,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;IAChC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,GAAG,CAAC,wBAAwB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACnG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC;IACD,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC;IACxB,MAAM,GAAG,GAAG,oBAAoB,CAAC,EAAE,OAAO,EAAE,GAAG,CAAC,qBAAqB,EAAE,KAAK,EAAE,GAAG,CAAC,uBAAuB,EAAE,CAAC,CAAC;IAC7G,MAAM,EAAE,GAAG,MAAM,0BAA0B,CAAC;QAC1C,GAAG,EAAE,GAAG,CAAC,YAAY;QACrB,kBAAkB,EAAE,OAAO,CAAC,MAAM,EAAE,EAAE,KAAK,CAAC;QAC5C,GAAG;KACJ,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,YAAY,CAAC;QAC1B,EAAE;QACF,IAAI,EAAE,GAAG,CAAC,2BAA2B;QACrC,gBAAgB,EAAE,GAAG,CAAC,gBAAgB;QACtC,WAAW,EAAE,GAAG,CAAC,WAAW;QAC5B,GAAG;KACJ,CAAC,CAAC;IACH,MAAM,MAAM,GAAG,mBAAmB,CAAC,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,EAAE,UAAU,EAAE,GAAG,CAAC,mBAAmB,EAAE,GAAG,EAAE,CAAC,CAAC;IACtG,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,CAAC;IAClC,MAAM,MAAM,CAAC,MAAM,EAAE,CAAC;IACtB,GAAG,CAAC,0BAA0B,IAAI,YAAY,GAAG,CAAC,mBAAmB,EAAE,CAAC,CAAC;IAEzE,8EAA8E;IAC9E,kFAAkF;IAClF,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;QAC1B,IAAI,QAAQ;YAAE,OAAO;QACrB,QAAQ,GAAG,IAAI,CAAC;QAChB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,EAAE,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;QAC1D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAClB,CAAC,CAAC;IACF,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC;IAC7C,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,KAAK,QAAQ,EAAE,CAAC,CAAC;AAC9C,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;IAC9B,GAAG,CAAC,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IAClF,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"socket-service.d.ts","sourceRoot":"","sources":["../../src/broker/socket-service.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAI7D,eAAO,MAAM,uBAAuB,QAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"socket-service.d.ts","sourceRoot":"","sources":["../../src/broker/socket-service.ts"],"names":[],"mappings":"AAcA,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAI7D,eAAO,MAAM,uBAAuB,QAAkB,CAAC;AA0BvD,MAAM,MAAM,aAAa,GAAG;IAAE,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IAAC,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE,CAAC;AA6BhF,wBAAgB,mBAAmB,CAAC,OAAO,EAAE;IAC3C,KAAK,EAAE,eAAe,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,sGAAsG;IACtG,UAAU,CAAC,EAAE;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE,CAAC;IACrD,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACjC,GAAG,aAAa,CA4ChB"}
|
|
@@ -15,7 +15,14 @@ import { z } from "zod";
|
|
|
15
15
|
import { scrub } from "./credential-text.js";
|
|
16
16
|
import { createTokenBucket } from "./token-bucket.js";
|
|
17
17
|
export const MAX_SCRUB_REQUEST_BYTES = 2 * 1024 * 1024;
|
|
18
|
-
|
|
18
|
+
/**
|
|
19
|
+
* Sized for dozens of agents reading at once, not for one: every agent's tool
|
|
20
|
+
* results share this budget. It only bounds runaway use; what keeps the socket
|
|
21
|
+
* from being an oracle is that it confirms no value a wordlist could reach
|
|
22
|
+
* (`resistsGuessing`). At 256 KB/s a 20-agent run withheld 40% of its results.
|
|
23
|
+
*/
|
|
24
|
+
const DEFAULT_SCRUB_BYTES = { capacity: 256 * 1024 * 1024, perSecond: 64 * 1024 * 1024 };
|
|
25
|
+
const DEFAULT_REQUESTS_PER_SECOND = 1_000;
|
|
19
26
|
const scrubRequestSchema = z.object({ text: z.string() });
|
|
20
27
|
/** Bits a value must carry — its length times its per-character entropy — before `/scrub` may confirm it. */
|
|
21
28
|
const MIN_SCRUB_BITS = 64;
|
|
@@ -57,7 +64,10 @@ function send(res, status, body) {
|
|
|
57
64
|
res.end(JSON.stringify(body));
|
|
58
65
|
}
|
|
59
66
|
export function createSocketService(options) {
|
|
60
|
-
const bucket = createTokenBucket({
|
|
67
|
+
const bucket = createTokenBucket({
|
|
68
|
+
ratePerSecond: options.ratePerSecond ?? DEFAULT_REQUESTS_PER_SECOND,
|
|
69
|
+
burst: options.ratePerSecond ?? DEFAULT_REQUESTS_PER_SECOND,
|
|
70
|
+
});
|
|
61
71
|
const scrubBytes = options.scrubBytes ?? DEFAULT_SCRUB_BYTES;
|
|
62
72
|
const bytes = createTokenBucket({ ratePerSecond: scrubBytes.perSecond, burst: scrubBytes.capacity });
|
|
63
73
|
async function handle(req, res) {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"socket-service.js","sourceRoot":"","sources":["../../src/broker/socket-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,YAAY,EAA6C,MAAM,WAAW,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD,MAAM,mBAAmB,GAAG,EAAE,QAAQ,EAAE,
|
|
1
|
+
{"version":3,"file":"socket-service.js","sourceRoot":"","sources":["../../src/broker/socket-service.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,EAAE,EAAE,MAAM,kBAAkB,CAAC;AACpD,OAAO,EAAE,YAAY,EAA6C,MAAM,WAAW,CAAC;AACpF,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAE,KAAK,EAAE,MAAM,sBAAsB,CAAC;AAC7C,OAAO,EAAE,iBAAiB,EAAE,MAAM,mBAAmB,CAAC;AAEtD,MAAM,CAAC,MAAM,uBAAuB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAC;AACvD;;;;;GAKG;AACH,MAAM,mBAAmB,GAAG,EAAE,QAAQ,EAAE,GAAG,GAAG,IAAI,GAAG,IAAI,EAAE,SAAS,EAAE,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC;AACzF,MAAM,2BAA2B,GAAG,KAAK,CAAC;AAE1C,MAAM,kBAAkB,GAAG,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;AAE1D,6GAA6G;AAC7G,MAAM,cAAc,GAAG,EAAE,CAAC;AAE1B,SAAS,eAAe,CAAC,KAAa;IACpC,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,KAAK;QAAE,MAAM,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;IACxE,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,KAAK,MAAM,KAAK,IAAI,MAAM,CAAC,MAAM,EAAE,EAAE,CAAC;QACpC,MAAM,CAAC,GAAG,KAAK,GAAG,KAAK,CAAC,MAAM,CAAC;QAC/B,OAAO,IAAI,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC9B,CAAC;IACD,OAAO,OAAO,GAAG,KAAK,CAAC,MAAM,IAAI,cAAc,CAAC;AAClD,CAAC;AAID,MAAM,YAAa,SAAQ,KAAK;CAAG;AAEnC,KAAK,UAAU,QAAQ,CAAC,GAAoB,EAAE,KAAa;IACzD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,EAAE,CAAC;QAC9B,MAAM,MAAM,GAAG,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3E,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC;QAC1B,IAAI,IAAI,GAAG,KAAK;YAAE,MAAM,IAAI,YAAY,EAAE,CAAC;QAC3C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACtB,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,SAAS,SAAS,CAAC,GAAW;IAC5B,IAAI,CAAC;QACH,OAAO,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACzB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AAED,SAAS,IAAI,CAAC,GAAmB,EAAE,MAAc,EAAE,IAAa;IAC9D,GAAG,CAAC,SAAS,CAAC,MAAM,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,CAAC,CAAC;IAC9D,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,OAOnC;IACC,MAAM,MAAM,GAAG,iBAAiB,CAAC;QAC/B,aAAa,EAAE,OAAO,CAAC,aAAa,IAAI,2BAA2B;QACnE,KAAK,EAAE,OAAO,CAAC,aAAa,IAAI,2BAA2B;KAC5D,CAAC,CAAC;IACH,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,IAAI,mBAAmB,CAAC;IAC7D,MAAM,KAAK,GAAG,iBAAiB,CAAC,EAAE,aAAa,EAAE,UAAU,CAAC,SAAS,EAAE,KAAK,EAAE,UAAU,CAAC,QAAQ,EAAE,CAAC,CAAC;IAErG,KAAK,UAAU,MAAM,CAAC,GAAoB,EAAE,GAAmB;QAC7D,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;YAClD,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;QACzE,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,MAAM,IAAI,GAAG,CAAC,GAAG,KAAK,QAAQ;YAAE,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,WAAW,EAAE,CAAC,CAAC;QACjG,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE;YAAE,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;QAChF,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,GAAG,EAAE,uBAAuB,CAAC,CAAC;QACzD,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,mCAAmC,EAAE,CAAC,CAAC;QAC/G,MAAM,MAAM,GAAG,kBAAkB,CAAC,SAAS,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QAC5D,IAAI,CAAC,MAAM,CAAC,OAAO;YAAE,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,iBAAiB,EAAE,CAAC,CAAC;QACzE,MAAM,WAAW,GAAG,CAAC,MAAM,OAAO,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,eAAe,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,CAAC;QACjH,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,WAAW,CAAC,EAAE,CAAC,CAAC;IACxE,CAAC;IAED,MAAM,MAAM,GAAG,YAAY,CAAC,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE;QACvC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;YACxC,IAAI,KAAK,YAAY,YAAY;gBAAE,OAAO,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,yBAAyB,EAAE,CAAC,CAAC;YAC/F,OAAO,CAAC,GAAG,EAAE,CAAC,0BAA0B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC;YAC3F,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,KAAK,EAAE,cAAc,EAAE,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,KAAK,CAAC,MAAM;YACV,MAAM,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9D,MAAM,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;YAC9C,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;gBAC1C,MAAM,CAAC,IAAI,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;gBAC7B,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC;YACrD,CAAC,CAAC,CAAC;YACH,MAAM,KAAK,CAAC,OAAO,CAAC,UAAU,EAAE,KAAK,CAAC,CAAC;QACzC,CAAC;QACD,KAAK,CAAC,KAAK;YACT,MAAM,IAAI,OAAO,CAAO,CAAC,OAAO,EAAE,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACtE,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -59,6 +59,13 @@ describe("broker socket service", () => {
|
|
|
59
59
|
expect(JSON.parse(health)).toEqual({ ok: true, credentials: 2 });
|
|
60
60
|
expect(health).not.toContain(VALUE);
|
|
61
61
|
});
|
|
62
|
+
it("keeps up with dozens of agents whose tool results all arrive at once", async () => {
|
|
63
|
+
// A 20-researcher run withheld 40% of its results on a 256 KB/s budget, with the broker healthy.
|
|
64
|
+
const { socketPath } = await start();
|
|
65
|
+
const result = `${"r".repeat(1024 * 1024 - 64)} ${VALUE}`;
|
|
66
|
+
const scrubbed = await Promise.all(Array.from({ length: 80 }, () => scrubViaBroker(result, { socketPath, timeoutMs: 30_000 })));
|
|
67
|
+
expect(scrubbed.every((text) => text.endsWith(" {{credential:shop}}"))).toBe(true);
|
|
68
|
+
});
|
|
62
69
|
it("answers 429 once a caller has sent more text than its byte budget allows", async () => {
|
|
63
70
|
const { socketPath } = await start({ scrubBytes: { capacity: 100, perSecond: 1 } });
|
|
64
71
|
const text = "x".repeat(60);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"socket-service.test.js","sourceRoot":"","sources":["../../src/broker/socket-service.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAsB,MAAM,qBAAqB,CAAC;AAE9E,MAAM,KAAK,GAAG,wBAAwB,CAAC;AACvC,MAAM,aAAa,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACnD,IAAI,OAAO,GAAyB,IAAI,CAAC;AAEzC,SAAS,CAAC,KAAK,IAAI,EAAE;IACnB,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC;IACvB,OAAO,GAAG,IAAI,CAAC;AACjB,CAAC,CAAC,CAAC;AAEH,KAAK,UAAU,KAAK,CAAC,MAAM,GAA6D,EAAE;IACxF,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAG,qBAAqB,CAAC;QAClC,gBAAgB,EAAE,GAAG,EAAE,CACrB,OAAO,CAAC,OAAO,CAAC;YACd,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,oBAAoB,CAAC,EAAE;YACpE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,iBAAiB,CAAC,EAAE;SAC3E,CAAC;KACL,CAAC,CAAC;IACH,OAAO,GAAG,mBAAmB,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAChE,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC;IACvB,OAAO,EAAE,UAAU,EAAE,CAAC;AACxB,CAAC;AAED,SAAS,GAAG,CAAC,UAAkB,EAAE,IAAY;IAC3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE;YAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACxB,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAC/B,IAAI,IAAI,KAAK,CAAC;YAChB,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxB,GAAG,CAAC,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QAErC,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAE1F,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,4EAA4E;QAC5E,+DAA+D;QAC/D,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QAErC,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,sBAAsB,aAAa,UAAU,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAEhG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,sBAAsB,aAAa,UAAU,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QACrC,MAAM,cAAc,CAAC,gBAAgB,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAEvD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAEhD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0EAA0E,EAAE,KAAK,IAAI,EAAE;QACxF,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,KAAK,CAAC,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACpF,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAE5B,MAAM,cAAc,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAE3C,MAAM,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAEhF,MAAM,CAAC,MAAM,cAAc,CAAC,OAAO,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
1
|
+
{"version":3,"file":"socket-service.test.js","sourceRoot":"","sources":["../../src/broker/socket-service.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAC;AACjC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AACzD,OAAO,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AACpD,OAAO,EAAE,qBAAqB,EAAE,MAAM,uBAAuB,CAAC;AAC9D,OAAO,EAAE,mBAAmB,EAAsB,MAAM,qBAAqB,CAAC;AAE9E,MAAM,KAAK,GAAG,wBAAwB,CAAC;AACvC,MAAM,aAAa,GAAG,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACnD,IAAI,OAAO,GAAyB,IAAI,CAAC;AAEzC,SAAS,CAAC,KAAK,IAAI,EAAE;IACnB,MAAM,OAAO,EAAE,KAAK,EAAE,CAAC;IACvB,OAAO,GAAG,IAAI,CAAC;AACjB,CAAC,CAAC,CAAC;AAEH,KAAK,UAAU,KAAK,CAAC,MAAM,GAA6D,EAAE;IACxF,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,EAAE,aAAa,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAG,qBAAqB,CAAC;QAClC,gBAAgB,EAAE,GAAG,EAAE,CACrB,OAAO,CAAC,OAAO,CAAC;YACd,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,CAAC,oBAAoB,CAAC,EAAE;YACpE,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,CAAC,iBAAiB,CAAC,EAAE;SAC3E,CAAC;KACL,CAAC,CAAC;IACH,OAAO,GAAG,mBAAmB,CAAC,EAAE,KAAK,EAAE,UAAU,EAAE,GAAG,MAAM,EAAE,CAAC,CAAC;IAChE,MAAM,OAAO,CAAC,MAAM,EAAE,CAAC;IACvB,OAAO,EAAE,UAAU,EAAE,CAAC;AACxB,CAAC;AAED,SAAS,GAAG,CAAC,UAAkB,EAAE,IAAY;IAC3C,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE;YAC/D,IAAI,IAAI,GAAG,EAAE,CAAC;YACd,GAAG,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;YACxB,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;gBAC/B,IAAI,IAAI,KAAK,CAAC;YAChB,CAAC,CAAC,CAAC;YACH,GAAG,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC;QACrC,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;QACxB,GAAG,CAAC,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,QAAQ,CAAC,uBAAuB,EAAE,GAAG,EAAE;IACrC,EAAE,CAAC,uDAAuD,EAAE,KAAK,IAAI,EAAE;QACrE,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QAErC,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,aAAa,KAAK,QAAQ,IAAI,CAAC,KAAK,CAAC,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAE1F,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,uDAAuD,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,KAAK,IAAI,EAAE;QACxE,4EAA4E;QAC5E,+DAA+D;QAC/D,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QAErC,MAAM,GAAG,GAAG,MAAM,cAAc,CAAC,sBAAsB,aAAa,UAAU,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAEhG,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,sBAAsB,aAAa,UAAU,CAAC,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4DAA4D,EAAE,KAAK,IAAI,EAAE;QAC1E,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QACrC,MAAM,cAAc,CAAC,gBAAgB,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAEvD,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAEhD,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;QACjE,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IACtC,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,KAAK,IAAI,EAAE;QACpF,iGAAiG;QACjG,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,KAAK,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,GAAG,GAAG,CAAC,MAAM,CAAC,IAAI,GAAG,IAAI,GAAG,EAAE,CAAC,IAAI,KAAK,EAAE,CAAC;QAE1D,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,EAAE,EAAE,EAAE,GAAG,EAAE,CAAC,cAAc,CAAC,MAAM,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC,CAAC,CAC5F,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0EAA0E,EAAE,KAAK,IAAI,EAAE;QACxF,MAAM,EAAE,UAAU,EAAE,GAAG,MAAM,KAAK,CAAC,EAAE,UAAU,EAAE,EAAE,QAAQ,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACpF,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;QAE5B,MAAM,cAAc,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC;QAE3C,MAAM,MAAM,CAAC,cAAc,CAAC,IAAI,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAC5E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,KAAK,IAAI,EAAE;QACpE,MAAM,UAAU,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,SAAS,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC;QAEhF,MAAM,CAAC,MAAM,cAAc,CAAC,OAAO,KAAK,EAAE,EAAE,EAAE,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,EAAE,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervisor-main.d.ts","sourceRoot":"","sources":["../../src/broker/supervisor-main.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* `baker-credential-broker-supervisor` — what Runtime activation starts, as
|
|
4
|
+
* root: it runs `baker-credential-broker` and starts it again whenever it
|
|
5
|
+
* exits or stops answering on its socket (`superviseBroker`). Its command line
|
|
6
|
+
* names the same install as the broker's, so the activation's stop reaches both.
|
|
7
|
+
*/
|
|
8
|
+
import { writeFileSync } from "node:fs";
|
|
9
|
+
import { request } from "node:http";
|
|
10
|
+
import { fileURLToPath } from "node:url";
|
|
11
|
+
import { spawnBroker } from "./broker-process.js";
|
|
12
|
+
import { brokerChildEnv, parseBrokerEnv } from "./env.js";
|
|
13
|
+
import { superviseBroker } from "./supervisor.js";
|
|
14
|
+
const BROKER_MAIN = fileURLToPath(new URL("./main.js", import.meta.url));
|
|
15
|
+
const HEALTH_TIMEOUT_MS = 2_000;
|
|
16
|
+
function log(message) {
|
|
17
|
+
console.error(`[credential-broker-supervisor] ${message}`);
|
|
18
|
+
}
|
|
19
|
+
// Parsed only for the socket path; the broker it starts inherits this environment and parses it again.
|
|
20
|
+
const parsed = parseBrokerEnv();
|
|
21
|
+
if (!parsed.success) {
|
|
22
|
+
log(`invalid environment: ${parsed.error.issues.map((issue) => issue.path.join(".")).join(", ")}`);
|
|
23
|
+
process.exit(1);
|
|
24
|
+
}
|
|
25
|
+
const SOCKET = parsed.data.BAKER_BROKER_SOCKET;
|
|
26
|
+
/**
|
|
27
|
+
* Out of the OOM killer's reach, and its brokers with it (the setting is
|
|
28
|
+
* inherited). With every process at the same score the kernel kills the
|
|
29
|
+
* biggest, and a busy broker was it: a 20-agent run filled the sandbox's 8 GB
|
|
30
|
+
* with the agent's own commands, the broker died, and the turn could read
|
|
31
|
+
* nothing more. The bridge marks the agent's commands as the ones to kill.
|
|
32
|
+
*/
|
|
33
|
+
try {
|
|
34
|
+
writeFileSync("/proc/self/oom_score_adj", "-1000");
|
|
35
|
+
}
|
|
36
|
+
catch (error) {
|
|
37
|
+
log(`could not leave the OOM killer's reach: ${error instanceof Error ? error.message : String(error)}`);
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Its heap, capped: V8 otherwise lets garbage pile up to hundreds of MB before
|
|
41
|
+
* collecting, and memory the broker holds is memory the agent's commands lack.
|
|
42
|
+
* 40 agents scrubbing at once fit in a 64 MB heap; a 10 MB proxied body needs room too.
|
|
43
|
+
*/
|
|
44
|
+
const BROKER_NODE_OPTIONS = "--max-old-space-size=192";
|
|
45
|
+
function healthy() {
|
|
46
|
+
return new Promise((resolve) => {
|
|
47
|
+
const req = request({ socketPath: SOCKET, path: "/health", method: "GET", timeout: HEALTH_TIMEOUT_MS }, (res) => {
|
|
48
|
+
res.resume();
|
|
49
|
+
resolve(res.statusCode === 200);
|
|
50
|
+
});
|
|
51
|
+
req.on("timeout", () => req.destroy());
|
|
52
|
+
req.on("error", () => resolve(false));
|
|
53
|
+
req.end();
|
|
54
|
+
});
|
|
55
|
+
}
|
|
56
|
+
const supervisor = superviseBroker({
|
|
57
|
+
start: () => spawnBroker(process.execPath, [BROKER_MAIN], brokerChildEnv(BROKER_NODE_OPTIONS), log),
|
|
58
|
+
healthy,
|
|
59
|
+
log,
|
|
60
|
+
});
|
|
61
|
+
let stopping = false;
|
|
62
|
+
for (const signal of ["SIGTERM", "SIGINT"]) {
|
|
63
|
+
process.on(signal, () => {
|
|
64
|
+
if (stopping)
|
|
65
|
+
return;
|
|
66
|
+
stopping = true;
|
|
67
|
+
void supervisor.stop(signal).then(() => process.exit(0));
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
//# sourceMappingURL=supervisor-main.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervisor-main.js","sourceRoot":"","sources":["../../src/broker/supervisor-main.ts"],"names":[],"mappings":";AACA;;;;;GAKG;AACH,OAAO,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACxC,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpC,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,cAAc,EAAE,MAAM,UAAU,CAAC;AAC1D,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAElD,MAAM,WAAW,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,WAAW,EAAE,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACzE,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAEhC,SAAS,GAAG,CAAC,OAAe;IAC1B,OAAO,CAAC,KAAK,CAAC,kCAAkC,OAAO,EAAE,CAAC,CAAC;AAC7D,CAAC;AAED,uGAAuG;AACvG,MAAM,MAAM,GAAG,cAAc,EAAE,CAAC;AAChC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;IACpB,GAAG,CAAC,wBAAwB,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACnG,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC;AACD,MAAM,MAAM,GAAG,MAAM,CAAC,IAAI,CAAC,mBAAmB,CAAC;AAE/C;;;;;;GAMG;AACH,IAAI,CAAC;IACH,aAAa,CAAC,0BAA0B,EAAE,OAAO,CAAC,CAAC;AACrD,CAAC;AAAC,OAAO,KAAK,EAAE,CAAC;IACf,GAAG,CAAC,2CAA2C,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;AAC3G,CAAC;AAED;;;;GAIG;AACH,MAAM,mBAAmB,GAAG,0BAA0B,CAAC;AAEvD,SAAS,OAAO;IACd,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE;QAC7B,MAAM,GAAG,GAAG,OAAO,CAAC,EAAE,UAAU,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,iBAAiB,EAAE,EAAE,CAAC,GAAG,EAAE,EAAE;YAC9G,GAAG,CAAC,MAAM,EAAE,CAAC;YACb,OAAO,CAAC,GAAG,CAAC,UAAU,KAAK,GAAG,CAAC,CAAC;QAClC,CAAC,CAAC,CAAC;QACH,GAAG,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;QACvC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC;QACtC,GAAG,CAAC,GAAG,EAAE,CAAC;IACZ,CAAC,CAAC,CAAC;AACL,CAAC;AAED,MAAM,UAAU,GAAG,eAAe,CAAC;IACjC,KAAK,EAAE,GAAG,EAAE,CAAC,WAAW,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC,WAAW,CAAC,EAAE,cAAc,CAAC,mBAAmB,CAAC,EAAE,GAAG,CAAC;IACnG,OAAO;IACP,GAAG;CACJ,CAAC,CAAC;AAEH,IAAI,QAAQ,GAAG,KAAK,CAAC;AACrB,KAAK,MAAM,MAAM,IAAI,CAAC,SAAS,EAAE,QAAQ,CAAU,EAAE,CAAC;IACpD,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,GAAG,EAAE;QACtB,IAAI,QAAQ;YAAE,OAAO;QACrB,QAAQ,GAAG,IAAI,CAAC;QAChB,KAAK,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAC3D,CAAC,CAAC,CAAC;AACL,CAAC"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keeps the credential broker running for the life of the Runtime.
|
|
3
|
+
*
|
|
4
|
+
* The bridge withholds every tool result it cannot have scrubbed, and it takes
|
|
5
|
+
* the socket file's existence to mean a broker is running. A broker that dies
|
|
6
|
+
* hard (the kernel's OOM killer, a crash) leaves that file behind, and one that
|
|
7
|
+
* hangs keeps it open; either way, without a supervisor every result is
|
|
8
|
+
* withheld until the next activation — a whole turn.
|
|
9
|
+
*/
|
|
10
|
+
export type BrokerExit = {
|
|
11
|
+
code: number | null;
|
|
12
|
+
signal: NodeJS.Signals | null;
|
|
13
|
+
};
|
|
14
|
+
export type BrokerProcess = {
|
|
15
|
+
exited: Promise<BrokerExit>;
|
|
16
|
+
kill(signal: NodeJS.Signals): void;
|
|
17
|
+
};
|
|
18
|
+
export declare function superviseBroker(options: {
|
|
19
|
+
start: () => BrokerProcess;
|
|
20
|
+
healthy: () => Promise<boolean>;
|
|
21
|
+
log?: (message: string) => void;
|
|
22
|
+
}): {
|
|
23
|
+
stop(signal: NodeJS.Signals): Promise<BrokerExit>;
|
|
24
|
+
};
|
|
25
|
+
//# sourceMappingURL=supervisor.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervisor.d.ts","sourceRoot":"","sources":["../../src/broker/supervisor.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AACH,MAAM,MAAM,UAAU,GAAG;IAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAA;CAAE,CAAC;AAChF,MAAM,MAAM,aAAa,GAAG;IAAE,MAAM,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;IAAC,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAA;CAAE,CAAC;AAkBhG,wBAAgB,eAAe,CAAC,OAAO,EAAE;IACvC,KAAK,EAAE,MAAM,aAAa,CAAC;IAC3B,OAAO,EAAE,MAAM,OAAO,CAAC,OAAO,CAAC,CAAC;IAChC,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,MAAM,KAAK,IAAI,CAAC;CACjC,GAAG;IAAE,IAAI,CAAC,MAAM,EAAE,MAAM,CAAC,OAAO,GAAG,OAAO,CAAC,UAAU,CAAC,CAAA;CAAE,CAuExD"}
|
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
const HEALTH_INTERVAL_MS = 5_000;
|
|
2
|
+
/** Consecutive missed health checks before a live broker is taken for hung. */
|
|
3
|
+
const UNHEALTHY_AFTER = 3;
|
|
4
|
+
/** A broker that has never answered gets this long to start (a first start writes its CA). */
|
|
5
|
+
const STARTUP_GRACE_MS = 30_000;
|
|
6
|
+
/** Pauses before each restart in a row; a broker that exits as soon as it starts is retried ever more slowly, never given up on. */
|
|
7
|
+
const RESTART_DELAYS_MS = [250, 1_000, 2_000, 5_000, 10_000, 30_000];
|
|
8
|
+
/** A broker still running this long after the stop signal is killed. */
|
|
9
|
+
const STOP_GRACE_MS = 5_000;
|
|
10
|
+
/** A broker that ran this long was not crash-looping: the next restart is quick again. */
|
|
11
|
+
const STABLE_AFTER_MS = 60_000;
|
|
12
|
+
function describeExit(exit) {
|
|
13
|
+
return exit.signal === null ? `code ${exit.code}` : `signal ${exit.signal}`;
|
|
14
|
+
}
|
|
15
|
+
export function superviseBroker(options) {
|
|
16
|
+
let stopping = false;
|
|
17
|
+
let restarts = 0;
|
|
18
|
+
/** A broker that could not be spawned (ENOMEM, EAGAIN under memory pressure) is one that exited at once. */
|
|
19
|
+
function startOrFail() {
|
|
20
|
+
try {
|
|
21
|
+
return options.start();
|
|
22
|
+
}
|
|
23
|
+
catch (error) {
|
|
24
|
+
options.log?.(`could not start broker: ${error instanceof Error ? error.message : String(error)}`);
|
|
25
|
+
return { exited: Promise.resolve({ code: null, signal: null }), kill: () => undefined };
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
function run() {
|
|
29
|
+
const broker = startOrFail();
|
|
30
|
+
const startedAt = Date.now();
|
|
31
|
+
let answered = false;
|
|
32
|
+
let missed = 0;
|
|
33
|
+
let probing = false;
|
|
34
|
+
const watchdog = setInterval(() => {
|
|
35
|
+
if (probing)
|
|
36
|
+
return;
|
|
37
|
+
probing = true;
|
|
38
|
+
void options
|
|
39
|
+
.healthy()
|
|
40
|
+
.catch(() => false)
|
|
41
|
+
.then((ok) => {
|
|
42
|
+
probing = false;
|
|
43
|
+
if (ok) {
|
|
44
|
+
answered = true;
|
|
45
|
+
missed = 0;
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
if (!answered && Date.now() - startedAt < STARTUP_GRACE_MS)
|
|
49
|
+
return;
|
|
50
|
+
missed += 1;
|
|
51
|
+
if (missed >= UNHEALTHY_AFTER) {
|
|
52
|
+
options.log?.(`broker missed ${missed} health checks in a row; killing it`);
|
|
53
|
+
broker.kill("SIGKILL");
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
}, HEALTH_INTERVAL_MS);
|
|
57
|
+
void broker.exited.then((exit) => {
|
|
58
|
+
clearInterval(watchdog);
|
|
59
|
+
if (stopping)
|
|
60
|
+
return;
|
|
61
|
+
const ranMs = Date.now() - startedAt;
|
|
62
|
+
if (ranMs >= STABLE_AFTER_MS)
|
|
63
|
+
restarts = 0;
|
|
64
|
+
const delay = RESTART_DELAYS_MS[Math.min(restarts, RESTART_DELAYS_MS.length - 1)];
|
|
65
|
+
restarts += 1;
|
|
66
|
+
options.log?.(`broker exited (${describeExit(exit)}) after ${Math.round(ranMs / 1000)}s; restarting in ${delay}ms`);
|
|
67
|
+
setTimeout(() => {
|
|
68
|
+
if (!stopping)
|
|
69
|
+
current = run();
|
|
70
|
+
}, delay);
|
|
71
|
+
});
|
|
72
|
+
return broker;
|
|
73
|
+
}
|
|
74
|
+
let current = run();
|
|
75
|
+
return {
|
|
76
|
+
stop(signal) {
|
|
77
|
+
stopping = true;
|
|
78
|
+
const broker = current;
|
|
79
|
+
broker.kill(signal);
|
|
80
|
+
const escalate = setTimeout(() => {
|
|
81
|
+
options.log?.(`broker still running ${STOP_GRACE_MS}ms after ${signal}; killing it`);
|
|
82
|
+
broker.kill("SIGKILL");
|
|
83
|
+
}, STOP_GRACE_MS);
|
|
84
|
+
return broker.exited.finally(() => clearTimeout(escalate));
|
|
85
|
+
},
|
|
86
|
+
};
|
|
87
|
+
}
|
|
88
|
+
//# sourceMappingURL=supervisor.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervisor.js","sourceRoot":"","sources":["../../src/broker/supervisor.ts"],"names":[],"mappings":"AAYA,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,+EAA+E;AAC/E,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,8FAA8F;AAC9F,MAAM,gBAAgB,GAAG,MAAM,CAAC;AAChC,oIAAoI;AACpI,MAAM,iBAAiB,GAAG,CAAC,GAAG,EAAE,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,MAAM,CAAU,CAAC;AAC9E,wEAAwE;AACxE,MAAM,aAAa,GAAG,KAAK,CAAC;AAC5B,0FAA0F;AAC1F,MAAM,eAAe,GAAG,MAAM,CAAC;AAE/B,SAAS,YAAY,CAAC,IAAgB;IACpC,OAAO,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,CAAC,CAAC,QAAQ,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,CAAC,MAAM,EAAE,CAAC;AAC9E,CAAC;AAED,MAAM,UAAU,eAAe,CAAC,OAI/B;IACC,IAAI,QAAQ,GAAG,KAAK,CAAC;IACrB,IAAI,QAAQ,GAAG,CAAC,CAAC;IAEjB,4GAA4G;IAC5G,SAAS,WAAW;QAClB,IAAI,CAAC;YACH,OAAO,OAAO,CAAC,KAAK,EAAE,CAAC;QACzB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,GAAG,EAAE,CAAC,2BAA2B,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;YACnG,OAAO,EAAE,MAAM,EAAE,OAAO,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,CAAC,SAAS,EAAE,CAAC;QAC1F,CAAC;IACH,CAAC;IAED,SAAS,GAAG;QACV,MAAM,MAAM,GAAG,WAAW,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC7B,IAAI,QAAQ,GAAG,KAAK,CAAC;QACrB,IAAI,MAAM,GAAG,CAAC,CAAC;QACf,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,EAAE;YAChC,IAAI,OAAO;gBAAE,OAAO;YACpB,OAAO,GAAG,IAAI,CAAC;YACf,KAAK,OAAO;iBACT,OAAO,EAAE;iBACT,KAAK,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC;iBAClB,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE;gBACX,OAAO,GAAG,KAAK,CAAC;gBAChB,IAAI,EAAE,EAAE,CAAC;oBACP,QAAQ,GAAG,IAAI,CAAC;oBAChB,MAAM,GAAG,CAAC,CAAC;oBACX,OAAO;gBACT,CAAC;gBACD,IAAI,CAAC,QAAQ,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,GAAG,gBAAgB;oBAAE,OAAO;gBACnE,MAAM,IAAI,CAAC,CAAC;gBACZ,IAAI,MAAM,IAAI,eAAe,EAAE,CAAC;oBAC9B,OAAO,CAAC,GAAG,EAAE,CAAC,iBAAiB,MAAM,qCAAqC,CAAC,CAAC;oBAC5E,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACzB,CAAC;YACH,CAAC,CAAC,CAAC;QACP,CAAC,EAAE,kBAAkB,CAAC,CAAC;QACvB,KAAK,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YAC/B,aAAa,CAAC,QAAQ,CAAC,CAAC;YACxB,IAAI,QAAQ;gBAAE,OAAO;YACrB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;YACrC,IAAI,KAAK,IAAI,eAAe;gBAAE,QAAQ,GAAG,CAAC,CAAC;YAC3C,MAAM,KAAK,GAAG,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;YAClF,QAAQ,IAAI,CAAC,CAAC;YACd,OAAO,CAAC,GAAG,EAAE,CACX,kBAAkB,YAAY,CAAC,IAAI,CAAC,WAAW,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,oBAAoB,KAAK,IAAI,CACrG,CAAC;YACF,UAAU,CAAC,GAAG,EAAE;gBACd,IAAI,CAAC,QAAQ;oBAAE,OAAO,GAAG,GAAG,EAAE,CAAC;YACjC,CAAC,EAAE,KAAK,CAAC,CAAC;QACZ,CAAC,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,IAAI,OAAO,GAAG,GAAG,EAAE,CAAC;IACpB,OAAO;QACL,IAAI,CAAC,MAAM;YACT,QAAQ,GAAG,IAAI,CAAC;YAChB,MAAM,MAAM,GAAG,OAAO,CAAC;YACvB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACpB,MAAM,QAAQ,GAAG,UAAU,CAAC,GAAG,EAAE;gBAC/B,OAAO,CAAC,GAAG,EAAE,CAAC,wBAAwB,aAAa,YAAY,MAAM,cAAc,CAAC,CAAC;gBACrF,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;YACzB,CAAC,EAAE,aAAa,CAAC,CAAC;YAClB,OAAO,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,EAAE,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC7D,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"supervisor.test.d.ts","sourceRoot":"","sources":["../../src/broker/supervisor.test.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
|
2
|
+
import { superviseBroker } from "./supervisor.js";
|
|
3
|
+
function fakeBroker() {
|
|
4
|
+
const settle = [];
|
|
5
|
+
const exited = new Promise((resolve) => settle.push(resolve));
|
|
6
|
+
const resolveExit = (exit) => settle[0]?.(exit);
|
|
7
|
+
const signals = [];
|
|
8
|
+
return {
|
|
9
|
+
exited,
|
|
10
|
+
signals,
|
|
11
|
+
kill(signal) {
|
|
12
|
+
signals.push(signal);
|
|
13
|
+
resolveExit({ code: null, signal });
|
|
14
|
+
},
|
|
15
|
+
die: (exit) => resolveExit(exit),
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
function launcher() {
|
|
19
|
+
const started = [];
|
|
20
|
+
return {
|
|
21
|
+
started,
|
|
22
|
+
start: () => {
|
|
23
|
+
const broker = fakeBroker();
|
|
24
|
+
started.push(broker);
|
|
25
|
+
return broker;
|
|
26
|
+
},
|
|
27
|
+
};
|
|
28
|
+
}
|
|
29
|
+
beforeEach(() => {
|
|
30
|
+
vi.useFakeTimers();
|
|
31
|
+
});
|
|
32
|
+
afterEach(() => {
|
|
33
|
+
vi.useRealTimers();
|
|
34
|
+
});
|
|
35
|
+
describe("superviseBroker", () => {
|
|
36
|
+
it("starts a new broker when the running one is killed", async () => {
|
|
37
|
+
const brokers = launcher();
|
|
38
|
+
superviseBroker({ start: brokers.start, healthy: async () => true });
|
|
39
|
+
brokers.started[0]?.die({ code: null, signal: "SIGKILL" });
|
|
40
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
41
|
+
expect(brokers.started).toHaveLength(2);
|
|
42
|
+
});
|
|
43
|
+
it("kills and replaces a broker that stays alive but stops answering", async () => {
|
|
44
|
+
const brokers = launcher();
|
|
45
|
+
let answering = true;
|
|
46
|
+
superviseBroker({ start: brokers.start, healthy: async () => answering });
|
|
47
|
+
await vi.advanceTimersByTimeAsync(10_000);
|
|
48
|
+
answering = false;
|
|
49
|
+
await vi.advanceTimersByTimeAsync(20_000);
|
|
50
|
+
expect(brokers.started[0]?.signals).toEqual(["SIGKILL"]);
|
|
51
|
+
expect(brokers.started).toHaveLength(2);
|
|
52
|
+
});
|
|
53
|
+
// A first start writes the broker's CA before it can answer.
|
|
54
|
+
it("gives a broker that has never answered 30 seconds to start before taking it for hung", async () => {
|
|
55
|
+
const brokers = launcher();
|
|
56
|
+
superviseBroker({ start: brokers.start, healthy: async () => false });
|
|
57
|
+
await vi.advanceTimersByTimeAsync(29_000);
|
|
58
|
+
const killedDuringStartup = [...(brokers.started[0]?.signals ?? [])];
|
|
59
|
+
await vi.advanceTimersByTimeAsync(20_000);
|
|
60
|
+
expect(killedDuringStartup).toEqual([]);
|
|
61
|
+
expect(brokers.started[0]?.signals).toEqual(["SIGKILL"]);
|
|
62
|
+
});
|
|
63
|
+
it("stops the broker and starts no other once told to stop — an activation's stop revokes its token", async () => {
|
|
64
|
+
const brokers = launcher();
|
|
65
|
+
const supervisor = superviseBroker({ start: brokers.start, healthy: async () => true });
|
|
66
|
+
await supervisor.stop("SIGTERM");
|
|
67
|
+
await vi.advanceTimersByTimeAsync(60_000);
|
|
68
|
+
expect(brokers.started).toHaveLength(1);
|
|
69
|
+
expect(brokers.started[0]?.signals).toEqual(["SIGTERM"]);
|
|
70
|
+
});
|
|
71
|
+
it("starts no broker when told to stop while waiting to restart one", async () => {
|
|
72
|
+
const brokers = launcher();
|
|
73
|
+
const supervisor = superviseBroker({ start: brokers.start, healthy: async () => true });
|
|
74
|
+
brokers.started[0]?.die({ code: 1, signal: null });
|
|
75
|
+
await vi.advanceTimersByTimeAsync(100);
|
|
76
|
+
await supervisor.stop("SIGTERM");
|
|
77
|
+
await vi.advanceTimersByTimeAsync(60_000);
|
|
78
|
+
expect(brokers.started).toHaveLength(1);
|
|
79
|
+
});
|
|
80
|
+
// A broker hung in its own shutdown would keep the supervisor, a root process, alive past its token's revocation.
|
|
81
|
+
it("kills a broker that has not stopped five seconds after being told to", async () => {
|
|
82
|
+
const signals = [];
|
|
83
|
+
let settle = () => undefined;
|
|
84
|
+
const exited = new Promise((resolve) => {
|
|
85
|
+
settle = resolve;
|
|
86
|
+
});
|
|
87
|
+
const supervisor = superviseBroker({
|
|
88
|
+
start: () => ({
|
|
89
|
+
exited,
|
|
90
|
+
kill(signal) {
|
|
91
|
+
signals.push(signal);
|
|
92
|
+
if (signal === "SIGKILL")
|
|
93
|
+
settle({ code: null, signal });
|
|
94
|
+
},
|
|
95
|
+
}),
|
|
96
|
+
healthy: async () => true,
|
|
97
|
+
});
|
|
98
|
+
const stopped = supervisor.stop("SIGTERM");
|
|
99
|
+
await vi.advanceTimersByTimeAsync(5_000);
|
|
100
|
+
await expect(stopped).resolves.toEqual({ code: null, signal: "SIGKILL" });
|
|
101
|
+
expect(signals).toEqual(["SIGTERM", "SIGKILL"]);
|
|
102
|
+
});
|
|
103
|
+
it("backs off from a broker that exits as soon as it starts, and keeps trying", async () => {
|
|
104
|
+
const started = [];
|
|
105
|
+
superviseBroker({
|
|
106
|
+
start: () => {
|
|
107
|
+
const broker = fakeBroker();
|
|
108
|
+
started.push(broker);
|
|
109
|
+
broker.die({ code: 1, signal: null });
|
|
110
|
+
return broker;
|
|
111
|
+
},
|
|
112
|
+
healthy: async () => false,
|
|
113
|
+
});
|
|
114
|
+
await vi.advanceTimersByTimeAsync(120_000);
|
|
115
|
+
const afterTwoMinutes = started.length;
|
|
116
|
+
await vi.advanceTimersByTimeAsync(120_000);
|
|
117
|
+
expect(afterTwoMinutes).toBeLessThan(15);
|
|
118
|
+
expect(started.length).toBeGreaterThan(afterTwoMinutes);
|
|
119
|
+
});
|
|
120
|
+
// Memory pressure, the case the supervisor exists for, is when a spawn fails.
|
|
121
|
+
it("retries a broker that could not even be started", async () => {
|
|
122
|
+
const brokers = launcher();
|
|
123
|
+
let attempts = 0;
|
|
124
|
+
const log = [];
|
|
125
|
+
superviseBroker({
|
|
126
|
+
start: () => {
|
|
127
|
+
attempts += 1;
|
|
128
|
+
if (attempts === 1)
|
|
129
|
+
throw new Error("spawn ENOMEM");
|
|
130
|
+
return brokers.start();
|
|
131
|
+
},
|
|
132
|
+
healthy: async () => true,
|
|
133
|
+
log: (line) => log.push(line),
|
|
134
|
+
});
|
|
135
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
136
|
+
expect(brokers.started).toHaveLength(1);
|
|
137
|
+
expect(log[0]).toContain("spawn ENOMEM");
|
|
138
|
+
});
|
|
139
|
+
it("restarts quickly again after a broker that ran for a minute, however slow the restarts before it", async () => {
|
|
140
|
+
const brokers = launcher();
|
|
141
|
+
const log = [];
|
|
142
|
+
superviseBroker({ start: brokers.start, healthy: async () => true, log: (line) => log.push(line) });
|
|
143
|
+
for (const index of [0, 1, 2]) {
|
|
144
|
+
brokers.started[index]?.die({ code: 1, signal: null });
|
|
145
|
+
await vi.advanceTimersByTimeAsync(3_000);
|
|
146
|
+
}
|
|
147
|
+
await vi.advanceTimersByTimeAsync(60_000);
|
|
148
|
+
brokers.started[3]?.die({ code: null, signal: "SIGKILL" });
|
|
149
|
+
await vi.advanceTimersByTimeAsync(0);
|
|
150
|
+
expect(log.map((line) => line.split("restarting in ")[1])).toEqual(["250ms", "1000ms", "2000ms", "250ms"]);
|
|
151
|
+
});
|
|
152
|
+
it("logs how each broker ended and when the next one starts", async () => {
|
|
153
|
+
const brokers = launcher();
|
|
154
|
+
const log = [];
|
|
155
|
+
superviseBroker({ start: brokers.start, healthy: async () => true, log: (line) => log.push(line) });
|
|
156
|
+
brokers.started[0]?.die({ code: null, signal: "SIGKILL" });
|
|
157
|
+
await vi.advanceTimersByTimeAsync(1_000);
|
|
158
|
+
expect(log).toEqual(["broker exited (signal SIGKILL) after 0s; restarting in 250ms"]);
|
|
159
|
+
});
|
|
160
|
+
});
|
|
161
|
+
//# sourceMappingURL=supervisor.test.js.map
|