@mycadet/assistant 0.13.0-dev.202611141959.760a53b → 0.13.0-dev.202611142107.f5e00a2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/credential-execution-api-key-propagation.test.ts +5 -6
- package/src/__tests__/credential-execution-client.test.ts +7 -7
- package/src/credential-execution/ces-runtime.ts +11 -142
- package/src/credential-execution/client.test.ts +22 -0
- package/src/credential-execution/client.ts +197 -30
- package/src/daemon/lifecycle.ts +1 -5
- package/src/security/ces-rpc-record-backend.ts +28 -7
- package/src/credential-execution/ces-connect.test.ts +0 -35
- package/src/credential-execution/ces-connect.ts +0 -93
package/package.json
CHANGED
|
@@ -12,7 +12,7 @@
|
|
|
12
12
|
* ## What this file tests (and what it does NOT)
|
|
13
13
|
*
|
|
14
14
|
* This file exercises **production code** exclusively:
|
|
15
|
-
* - `
|
|
15
|
+
* - `createCesRpcClient` from `@mycadet/ces-client/rpc-client`
|
|
16
16
|
* - Zod schemas from `@mycadet/service-contracts/credential-rpc` (HandshakeRequestSchema,
|
|
17
17
|
* CesRpcSchemas, CesRpcMethod)
|
|
18
18
|
*
|
|
@@ -30,6 +30,7 @@ import { describe, expect, test } from "bun:test";
|
|
|
30
30
|
|
|
31
31
|
import type { CesTransport } from "@mycadet/ces-client/rpc-client";
|
|
32
32
|
import type { CesRpcClient } from "@mycadet/ces-client/rpc-client";
|
|
33
|
+
import { createCesRpcClient } from "@mycadet/ces-client/rpc-client";
|
|
33
34
|
import {
|
|
34
35
|
CES_PROTOCOL_VERSION,
|
|
35
36
|
CesRpcMethod,
|
|
@@ -37,8 +38,6 @@ import {
|
|
|
37
38
|
HandshakeRequestSchema,
|
|
38
39
|
} from "@mycadet/service-contracts/credential-rpc";
|
|
39
40
|
|
|
40
|
-
import { createCesClient } from "../credential-execution/client.js";
|
|
41
|
-
|
|
42
41
|
// ---------------------------------------------------------------------------
|
|
43
42
|
// Helpers
|
|
44
43
|
// ---------------------------------------------------------------------------
|
|
@@ -172,7 +171,7 @@ describe("update_managed_credential RPC contract", () => {
|
|
|
172
171
|
describe("CesClient.updateAssistantApiKey()", () => {
|
|
173
172
|
test("sends update_managed_credential RPC with the correct payload", async () => {
|
|
174
173
|
const transport = createMockTransport();
|
|
175
|
-
const client =
|
|
174
|
+
const client = createCesRpcClient(transport, {
|
|
176
175
|
handshakeTimeoutMs: 1_000,
|
|
177
176
|
requestTimeoutMs: 1_000,
|
|
178
177
|
});
|
|
@@ -210,7 +209,7 @@ describe("CesClient.updateAssistantApiKey()", () => {
|
|
|
210
209
|
|
|
211
210
|
test("propagation flow: handshake without key then update", async () => {
|
|
212
211
|
const transport = createMockTransport();
|
|
213
|
-
const client =
|
|
212
|
+
const client = createCesRpcClient(transport, {
|
|
214
213
|
handshakeTimeoutMs: 1_000,
|
|
215
214
|
requestTimeoutMs: 1_000,
|
|
216
215
|
});
|
|
@@ -254,7 +253,7 @@ describe("CesClient.updateAssistantApiKey()", () => {
|
|
|
254
253
|
|
|
255
254
|
test("throws if called before handshake", async () => {
|
|
256
255
|
const transport = createMockTransport();
|
|
257
|
-
const client =
|
|
256
|
+
const client = createCesRpcClient(transport);
|
|
258
257
|
|
|
259
258
|
try {
|
|
260
259
|
await client.updateAssistantApiKey("key");
|
|
@@ -24,10 +24,10 @@ import type { CesTransport } from "@mycadet/ces-client/rpc-client";
|
|
|
24
24
|
import { CesClientError } from "@mycadet/ces-client/rpc-client";
|
|
25
25
|
import { CesHandshakeError } from "@mycadet/ces-client/rpc-client";
|
|
26
26
|
import { CesTransportError } from "@mycadet/ces-client/rpc-client";
|
|
27
|
+
import { createCesRpcClient } from "@mycadet/ces-client/rpc-client";
|
|
27
28
|
import { resolveIpcEndpoint } from "@mycadet/ipc-server-utils";
|
|
28
29
|
import { CES_PROTOCOL_VERSION } from "@mycadet/service-contracts/credential-rpc";
|
|
29
30
|
|
|
30
|
-
import { createCesClient } from "../credential-execution/client.js";
|
|
31
31
|
import {
|
|
32
32
|
discoverCes,
|
|
33
33
|
discoverCesWithRetry,
|
|
@@ -207,7 +207,7 @@ describe("discoverCesWithRetry", () => {
|
|
|
207
207
|
describe("CES client", () => {
|
|
208
208
|
test("handshake sends correct protocol version", async () => {
|
|
209
209
|
const transport = createMockTransport();
|
|
210
|
-
const client =
|
|
210
|
+
const client = createCesRpcClient(transport, {
|
|
211
211
|
handshakeTimeoutMs: 1_000,
|
|
212
212
|
});
|
|
213
213
|
|
|
@@ -240,7 +240,7 @@ describe("CES client", () => {
|
|
|
240
240
|
|
|
241
241
|
test("handshake times out when CES does not respond", async () => {
|
|
242
242
|
const transport = createMockTransport();
|
|
243
|
-
const client =
|
|
243
|
+
const client = createCesRpcClient(transport, {
|
|
244
244
|
handshakeTimeoutMs: 100, // Very short timeout for test
|
|
245
245
|
});
|
|
246
246
|
|
|
@@ -258,7 +258,7 @@ describe("CES client", () => {
|
|
|
258
258
|
|
|
259
259
|
test("handshake reports rejection from CES", async () => {
|
|
260
260
|
const transport = createMockTransport();
|
|
261
|
-
const client =
|
|
261
|
+
const client = createCesRpcClient(transport, {
|
|
262
262
|
handshakeTimeoutMs: 1_000,
|
|
263
263
|
});
|
|
264
264
|
|
|
@@ -285,7 +285,7 @@ describe("CES client", () => {
|
|
|
285
285
|
|
|
286
286
|
test("call() throws before handshake", async () => {
|
|
287
287
|
const transport = createMockTransport();
|
|
288
|
-
const client =
|
|
288
|
+
const client = createCesRpcClient(transport);
|
|
289
289
|
|
|
290
290
|
try {
|
|
291
291
|
await client.call("list_credentials", {});
|
|
@@ -300,7 +300,7 @@ describe("CES client", () => {
|
|
|
300
300
|
|
|
301
301
|
test("call() throws when transport is dead", async () => {
|
|
302
302
|
const transport = createMockTransport();
|
|
303
|
-
const client =
|
|
303
|
+
const client = createCesRpcClient(transport, {
|
|
304
304
|
handshakeTimeoutMs: 1_000,
|
|
305
305
|
});
|
|
306
306
|
|
|
@@ -332,7 +332,7 @@ describe("CES client", () => {
|
|
|
332
332
|
|
|
333
333
|
test("close() cancels pending requests", async () => {
|
|
334
334
|
const transport = createMockTransport();
|
|
335
|
-
const client =
|
|
335
|
+
const client = createCesRpcClient(transport, {
|
|
336
336
|
handshakeTimeoutMs: 1_000,
|
|
337
337
|
requestTimeoutMs: 10_000,
|
|
338
338
|
});
|
|
@@ -2,12 +2,10 @@ import type { CesRpcClient as CesClient } from "@mycadet/ces-client/rpc-client";
|
|
|
2
2
|
import type { CesRpcHandshakeOptions as CesClientHandshakeOptions } from "@mycadet/ces-client/rpc-client";
|
|
3
3
|
|
|
4
4
|
import { getCloudAssistantId } from "../config/env.js";
|
|
5
|
-
import type { AssistantConfig } from "../config/schema.js";
|
|
6
5
|
import { resolveCloudProxyContext } from "../providers/cloud-proxy/context.js";
|
|
7
6
|
import { cesRecordBackend } from "../security/ces-rpc-record-backend.js";
|
|
8
7
|
import { getLogger } from "../util/logger.js";
|
|
9
|
-
import {
|
|
10
|
-
import type { CesSessionClient } from "./client.js";
|
|
8
|
+
import { type CesSessionClient, createCesClient } from "./client.js";
|
|
11
9
|
|
|
12
10
|
const log = getLogger("ces-runtime");
|
|
13
11
|
|
|
@@ -16,15 +14,16 @@ const log = getLogger("ces-runtime");
|
|
|
16
14
|
* Service) connection. `initializeCesClient()` connects and handshakes at
|
|
17
15
|
* daemon startup; `getCesClient()` exposes the live client to routes;
|
|
18
16
|
* `stopCes()` tears it down at shutdown.
|
|
17
|
+
*
|
|
18
|
+
* Reconnection is owned by the session itself: a dead transport triggers the
|
|
19
|
+
* client's in-session reconnect loop (same handshake, bounded backoff), so
|
|
20
|
+
* the `clientRef` reference stays stable across CES restarts.
|
|
19
21
|
*/
|
|
20
22
|
|
|
21
23
|
let clientPromise: Promise<CesClient | undefined> | undefined;
|
|
22
|
-
let initAbortController: AbortController | undefined;
|
|
23
24
|
let clientRef: CesSessionClient | undefined;
|
|
24
25
|
/** Monotonically increasing counter to detect stale client updates. */
|
|
25
26
|
let clientGeneration = 0;
|
|
26
|
-
/** Boot-session reconnect, set by initializeCesClient and used by the proactive loop. */
|
|
27
|
-
let reconnectBootSession: (() => Promise<CesClient | undefined>) | undefined;
|
|
28
27
|
|
|
29
28
|
/**
|
|
30
29
|
* Initialize the assistant's CES RPC client: discover the socket, connect,
|
|
@@ -34,32 +33,10 @@ let reconnectBootSession: (() => Promise<CesClient | undefined>) | undefined;
|
|
|
34
33
|
* reads route through the record backend's own lazily opened session, and
|
|
35
34
|
* `clientRef` is updated once the handshake resolves.
|
|
36
35
|
*/
|
|
37
|
-
export async function initializeCesClient(
|
|
38
|
-
_config: AssistantConfig,
|
|
39
|
-
): Promise<void> {
|
|
40
|
-
const abortController = new AbortController();
|
|
41
|
-
initAbortController = abortController;
|
|
42
|
-
let handshake: CesClientHandshakeOptions = {};
|
|
36
|
+
export async function initializeCesClient(): Promise<void> {
|
|
43
37
|
const generation = ++clientGeneration;
|
|
44
38
|
|
|
45
|
-
|
|
46
|
-
// Reset the dead session's transport, then open a fresh one — the
|
|
47
|
-
// replacement re-registers its own transport-close hook.
|
|
48
|
-
await clientRef?.stop().catch(() => {});
|
|
49
|
-
const client = await openCesRpcSession({
|
|
50
|
-
handshake,
|
|
51
|
-
onTransportClose: () => startProactiveReconnectLoop(),
|
|
52
|
-
});
|
|
53
|
-
if (client) {
|
|
54
|
-
log.info("CES reconnection handshake accepted");
|
|
55
|
-
clientGeneration++;
|
|
56
|
-
clientRef = client;
|
|
57
|
-
}
|
|
58
|
-
return client;
|
|
59
|
-
};
|
|
60
|
-
|
|
61
|
-
// Reconnect ownership is claimed above, before the identity read.
|
|
62
|
-
// Snapshotting the API key inside this promise (not before it) avoids a
|
|
39
|
+
// Snapshot the API key inside this promise (not before it) to avoid a
|
|
63
40
|
// second resolveCloudProxyContext() after the handshake, which would read
|
|
64
41
|
// the key through CES while reconnecting.
|
|
65
42
|
clientPromise = (async (): Promise<CesClient | undefined> => {
|
|
@@ -69,21 +46,14 @@ export async function initializeCesClient(
|
|
|
69
46
|
// after hatch and stored in the credential store. CES can't read
|
|
70
47
|
// the env var, so we pass it via the handshake.
|
|
71
48
|
const proxyCtx = await resolveCloudProxyContext();
|
|
72
|
-
if (abortController.signal.aborted) {
|
|
73
|
-
return undefined;
|
|
74
|
-
}
|
|
75
49
|
const assistantId = getCloudAssistantId();
|
|
76
|
-
handshake = {
|
|
50
|
+
const handshake: CesClientHandshakeOptions = {
|
|
77
51
|
...(proxyCtx.assistantApiKey
|
|
78
52
|
? { assistantApiKey: proxyCtx.assistantApiKey }
|
|
79
53
|
: {}),
|
|
80
54
|
...(assistantId ? { assistantId } : {}),
|
|
81
55
|
};
|
|
82
|
-
const client = await
|
|
83
|
-
signal: abortController.signal,
|
|
84
|
-
handshake,
|
|
85
|
-
onTransportClose: () => startProactiveReconnectLoop(),
|
|
86
|
-
});
|
|
56
|
+
const client = await createCesClient({ handshake });
|
|
87
57
|
if (client && clientGeneration === generation) {
|
|
88
58
|
clientRef = client;
|
|
89
59
|
log.info(
|
|
@@ -114,20 +84,11 @@ export function getCesClient(): CesClient | undefined {
|
|
|
114
84
|
|
|
115
85
|
/** Tear down the CES connection during daemon shutdown. */
|
|
116
86
|
export async function stopCes(): Promise<void> {
|
|
117
|
-
// Suppress the proactive reconnect loop — forceStop() closes the
|
|
118
|
-
// transport, which would otherwise trigger onTransportClose and start
|
|
119
|
-
// a reconnect loop racing against this shutdown.
|
|
120
|
-
shuttingDown = true;
|
|
121
87
|
// Drop the record backend's own session too.
|
|
122
88
|
await cesRecordBackend.close().catch(() => {});
|
|
123
|
-
// Abort any in-flight CES initialization so it fails fast instead of
|
|
124
|
-
// blocking shutdown for up to ~15s (socket connect + handshake timeouts).
|
|
125
|
-
if (initAbortController) {
|
|
126
|
-
initAbortController.abort();
|
|
127
|
-
initAbortController = undefined;
|
|
128
|
-
}
|
|
129
89
|
// Force-stop the live session's transport — forceStop() works even if
|
|
130
|
-
// start() hasn't finished (unlike stop() which is a no-op when
|
|
90
|
+
// start() hasn't finished (unlike stop() which is a no-op when not
|
|
91
|
+
// connected), and it disables the session's own reconnection loop.
|
|
131
92
|
if (clientRef) {
|
|
132
93
|
await clientRef.forceStop().catch(() => {});
|
|
133
94
|
// Cancel in-flight handshake/RPC timers by closing the client directly.
|
|
@@ -142,96 +103,4 @@ export async function stopCes(): Promise<void> {
|
|
|
142
103
|
await clientPromise.catch(() => undefined);
|
|
143
104
|
clientPromise = undefined;
|
|
144
105
|
}
|
|
145
|
-
reconnectBootSession = undefined;
|
|
146
|
-
proactiveReconnectInFlight = false;
|
|
147
|
-
}
|
|
148
|
-
|
|
149
|
-
// ---------------------------------------------------------------------------
|
|
150
|
-
// Proactive reconnect loop
|
|
151
|
-
// ---------------------------------------------------------------------------
|
|
152
|
-
|
|
153
|
-
/**
|
|
154
|
-
* Backoff schedule for proactive reconnection attempts (milliseconds).
|
|
155
|
-
* The first attempt fires immediately (0ms); subsequent attempts use
|
|
156
|
-
* exponential backoff capped at 30s.
|
|
157
|
-
*/
|
|
158
|
-
const RECONNECT_BACKOFF_MS = [0, 2_000, 5_000, 10_000, 20_000, 30_000];
|
|
159
|
-
|
|
160
|
-
/** Maximum reconnection attempts before giving up and relying on the lazy path. */
|
|
161
|
-
const MAX_RECONNECT_ATTEMPTS = RECONNECT_BACKOFF_MS.length;
|
|
162
|
-
|
|
163
|
-
/**
|
|
164
|
-
* Track the active reconnect loop so a new transport-close event (e.g. from
|
|
165
|
-
* a failed reconnect's own transport dying) doesn't start a second loop.
|
|
166
|
-
*/
|
|
167
|
-
let proactiveReconnectInFlight = false;
|
|
168
|
-
/** Set to true during stopCes() so the proactive reconnect loop doesn't fire
|
|
169
|
-
* when forceStop() intentionally closes the transport. */
|
|
170
|
-
let shuttingDown = false;
|
|
171
|
-
|
|
172
|
-
/**
|
|
173
|
-
* Start a retry-with-backoff loop that proactively re-establishes the boot
|
|
174
|
-
* CES session after the transport dies. Each attempt calls the boot-session
|
|
175
|
-
* reconnect (which re-handshakes with the assistant API key and updates
|
|
176
|
-
* `clientRef`). The record backend reconnects its own session independently,
|
|
177
|
-
* cooldown-gated, on the next credential/record operation.
|
|
178
|
-
*/
|
|
179
|
-
function startProactiveReconnectLoop(): void {
|
|
180
|
-
if (shuttingDown) {
|
|
181
|
-
log.debug("CES shutting down — not starting proactive reconnect loop");
|
|
182
|
-
return;
|
|
183
|
-
}
|
|
184
|
-
if (proactiveReconnectInFlight) {
|
|
185
|
-
log.debug("Proactive reconnect loop already running — skipping");
|
|
186
|
-
return;
|
|
187
|
-
}
|
|
188
|
-
proactiveReconnectInFlight = true;
|
|
189
|
-
|
|
190
|
-
void (async () => {
|
|
191
|
-
log.warn("CES transport died — starting proactive reconnect loop");
|
|
192
|
-
|
|
193
|
-
for (let attempt = 0; attempt < MAX_RECONNECT_ATTEMPTS; attempt++) {
|
|
194
|
-
const delay = RECONNECT_BACKOFF_MS[attempt];
|
|
195
|
-
if (delay > 0) {
|
|
196
|
-
await sleep(delay);
|
|
197
|
-
}
|
|
198
|
-
|
|
199
|
-
// Stop if the daemon is shutting down.
|
|
200
|
-
if (shuttingDown) {
|
|
201
|
-
log.info("Proactive reconnect loop stopped — shutdown in progress");
|
|
202
|
-
break;
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
try {
|
|
206
|
-
const client = await reconnectBootSession?.();
|
|
207
|
-
if (client) {
|
|
208
|
-
log.info(
|
|
209
|
-
{ attempt: attempt + 1 },
|
|
210
|
-
"Proactive CES reconnection succeeded",
|
|
211
|
-
);
|
|
212
|
-
break;
|
|
213
|
-
}
|
|
214
|
-
log.warn(
|
|
215
|
-
{ attempt: attempt + 1, max: MAX_RECONNECT_ATTEMPTS },
|
|
216
|
-
"Proactive CES reconnection attempt did not succeed",
|
|
217
|
-
);
|
|
218
|
-
} catch (err) {
|
|
219
|
-
log.warn(
|
|
220
|
-
{
|
|
221
|
-
attempt: attempt + 1,
|
|
222
|
-
max: MAX_RECONNECT_ATTEMPTS,
|
|
223
|
-
error: err instanceof Error ? err.message : String(err),
|
|
224
|
-
},
|
|
225
|
-
"Proactive CES reconnection attempt threw",
|
|
226
|
-
);
|
|
227
|
-
}
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
proactiveReconnectInFlight = false;
|
|
231
|
-
log.info("Proactive reconnect loop finished");
|
|
232
|
-
})();
|
|
233
|
-
}
|
|
234
|
-
|
|
235
|
-
function sleep(ms: number): Promise<void> {
|
|
236
|
-
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
237
106
|
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for the shared CES RPC session factory used by assistant boot and
|
|
3
|
+
* child-process credential resolution.
|
|
4
|
+
*/
|
|
5
|
+
import { describe, expect, test } from "bun:test";
|
|
6
|
+
|
|
7
|
+
import { createCesClient } from "./client.js";
|
|
8
|
+
|
|
9
|
+
describe("createCesClient", () => {
|
|
10
|
+
test("returns undefined when discovery fails without polling", async () => {
|
|
11
|
+
const start = Date.now();
|
|
12
|
+
const client = await createCesClient({
|
|
13
|
+
discover: async () => ({
|
|
14
|
+
mode: "unavailable",
|
|
15
|
+
reason: "missing test socket",
|
|
16
|
+
}),
|
|
17
|
+
});
|
|
18
|
+
|
|
19
|
+
expect(client).toBeUndefined();
|
|
20
|
+
expect(Date.now() - start).toBeLessThan(1_000);
|
|
21
|
+
});
|
|
22
|
+
});
|
|
@@ -1,61 +1,228 @@
|
|
|
1
1
|
/**
|
|
2
2
|
* CES RPC client — assistant-local entry point.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
* `
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
4
|
+
* The session fully owns its transport lifecycle: the process manager,
|
|
5
|
+
* socket discovery, handshake, and reconnection all live here. Consumers
|
|
6
|
+
* call `createCesClient()` and get back a stable client reference whose
|
|
7
|
+
* transport self-heals — when the socket dies the session reconnects with
|
|
8
|
+
* the same handshake (exponential backoff, capped) without the caller
|
|
9
|
+
* seeing a new object. `client.stop()` / `client.forceStop()` tear the
|
|
10
|
+
* session down and disable reconnection.
|
|
11
|
+
*
|
|
12
|
+
* Only the protocol/envelope logic is delegated to the shared
|
|
13
|
+
* `@mycadet/ces-client` package; the process-manager wiring stays in the
|
|
14
|
+
* assistant codebase.
|
|
10
15
|
*/
|
|
11
16
|
|
|
12
17
|
import {
|
|
18
|
+
CesClientError,
|
|
13
19
|
type CesRpcClient,
|
|
14
20
|
type CesRpcClientConfig,
|
|
15
|
-
type
|
|
21
|
+
type CesRpcHandshakeOptions,
|
|
16
22
|
createCesRpcClient,
|
|
17
23
|
} from "@mycadet/ces-client/rpc-client";
|
|
18
24
|
|
|
19
25
|
import { getLogger } from "../util/logger.js";
|
|
26
|
+
import {
|
|
27
|
+
CesUnavailableError,
|
|
28
|
+
createCesProcessManager,
|
|
29
|
+
} from "./process-manager.js";
|
|
20
30
|
|
|
21
31
|
const log = getLogger("ces-rpc-client");
|
|
22
32
|
|
|
33
|
+
/**
|
|
34
|
+
* Backoff schedule for a session's proactive reconnection attempts
|
|
35
|
+
* (milliseconds). The first attempt fires immediately (0ms); subsequent
|
|
36
|
+
* attempts use exponential backoff capped at 30s.
|
|
37
|
+
*/
|
|
38
|
+
const RECONNECT_BACKOFF_MS = [0, 2_000, 5_000, 10_000, 20_000, 30_000];
|
|
39
|
+
|
|
40
|
+
/** Maximum reconnection attempts before the session gives up and relies on
|
|
41
|
+
* callers lazily opening a fresh one. */
|
|
42
|
+
const MAX_RECONNECT_ATTEMPTS = RECONNECT_BACKOFF_MS.length;
|
|
43
|
+
|
|
23
44
|
/**
|
|
24
45
|
* The transport lifecycle of one CES RPC session. The lifecycle rides on the
|
|
25
46
|
* client itself (`client.stop()` / `client.forceStop()`) — there is no
|
|
26
47
|
* separate session object to keep alive alongside it.
|
|
27
48
|
*/
|
|
28
49
|
export interface CesClientLifecycle {
|
|
29
|
-
/** Disconnect this session's transport. Idempotent. */
|
|
50
|
+
/** Disconnect this session's transport. Idempotent. Disables reconnection. */
|
|
30
51
|
stop(): Promise<void>;
|
|
31
|
-
/** Shutdown path: disconnects even while
|
|
52
|
+
/** Shutdown path: disconnects even while a connect is still in flight. */
|
|
32
53
|
forceStop(): Promise<void>;
|
|
33
54
|
}
|
|
34
55
|
|
|
35
|
-
/** A CES RPC client that owns its transport lifecycle. */
|
|
56
|
+
/** A CES RPC client that owns its transport lifecycle and reconnection. */
|
|
36
57
|
export interface CesSessionClient extends CesRpcClient, CesClientLifecycle {}
|
|
37
58
|
|
|
59
|
+
export interface CesSessionClientOptions {
|
|
60
|
+
/** Handshake material reused verbatim across reconnections. */
|
|
61
|
+
handshake?: CesRpcHandshakeOptions;
|
|
62
|
+
/** Socket discovery override. Tests inject a stub. */
|
|
63
|
+
discover?: Parameters<typeof createCesProcessManager>[0]["discover"];
|
|
64
|
+
/** RPC/handshake timeouts forwarded to the shared client. */
|
|
65
|
+
config?: CesRpcClientConfig;
|
|
66
|
+
}
|
|
67
|
+
|
|
38
68
|
/**
|
|
39
|
-
*
|
|
40
|
-
*
|
|
69
|
+
* Open a CES RPC session: discover the socket, connect, handshake, and own
|
|
70
|
+
* the transport from then on — a dead transport triggers an in-session
|
|
71
|
+
* reconnection (same handshake, bounded backoff) instead of surfacing a
|
|
72
|
+
* close callback to the caller.
|
|
73
|
+
*
|
|
74
|
+
* Returns undefined when CES is missing or the handshake is rejected. The
|
|
75
|
+
* transport is stopped on those paths so a later call can open a fresh
|
|
76
|
+
* session. Shutdown goes through `client.forceStop()`, which also stops any
|
|
77
|
+
* in-session reconnect loop.
|
|
41
78
|
*/
|
|
42
|
-
export function createCesClient(
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
79
|
+
export async function createCesClient(
|
|
80
|
+
options: CesSessionClientOptions = {},
|
|
81
|
+
): Promise<CesSessionClient | undefined> {
|
|
82
|
+
const pm = createCesProcessManager({ discover: options.discover });
|
|
83
|
+
const handshake = options.handshake;
|
|
84
|
+
|
|
85
|
+
let inner: CesRpcClient | undefined;
|
|
86
|
+
let stopping = false;
|
|
87
|
+
let reconnecting = false;
|
|
88
|
+
|
|
89
|
+
const stopInternal = async (force: boolean): Promise<void> => {
|
|
90
|
+
stopping = true;
|
|
91
|
+
inner?.close();
|
|
92
|
+
inner = undefined;
|
|
93
|
+
await (force ? pm.forceStop() : pm.stop()).catch(() => {});
|
|
94
|
+
};
|
|
95
|
+
|
|
96
|
+
/** Connect + handshake one transport. Returns the ready inner client. */
|
|
97
|
+
const connectOnce = async (): Promise<CesRpcClient> => {
|
|
98
|
+
const transport = await pm.start();
|
|
99
|
+
const rpc = createCesRpcClient(transport, {
|
|
100
|
+
logger: log,
|
|
101
|
+
...options.config,
|
|
102
|
+
});
|
|
103
|
+
const { accepted, reason } = await rpc.handshake(handshake);
|
|
104
|
+
if (!accepted) {
|
|
105
|
+
rpc.close();
|
|
106
|
+
throw new Error(
|
|
107
|
+
reason ? `CES handshake rejected: ${reason}` : "CES handshake rejected",
|
|
108
|
+
);
|
|
109
|
+
}
|
|
110
|
+
return rpc;
|
|
111
|
+
};
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Reconnect with bounded backoff after the transport dies. The live inner
|
|
115
|
+
* client is swapped in place, so consumers holding the session reference
|
|
116
|
+
* keep calling the same object.
|
|
117
|
+
*/
|
|
118
|
+
const scheduleReconnect = (): void => {
|
|
119
|
+
if (stopping) {
|
|
120
|
+
return;
|
|
121
|
+
}
|
|
122
|
+
if (reconnecting) {
|
|
123
|
+
return;
|
|
124
|
+
}
|
|
125
|
+
reconnecting = true;
|
|
126
|
+
|
|
127
|
+
void (async () => {
|
|
128
|
+
log.warn("CES transport died — starting in-session reconnect loop");
|
|
129
|
+
|
|
130
|
+
for (let attempt = 0; attempt < MAX_RECONNECT_ATTEMPTS; attempt++) {
|
|
131
|
+
const delay = RECONNECT_BACKOFF_MS[attempt];
|
|
132
|
+
if (delay > 0) {
|
|
133
|
+
await sleep(delay);
|
|
134
|
+
}
|
|
135
|
+
if (stopping) {
|
|
136
|
+
reconnecting = false;
|
|
137
|
+
log.info("CES reconnect loop stopped — session is closing");
|
|
138
|
+
return;
|
|
139
|
+
}
|
|
140
|
+
try {
|
|
141
|
+
// The transport died but the pm still counts as running; reset it
|
|
142
|
+
// so start() opens a fresh socket.
|
|
143
|
+
await pm.stop().catch(() => {});
|
|
144
|
+
inner = await connectOnce();
|
|
145
|
+
reconnecting = false;
|
|
146
|
+
log.info(
|
|
147
|
+
{ attempt: attempt + 1 },
|
|
148
|
+
"CES session reconnected and handshake accepted",
|
|
149
|
+
);
|
|
150
|
+
return;
|
|
151
|
+
} catch (err) {
|
|
152
|
+
log.warn(
|
|
153
|
+
{
|
|
154
|
+
attempt: attempt + 1,
|
|
155
|
+
max: MAX_RECONNECT_ATTEMPTS,
|
|
156
|
+
error: err instanceof Error ? err.message : String(err),
|
|
157
|
+
},
|
|
158
|
+
"CES session reconnection attempt did not succeed",
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
reconnecting = false;
|
|
164
|
+
log.info("CES reconnect loop finished — later ops reopen lazily");
|
|
165
|
+
})();
|
|
166
|
+
};
|
|
167
|
+
|
|
168
|
+
pm.onTransportClose(() => scheduleReconnect());
|
|
169
|
+
|
|
170
|
+
// Stable delegating surface: consumers keep one reference while the inner
|
|
171
|
+
// client is swapped underneath it across reconnections.
|
|
172
|
+
const session: CesSessionClient = {
|
|
173
|
+
handshake(sessionOptions) {
|
|
174
|
+
return inner
|
|
175
|
+
? inner.handshake(sessionOptions)
|
|
176
|
+
: Promise.resolve({ accepted: false, reason: "CES session is closed" });
|
|
177
|
+
},
|
|
178
|
+
call(method, request) {
|
|
179
|
+
if (!inner) {
|
|
180
|
+
return Promise.reject(new CesClientError("CES session is closed"));
|
|
181
|
+
}
|
|
182
|
+
return inner.call(method, request);
|
|
183
|
+
},
|
|
184
|
+
updateAssistantApiKey(assistantApiKey, assistantId) {
|
|
185
|
+
if (!inner) {
|
|
186
|
+
return Promise.resolve({ updated: false });
|
|
187
|
+
}
|
|
188
|
+
return inner.updateAssistantApiKey(assistantApiKey, assistantId);
|
|
189
|
+
},
|
|
190
|
+
isReady() {
|
|
191
|
+
return inner?.isReady() ?? false;
|
|
192
|
+
},
|
|
193
|
+
close() {
|
|
194
|
+
stopping = true;
|
|
195
|
+
inner?.close();
|
|
196
|
+
inner = undefined;
|
|
197
|
+
},
|
|
198
|
+
stop() {
|
|
199
|
+
return stopInternal(false);
|
|
200
|
+
},
|
|
201
|
+
forceStop() {
|
|
202
|
+
return stopInternal(true);
|
|
203
|
+
},
|
|
204
|
+
};
|
|
205
|
+
|
|
206
|
+
try {
|
|
207
|
+
inner = await connectOnce();
|
|
208
|
+
return session;
|
|
209
|
+
} catch (err) {
|
|
210
|
+
if (err instanceof CesUnavailableError) {
|
|
211
|
+
log.info({ reason: err.message }, "CES is not available");
|
|
212
|
+
} else {
|
|
213
|
+
log.warn(
|
|
214
|
+
{ error: err instanceof Error ? err.message : String(err) },
|
|
215
|
+
"Failed to open CES RPC session",
|
|
216
|
+
);
|
|
217
|
+
}
|
|
218
|
+
stopping = true;
|
|
219
|
+
inner?.close();
|
|
220
|
+
inner = undefined;
|
|
221
|
+
await pm.stop().catch(() => {});
|
|
222
|
+
return undefined;
|
|
59
223
|
}
|
|
60
|
-
|
|
224
|
+
}
|
|
225
|
+
|
|
226
|
+
function sleep(ms: number): Promise<void> {
|
|
227
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
61
228
|
}
|
package/src/daemon/lifecycle.ts
CHANGED
|
@@ -547,11 +547,7 @@ export async function runDaemon(): Promise<void> {
|
|
|
547
547
|
applyModuleLogLevels(config.logFile?.levels);
|
|
548
548
|
|
|
549
549
|
// Open the assistant's CES RPC client (handshake + reconnect wiring).
|
|
550
|
-
|
|
551
|
-
// provider init; non-fatal, falls back to the direct credential store on
|
|
552
|
-
// failure. CES serves a multi-connection IPC socket, so child
|
|
553
|
-
// processes can open the same `openCesRpcSession` path independently.
|
|
554
|
-
await initializeCesClient(config);
|
|
550
|
+
await initializeCesClient();
|
|
555
551
|
|
|
556
552
|
// Bring up the plugin layer: install the runtime bridge, register the
|
|
557
553
|
// first-party defaults, load user plugins, and run every plugin's
|
|
@@ -14,8 +14,8 @@ import { CesRpcClient as CesClient } from "@mycadet/ces-client/rpc-client";
|
|
|
14
14
|
import type { CredentialRecord } from "@mycadet/service-contracts/credential-rpc";
|
|
15
15
|
import { CesRpcMethod } from "@mycadet/service-contracts/credential-rpc";
|
|
16
16
|
|
|
17
|
-
import { openCesRpcSession } from "../credential-execution/ces-connect.js";
|
|
18
17
|
import type { CesSessionClient } from "../credential-execution/client.js";
|
|
18
|
+
import { createCesClient } from "../credential-execution/client.js";
|
|
19
19
|
import { CesUnavailableError } from "../credential-execution/process-manager.js";
|
|
20
20
|
import { getLogger } from "../util/logger.js";
|
|
21
21
|
|
|
@@ -24,10 +24,19 @@ const log = getLogger("ces-rpc-record-backend");
|
|
|
24
24
|
/** Minimum interval between CES connection attempts after a failure. */
|
|
25
25
|
const LAZY_SESSION_COOLDOWN_MS = 3_000;
|
|
26
26
|
|
|
27
|
+
/** How long ensureClient waits on a self-healing session before giving up. */
|
|
28
|
+
const RECONNECT_WAIT_MS = 15_000;
|
|
29
|
+
|
|
30
|
+
function sleep(ms: number): Promise<void> {
|
|
31
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
32
|
+
}
|
|
33
|
+
|
|
27
34
|
class CesRpcRecordBackend {
|
|
28
35
|
/**
|
|
29
|
-
* The live IPC
|
|
30
|
-
*
|
|
36
|
+
* The live IPC session, opened by this backend's first record operation.
|
|
37
|
+
* The session owns its transport and reconnection: after a CES restart the
|
|
38
|
+
* same reference becomes ready again, so this backend never re-opens or
|
|
39
|
+
* swaps clients itself.
|
|
31
40
|
*/
|
|
32
41
|
private client: CesSessionClient | undefined;
|
|
33
42
|
|
|
@@ -48,21 +57,33 @@ class CesRpcRecordBackend {
|
|
|
48
57
|
* surface that rather than degrading, so a lost write is never mistaken
|
|
49
58
|
* for a successful one. Failed attempts are not re-attempted within the
|
|
50
59
|
* cooldown window: every operation would otherwise re-pay the discovery
|
|
51
|
-
* polling while CES is down.
|
|
60
|
+
* polling while CES is down. A held session whose transport died is NOT
|
|
61
|
+
* dropped here — the session reconnects itself; we just wait for it.
|
|
52
62
|
*/
|
|
53
63
|
async ensureClient(): Promise<CesClient> {
|
|
54
64
|
if (this.client?.isReady()) {
|
|
55
65
|
return this.client;
|
|
56
66
|
}
|
|
57
|
-
|
|
58
|
-
|
|
67
|
+
if (this.client && !this.client.isReady()) {
|
|
68
|
+
// The session is self-healing (in-session reconnect loop) — wait for
|
|
69
|
+
// it to come back rather than opening a competing session.
|
|
70
|
+
for (let waited = 0; waited < RECONNECT_WAIT_MS; waited += 250) {
|
|
71
|
+
await sleep(250);
|
|
72
|
+
if (this.client.isReady()) {
|
|
73
|
+
return this.client;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
throw new CesUnavailableError(
|
|
77
|
+
"CES is unreachable (session reconnect did not become ready)",
|
|
78
|
+
);
|
|
79
|
+
}
|
|
59
80
|
if (Date.now() - this.lastFailedAttemptAt < LAZY_SESSION_COOLDOWN_MS) {
|
|
60
81
|
throw new CesUnavailableError(
|
|
61
82
|
"CES is unreachable (recent connection attempt failed)",
|
|
62
83
|
);
|
|
63
84
|
}
|
|
64
85
|
if (!this.sessionAttempt) {
|
|
65
|
-
this.sessionAttempt =
|
|
86
|
+
this.sessionAttempt = createCesClient()
|
|
66
87
|
.then((client) => {
|
|
67
88
|
if (!client) {
|
|
68
89
|
throw new CesUnavailableError(
|
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Tests for the shared CES RPC session helper used by assistant boot and
|
|
3
|
-
* child-process credential resolution.
|
|
4
|
-
*/
|
|
5
|
-
import { describe, expect, test } from "bun:test";
|
|
6
|
-
|
|
7
|
-
import { openCesRpcSession } from "./ces-connect.js";
|
|
8
|
-
|
|
9
|
-
describe("openCesRpcSession", () => {
|
|
10
|
-
test("returns undefined when discovery fails without polling", async () => {
|
|
11
|
-
const start = Date.now();
|
|
12
|
-
const client = await openCesRpcSession({
|
|
13
|
-
discover: async () => ({
|
|
14
|
-
mode: "unavailable",
|
|
15
|
-
reason: "missing test socket",
|
|
16
|
-
}),
|
|
17
|
-
});
|
|
18
|
-
|
|
19
|
-
expect(client).toBeUndefined();
|
|
20
|
-
expect(Date.now() - start).toBeLessThan(1_000);
|
|
21
|
-
});
|
|
22
|
-
|
|
23
|
-
test("returns undefined when the abort signal is already fired", async () => {
|
|
24
|
-
const abort = new AbortController();
|
|
25
|
-
abort.abort();
|
|
26
|
-
const client = await openCesRpcSession({
|
|
27
|
-
discover: async () => {
|
|
28
|
-
throw new Error("discover should not run after abort");
|
|
29
|
-
},
|
|
30
|
-
signal: abort.signal,
|
|
31
|
-
});
|
|
32
|
-
|
|
33
|
-
expect(client).toBeUndefined();
|
|
34
|
-
});
|
|
35
|
-
});
|
|
@@ -1,93 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Open a CES RPC client over the shared IPC socket.
|
|
3
|
-
*
|
|
4
|
-
* Assistant boot and child-process credential reads use this helper. Both
|
|
5
|
-
* are CES API clients: discover `ces.sock`, connect, handshake. Each opened
|
|
6
|
-
* session owns its transport lifecycle through the returned client
|
|
7
|
-
* (`client.stop()` / `client.forceStop()`); reconnect opens a fresh session.
|
|
8
|
-
*/
|
|
9
|
-
|
|
10
|
-
import type { CesRpcHandshakeOptions } from "@mycadet/ces-client/rpc-client";
|
|
11
|
-
|
|
12
|
-
import type { AssistantConfig } from "../config/schema.js";
|
|
13
|
-
import { getLogger } from "../util/logger.js";
|
|
14
|
-
import { type CesSessionClient, createCesClient } from "./client.js";
|
|
15
|
-
import {
|
|
16
|
-
type CesProcessManagerConfig,
|
|
17
|
-
CesUnavailableError,
|
|
18
|
-
createCesProcessManager,
|
|
19
|
-
} from "./process-manager.js";
|
|
20
|
-
|
|
21
|
-
const log = getLogger("ces-connect");
|
|
22
|
-
|
|
23
|
-
export interface OpenCesRpcSessionOptions {
|
|
24
|
-
handshake?: CesRpcHandshakeOptions;
|
|
25
|
-
signal?: AbortSignal;
|
|
26
|
-
assistantConfig?: AssistantConfig;
|
|
27
|
-
discover?: CesProcessManagerConfig["discover"];
|
|
28
|
-
/** Fired when the session's transport dies (the reconnect trigger). */
|
|
29
|
-
onTransportClose?: () => void;
|
|
30
|
-
}
|
|
31
|
-
|
|
32
|
-
/**
|
|
33
|
-
* Discover the CES socket, connect, and complete the RPC handshake.
|
|
34
|
-
*
|
|
35
|
-
* Returns undefined when CES is missing, the handshake is rejected, the
|
|
36
|
-
* abort signal fires, or the transport fails. The session's transport is
|
|
37
|
-
* stopped on those paths so a later call can open a fresh session.
|
|
38
|
-
*/
|
|
39
|
-
export async function openCesRpcSession(
|
|
40
|
-
options: OpenCesRpcSessionOptions = {},
|
|
41
|
-
): Promise<CesSessionClient | undefined> {
|
|
42
|
-
const pm = createCesProcessManager({
|
|
43
|
-
assistantConfig: options.assistantConfig,
|
|
44
|
-
discover: options.discover,
|
|
45
|
-
});
|
|
46
|
-
if (options.onTransportClose) {
|
|
47
|
-
pm.onTransportClose(options.onTransportClose);
|
|
48
|
-
}
|
|
49
|
-
|
|
50
|
-
const fail = async (): Promise<undefined> => {
|
|
51
|
-
await pm.stop().catch(() => {});
|
|
52
|
-
return undefined;
|
|
53
|
-
};
|
|
54
|
-
|
|
55
|
-
if (options.signal?.aborted) {
|
|
56
|
-
return fail();
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
try {
|
|
60
|
-
const transport = await pm.start();
|
|
61
|
-
if (options.signal?.aborted) {
|
|
62
|
-
return fail();
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
const client = createCesClient(transport, undefined, {
|
|
66
|
-
stop: () => pm.stop(),
|
|
67
|
-
forceStop: () => pm.forceStop(),
|
|
68
|
-
});
|
|
69
|
-
|
|
70
|
-
const { accepted, reason } = await client.handshake(options.handshake);
|
|
71
|
-
if (options.signal?.aborted) {
|
|
72
|
-
client.close();
|
|
73
|
-
return fail();
|
|
74
|
-
}
|
|
75
|
-
if (!accepted) {
|
|
76
|
-
log.warn({ reason }, "CES handshake rejected");
|
|
77
|
-
client.close();
|
|
78
|
-
return fail();
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
return client;
|
|
82
|
-
} catch (err) {
|
|
83
|
-
if (err instanceof CesUnavailableError) {
|
|
84
|
-
log.info({ reason: err.message }, "CES is not available");
|
|
85
|
-
} else {
|
|
86
|
-
log.warn(
|
|
87
|
-
{ error: err instanceof Error ? err.message : String(err) },
|
|
88
|
-
"Failed to open CES RPC session",
|
|
89
|
-
);
|
|
90
|
-
}
|
|
91
|
-
return fail();
|
|
92
|
-
}
|
|
93
|
-
}
|