@vellumai/assistant 0.12.2-dev.202609171820.9ecfa87 → 0.12.2-dev.202609171913.b5e95d3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/__tests__/credential-execution-client.test.ts +1 -1
- package/src/__tests__/secure-keys.test.ts +43 -1
- package/src/credential-execution/ces-connect.test.ts +69 -0
- package/src/credential-execution/ces-connect.ts +115 -0
- package/src/credential-execution/ces-runtime.ts +55 -93
- package/src/daemon/lifecycle.ts +5 -5
- package/src/security/secure-keys.ts +53 -103
package/package.json
CHANGED
|
@@ -95,7 +95,7 @@ function withBootstrapDir(dir: string): () => void {
|
|
|
95
95
|
};
|
|
96
96
|
}
|
|
97
97
|
|
|
98
|
-
describe("
|
|
98
|
+
describe("CES discovery", () => {
|
|
99
99
|
test("returns unavailable when bootstrap socket does not exist", () => {
|
|
100
100
|
const bootstrapDir = mkdtempSync(join(tmpdir(), "ces-missing-"));
|
|
101
101
|
const restore = withBootstrapDir(bootstrapDir);
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { randomBytes } from "node:crypto";
|
|
2
|
-
import { existsSync, mkdirSync, rmSync } from "node:fs";
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, rmSync } from "node:fs";
|
|
3
|
+
import { createServer, type Server } from "node:net";
|
|
3
4
|
import { tmpdir } from "node:os";
|
|
4
5
|
import { join } from "node:path";
|
|
5
6
|
import {
|
|
@@ -11,6 +12,8 @@ import {
|
|
|
11
12
|
test,
|
|
12
13
|
} from "bun:test";
|
|
13
14
|
|
|
15
|
+
import { resolveIpcEndpoint } from "@vellumai/ipc-server-utils";
|
|
16
|
+
|
|
14
17
|
// ---------------------------------------------------------------------------
|
|
15
18
|
// Mock logger (no-op — compatible with other test files' identical mock)
|
|
16
19
|
// ---------------------------------------------------------------------------
|
|
@@ -602,4 +605,43 @@ describe("secure-keys", () => {
|
|
|
602
605
|
expect(result.unreachable).toBe(true);
|
|
603
606
|
});
|
|
604
607
|
});
|
|
608
|
+
|
|
609
|
+
// -----------------------------------------------------------------------
|
|
610
|
+
// Session ownership: boot claims reconnect before identity reads
|
|
611
|
+
// -----------------------------------------------------------------------
|
|
612
|
+
describe("CES session owner skips a second open", () => {
|
|
613
|
+
test("a registered reconnect owner does not handshake a live CES socket", async () => {
|
|
614
|
+
const dir = mkdtempSync(join(tmpdir(), "ces-owner-"));
|
|
615
|
+
const socketPath = resolveIpcEndpoint("ces", { workspaceDir: dir }).path;
|
|
616
|
+
const savedDir = process.env.CES_BOOTSTRAP_SOCKET_DIR;
|
|
617
|
+
process.env.CES_BOOTSTRAP_SOCKET_DIR = dir;
|
|
618
|
+
|
|
619
|
+
const connections: Array<import("node:net").Socket> = [];
|
|
620
|
+
const server: Server = createServer((socket) => {
|
|
621
|
+
connections.push(socket);
|
|
622
|
+
socket.on("error", () => {});
|
|
623
|
+
});
|
|
624
|
+
await new Promise<void>((resolve) => server.listen(socketPath, resolve));
|
|
625
|
+
|
|
626
|
+
try {
|
|
627
|
+
setCesReconnect(async () => undefined);
|
|
628
|
+
const start = Date.now();
|
|
629
|
+
await getSecureKeyAsync("openai");
|
|
630
|
+
expect(Date.now() - start).toBeLessThan(1_000);
|
|
631
|
+
expect(getActiveBackendName()).toBe("encrypted-store");
|
|
632
|
+
expect(connections.length).toBe(0);
|
|
633
|
+
} finally {
|
|
634
|
+
for (const sock of connections) {
|
|
635
|
+
sock.destroy();
|
|
636
|
+
}
|
|
637
|
+
await new Promise<void>((resolve) => server.close(() => resolve()));
|
|
638
|
+
if (savedDir !== undefined) {
|
|
639
|
+
process.env.CES_BOOTSTRAP_SOCKET_DIR = savedDir;
|
|
640
|
+
} else {
|
|
641
|
+
delete process.env.CES_BOOTSTRAP_SOCKET_DIR;
|
|
642
|
+
}
|
|
643
|
+
rmSync(dir, { recursive: true, force: true });
|
|
644
|
+
}
|
|
645
|
+
});
|
|
646
|
+
});
|
|
605
647
|
});
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Tests for the shared CES RPC session helper used by assistant boot and
|
|
3
|
+
* child-process credential resolution.
|
|
4
|
+
*/
|
|
5
|
+
import { describe, expect, test } from "bun:test";
|
|
6
|
+
|
|
7
|
+
import { openCesRpcSession, reconnectCesRpcSession } from "./ces-connect.js";
|
|
8
|
+
import { createCesProcessManager } from "./process-manager.js";
|
|
9
|
+
|
|
10
|
+
describe("openCesRpcSession", () => {
|
|
11
|
+
test("returns undefined when discovery fails without polling", async () => {
|
|
12
|
+
const start = Date.now();
|
|
13
|
+
const pm = createCesProcessManager({
|
|
14
|
+
discover: async () => ({
|
|
15
|
+
mode: "unavailable",
|
|
16
|
+
reason: "missing test socket",
|
|
17
|
+
}),
|
|
18
|
+
});
|
|
19
|
+
|
|
20
|
+
const session = await openCesRpcSession({ processManager: pm });
|
|
21
|
+
|
|
22
|
+
expect(session).toBeUndefined();
|
|
23
|
+
expect(pm.isRunning()).toBe(false);
|
|
24
|
+
expect(Date.now() - start).toBeLessThan(1_000);
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
test("returns undefined when the abort signal is already fired", async () => {
|
|
28
|
+
const abort = new AbortController();
|
|
29
|
+
abort.abort();
|
|
30
|
+
const pm = createCesProcessManager({
|
|
31
|
+
discover: async () => {
|
|
32
|
+
throw new Error("discover should not run after abort");
|
|
33
|
+
},
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
const session = await openCesRpcSession({
|
|
37
|
+
processManager: pm,
|
|
38
|
+
signal: abort.signal,
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
expect(session).toBeUndefined();
|
|
42
|
+
expect(pm.isRunning()).toBe(false);
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
test("creates a process manager when none is provided", async () => {
|
|
46
|
+
const session = await openCesRpcSession({
|
|
47
|
+
discover: async () => ({
|
|
48
|
+
mode: "unavailable",
|
|
49
|
+
reason: "missing test socket",
|
|
50
|
+
}),
|
|
51
|
+
});
|
|
52
|
+
|
|
53
|
+
expect(session).toBeUndefined();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
test("reconnectCesRpcSession stops the manager and returns undefined when discovery fails", async () => {
|
|
57
|
+
const pm = createCesProcessManager({
|
|
58
|
+
discover: async () => ({
|
|
59
|
+
mode: "unavailable",
|
|
60
|
+
reason: "missing test socket",
|
|
61
|
+
}),
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
const client = await reconnectCesRpcSession(pm);
|
|
65
|
+
|
|
66
|
+
expect(client).toBeUndefined();
|
|
67
|
+
expect(pm.isRunning()).toBe(false);
|
|
68
|
+
});
|
|
69
|
+
});
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Open a CES RPC client over the shared bootstrap socket.
|
|
3
|
+
*
|
|
4
|
+
* Assistant boot and child-process credential reads use this helper. Both
|
|
5
|
+
* are CES API clients: discover `ces.sock`, connect, handshake, reconnect.
|
|
6
|
+
*
|
|
7
|
+
* They stay two entry points because boot must load handshake identity
|
|
8
|
+
* without talking to CES, then hand that identity to CES. Children open a
|
|
9
|
+
* session only when this process has not already claimed one (no live
|
|
10
|
+
* client and no reconnect owner).
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import type { AssistantConfig } from "../config/schema.js";
|
|
14
|
+
import { getLogger } from "../util/logger.js";
|
|
15
|
+
import {
|
|
16
|
+
type CesClient,
|
|
17
|
+
type CesClientHandshakeOptions,
|
|
18
|
+
createCesClient,
|
|
19
|
+
} from "./client.js";
|
|
20
|
+
import {
|
|
21
|
+
type CesProcessManager,
|
|
22
|
+
type CesProcessManagerConfig,
|
|
23
|
+
CesUnavailableError,
|
|
24
|
+
createCesProcessManager,
|
|
25
|
+
} from "./process-manager.js";
|
|
26
|
+
|
|
27
|
+
const log = getLogger("ces-connect");
|
|
28
|
+
|
|
29
|
+
export interface CesRpcSession {
|
|
30
|
+
client: CesClient;
|
|
31
|
+
processManager: CesProcessManager;
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
export interface OpenCesRpcSessionOptions {
|
|
35
|
+
/**
|
|
36
|
+
* Reuse an existing process manager (reconnect after `stop()`). When
|
|
37
|
+
* omitted, a new manager is created.
|
|
38
|
+
*/
|
|
39
|
+
processManager?: CesProcessManager;
|
|
40
|
+
handshake?: CesClientHandshakeOptions;
|
|
41
|
+
signal?: AbortSignal;
|
|
42
|
+
assistantConfig?: AssistantConfig;
|
|
43
|
+
discover?: CesProcessManagerConfig["discover"];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
/**
|
|
47
|
+
* Discover the CES socket, connect, and complete the RPC handshake.
|
|
48
|
+
*
|
|
49
|
+
* Returns undefined when CES is missing, the handshake is rejected, the
|
|
50
|
+
* abort signal fires, or the transport fails. The process manager is
|
|
51
|
+
* stopped on those paths so a later reconnect can call `start()` again.
|
|
52
|
+
*/
|
|
53
|
+
export async function openCesRpcSession(
|
|
54
|
+
options: OpenCesRpcSessionOptions = {},
|
|
55
|
+
): Promise<CesRpcSession | undefined> {
|
|
56
|
+
const pm =
|
|
57
|
+
options.processManager ??
|
|
58
|
+
createCesProcessManager({
|
|
59
|
+
assistantConfig: options.assistantConfig,
|
|
60
|
+
discover: options.discover,
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
const fail = async (): Promise<undefined> => {
|
|
64
|
+
await pm.stop().catch(() => {});
|
|
65
|
+
return undefined;
|
|
66
|
+
};
|
|
67
|
+
|
|
68
|
+
if (options.signal?.aborted) {
|
|
69
|
+
return fail();
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
try {
|
|
73
|
+
const transport = await pm.start();
|
|
74
|
+
if (options.signal?.aborted) {
|
|
75
|
+
return fail();
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
const client = createCesClient(transport);
|
|
79
|
+
const { accepted, reason } = await client.handshake(options.handshake);
|
|
80
|
+
if (options.signal?.aborted) {
|
|
81
|
+
client.close();
|
|
82
|
+
return fail();
|
|
83
|
+
}
|
|
84
|
+
if (!accepted) {
|
|
85
|
+
log.warn({ reason }, "CES handshake rejected");
|
|
86
|
+
client.close();
|
|
87
|
+
return fail();
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
return { client, processManager: pm };
|
|
91
|
+
} catch (err) {
|
|
92
|
+
if (err instanceof CesUnavailableError) {
|
|
93
|
+
log.info({ reason: err.message }, "CES is not available");
|
|
94
|
+
} else {
|
|
95
|
+
log.warn(
|
|
96
|
+
{ error: err instanceof Error ? err.message : String(err) },
|
|
97
|
+
"Failed to open CES RPC session",
|
|
98
|
+
);
|
|
99
|
+
}
|
|
100
|
+
return fail();
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/**
|
|
105
|
+
* Stop the current transport and open a new session on the same process
|
|
106
|
+
* manager. Used by assistant boot and child-process reconnect callbacks.
|
|
107
|
+
*/
|
|
108
|
+
export async function reconnectCesRpcSession(
|
|
109
|
+
processManager: CesProcessManager,
|
|
110
|
+
handshake?: CesClientHandshakeOptions,
|
|
111
|
+
): Promise<CesClient | undefined> {
|
|
112
|
+
await processManager.stop();
|
|
113
|
+
const session = await openCesRpcSession({ processManager, handshake });
|
|
114
|
+
return session?.client;
|
|
115
|
+
}
|
|
@@ -9,10 +9,16 @@ import {
|
|
|
9
9
|
setCesReconnect,
|
|
10
10
|
} from "../security/secure-keys.js";
|
|
11
11
|
import { getLogger } from "../util/logger.js";
|
|
12
|
-
import {
|
|
12
|
+
import {
|
|
13
|
+
openCesRpcSession,
|
|
14
|
+
reconnectCesRpcSession,
|
|
15
|
+
} from "./ces-connect.js";
|
|
16
|
+
import {
|
|
17
|
+
type CesClient,
|
|
18
|
+
type CesClientHandshakeOptions,
|
|
19
|
+
} from "./client.js";
|
|
13
20
|
import {
|
|
14
21
|
type CesProcessManager,
|
|
15
|
-
CesUnavailableError,
|
|
16
22
|
createCesProcessManager,
|
|
17
23
|
} from "./process-manager.js";
|
|
18
24
|
import {
|
|
@@ -45,70 +51,69 @@ interface CesStartupResult {
|
|
|
45
51
|
}
|
|
46
52
|
|
|
47
53
|
/**
|
|
48
|
-
*
|
|
49
|
-
* handles to the in-flight initialization
|
|
50
|
-
* for startup to continue.
|
|
54
|
+
* Open the assistant's CES RPC client and perform the handshake. Returns
|
|
55
|
+
* immediately with handles to the in-flight initialization: callers don't
|
|
56
|
+
* need to await this for startup to continue.
|
|
51
57
|
*
|
|
52
|
-
*
|
|
53
|
-
*
|
|
58
|
+
* Claims reconnect ownership before any credential read so boot identity
|
|
59
|
+
* loading cannot open a second, identity-less session through the child
|
|
60
|
+
* entry point. CES serves a multi-connection Unix socket, so child
|
|
61
|
+
* processes in other address spaces still open their own connections.
|
|
54
62
|
*/
|
|
55
63
|
function startCesProcess(config: AssistantConfig): CesStartupResult {
|
|
56
64
|
const pm = createCesProcessManager({ assistantConfig: config });
|
|
57
65
|
const abortController = new AbortController();
|
|
58
66
|
let currentClient: CesClient | undefined;
|
|
67
|
+
let handshake: CesClientHandshakeOptions = {};
|
|
68
|
+
|
|
69
|
+
// Own this process's CES session before resolveManagedProxyContext()
|
|
70
|
+
// reads the API key. That read must not take the child open path.
|
|
71
|
+
setCesReconnect(async () => {
|
|
72
|
+
const client = await reconnectCesRpcSession(pm, handshake);
|
|
73
|
+
if (client) {
|
|
74
|
+
log.info("CES reconnection handshake accepted");
|
|
75
|
+
}
|
|
76
|
+
return client;
|
|
77
|
+
});
|
|
59
78
|
|
|
60
79
|
const handshakePromise = (async (): Promise<CesClient | undefined> => {
|
|
61
80
|
try {
|
|
62
|
-
const transport = await pm.start();
|
|
63
|
-
if (abortController.signal.aborted) {
|
|
64
|
-
throw new Error("CES initialization aborted during shutdown");
|
|
65
|
-
}
|
|
66
|
-
const client = createCesClient(transport);
|
|
67
|
-
currentClient = client;
|
|
68
81
|
// Resolve the assistant API key so CES can use it for platform
|
|
69
82
|
// credential materialisation. In managed mode the key is provisioned
|
|
70
|
-
// after hatch and stored in the credential store
|
|
71
|
-
// the env var, so we pass it via the handshake.
|
|
83
|
+
// after hatch and stored in the credential store. CES can't read
|
|
84
|
+
// the env var, so we pass it via the handshake. Reconnect ownership
|
|
85
|
+
// is already claimed, so this read uses HTTP or the encrypted store,
|
|
86
|
+
// not a second RPC session.
|
|
72
87
|
const proxyCtx = await resolveManagedProxyContext();
|
|
88
|
+
if (abortController.signal.aborted) {
|
|
89
|
+
return undefined;
|
|
90
|
+
}
|
|
73
91
|
const assistantId = getPlatformAssistantId();
|
|
74
|
-
|
|
92
|
+
handshake = {
|
|
75
93
|
...(proxyCtx.assistantApiKey
|
|
76
94
|
? { assistantApiKey: proxyCtx.assistantApiKey }
|
|
77
95
|
: {}),
|
|
78
96
|
...(assistantId ? { assistantId } : {}),
|
|
97
|
+
};
|
|
98
|
+
const session = await openCesRpcSession({
|
|
99
|
+
processManager: pm,
|
|
100
|
+
signal: abortController.signal,
|
|
101
|
+
handshake,
|
|
79
102
|
});
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
throw new Error("CES initialization aborted during shutdown");
|
|
83
|
-
}
|
|
84
|
-
if (accepted) {
|
|
103
|
+
currentClient = session?.client;
|
|
104
|
+
if (session) {
|
|
85
105
|
log.info(
|
|
86
106
|
"CES client initialized and handshake accepted (server-level)",
|
|
87
107
|
);
|
|
88
|
-
return client;
|
|
89
108
|
}
|
|
109
|
+
return session?.client;
|
|
110
|
+
} catch (err) {
|
|
90
111
|
log.warn(
|
|
91
|
-
{
|
|
92
|
-
"
|
|
112
|
+
{ error: err instanceof Error ? err.message : String(err) },
|
|
113
|
+
"Failed to initialize CES client",
|
|
93
114
|
);
|
|
94
|
-
client.close();
|
|
95
115
|
currentClient = undefined;
|
|
96
|
-
await pm.stop();
|
|
97
|
-
return undefined;
|
|
98
|
-
} catch (err) {
|
|
99
|
-
if (err instanceof CesUnavailableError) {
|
|
100
|
-
log.info(
|
|
101
|
-
{ reason: err.message },
|
|
102
|
-
"CES is not available — CES tools will be unavailable",
|
|
103
|
-
);
|
|
104
|
-
} else {
|
|
105
|
-
log.warn(
|
|
106
|
-
{ error: err instanceof Error ? err.message : String(err) },
|
|
107
|
-
"Failed to initialize CES client — CES tools will be unavailable",
|
|
108
|
-
);
|
|
109
|
-
}
|
|
110
116
|
await pm.stop().catch(() => {});
|
|
111
|
-
currentClient = undefined;
|
|
112
117
|
return undefined;
|
|
113
118
|
}
|
|
114
119
|
})();
|
|
@@ -154,11 +159,11 @@ function updateClientRef(client: CesClient | undefined): void {
|
|
|
154
159
|
}
|
|
155
160
|
|
|
156
161
|
/**
|
|
157
|
-
*
|
|
158
|
-
*
|
|
159
|
-
*
|
|
160
|
-
*
|
|
161
|
-
* direct credential store.
|
|
162
|
+
* Open the assistant's CES RPC client: handshake (blocking up to a 20s
|
|
163
|
+
* timeout so credential reads can route through CES before provider init)
|
|
164
|
+
* and keep the live client reference in sync. Reconnect ownership is
|
|
165
|
+
* claimed before the identity read. Non-fatal: on failure the assistant
|
|
166
|
+
* falls back to the direct credential store.
|
|
162
167
|
*/
|
|
163
168
|
export async function startCes(config: AssistantConfig): Promise<void> {
|
|
164
169
|
const cesResult = startCesProcess(config);
|
|
@@ -171,7 +176,7 @@ export async function startCes(config: AssistantConfig): Promise<void> {
|
|
|
171
176
|
timeoutMs: DEFAULT_CES_STARTUP_TIMEOUT_MS,
|
|
172
177
|
onTimeout: () => {
|
|
173
178
|
log.warn(
|
|
174
|
-
"CES handshake timed out after 20s
|
|
179
|
+
"CES handshake timed out after 20s, falling back to direct credential store",
|
|
175
180
|
);
|
|
176
181
|
},
|
|
177
182
|
});
|
|
@@ -193,56 +198,13 @@ export async function startCes(config: AssistantConfig): Promise<void> {
|
|
|
193
198
|
}
|
|
194
199
|
}
|
|
195
200
|
|
|
196
|
-
//
|
|
197
|
-
//
|
|
198
|
-
//
|
|
201
|
+
// Reconnect ownership is claimed inside startCesProcess before the
|
|
202
|
+
// identity read. Snapshotting the API key there (not here) avoids a
|
|
203
|
+
// second resolveManagedProxyContext() after setCesClient, which would
|
|
204
|
+
// read the key through CES while reconnecting.
|
|
199
205
|
if (cesResult.processManager) {
|
|
200
206
|
const pm = cesResult.processManager;
|
|
201
207
|
|
|
202
|
-
// Snapshot the managed-proxy context and assistant ID at CES startup so the
|
|
203
|
-
// reconnect closure below never calls back into `resolveManagedProxyContext()`.
|
|
204
|
-
// That function reads the assistant API key via `getSecureKeyAsync()`, which
|
|
205
|
-
// — once `setCesClient()` has resolved the backend to CES RPC — routes the
|
|
206
|
-
// read through CES itself. During a reconnect the old transport is dead and
|
|
207
|
-
// a new one is being set up by this very closure, so the nested credential
|
|
208
|
-
// read recursively awaits its own in-flight reconnection and deadlocks until
|
|
209
|
-
// `CREDENTIAL_OP_TIMEOUT_MS` (45s) fires. That 45-second stall delays every
|
|
210
|
-
// CES restart and causes dependent credential reads (e.g. Meet's STT
|
|
211
|
-
// provider resolution) to return `undefined` during the window. API key
|
|
212
|
-
// rotation uses the `updateAssistantApiKey` RPC on the live client, not a
|
|
213
|
-
// reconnect, so caching at startup is safe.
|
|
214
|
-
const startupProxyCtx = await resolveManagedProxyContext();
|
|
215
|
-
const startupAssistantId = getPlatformAssistantId();
|
|
216
|
-
|
|
217
|
-
setCesReconnect(async () => {
|
|
218
|
-
try {
|
|
219
|
-
await pm.stop();
|
|
220
|
-
const transport = await pm.start();
|
|
221
|
-
const newClient = createCesClient(transport);
|
|
222
|
-
const { accepted, reason } = await newClient.handshake({
|
|
223
|
-
...(startupProxyCtx.assistantApiKey
|
|
224
|
-
? { assistantApiKey: startupProxyCtx.assistantApiKey }
|
|
225
|
-
: {}),
|
|
226
|
-
...(startupAssistantId ? { assistantId: startupAssistantId } : {}),
|
|
227
|
-
});
|
|
228
|
-
if (accepted) {
|
|
229
|
-
log.info("CES reconnection handshake accepted");
|
|
230
|
-
return newClient;
|
|
231
|
-
}
|
|
232
|
-
log.warn({ reason }, "CES reconnection handshake rejected");
|
|
233
|
-
newClient.close();
|
|
234
|
-
await pm.stop().catch(() => {});
|
|
235
|
-
return undefined;
|
|
236
|
-
} catch (err) {
|
|
237
|
-
log.warn(
|
|
238
|
-
{ error: err instanceof Error ? err.message : String(err) },
|
|
239
|
-
"CES reconnection attempt failed",
|
|
240
|
-
);
|
|
241
|
-
await pm.stop().catch(() => {});
|
|
242
|
-
return undefined;
|
|
243
|
-
}
|
|
244
|
-
});
|
|
245
|
-
|
|
246
208
|
// Proactive reconnect: when the transport dies (socket close, process
|
|
247
209
|
// exit), start a retry-with-backoff loop immediately instead of waiting
|
|
248
210
|
// for the next credential operation to trigger the lazy reconnect path.
|
package/src/daemon/lifecycle.ts
CHANGED
|
@@ -673,11 +673,11 @@ export async function runDaemon(): Promise<void> {
|
|
|
673
673
|
// blocked.
|
|
674
674
|
startConsentRefresh();
|
|
675
675
|
|
|
676
|
-
//
|
|
677
|
-
//
|
|
678
|
-
//
|
|
679
|
-
//
|
|
680
|
-
//
|
|
676
|
+
// Open the assistant's CES RPC client (handshake + reconnect wiring).
|
|
677
|
+
// Blocks up to a 20s timeout so credential reads route through CES before
|
|
678
|
+
// provider init; non-fatal, falls back to the direct credential store on
|
|
679
|
+
// failure. CES serves a multi-connection bootstrap socket, so child
|
|
680
|
+
// processes can open the same `openCesRpcSession` path independently.
|
|
681
681
|
await startCes(config);
|
|
682
682
|
|
|
683
683
|
// Bring up the plugin layer: install the runtime bridge, register the
|
|
@@ -1,22 +1,18 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Unified secure key storage
|
|
2
|
+
* Unified secure key storage: single-backend routing through CredentialBackend
|
|
3
3
|
* adapters.
|
|
4
4
|
*
|
|
5
5
|
* Backend selection (`resolveBackendAsync`) is the single async decision point:
|
|
6
|
-
* 1. CES RPC (primary) -
|
|
7
|
-
*
|
|
8
|
-
*
|
|
9
|
-
*
|
|
10
|
-
*
|
|
11
|
-
*
|
|
12
|
-
* resolution they discover the CES bootstrap socket
|
|
13
|
-
* (`CES_BOOTSTRAP_SOCKET_DIR`) and cache
|
|
14
|
-
* the connection.
|
|
15
|
-
* 3. CES HTTP - containerized failover when IPC is unavailable
|
|
6
|
+
* 1. CES RPC (primary) - connect to the CES bootstrap socket and talk RPC.
|
|
7
|
+
* `openCesRpcSession` is the one client constructor. The assistant
|
|
8
|
+
* claims session ownership at boot (`startCes`) and hands CES the
|
|
9
|
+
* assistant API key. Child processes open a session on first credential
|
|
10
|
+
* read only when this process has no client and no reconnect owner.
|
|
11
|
+
* 2. CES HTTP - containerized failover when IPC is unavailable
|
|
16
12
|
* (`IS_CONTAINERIZED` + `CES_CREDENTIAL_URL`). Used if the assistant's
|
|
17
13
|
* bootstrap RPC transport is down, or if a process with HTTP env could
|
|
18
14
|
* not open the socket.
|
|
19
|
-
*
|
|
15
|
+
* 3. Encrypted file store (fallback) - used when CES is unavailable locally.
|
|
20
16
|
*
|
|
21
17
|
* All operations (reads, writes, lists, deletes) go to exactly one backend.
|
|
22
18
|
* There are no cross-store fallbacks or merges. The only transport failover is
|
|
@@ -35,14 +31,11 @@ import type {
|
|
|
35
31
|
|
|
36
32
|
import { getIsContainerized } from "../config/env-registry.js";
|
|
37
33
|
import {
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
} from "../credential-execution/
|
|
34
|
+
openCesRpcSession,
|
|
35
|
+
reconnectCesRpcSession,
|
|
36
|
+
} from "../credential-execution/ces-connect.js";
|
|
37
|
+
import { type CesClient } from "../credential-execution/client.js";
|
|
41
38
|
import { discoverCes } from "../credential-execution/executable-discovery.js";
|
|
42
|
-
import {
|
|
43
|
-
CesUnavailableError,
|
|
44
|
-
createCesProcessManager,
|
|
45
|
-
} from "../credential-execution/process-manager.js";
|
|
46
39
|
import { getAnyProviderEnvVar } from "../providers/provider-env-vars.js";
|
|
47
40
|
import { getLogger } from "../util/logger.js";
|
|
48
41
|
import { getProtectedDir } from "../util/platform.js";
|
|
@@ -85,11 +78,11 @@ let _resolvedBackend: CredentialBackend | undefined;
|
|
|
85
78
|
let _resolvePromise: Promise<CredentialBackend> | undefined;
|
|
86
79
|
|
|
87
80
|
/**
|
|
88
|
-
* In-flight
|
|
81
|
+
* In-flight CES RPC session promise for processes that did not call startCes().
|
|
89
82
|
*
|
|
90
83
|
* Workers and CLI subprocesses never call startCes(). When they hit
|
|
91
84
|
* resolveBackendAsync() with no _cesClient and no _cesReconnect
|
|
92
|
-
* (
|
|
85
|
+
* (assistant-boot only), this promise memoizes `openCesRpcSession` so
|
|
93
86
|
* concurrent credential reads in the same process share a single
|
|
94
87
|
* connect+handshake rather than racing.
|
|
95
88
|
*/
|
|
@@ -206,10 +199,9 @@ function getEncryptedStoreBackend(): CredentialBackend {
|
|
|
206
199
|
* Resolve the primary credential backend for this process (async).
|
|
207
200
|
*
|
|
208
201
|
* Priority:
|
|
209
|
-
* 1. CES RPC client
|
|
210
|
-
* 2.
|
|
211
|
-
* 3.
|
|
212
|
-
* 4. Encrypted file store: local fallback when CES is unavailable.
|
|
202
|
+
* 1. CES RPC: live client, or open one via `openCesRpcSession`.
|
|
203
|
+
* 2. Containerized + CES_CREDENTIAL_URL: CES HTTP, only if IPC is down.
|
|
204
|
+
* 3. Encrypted file store: local fallback when CES is unavailable.
|
|
213
205
|
*
|
|
214
206
|
* Once resolved, the backend is cached. If it becomes unavailable (e.g. the
|
|
215
207
|
* CES transport dies), we attempt to reconnect via `_cesReconnect` rather
|
|
@@ -415,19 +407,19 @@ export async function attemptCesReconnection(
|
|
|
415
407
|
}
|
|
416
408
|
|
|
417
409
|
/**
|
|
418
|
-
*
|
|
410
|
+
* Open a CES RPC session from a process that did not call startCes().
|
|
419
411
|
*
|
|
420
412
|
* Workers and CLI subprocesses never call startCes(). This function
|
|
421
|
-
*
|
|
413
|
+
* opens the same `openCesRpcSession` path the assistant uses at boot,
|
|
422
414
|
* memoizing the in-flight promise so concurrent callers share a single
|
|
423
415
|
* connect+handshake. Discovery uses the shared CES bootstrap socket; a
|
|
424
416
|
* missing socket fails immediately so callers can fall through without
|
|
425
417
|
* polling.
|
|
426
418
|
*
|
|
427
419
|
* On success, the client is injected via setCesClient() so subsequent
|
|
428
|
-
* resolveBackendAsync() calls take the
|
|
429
|
-
*
|
|
430
|
-
*
|
|
420
|
+
* resolveBackendAsync() calls take the live CES RPC path. A reconnect
|
|
421
|
+
* callback is also registered so the session can heal if the transport
|
|
422
|
+
* drops mid-process.
|
|
431
423
|
*
|
|
432
424
|
* Returns undefined on any failure (socket not found, handshake rejected,
|
|
433
425
|
* timeout) so the caller falls through to the encrypted file store.
|
|
@@ -438,70 +430,30 @@ async function tryLazyCesConnect(): Promise<CesClient | undefined> {
|
|
|
438
430
|
}
|
|
439
431
|
|
|
440
432
|
_lazyConnectPromise = (async () => {
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
if (discovery.mode === "unavailable") {
|
|
444
|
-
log.info(
|
|
445
|
-
{ reason: discovery.reason },
|
|
446
|
-
"CES socket not reachable for lazy connect, falling back to encrypted file store",
|
|
447
|
-
);
|
|
448
|
-
return undefined;
|
|
449
|
-
}
|
|
450
|
-
const pm = createCesProcessManager({});
|
|
451
|
-
const transport = await pm.start();
|
|
452
|
-
const client = createCesClient(transport);
|
|
453
|
-
const { accepted, reason } = await client.handshake();
|
|
454
|
-
if (!accepted) {
|
|
455
|
-
log.warn(
|
|
456
|
-
{ reason },
|
|
457
|
-
"Lazy CES connection handshake rejected — falling back to encrypted file store",
|
|
458
|
-
);
|
|
459
|
-
client.close();
|
|
460
|
-
await pm.stop().catch(() => {});
|
|
461
|
-
return undefined;
|
|
462
|
-
}
|
|
433
|
+
const discovery = discoverCes();
|
|
434
|
+
if (discovery.mode === "unavailable") {
|
|
463
435
|
log.info(
|
|
464
|
-
|
|
436
|
+
{ reason: discovery.reason },
|
|
437
|
+
"CES socket not reachable for lazy connect, falling back to encrypted file store",
|
|
465
438
|
);
|
|
466
|
-
setCesClient(client);
|
|
467
|
-
// Register a reconnect callback so the lazy connection self-heals
|
|
468
|
-
// if the transport drops, mirroring the daemon's proactive reconnect.
|
|
469
|
-
setCesReconnect(async () => {
|
|
470
|
-
try {
|
|
471
|
-
await pm.stop();
|
|
472
|
-
const newTransport = await pm.start();
|
|
473
|
-
const newClient = createCesClient(newTransport);
|
|
474
|
-
const { accepted: ok } = await newClient.handshake();
|
|
475
|
-
if (ok) {
|
|
476
|
-
log.info("Lazy CES reconnection successful");
|
|
477
|
-
return newClient;
|
|
478
|
-
}
|
|
479
|
-
newClient.close();
|
|
480
|
-
await pm.stop().catch(() => {});
|
|
481
|
-
return undefined;
|
|
482
|
-
} catch (err) {
|
|
483
|
-
log.warn(
|
|
484
|
-
{ error: err instanceof Error ? err.message : String(err) },
|
|
485
|
-
"Lazy CES reconnection failed",
|
|
486
|
-
);
|
|
487
|
-
return undefined;
|
|
488
|
-
}
|
|
489
|
-
});
|
|
490
|
-
return client;
|
|
491
|
-
} catch (err) {
|
|
492
|
-
if (err instanceof CesUnavailableError) {
|
|
493
|
-
log.info(
|
|
494
|
-
{ reason: err.message },
|
|
495
|
-
"CES socket not reachable for lazy connect — falling back to encrypted file store",
|
|
496
|
-
);
|
|
497
|
-
} else {
|
|
498
|
-
log.warn(
|
|
499
|
-
{ error: err instanceof Error ? err.message : String(err) },
|
|
500
|
-
"Lazy CES connection failed — falling back to encrypted file store",
|
|
501
|
-
);
|
|
502
|
-
}
|
|
503
439
|
return undefined;
|
|
504
440
|
}
|
|
441
|
+
const session = await openCesRpcSession();
|
|
442
|
+
if (!session) {
|
|
443
|
+
return undefined;
|
|
444
|
+
}
|
|
445
|
+
log.info(
|
|
446
|
+
"CES RPC session established; credential operations route through CES RPC",
|
|
447
|
+
);
|
|
448
|
+
setCesClient(session.client);
|
|
449
|
+
setCesReconnect(async () => {
|
|
450
|
+
const client = await reconnectCesRpcSession(session.processManager);
|
|
451
|
+
if (client) {
|
|
452
|
+
log.info("CES RPC reconnection successful");
|
|
453
|
+
}
|
|
454
|
+
return client;
|
|
455
|
+
});
|
|
456
|
+
return session.client;
|
|
505
457
|
})();
|
|
506
458
|
|
|
507
459
|
try {
|
|
@@ -512,7 +464,10 @@ async function tryLazyCesConnect(): Promise<CesClient | undefined> {
|
|
|
512
464
|
}
|
|
513
465
|
|
|
514
466
|
async function doResolveBackend(): Promise<CredentialBackend> {
|
|
515
|
-
// 1. CES RPC. Primary credential backend in every environment.
|
|
467
|
+
// 1. CES RPC. Primary credential backend in every environment. Boot
|
|
468
|
+
// claims reconnect ownership before it reads handshake identity, so
|
|
469
|
+
// that read cannot open a second session. Children open here only
|
|
470
|
+
// when this process has no client and no reconnect owner.
|
|
516
471
|
if (_cesClient) {
|
|
517
472
|
const cesRpc = new CesRpcCredentialBackend(_cesClient);
|
|
518
473
|
if (cesRpc.isAvailable()) {
|
|
@@ -525,24 +480,19 @@ async function doResolveBackend(): Promise<CredentialBackend> {
|
|
|
525
480
|
);
|
|
526
481
|
}
|
|
527
482
|
|
|
528
|
-
// 2. Lazy CES RPC connect. Child processes never call startCes(). When
|
|
529
|
-
// the assistant's setCesReconnect() is NOT registered, attempt a
|
|
530
|
-
// direct connection to the CES bootstrap socket. On success, inject
|
|
531
|
-
// the client via setCesClient()
|
|
532
|
-
// and re-resolve through the CES RPC path. On failure, fall through.
|
|
533
483
|
if (!_cesClient && !_cesReconnect) {
|
|
534
|
-
const
|
|
535
|
-
if (
|
|
536
|
-
const cesRpc = new CesRpcCredentialBackend(
|
|
484
|
+
const client = await tryLazyCesConnect();
|
|
485
|
+
if (client) {
|
|
486
|
+
const cesRpc = new CesRpcCredentialBackend(client);
|
|
537
487
|
if (cesRpc.isAvailable()) {
|
|
538
488
|
_resolvedBackend = cesRpc;
|
|
539
|
-
log.info("Resolved credential backend: ces-rpc
|
|
489
|
+
log.info("Resolved credential backend: ces-rpc");
|
|
540
490
|
return cesRpc;
|
|
541
491
|
}
|
|
542
492
|
}
|
|
543
493
|
}
|
|
544
494
|
|
|
545
|
-
//
|
|
495
|
+
// 2. CES HTTP. Managed failover when IPC is unavailable.
|
|
546
496
|
if (getIsContainerized() && process.env.CES_CREDENTIAL_URL) {
|
|
547
497
|
const ces = createCesCredentialBackend();
|
|
548
498
|
if (ces.isAvailable()) {
|
|
@@ -556,7 +506,7 @@ async function doResolveBackend(): Promise<CredentialBackend> {
|
|
|
556
506
|
);
|
|
557
507
|
}
|
|
558
508
|
|
|
559
|
-
//
|
|
509
|
+
// 3. On a containerized pod the local encrypted store does not exist and CES
|
|
560
510
|
// owns credentials. Never resolve to the encrypted store here; it would
|
|
561
511
|
// report a provisioned credential as absent. Return an unreachable backend
|
|
562
512
|
// (presence indeterminate, which callers retry) WITHOUT caching it, so the
|
|
@@ -568,7 +518,7 @@ async function doResolveBackend(): Promise<CredentialBackend> {
|
|
|
568
518
|
return createUnavailableBackend();
|
|
569
519
|
}
|
|
570
520
|
|
|
571
|
-
//
|
|
521
|
+
// 4. Encrypted file store: the legitimate backend for local / self-hosted
|
|
572
522
|
// mode when CES is unavailable.
|
|
573
523
|
_resolvedBackend = getEncryptedStoreBackend();
|
|
574
524
|
log.info("Resolved credential backend: encrypted-store (local mode)");
|