@vellumai/assistant 0.11.2-dev.202608060138.f2a1896 → 0.11.2-dev.202608060508.a7ab7f7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/node_modules/@vellumai/ces-client/src/index.ts +1 -0
- package/node_modules/@vellumai/ces-client/src/rpc-client.ts +19 -2
- package/package.json +1 -1
- package/src/__tests__/conversation-error.test.ts +17 -1
- package/src/__tests__/secret-routes-platform-proxy.test.ts +29 -1
- package/src/__tests__/secure-keys.test.ts +91 -0
- package/src/credential-execution/ces-runtime.ts +3 -0
- package/src/credential-execution/client.ts +22 -1
- package/src/credential-execution/process-manager.test.ts +99 -0
- package/src/credential-execution/process-manager.ts +62 -16
- package/src/daemon/conversation-error.ts +7 -0
- package/src/providers/__tests__/preflight-resolved-config.test.ts +58 -1
- package/src/providers/connection-resolution.ts +35 -1
- package/src/runtime/routes/secret-routes.ts +40 -23
- package/src/security/credential-backend.ts +51 -4
- package/src/security/encrypted-store.ts +64 -42
- package/src/security/secure-keys.ts +55 -7
|
@@ -69,11 +69,23 @@ interface PendingRequest {
|
|
|
69
69
|
// Client configuration
|
|
70
70
|
// ---------------------------------------------------------------------------
|
|
71
71
|
|
|
72
|
+
/** Minimal logger interface. Callers can supply pino, console, etc. */
|
|
73
|
+
export interface CesRpcLogger {
|
|
74
|
+
info(obj: Record<string, unknown>, msg: string): void;
|
|
75
|
+
info(msg: string): void;
|
|
76
|
+
}
|
|
77
|
+
|
|
72
78
|
export interface CesRpcClientConfig {
|
|
73
79
|
/** Timeout for individual RPC requests (ms). Default: 30 000 */
|
|
74
80
|
requestTimeoutMs?: number;
|
|
75
81
|
/** Timeout for the initial handshake (ms). Default: 10 000 */
|
|
76
82
|
handshakeTimeoutMs?: number;
|
|
83
|
+
/**
|
|
84
|
+
* Optional logger for lifecycle transitions (handshake accepted, close).
|
|
85
|
+
* These transitions are otherwise invisible, which made a stuck credential
|
|
86
|
+
* backend hard to diagnose.
|
|
87
|
+
*/
|
|
88
|
+
logger?: CesRpcLogger;
|
|
77
89
|
}
|
|
78
90
|
|
|
79
91
|
const DEFAULT_REQUEST_TIMEOUT_MS = 30_000;
|
|
@@ -154,12 +166,15 @@ export function createCesRpcClient(
|
|
|
154
166
|
config?.requestTimeoutMs ?? DEFAULT_REQUEST_TIMEOUT_MS;
|
|
155
167
|
const handshakeTimeoutMs =
|
|
156
168
|
config?.handshakeTimeoutMs ?? DEFAULT_HANDSHAKE_TIMEOUT_MS;
|
|
169
|
+
const logger = config?.logger;
|
|
157
170
|
|
|
158
171
|
const sessionId = randomUUID();
|
|
159
172
|
let requestCounter = 0;
|
|
160
173
|
let ready = false;
|
|
161
|
-
let inflightHandshake: Promise<{
|
|
162
|
-
|
|
174
|
+
let inflightHandshake: Promise<{
|
|
175
|
+
accepted: boolean;
|
|
176
|
+
reason?: string;
|
|
177
|
+
}> | null = null;
|
|
163
178
|
|
|
164
179
|
const pending = new Map<string, PendingRequest>();
|
|
165
180
|
|
|
@@ -380,6 +395,7 @@ export function createCesRpcClient(
|
|
|
380
395
|
|
|
381
396
|
if (ack.accepted) {
|
|
382
397
|
ready = true;
|
|
398
|
+
logger?.info("CES RPC client handshake accepted; ready for calls");
|
|
383
399
|
}
|
|
384
400
|
|
|
385
401
|
return { accepted: ack.accepted, reason: ack.reason };
|
|
@@ -413,6 +429,7 @@ export function createCesRpcClient(
|
|
|
413
429
|
close(): void {
|
|
414
430
|
rejectAllPending(new CesTransportError("CES client closed"));
|
|
415
431
|
ready = false;
|
|
432
|
+
logger?.info("CES RPC client closed");
|
|
416
433
|
transport.close();
|
|
417
434
|
},
|
|
418
435
|
};
|
package/package.json
CHANGED
|
@@ -1613,7 +1613,7 @@ describe("ConnectionResolutionError classification", () => {
|
|
|
1613
1613
|
expect(result.userMessage).toContain('profile "custom-fast"');
|
|
1614
1614
|
});
|
|
1615
1615
|
|
|
1616
|
-
it("classifies platform_unauthenticated with a log-in fix", () => {
|
|
1616
|
+
it("classifies platform_unauthenticated with a log-in fix on a non-platform install", () => {
|
|
1617
1617
|
const err = new ConnectionResolutionError(
|
|
1618
1618
|
"vellum",
|
|
1619
1619
|
"platform_unauthenticated",
|
|
@@ -1624,6 +1624,22 @@ describe("ConnectionResolutionError classification", () => {
|
|
|
1624
1624
|
expect(result.userMessage).toContain("Log in");
|
|
1625
1625
|
});
|
|
1626
1626
|
|
|
1627
|
+
it("classifies platform_unauthenticated as a re-provision hint on a platform-managed assistant", () => {
|
|
1628
|
+
const err = new ConnectionResolutionError(
|
|
1629
|
+
"vellum",
|
|
1630
|
+
"platform_unauthenticated",
|
|
1631
|
+
"unavailable",
|
|
1632
|
+
);
|
|
1633
|
+
process.env.IS_PLATFORM = "true";
|
|
1634
|
+
try {
|
|
1635
|
+
const result = classifyConversationError(err, errCtx);
|
|
1636
|
+
expect(result.userMessage).toContain("re-provisioned");
|
|
1637
|
+
expect(result.userMessage).not.toContain("Log in");
|
|
1638
|
+
} finally {
|
|
1639
|
+
delete process.env.IS_PLATFORM;
|
|
1640
|
+
}
|
|
1641
|
+
});
|
|
1642
|
+
|
|
1627
1643
|
it("is a structured VellumError (ConfigError) for logging/monitoring", () => {
|
|
1628
1644
|
const err = new ConnectionResolutionError("x", "not_found", "m");
|
|
1629
1645
|
expect(err).toBeInstanceOf(VellumError);
|
|
@@ -108,7 +108,10 @@ import {
|
|
|
108
108
|
initializeProviders,
|
|
109
109
|
listProviders,
|
|
110
110
|
} from "../providers/registry.js";
|
|
111
|
-
import {
|
|
111
|
+
import {
|
|
112
|
+
notifyCesOfAssistantApiKeyUpdate,
|
|
113
|
+
ROUTES,
|
|
114
|
+
} from "../runtime/routes/secret-routes.js";
|
|
112
115
|
|
|
113
116
|
const addRoute = ROUTES.find(
|
|
114
117
|
(r) => r.method === "POST" && r.endpoint === "secrets",
|
|
@@ -176,6 +179,31 @@ describe("secret routes managed proxy registry sync", () => {
|
|
|
176
179
|
expect(lastGeminiConstructorOpts).toBeDefined();
|
|
177
180
|
});
|
|
178
181
|
|
|
182
|
+
test("the CES key-update notification with no CES client logs a warning", async () => {
|
|
183
|
+
// When getCesClient() is undefined the push is skipped, but the skip must
|
|
184
|
+
// be logged, not dropped silently. Exercise the extracted helper directly
|
|
185
|
+
// with an injected logger.
|
|
186
|
+
const warns: string[] = [];
|
|
187
|
+
const fakeLogger = {
|
|
188
|
+
info: () => {},
|
|
189
|
+
warn: (a: unknown, b?: unknown) => {
|
|
190
|
+
warns.push(typeof a === "string" ? a : String(b));
|
|
191
|
+
},
|
|
192
|
+
};
|
|
193
|
+
|
|
194
|
+
await notifyCesOfAssistantApiKeyUpdate(
|
|
195
|
+
"ast-managed-key",
|
|
196
|
+
undefined,
|
|
197
|
+
fakeLogger as unknown as Parameters<
|
|
198
|
+
typeof notifyCesOfAssistantApiKeyUpdate
|
|
199
|
+
>[2],
|
|
200
|
+
);
|
|
201
|
+
|
|
202
|
+
expect(warns.some((m) => m.includes("no CES client is connected"))).toBe(
|
|
203
|
+
true,
|
|
204
|
+
);
|
|
205
|
+
});
|
|
206
|
+
|
|
179
207
|
test("provider API key writes notify live-conversation refresh listeners", async () => {
|
|
180
208
|
await addApiKey("fireworks", "fw-key");
|
|
181
209
|
|
|
@@ -19,6 +19,7 @@ import * as encryptedStore from "../security/encrypted-store.js";
|
|
|
19
19
|
import {
|
|
20
20
|
_resetBackend,
|
|
21
21
|
deleteSecureKeyAsync,
|
|
22
|
+
getActiveBackendName,
|
|
22
23
|
getSecureKeyAsync,
|
|
23
24
|
getSecureKeyResultAsync,
|
|
24
25
|
listSecureKeysAsync,
|
|
@@ -58,6 +59,9 @@ describe("secure-keys", () => {
|
|
|
58
59
|
_resetBackend();
|
|
59
60
|
delete process.env.VELLUM_DEV;
|
|
60
61
|
delete process.env.VELLUM_DESKTOP_APP;
|
|
62
|
+
delete process.env.IS_CONTAINERIZED;
|
|
63
|
+
delete process.env.CES_CREDENTIAL_URL;
|
|
64
|
+
delete process.env.CES_LOCAL_SOCKET;
|
|
61
65
|
});
|
|
62
66
|
|
|
63
67
|
afterAll(() => {
|
|
@@ -507,4 +511,91 @@ describe("secure-keys", () => {
|
|
|
507
511
|
).toBeUndefined();
|
|
508
512
|
});
|
|
509
513
|
});
|
|
514
|
+
|
|
515
|
+
// -----------------------------------------------------------------------
|
|
516
|
+
// Encrypted-store → CES upgrade without a reconnect callback
|
|
517
|
+
// -----------------------------------------------------------------------
|
|
518
|
+
//
|
|
519
|
+
// Worker/CLI processes inject a CES client whose handshake is still in
|
|
520
|
+
// flight, so the first credential read falls back to the encrypted store.
|
|
521
|
+
// These processes never register `_cesReconnect`. Once the handshake
|
|
522
|
+
// completes and the client reports ready, the next read must upgrade to
|
|
523
|
+
// CES RPC, since the live client already exists and no reconnect is needed.
|
|
524
|
+
describe("encrypted-store → CES upgrade without a reconnect callback", () => {
|
|
525
|
+
test("upgrades to CES RPC once the injected client becomes ready", async () => {
|
|
526
|
+
const client = makeMockCesClient(false);
|
|
527
|
+
setCesClient(client);
|
|
528
|
+
|
|
529
|
+
// Not ready yet → falls back to the encrypted store (local mode).
|
|
530
|
+
encryptedStore.setKey("openai", "local-value");
|
|
531
|
+
expect(await getSecureKeyAsync("openai")).toBe("local-value");
|
|
532
|
+
expect(getActiveBackendName()).toBe("encrypted-store");
|
|
533
|
+
|
|
534
|
+
// Handshake completes; no `_cesReconnect` was ever registered.
|
|
535
|
+
client.setReady(true);
|
|
536
|
+
|
|
537
|
+
await getSecureKeyResultAsync("openai");
|
|
538
|
+
expect(getActiveBackendName()).toBe("ces-rpc");
|
|
539
|
+
});
|
|
540
|
+
});
|
|
541
|
+
|
|
542
|
+
// -----------------------------------------------------------------------
|
|
543
|
+
// Containerized pod with no ready CES backend
|
|
544
|
+
// -----------------------------------------------------------------------
|
|
545
|
+
//
|
|
546
|
+
// On a pod the local encrypted store does not exist. When no CES path is
|
|
547
|
+
// ready, reads must report the store as unreachable (presence
|
|
548
|
+
// indeterminate) rather than absent. A provisioned credential is never
|
|
549
|
+
// reported as "no key". The unreachable result must not be pinned: once a
|
|
550
|
+
// CES client arrives, the next read routes through it.
|
|
551
|
+
describe("containerized pod with no ready CES backend", () => {
|
|
552
|
+
test("reads report unreachable (indeterminate), never absent", async () => {
|
|
553
|
+
process.env.IS_CONTAINERIZED = "1";
|
|
554
|
+
delete process.env.CES_CREDENTIAL_URL;
|
|
555
|
+
delete process.env.CES_LOCAL_SOCKET;
|
|
556
|
+
_resetBackend();
|
|
557
|
+
|
|
558
|
+
const result = await getSecureKeyResultAsync("openai");
|
|
559
|
+
expect(result.value).toBeUndefined();
|
|
560
|
+
// unreachable → presence indeterminate, NEVER absent.
|
|
561
|
+
expect(result.unreachable).toBe(true);
|
|
562
|
+
});
|
|
563
|
+
|
|
564
|
+
test("does not pin the unreachable result: a ready CES client wins the next read", async () => {
|
|
565
|
+
process.env.IS_CONTAINERIZED = "1";
|
|
566
|
+
delete process.env.CES_CREDENTIAL_URL;
|
|
567
|
+
delete process.env.CES_LOCAL_SOCKET;
|
|
568
|
+
_resetBackend();
|
|
569
|
+
|
|
570
|
+
expect((await getSecureKeyResultAsync("openai")).unreachable).toBe(true);
|
|
571
|
+
|
|
572
|
+
setCesClient(makeMockCesClient(true));
|
|
573
|
+
|
|
574
|
+
const afterAttach = await getSecureKeyResultAsync("openai");
|
|
575
|
+
expect(afterAttach.unreachable).toBe(false);
|
|
576
|
+
expect(getActiveBackendName()).toBe("ces-rpc");
|
|
577
|
+
});
|
|
578
|
+
});
|
|
579
|
+
|
|
580
|
+
// -----------------------------------------------------------------------
|
|
581
|
+
// Encrypted-store read error vs genuinely-absent (local mode)
|
|
582
|
+
// -----------------------------------------------------------------------
|
|
583
|
+
describe("encrypted-store read error vs absent", () => {
|
|
584
|
+
test("genuinely-missing store → absent (unreachable false)", async () => {
|
|
585
|
+
const result = await getSecureKeyResultAsync("nope");
|
|
586
|
+
expect(result.value).toBeUndefined();
|
|
587
|
+
expect(result.unreachable).toBe(false);
|
|
588
|
+
});
|
|
589
|
+
|
|
590
|
+
test("store exists but key material is missing → unreachable", async () => {
|
|
591
|
+
encryptedStore.setKey("openai", "sk-1");
|
|
592
|
+
// Remove the store key so the existing entry can no longer be decrypted.
|
|
593
|
+
rmSync(join(TEST_DIR, "store.key"));
|
|
594
|
+
_resetBackend();
|
|
595
|
+
|
|
596
|
+
const result = await getSecureKeyResultAsync("openai");
|
|
597
|
+
expect(result.value).toBeUndefined();
|
|
598
|
+
expect(result.unreachable).toBe(true);
|
|
599
|
+
});
|
|
600
|
+
});
|
|
510
601
|
});
|
|
@@ -176,6 +176,9 @@ export async function startCes(config: AssistantConfig): Promise<void> {
|
|
|
176
176
|
},
|
|
177
177
|
});
|
|
178
178
|
if (client) {
|
|
179
|
+
log.info(
|
|
180
|
+
"CES client injected into credential resolver at startup; credential reads route through CES RPC",
|
|
181
|
+
);
|
|
179
182
|
setCesClient(client);
|
|
180
183
|
} else {
|
|
181
184
|
// The handshake lost the startup race, so provider init proceeds on the
|
|
@@ -9,6 +9,15 @@
|
|
|
9
9
|
* delegated to the shared package.
|
|
10
10
|
*/
|
|
11
11
|
|
|
12
|
+
import {
|
|
13
|
+
type CesRpcClient,
|
|
14
|
+
type CesRpcClientConfig,
|
|
15
|
+
type CesTransport,
|
|
16
|
+
createCesRpcClient,
|
|
17
|
+
} from "@vellumai/ces-client/rpc-client";
|
|
18
|
+
|
|
19
|
+
import { getLogger } from "../util/logger.js";
|
|
20
|
+
|
|
12
21
|
export {
|
|
13
22
|
type CesRpcClient as CesClient,
|
|
14
23
|
type CesRpcClientConfig as CesClientConfig,
|
|
@@ -17,5 +26,17 @@ export {
|
|
|
17
26
|
CesHandshakeError,
|
|
18
27
|
type CesTransport,
|
|
19
28
|
CesTransportError,
|
|
20
|
-
createCesRpcClient as createCesClient,
|
|
21
29
|
} from "@vellumai/ces-client/rpc-client";
|
|
30
|
+
|
|
31
|
+
const log = getLogger("ces-rpc-client");
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Construct a CES RPC client wired to the assistant logger so handshake-ready
|
|
35
|
+
* and close transitions are visible in the daemon logs.
|
|
36
|
+
*/
|
|
37
|
+
export function createCesClient(
|
|
38
|
+
transport: CesTransport,
|
|
39
|
+
config?: CesRpcClientConfig,
|
|
40
|
+
): CesRpcClient {
|
|
41
|
+
return createCesRpcClient(transport, { logger: log, ...config });
|
|
42
|
+
}
|
|
@@ -125,3 +125,102 @@ describe("CesProcessManager.onTransportClose", () => {
|
|
|
125
125
|
await pm.stop();
|
|
126
126
|
});
|
|
127
127
|
});
|
|
128
|
+
|
|
129
|
+
// ---------------------------------------------------------------------------
|
|
130
|
+
// Transport-death logging: WARN only for genuinely-unexpected death
|
|
131
|
+
// ---------------------------------------------------------------------------
|
|
132
|
+
|
|
133
|
+
type LogCall = { level: string; msg: string };
|
|
134
|
+
|
|
135
|
+
function makeRecordingLogger(): {
|
|
136
|
+
calls: LogCall[];
|
|
137
|
+
logger: NonNullable<Parameters<typeof createCesProcessManager>[0]["logger"]>;
|
|
138
|
+
} {
|
|
139
|
+
const calls: LogCall[] = [];
|
|
140
|
+
const rec =
|
|
141
|
+
(level: string) =>
|
|
142
|
+
(...args: unknown[]) => {
|
|
143
|
+
const msg = typeof args[0] === "string" ? args[0] : args[1];
|
|
144
|
+
calls.push({ level, msg: typeof msg === "string" ? msg : "" });
|
|
145
|
+
};
|
|
146
|
+
const logger = {
|
|
147
|
+
info: rec("info"),
|
|
148
|
+
warn: rec("warn"),
|
|
149
|
+
debug: rec("debug"),
|
|
150
|
+
error: rec("error"),
|
|
151
|
+
} as unknown as NonNullable<
|
|
152
|
+
Parameters<typeof createCesProcessManager>[0]["logger"]
|
|
153
|
+
>;
|
|
154
|
+
return { calls, logger };
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
describe("CesProcessManager transport-death logging", () => {
|
|
158
|
+
let tempDir: string;
|
|
159
|
+
let server: Server;
|
|
160
|
+
let connections: Array<import("node:net").Socket>;
|
|
161
|
+
|
|
162
|
+
beforeEach(async () => {
|
|
163
|
+
tempDir = mkdtempSync(join(tmpdir(), "ces-pm-log-test-"));
|
|
164
|
+
mockSocketPath = join(tempDir, "ces.sock");
|
|
165
|
+
connections = [];
|
|
166
|
+
server = createServer((socket) => {
|
|
167
|
+
connections.push(socket);
|
|
168
|
+
socket.on("error", () => {});
|
|
169
|
+
});
|
|
170
|
+
await new Promise<void>((resolve) =>
|
|
171
|
+
server.listen(mockSocketPath, resolve),
|
|
172
|
+
);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
afterEach(async () => {
|
|
176
|
+
for (const sock of connections) {
|
|
177
|
+
sock.destroy();
|
|
178
|
+
}
|
|
179
|
+
connections = [];
|
|
180
|
+
await new Promise<void>((resolve) => server.close(() => resolve()));
|
|
181
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
182
|
+
});
|
|
183
|
+
|
|
184
|
+
test("logs WARN when the transport dies unexpectedly (remote close)", async () => {
|
|
185
|
+
const { calls, logger } = makeRecordingLogger();
|
|
186
|
+
const pm = createCesProcessManager({ logger });
|
|
187
|
+
const transport = await pm.start();
|
|
188
|
+
|
|
189
|
+
// The remote (server) closes the connection: genuinely-unexpected death.
|
|
190
|
+
for (const sock of connections) {
|
|
191
|
+
sock.destroy();
|
|
192
|
+
}
|
|
193
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
194
|
+
expect(transport.isAlive()).toBe(false);
|
|
195
|
+
|
|
196
|
+
expect(
|
|
197
|
+
calls.some(
|
|
198
|
+
(c) => c.level === "warn" && c.msg.includes("died unexpectedly"),
|
|
199
|
+
),
|
|
200
|
+
).toBe(true);
|
|
201
|
+
|
|
202
|
+
await pm.stop();
|
|
203
|
+
});
|
|
204
|
+
|
|
205
|
+
test("does not log WARN on an intentional stop()", async () => {
|
|
206
|
+
const { calls, logger } = makeRecordingLogger();
|
|
207
|
+
const pm = createCesProcessManager({ logger });
|
|
208
|
+
await pm.start();
|
|
209
|
+
|
|
210
|
+
await pm.stop();
|
|
211
|
+
// Let the socket close event propagate after destroy().
|
|
212
|
+
await new Promise((resolve) => setTimeout(resolve, 50));
|
|
213
|
+
|
|
214
|
+
expect(
|
|
215
|
+
calls.some(
|
|
216
|
+
(c) => c.level === "warn" && c.msg.includes("died unexpectedly"),
|
|
217
|
+
),
|
|
218
|
+
).toBe(false);
|
|
219
|
+
expect(
|
|
220
|
+
calls.some(
|
|
221
|
+
(c) =>
|
|
222
|
+
c.level === "debug" && c.msg.includes("CES socket transport closed"),
|
|
223
|
+
),
|
|
224
|
+
).toBe(true);
|
|
225
|
+
});
|
|
226
|
+
});
|
|
@@ -29,6 +29,8 @@ import {
|
|
|
29
29
|
|
|
30
30
|
const log = getLogger("ces-process-manager");
|
|
31
31
|
|
|
32
|
+
type PmLogger = ReturnType<typeof getLogger>;
|
|
33
|
+
|
|
32
34
|
const SOCKET_CONNECT_TIMEOUT_MS = 5_000;
|
|
33
35
|
|
|
34
36
|
// ---------------------------------------------------------------------------
|
|
@@ -57,6 +59,9 @@ export interface CesProcessManagerConfig {
|
|
|
57
59
|
* Reserved for future feature-flag checks or config-driven behavior.
|
|
58
60
|
*/
|
|
59
61
|
assistantConfig?: AssistantConfig;
|
|
62
|
+
|
|
63
|
+
/** Logger override. Defaults to the module logger; injected in tests. */
|
|
64
|
+
logger?: PmLogger;
|
|
60
65
|
}
|
|
61
66
|
|
|
62
67
|
// ---------------------------------------------------------------------------
|
|
@@ -99,12 +104,13 @@ export interface CesProcessManager {
|
|
|
99
104
|
// ---------------------------------------------------------------------------
|
|
100
105
|
|
|
101
106
|
export function createCesProcessManager(
|
|
102
|
-
|
|
107
|
+
config: CesProcessManagerConfig,
|
|
103
108
|
): CesProcessManager {
|
|
109
|
+
const pmLog = config.logger ?? log;
|
|
104
110
|
let managedSocket: Socket | null = null;
|
|
105
111
|
let discoveryResult: DiscoveryResult | null = null;
|
|
106
112
|
let running = false;
|
|
107
|
-
let currentTransport:
|
|
113
|
+
let currentTransport: SocketTransport | null = null;
|
|
108
114
|
const transportCloseHandlers: Array<() => void> = [];
|
|
109
115
|
|
|
110
116
|
return {
|
|
@@ -136,24 +142,28 @@ export function createCesProcessManager(
|
|
|
136
142
|
}
|
|
137
143
|
|
|
138
144
|
if (managedSocket) {
|
|
145
|
+
// Mark the close as intentional so the transport-death log is debug,
|
|
146
|
+
// not a misleading WARN, on routine shutdown/reconnect.
|
|
147
|
+
currentTransport?.markClosing();
|
|
139
148
|
managedSocket.destroy();
|
|
140
149
|
managedSocket = null;
|
|
141
150
|
}
|
|
142
151
|
|
|
143
152
|
currentTransport = null;
|
|
144
153
|
running = false;
|
|
145
|
-
|
|
154
|
+
pmLog.info("CES process manager stopped");
|
|
146
155
|
},
|
|
147
156
|
|
|
148
157
|
async forceStop(): Promise<void> {
|
|
149
158
|
if (managedSocket) {
|
|
159
|
+
currentTransport?.markClosing();
|
|
150
160
|
managedSocket.destroy();
|
|
151
161
|
managedSocket = null;
|
|
152
162
|
}
|
|
153
163
|
|
|
154
164
|
currentTransport = null;
|
|
155
165
|
running = false;
|
|
156
|
-
|
|
166
|
+
pmLog.info("CES process manager force-stopped");
|
|
157
167
|
},
|
|
158
168
|
|
|
159
169
|
getDiscoveryResult(): DiscoveryResult | null {
|
|
@@ -195,8 +205,8 @@ export function createCesProcessManager(
|
|
|
195
205
|
|
|
196
206
|
async function connectManagedSocket(
|
|
197
207
|
discovery: ManagedDiscoverySuccess | SiblingDiscoverySuccess,
|
|
198
|
-
): Promise<
|
|
199
|
-
|
|
208
|
+
): Promise<SocketTransport> {
|
|
209
|
+
pmLog.info(
|
|
200
210
|
{ socketPath: discovery.socketPath, mode: discovery.mode },
|
|
201
211
|
"Connecting to CES over socket",
|
|
202
212
|
);
|
|
@@ -207,9 +217,14 @@ export function createCesProcessManager(
|
|
|
207
217
|
);
|
|
208
218
|
managedSocket = socket;
|
|
209
219
|
|
|
210
|
-
|
|
220
|
+
pmLog.info("Connected to CES over socket");
|
|
211
221
|
|
|
212
|
-
|
|
222
|
+
// Assign currentTransport synchronously here (no await between socket
|
|
223
|
+
// assignment and this line) so stop()/forceStop() can always mark an
|
|
224
|
+
// intentional close, even if they race the connect.
|
|
225
|
+
const transport = createSocketTransport(socket, pmLog);
|
|
226
|
+
currentTransport = transport;
|
|
227
|
+
return transport;
|
|
213
228
|
}
|
|
214
229
|
}
|
|
215
230
|
|
|
@@ -221,23 +236,43 @@ export function createCesProcessManager(
|
|
|
221
236
|
* Tracks the transport `alive` state and notifies registered handlers exactly
|
|
222
237
|
* once when the transport dies, so the RPC client can fail-fast any in-flight
|
|
223
238
|
* calls instead of waiting out their timeouts.
|
|
239
|
+
*
|
|
240
|
+
* `markClosing()` records that an imminent close was initiated by us (shutdown,
|
|
241
|
+
* reconnect, handshake rejection). A close so marked logs at debug; an
|
|
242
|
+
* unmarked death (remote socket close, socket error) logs at WARN.
|
|
224
243
|
*/
|
|
225
|
-
function createCloseNotifier(): {
|
|
244
|
+
function createCloseNotifier(log: PmLogger): {
|
|
226
245
|
isAlive: () => boolean;
|
|
227
|
-
|
|
246
|
+
markClosing: () => void;
|
|
247
|
+
markDead: (reason?: string) => void;
|
|
228
248
|
onClose: (handler: () => void) => void;
|
|
229
249
|
} {
|
|
230
250
|
let alive = true;
|
|
231
251
|
let notified = false;
|
|
252
|
+
let intentional = false;
|
|
232
253
|
const handlers: Array<() => void> = [];
|
|
233
254
|
return {
|
|
234
255
|
isAlive: () => alive,
|
|
235
|
-
|
|
256
|
+
markClosing() {
|
|
257
|
+
intentional = true;
|
|
258
|
+
},
|
|
259
|
+
markDead(reason?: string) {
|
|
236
260
|
alive = false;
|
|
237
261
|
if (notified) {
|
|
238
262
|
return;
|
|
239
263
|
}
|
|
240
264
|
notified = true;
|
|
265
|
+
if (intentional) {
|
|
266
|
+
log.debug(
|
|
267
|
+
{ reason: reason ?? "intentional" },
|
|
268
|
+
"CES socket transport closed",
|
|
269
|
+
);
|
|
270
|
+
} else {
|
|
271
|
+
log.warn(
|
|
272
|
+
{ reason: reason ?? "unknown" },
|
|
273
|
+
"CES socket transport died unexpectedly; credential ops will fail over or reconnect",
|
|
274
|
+
);
|
|
275
|
+
}
|
|
241
276
|
for (const handler of handlers) {
|
|
242
277
|
try {
|
|
243
278
|
handler();
|
|
@@ -256,10 +291,18 @@ function createCloseNotifier(): {
|
|
|
256
291
|
// Socket transport (managed mode)
|
|
257
292
|
// ---------------------------------------------------------------------------
|
|
258
293
|
|
|
259
|
-
|
|
294
|
+
/**
|
|
295
|
+
* A CesTransport plus `markClosing()`, which the process manager calls before
|
|
296
|
+
* an intentional socket destroy so the ensuing close logs at debug, not WARN.
|
|
297
|
+
*/
|
|
298
|
+
interface SocketTransport extends CesTransport {
|
|
299
|
+
markClosing(): void;
|
|
300
|
+
}
|
|
301
|
+
|
|
302
|
+
function createSocketTransport(socket: Socket, log: PmLogger): SocketTransport {
|
|
260
303
|
const messageHandlers: Array<(message: string) => void> = [];
|
|
261
304
|
let buffer = "";
|
|
262
|
-
const death = createCloseNotifier();
|
|
305
|
+
const death = createCloseNotifier(log);
|
|
263
306
|
|
|
264
307
|
const decoder = new StringDecoder("utf8");
|
|
265
308
|
|
|
@@ -278,12 +321,12 @@ function createSocketTransport(socket: Socket): CesTransport {
|
|
|
278
321
|
});
|
|
279
322
|
|
|
280
323
|
socket.on("close", () => {
|
|
281
|
-
death.markDead();
|
|
324
|
+
death.markDead("socket closed");
|
|
282
325
|
});
|
|
283
326
|
|
|
284
327
|
socket.on("error", (err) => {
|
|
285
328
|
log.warn({ err }, "CES socket transport error");
|
|
286
|
-
death.markDead();
|
|
329
|
+
death.markDead("socket error");
|
|
287
330
|
});
|
|
288
331
|
|
|
289
332
|
return {
|
|
@@ -304,8 +347,11 @@ function createSocketTransport(socket: Socket): CesTransport {
|
|
|
304
347
|
|
|
305
348
|
onClose: death.onClose,
|
|
306
349
|
|
|
350
|
+
markClosing: death.markClosing,
|
|
351
|
+
|
|
307
352
|
close(): void {
|
|
308
|
-
death.
|
|
353
|
+
death.markClosing();
|
|
354
|
+
death.markDead("transport.close() called");
|
|
309
355
|
socket.destroy();
|
|
310
356
|
},
|
|
311
357
|
};
|
|
@@ -6,6 +6,7 @@ import type {
|
|
|
6
6
|
ConversationErrorCode,
|
|
7
7
|
ConversationErrorEvent,
|
|
8
8
|
} from "../api/events/conversation-error.js";
|
|
9
|
+
import { getIsPlatform } from "../config/env-registry.js";
|
|
9
10
|
import {
|
|
10
11
|
isImageDimensionsTooLargeError,
|
|
11
12
|
isImageMediaTypeMismatchError,
|
|
@@ -361,6 +362,12 @@ function connectionResolutionUserMessage(
|
|
|
361
362
|
// doesn't.
|
|
362
363
|
return `${connection}${usedBy} has no stored credential. Add an API key or reconnect it in ${fixPath}.`;
|
|
363
364
|
case "platform_unauthenticated":
|
|
365
|
+
// A platform-managed assistant cannot log in or switch providers, and
|
|
366
|
+
// this classifier only sees the reason code, so one honest wording covers
|
|
367
|
+
// both the transient and the re-provision cases.
|
|
368
|
+
if (getIsPlatform()) {
|
|
369
|
+
return `${connection}${usedBy} is unavailable. If this persists, the platform credential may need to be re-provisioned on the Vellum platform.`;
|
|
370
|
+
}
|
|
364
371
|
return `${connection}${usedBy} requires a Vellum platform login. Log in, or pick a different provider in ${fixPath}.`;
|
|
365
372
|
case "model_incompatible":
|
|
366
373
|
return `${error.model ? `Model "${error.model}"` : "The requested model"} isn't available on ${connectionName ? `connection "${connectionName}"` : "the configured connection"}${usedBy}. Pick a different model or connection in ${fixPath}.`;
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { beforeEach, describe, expect, mock, test } from "bun:test";
|
|
1
|
+
import { afterEach, beforeEach, describe, expect, mock, test } from "bun:test";
|
|
2
2
|
|
|
3
3
|
let connectionsByName: Record<string, unknown> = {};
|
|
4
4
|
let secureKeys: Record<string, string | undefined> = {};
|
|
@@ -74,6 +74,11 @@ beforeEach(() => {
|
|
|
74
74
|
secureKeys = { "credential/anthropic/api_key": "sk-ant" };
|
|
75
75
|
cesUnreachable = false;
|
|
76
76
|
platformLoggedIn = false;
|
|
77
|
+
delete process.env.IS_PLATFORM;
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
afterEach(() => {
|
|
81
|
+
delete process.env.IS_PLATFORM;
|
|
77
82
|
});
|
|
78
83
|
|
|
79
84
|
describe("preflightResolvedConfig", () => {
|
|
@@ -300,3 +305,55 @@ describe("preflightResolvedConfig", () => {
|
|
|
300
305
|
expect(await preflightError(resolved())).toBeUndefined();
|
|
301
306
|
});
|
|
302
307
|
});
|
|
308
|
+
|
|
309
|
+
describe("preflightResolvedConfig platform-managed credential messaging", () => {
|
|
310
|
+
const platformManaged = () => {
|
|
311
|
+
connectionsByName = {
|
|
312
|
+
vellum: {
|
|
313
|
+
name: "vellum",
|
|
314
|
+
provider: "vellum",
|
|
315
|
+
auth: { type: "platform" },
|
|
316
|
+
},
|
|
317
|
+
};
|
|
318
|
+
process.env.IS_PLATFORM = "true";
|
|
319
|
+
};
|
|
320
|
+
const managedRoute = () => resolved({ provider_connection: "vellum" });
|
|
321
|
+
|
|
322
|
+
test("store unreachable surfaces a retriable message with no login text", async () => {
|
|
323
|
+
platformManaged();
|
|
324
|
+
cesUnreachable = true;
|
|
325
|
+
const err = await preflightError(managedRoute());
|
|
326
|
+
expect(err?.reason).toBe("platform_unauthenticated");
|
|
327
|
+
expect(err?.message).toContain("temporarily unavailable");
|
|
328
|
+
expect(err?.message).not.toContain("log in");
|
|
329
|
+
});
|
|
330
|
+
|
|
331
|
+
test("credential absent asks for platform re-provisioning, not a login", async () => {
|
|
332
|
+
platformManaged();
|
|
333
|
+
// Logged out with a reachable store means the credential is genuinely absent.
|
|
334
|
+
const err = await preflightError(managedRoute());
|
|
335
|
+
expect(err?.reason).toBe("platform_unauthenticated");
|
|
336
|
+
expect(err?.message).toContain("must be re-provisioned");
|
|
337
|
+
expect(err?.message).not.toContain("log in");
|
|
338
|
+
});
|
|
339
|
+
|
|
340
|
+
test("a healthy platform login passes silently", async () => {
|
|
341
|
+
platformManaged();
|
|
342
|
+
platformLoggedIn = true;
|
|
343
|
+
expect(await preflightError(managedRoute())).toBeUndefined();
|
|
344
|
+
});
|
|
345
|
+
|
|
346
|
+
test("non-platform install keeps the log-in-or-switch wording", async () => {
|
|
347
|
+
connectionsByName = {
|
|
348
|
+
vellum: {
|
|
349
|
+
name: "vellum",
|
|
350
|
+
provider: "vellum",
|
|
351
|
+
auth: { type: "platform" },
|
|
352
|
+
},
|
|
353
|
+
};
|
|
354
|
+
// IS_PLATFORM left unset covers both local and self-hosted Docker installs.
|
|
355
|
+
const err = await preflightError(managedRoute());
|
|
356
|
+
expect(err?.reason).toBe("platform_unauthenticated");
|
|
357
|
+
expect(err?.message).toContain("log in or pick a different provider");
|
|
358
|
+
});
|
|
359
|
+
});
|
|
@@ -27,6 +27,7 @@
|
|
|
27
27
|
* a conversation offline.
|
|
28
28
|
*/
|
|
29
29
|
|
|
30
|
+
import { getIsPlatform } from "../config/env-registry.js";
|
|
30
31
|
import {
|
|
31
32
|
resolveCallSiteConfig,
|
|
32
33
|
selectWinningProfile,
|
|
@@ -468,7 +469,40 @@ export async function preflightResolvedConfig(
|
|
|
468
469
|
errorOptions,
|
|
469
470
|
);
|
|
470
471
|
}
|
|
471
|
-
|
|
472
|
+
const presence = await platformLoginPresence();
|
|
473
|
+
if (presence === "ok") {
|
|
474
|
+
return;
|
|
475
|
+
}
|
|
476
|
+
if (getIsPlatform()) {
|
|
477
|
+
// A platform-managed assistant cannot present a login screen or switch
|
|
478
|
+
// providers, so the login-or-switch wording never fits. An unreachable
|
|
479
|
+
// store is transient and recovers on its own, so it reads as a retry. A
|
|
480
|
+
// reachable but empty store can only be fixed by re-provisioning the
|
|
481
|
+
// credential platform-side.
|
|
482
|
+
if (presence === "indeterminate") {
|
|
483
|
+
throw new ConnectionResolutionError(
|
|
484
|
+
connectionName,
|
|
485
|
+
"platform_unauthenticated",
|
|
486
|
+
"The assistant's platform credentials are temporarily unavailable; retrying automatically.",
|
|
487
|
+
errorOptions,
|
|
488
|
+
);
|
|
489
|
+
}
|
|
490
|
+
log.error(
|
|
491
|
+
{
|
|
492
|
+
connectionName,
|
|
493
|
+
model: resolved.model,
|
|
494
|
+
profileName: attribution.profileName,
|
|
495
|
+
},
|
|
496
|
+
"Managed platform credential is missing and must be re-provisioned",
|
|
497
|
+
);
|
|
498
|
+
throw new ConnectionResolutionError(
|
|
499
|
+
connectionName,
|
|
500
|
+
"platform_unauthenticated",
|
|
501
|
+
"This assistant's platform credential is missing and must be re-provisioned on the Vellum platform.",
|
|
502
|
+
errorOptions,
|
|
503
|
+
);
|
|
504
|
+
}
|
|
505
|
+
if (presence === "unauthenticated") {
|
|
472
506
|
throw new ConnectionResolutionError(
|
|
473
507
|
connectionName,
|
|
474
508
|
"platform_unauthenticated",
|
|
@@ -108,7 +108,7 @@ async function queueApiKeyPropagation(
|
|
|
108
108
|
getPlatformAssistantId() || undefined,
|
|
109
109
|
);
|
|
110
110
|
log.info(
|
|
111
|
-
"
|
|
111
|
+
"Notified CES of the queued assistant API key after handshake completed (ack only; CES reads the durable value from the credential store)",
|
|
112
112
|
);
|
|
113
113
|
} catch (err) {
|
|
114
114
|
log.warn(
|
|
@@ -124,6 +124,44 @@ async function queueApiKeyPropagation(
|
|
|
124
124
|
);
|
|
125
125
|
}
|
|
126
126
|
|
|
127
|
+
/**
|
|
128
|
+
* Notify CES that the assistant API key changed. This is advisory: the key is
|
|
129
|
+
* durably stored in the credential vault and CES reads it from there. When no
|
|
130
|
+
* CES client is connected the notification is skipped, but the skip is logged
|
|
131
|
+
* rather than dropped silently. Exported for direct testing.
|
|
132
|
+
*/
|
|
133
|
+
export async function notifyCesOfAssistantApiKeyUpdate(
|
|
134
|
+
value: string,
|
|
135
|
+
cesClient: CesClient | undefined,
|
|
136
|
+
logger: Pick<ReturnType<typeof getLogger>, "info" | "warn"> = log,
|
|
137
|
+
): Promise<void> {
|
|
138
|
+
const generation = ++apiKeyGeneration;
|
|
139
|
+
if (!cesClient) {
|
|
140
|
+
logger.warn(
|
|
141
|
+
"Assistant API key updated but no CES client is connected; skipping the CES key-update notification (advisory only; CES reads the durable value from the credential store)",
|
|
142
|
+
);
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
if (cesClient.isReady()) {
|
|
146
|
+
try {
|
|
147
|
+
await cesClient.updateAssistantApiKey(
|
|
148
|
+
value,
|
|
149
|
+
getPlatformAssistantId() || undefined,
|
|
150
|
+
);
|
|
151
|
+
logger.info(
|
|
152
|
+
"Notified CES of the updated assistant API key (ack only; CES reads the durable value from the credential store)",
|
|
153
|
+
);
|
|
154
|
+
} catch (err) {
|
|
155
|
+
logger.warn(
|
|
156
|
+
{ error: err instanceof Error ? err.message : String(err) },
|
|
157
|
+
"Failed to notify CES of the assistant API key update (non-fatal)",
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
return;
|
|
161
|
+
}
|
|
162
|
+
void queueApiKeyPropagation(cesClient, value, generation);
|
|
163
|
+
}
|
|
164
|
+
|
|
127
165
|
// ---------------------------------------------------------------------------
|
|
128
166
|
// Provider refresh after secret changes
|
|
129
167
|
// ---------------------------------------------------------------------------
|
|
@@ -381,28 +419,7 @@ async function handleAddSecret({ body }: RouteHandlerArgs) {
|
|
|
381
419
|
// pages unseeded until restart. Detached — must not block the response.
|
|
382
420
|
void maybeReseedCapabilitiesAfterManagedCredential(getConfig());
|
|
383
421
|
if (service === "vellum" && field === "assistant_api_key") {
|
|
384
|
-
|
|
385
|
-
const cesClient = getCesClient();
|
|
386
|
-
if (cesClient) {
|
|
387
|
-
if (cesClient.isReady()) {
|
|
388
|
-
try {
|
|
389
|
-
await cesClient.updateAssistantApiKey(
|
|
390
|
-
value,
|
|
391
|
-
getPlatformAssistantId() || undefined,
|
|
392
|
-
);
|
|
393
|
-
log.info(
|
|
394
|
-
"Pushed assistant API key to CES after managed proxy credential update",
|
|
395
|
-
);
|
|
396
|
-
} catch (err) {
|
|
397
|
-
log.warn(
|
|
398
|
-
{ error: err instanceof Error ? err.message : String(err) },
|
|
399
|
-
"Failed to push assistant API key to CES (non-fatal)",
|
|
400
|
-
);
|
|
401
|
-
}
|
|
402
|
-
} else {
|
|
403
|
-
void queueApiKeyPropagation(cesClient, value, generation);
|
|
404
|
-
}
|
|
405
|
-
}
|
|
422
|
+
await notifyCesOfAssistantApiKeyUpdate(value, getCesClient());
|
|
406
423
|
}
|
|
407
424
|
}
|
|
408
425
|
if (
|
|
@@ -4,7 +4,8 @@
|
|
|
4
4
|
* is in use.
|
|
5
5
|
*/
|
|
6
6
|
|
|
7
|
-
import {
|
|
7
|
+
import { getIsContainerized } from "../config/env-registry.js";
|
|
8
|
+
import { deleteKey, listKeys, readKey, setKey } from "./encrypted-store.js";
|
|
8
9
|
|
|
9
10
|
// ---------------------------------------------------------------------------
|
|
10
11
|
// Types
|
|
@@ -62,14 +63,21 @@ class EncryptedStoreBackend implements CredentialBackend {
|
|
|
62
63
|
readonly name = "encrypted-store";
|
|
63
64
|
|
|
64
65
|
isAvailable(): boolean {
|
|
65
|
-
|
|
66
|
+
// The local encrypted store is the legitimate backend only outside
|
|
67
|
+
// containers. On a pod `keys.enc` does not exist and CES owns credentials,
|
|
68
|
+
// so reporting available here would pin reads to a store that can never
|
|
69
|
+
// hold the key. Returning false lets the resolver recover to CES.
|
|
70
|
+
return !getIsContainerized();
|
|
66
71
|
}
|
|
67
72
|
|
|
68
73
|
async get(account: string): Promise<CredentialGetResult> {
|
|
69
74
|
try {
|
|
70
|
-
return { value:
|
|
75
|
+
return { value: readKey(account), unreachable: false };
|
|
71
76
|
} catch {
|
|
72
|
-
|
|
77
|
+
// A thrown error means the store exists but could not be read or
|
|
78
|
+
// decrypted, so it is unavailable, not absent. A genuinely-missing store
|
|
79
|
+
// or key returns undefined from readKey without throwing.
|
|
80
|
+
return { value: undefined, unreachable: true };
|
|
73
81
|
}
|
|
74
82
|
}
|
|
75
83
|
|
|
@@ -98,6 +106,41 @@ class EncryptedStoreBackend implements CredentialBackend {
|
|
|
98
106
|
}
|
|
99
107
|
}
|
|
100
108
|
|
|
109
|
+
// ---------------------------------------------------------------------------
|
|
110
|
+
// UnavailableBackend
|
|
111
|
+
// ---------------------------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* A backend whose every operation reports the store as unreachable. Used on
|
|
115
|
+
* containerized pods when no CES path is ready: resolving to the (nonexistent)
|
|
116
|
+
* local encrypted store would report provisioned credentials as absent, but
|
|
117
|
+
* the correct state is "temporarily unreachable" (presence indeterminate),
|
|
118
|
+
* which callers retry rather than treating as a missing key.
|
|
119
|
+
*/
|
|
120
|
+
class UnavailableBackend implements CredentialBackend {
|
|
121
|
+
readonly name = "unavailable";
|
|
122
|
+
|
|
123
|
+
isAvailable(): boolean {
|
|
124
|
+
return false;
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
async get(): Promise<CredentialGetResult> {
|
|
128
|
+
return { value: undefined, unreachable: true };
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
async set(): Promise<boolean> {
|
|
132
|
+
return false;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
async delete(): Promise<DeleteResult> {
|
|
136
|
+
return "error";
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async list(): Promise<CredentialListResult> {
|
|
140
|
+
return { accounts: [], unreachable: true };
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
101
144
|
// ---------------------------------------------------------------------------
|
|
102
145
|
// Factory functions
|
|
103
146
|
// ---------------------------------------------------------------------------
|
|
@@ -105,3 +148,7 @@ class EncryptedStoreBackend implements CredentialBackend {
|
|
|
105
148
|
export function createEncryptedStoreBackend(): EncryptedStoreBackend {
|
|
106
149
|
return new EncryptedStoreBackend();
|
|
107
150
|
}
|
|
151
|
+
|
|
152
|
+
export function createUnavailableBackend(): UnavailableBackend {
|
|
153
|
+
return new UnavailableBackend();
|
|
154
|
+
}
|
|
@@ -418,58 +418,80 @@ function decrypt(entry: EncryptedEntry, key: Buffer): string {
|
|
|
418
418
|
return decrypted.toString("utf-8");
|
|
419
419
|
}
|
|
420
420
|
|
|
421
|
+
// ---------------------------------------------------------------------------
|
|
422
|
+
// Errors
|
|
423
|
+
// ---------------------------------------------------------------------------
|
|
424
|
+
|
|
425
|
+
/**
|
|
426
|
+
* Thrown by `readKey()` when the store is UNAVAILABLE: the store file exists
|
|
427
|
+
* but cannot be read, the key material is missing, or an entry fails to
|
|
428
|
+
* decrypt. Distinct from a credential being ABSENT (no store file, or the
|
|
429
|
+
* store reads cleanly but lacks the requested key), which returns `undefined`.
|
|
430
|
+
*
|
|
431
|
+
* Mirrors the CES-side `StoreUnavailableError` so managed pods and local mode
|
|
432
|
+
* report the same absent-vs-unavailable distinction: an unavailable store must
|
|
433
|
+
* surface as `unreachable` (presence indeterminate), never as "no key".
|
|
434
|
+
*/
|
|
435
|
+
export class StoreUnavailableError extends Error {
|
|
436
|
+
constructor(message: string) {
|
|
437
|
+
super(message);
|
|
438
|
+
this.name = "StoreUnavailableError";
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
|
|
421
442
|
// ---------------------------------------------------------------------------
|
|
422
443
|
// Public API
|
|
423
444
|
// ---------------------------------------------------------------------------
|
|
424
445
|
|
|
425
446
|
/**
|
|
426
|
-
* Retrieve a secret
|
|
427
|
-
*
|
|
447
|
+
* Retrieve a secret, distinguishing a genuinely-absent credential (returns
|
|
448
|
+
* `undefined`) from a store that exists but cannot be read or decrypted
|
|
449
|
+
* (throws {@link StoreUnavailableError}).
|
|
428
450
|
*/
|
|
429
|
-
export function
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
451
|
+
export function readKey(account: string): string | undefined {
|
|
452
|
+
// `readStore()` returns null for a missing store (or a corrupt store it has
|
|
453
|
+
// just reset) and throws on transient I/O errors (EACCES, EMFILE, EIO). A
|
|
454
|
+
// missing store is genuinely absent; a transient read error is unavailable
|
|
455
|
+
// and propagates.
|
|
456
|
+
const store = readStore();
|
|
457
|
+
if (!store) {
|
|
458
|
+
return undefined;
|
|
459
|
+
}
|
|
435
460
|
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
if (!entry) {
|
|
443
|
-
return undefined;
|
|
444
|
-
}
|
|
445
|
-
const key = getKeyForStore(migrated);
|
|
446
|
-
if (!key) {
|
|
447
|
-
return undefined;
|
|
448
|
-
}
|
|
449
|
-
return decrypt(entry, key);
|
|
450
|
-
}
|
|
451
|
-
// Migration failed -- try reading with legacy key
|
|
452
|
-
const entry = store.entries[account];
|
|
453
|
-
if (!entry) {
|
|
454
|
-
return undefined;
|
|
455
|
-
}
|
|
456
|
-
const key = getKeyForStore(store);
|
|
457
|
-
if (!key) {
|
|
458
|
-
return undefined;
|
|
459
|
-
}
|
|
460
|
-
return decrypt(entry, key);
|
|
461
|
+
let effectiveStore: StoreFile = store;
|
|
462
|
+
if (store.version === 1) {
|
|
463
|
+
const migrated = migrateV1ToV2(store);
|
|
464
|
+
if (migrated) {
|
|
465
|
+
writeStore(migrated);
|
|
466
|
+
effectiveStore = migrated;
|
|
461
467
|
}
|
|
468
|
+
// Migration failed: fall through and read with the legacy PBKDF2 key.
|
|
469
|
+
}
|
|
462
470
|
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
471
|
+
const entry = effectiveStore.entries[account];
|
|
472
|
+
if (!entry) {
|
|
473
|
+
return undefined;
|
|
474
|
+
}
|
|
467
475
|
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
476
|
+
const key = getKeyForStore(effectiveStore);
|
|
477
|
+
if (!key) {
|
|
478
|
+
throw new StoreUnavailableError(
|
|
479
|
+
"credential store entry present but key material is unavailable",
|
|
480
|
+
);
|
|
481
|
+
}
|
|
482
|
+
// A decrypt failure means corrupt data or wrong key material, so the credential
|
|
483
|
+
// exists but its value cannot be produced. Let it propagate as unavailable
|
|
484
|
+
// rather than swallowing it into a false "not found".
|
|
485
|
+
return decrypt(entry, key);
|
|
486
|
+
}
|
|
487
|
+
|
|
488
|
+
/**
|
|
489
|
+
* Retrieve a secret from the encrypted store.
|
|
490
|
+
* Returns `undefined` if the key doesn't exist or decryption fails.
|
|
491
|
+
*/
|
|
492
|
+
export function getKey(account: string): string | undefined {
|
|
493
|
+
try {
|
|
494
|
+
return readKey(account);
|
|
473
495
|
} catch (err) {
|
|
474
496
|
log.debug({ err, account }, "Failed to read from encrypted store");
|
|
475
497
|
return undefined;
|
|
@@ -50,7 +50,10 @@ import type {
|
|
|
50
50
|
CredentialListResult,
|
|
51
51
|
DeleteResult,
|
|
52
52
|
} from "./credential-backend.js";
|
|
53
|
-
import {
|
|
53
|
+
import {
|
|
54
|
+
createEncryptedStoreBackend,
|
|
55
|
+
createUnavailableBackend,
|
|
56
|
+
} from "./credential-backend.js";
|
|
54
57
|
import { credentialKey } from "./credential-key.js";
|
|
55
58
|
|
|
56
59
|
export type {
|
|
@@ -148,6 +151,10 @@ export function setCesClient(client: CesClient | undefined): void {
|
|
|
148
151
|
_resolvedBackend = undefined;
|
|
149
152
|
_resolvePromise = undefined;
|
|
150
153
|
_cesHttpUnreachable = false;
|
|
154
|
+
log.info(
|
|
155
|
+
{ hasClient: !!client },
|
|
156
|
+
"CES client updated; resetting resolved credential backend cache",
|
|
157
|
+
);
|
|
151
158
|
_cesClientListener?.(client);
|
|
152
159
|
}
|
|
153
160
|
|
|
@@ -238,6 +245,21 @@ async function resolveBackendAsync(
|
|
|
238
245
|
// backends.
|
|
239
246
|
return _resolvedBackend;
|
|
240
247
|
}
|
|
248
|
+
} else if (
|
|
249
|
+
_resolvedBackend.name === "encrypted-store" &&
|
|
250
|
+
_cesClient?.isReady()
|
|
251
|
+
) {
|
|
252
|
+
// A CES client became ready after we fell back to the encrypted store
|
|
253
|
+
// (e.g. a worker/CLI whose handshake completed post-fallback, or a
|
|
254
|
+
// client injected before its handshake finished). No reconnect callback
|
|
255
|
+
// is needed, since the live client already exists. Drop the fallback and
|
|
256
|
+
// re-resolve so the CES RPC path wins.
|
|
257
|
+
log.info(
|
|
258
|
+
"CES client is ready; upgrading credential backend from encrypted store to CES RPC",
|
|
259
|
+
);
|
|
260
|
+
_resolvedBackend = undefined;
|
|
261
|
+
_resolvePromise = undefined;
|
|
262
|
+
// Fall through to re-resolve.
|
|
241
263
|
} else if (
|
|
242
264
|
_cesReconnect &&
|
|
243
265
|
(_resolvedBackend.name === "encrypted-store" ||
|
|
@@ -264,7 +286,16 @@ async function resolveBackendAsync(
|
|
|
264
286
|
}
|
|
265
287
|
}
|
|
266
288
|
if (!_resolvePromise) {
|
|
267
|
-
_resolvePromise = doResolveBackend()
|
|
289
|
+
_resolvePromise = doResolveBackend().then((backend) => {
|
|
290
|
+
// The containerized guard returns an unreachable backend without caching
|
|
291
|
+
// it (`_resolvedBackend` stays unset). Clear the memoized promise so the
|
|
292
|
+
// next credential op re-attempts CES rather than pinning the unreachable
|
|
293
|
+
// result to a pod whose CES may since have come up.
|
|
294
|
+
if (backend.name === "unavailable") {
|
|
295
|
+
_resolvePromise = undefined;
|
|
296
|
+
}
|
|
297
|
+
return backend;
|
|
298
|
+
});
|
|
268
299
|
}
|
|
269
300
|
return _resolvePromise;
|
|
270
301
|
}
|
|
@@ -456,10 +487,11 @@ async function doResolveBackend(): Promise<CredentialBackend> {
|
|
|
456
487
|
const cesRpc = new CesRpcCredentialBackend(_cesClient);
|
|
457
488
|
if (cesRpc.isAvailable()) {
|
|
458
489
|
_resolvedBackend = cesRpc;
|
|
490
|
+
log.info("Resolved credential backend: ces-rpc");
|
|
459
491
|
return cesRpc;
|
|
460
492
|
}
|
|
461
493
|
log.warn(
|
|
462
|
-
"CES RPC client is set but not ready
|
|
494
|
+
"CES RPC client is set but not ready; trying the next credential backend",
|
|
463
495
|
);
|
|
464
496
|
}
|
|
465
497
|
|
|
@@ -468,11 +500,12 @@ async function doResolveBackend(): Promise<CredentialBackend> {
|
|
|
468
500
|
const ces = createCesCredentialBackend();
|
|
469
501
|
if (ces.isAvailable()) {
|
|
470
502
|
_resolvedBackend = ces;
|
|
503
|
+
log.info("Resolved credential backend: ces-http");
|
|
471
504
|
return ces;
|
|
472
505
|
}
|
|
473
506
|
log.warn(
|
|
474
|
-
"CES_CREDENTIAL_URL is set but CES backend is not available
|
|
475
|
-
"
|
|
507
|
+
"CES_CREDENTIAL_URL is set but CES backend is not available; " +
|
|
508
|
+
"trying the next credential backend",
|
|
476
509
|
);
|
|
477
510
|
}
|
|
478
511
|
|
|
@@ -488,14 +521,29 @@ async function doResolveBackend(): Promise<CredentialBackend> {
|
|
|
488
521
|
const cesRpc = new CesRpcCredentialBackend(lazyClient);
|
|
489
522
|
if (cesRpc.isAvailable()) {
|
|
490
523
|
_resolvedBackend = cesRpc;
|
|
524
|
+
log.info("Resolved credential backend: ces-rpc (lazy connect)");
|
|
491
525
|
return cesRpc;
|
|
492
526
|
}
|
|
493
527
|
}
|
|
494
|
-
// Lazy connect failed
|
|
528
|
+
// Lazy connect failed; fall through.
|
|
529
|
+
}
|
|
530
|
+
|
|
531
|
+
// 3. On a containerized pod the local encrypted store does not exist and CES
|
|
532
|
+
// owns credentials. Never resolve to the encrypted store here; it would
|
|
533
|
+
// report a provisioned credential as absent. Return an unreachable backend
|
|
534
|
+
// (presence indeterminate, which callers retry) WITHOUT caching it, so the
|
|
535
|
+
// next credential op re-attempts CES rather than pinning the dead answer.
|
|
536
|
+
if (getIsContainerized()) {
|
|
537
|
+
log.error(
|
|
538
|
+
"No CES credential backend is ready on a containerized pod; reporting credentials as unreachable (indeterminate). The local encrypted store is not used in managed mode.",
|
|
539
|
+
);
|
|
540
|
+
return createUnavailableBackend();
|
|
495
541
|
}
|
|
496
542
|
|
|
497
|
-
//
|
|
543
|
+
// 4. Encrypted file store: the legitimate backend for local / self-hosted
|
|
544
|
+
// mode when CES is unavailable.
|
|
498
545
|
_resolvedBackend = getEncryptedStoreBackend();
|
|
546
|
+
log.info("Resolved credential backend: encrypted-store (local mode)");
|
|
499
547
|
return _resolvedBackend;
|
|
500
548
|
}
|
|
501
549
|
|