dsh-keys-manager 0.0.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +203 -16
- package/README.zh-CN.md +125 -0
- package/bundle/cordis.patch.yml +5 -0
- package/icon.svg +12 -0
- package/lib/client/format.d.ts +8 -0
- package/lib/client/index.d.ts +3 -0
- package/lib/client/locales.d.ts +4 -0
- package/lib/client/runtime.d.ts +21 -0
- package/lib/client/store.d.ts +4 -0
- package/lib/client/styles.d.ts +2 -0
- package/lib/client/types.d.ts +130 -0
- package/lib/client/ui.d.ts +18 -0
- package/lib/client.js +2106 -0
- package/lib/core.d.ts +115 -0
- package/lib/core.js +282 -0
- package/lib/core.js.map +1 -0
- package/lib/domain.d.ts +246 -0
- package/lib/domain.js +9 -0
- package/lib/domain.js.map +1 -0
- package/lib/host-types.d.ts +240 -0
- package/lib/host-types.js +2 -0
- package/lib/host-types.js.map +1 -0
- package/lib/index.d.ts +860 -0
- package/lib/index.js +4157 -0
- package/lib/index.js.map +1 -0
- package/lib/providers.d.ts +28 -0
- package/lib/providers.js +1308 -0
- package/lib/providers.js.map +1 -0
- package/lib/store.d.ts +62 -0
- package/lib/store.js +432 -0
- package/lib/store.js.map +1 -0
- package/locale/en.json +25 -0
- package/locale/zh.json +25 -0
- package/package.json +79 -14
- package/scripts/build-client.mjs +53 -0
- package/scripts/prepare-release.mjs +243 -0
- package/scripts/verify-dist.mjs +30 -0
- package/index.js +0 -30
package/lib/index.js
ADDED
|
@@ -0,0 +1,4157 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Keys Manager Host half.
|
|
3
|
+
*
|
|
4
|
+
* Responsibilities:
|
|
5
|
+
* - Per-provider API-key CRUD, with key literals in the Harness credential
|
|
6
|
+
* store and secret-free metadata in `<DSH_HOME>/keys-manager/state.json`.
|
|
7
|
+
* - Extend an existing provider's `apiKeyEnv` to a broker reference and
|
|
8
|
+
* answer that reference per request, so the original pi-ai / DeepSeek
|
|
9
|
+
* adapters keep working without patching their transports.
|
|
10
|
+
* - Sticky per-session round-robin selection, bounded retry, key failover,
|
|
11
|
+
* and exponential cooldown recovery.
|
|
12
|
+
* - Install searchable extra built-in providers as normal pi-ai routes.
|
|
13
|
+
* - A small Typert Remote namespace that the Web settings page calls through
|
|
14
|
+
* `ctx.connection.rpc`.
|
|
15
|
+
*/
|
|
16
|
+
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
17
|
+
import { createHash, randomBytes } from 'node:crypto';
|
|
18
|
+
import { KEY_LIMIT, accountRef, applyFailureCooldown, bindingKey, classifyOAuthFailure, isSelectable, markKeySuccess, markKeyUsed, normalizeSettings, patchSettings, pickHealthyExcluding, pickRoundRobin, shouldFailover, statusOfKey, statusOfProvider, validateKeyInput, validateKeyPatch, } from './core.js';
|
|
19
|
+
import { EXTRA_PROVIDER_MAP, EXTRA_PROVIDERS } from './providers.js';
|
|
20
|
+
import { KeysManagerStore, ensureStateFile, statePath } from './store.js';
|
|
21
|
+
export const name = 'keys-manager';
|
|
22
|
+
export const inject = ['llm', 'credentials', 'typert'];
|
|
23
|
+
export const provide = ['keysManager'];
|
|
24
|
+
const SERVICE = 'keysManager';
|
|
25
|
+
const NAMESPACE = 'keysManager';
|
|
26
|
+
const BROKER_PREFIX = 'KEYSMGR_';
|
|
27
|
+
const ORIGINAL = Symbol.for('cordis.original');
|
|
28
|
+
const MAX_SWITCHES = 6;
|
|
29
|
+
const ATTEMPT_TTL_MS = 60 * 60 * 1000;
|
|
30
|
+
/** A retry pin only lives for this long; an expired pin is treated as absent. */
|
|
31
|
+
const PIN_TTL_MS = 60_000;
|
|
32
|
+
/**
|
|
33
|
+
* An interactive sign-in that has not settled within this window is failed
|
|
34
|
+
* and withdrawn: real flows finish in minutes, and a wedged attempt would
|
|
35
|
+
* otherwise hold the provider's single authorization slot forever.
|
|
36
|
+
*/
|
|
37
|
+
const LOGIN_ATTEMPT_TTL_MS = 10 * 60 * 1000;
|
|
38
|
+
/**
|
|
39
|
+
* Same shape of bound the api-key attempt cache uses (pruneAttempts): the
|
|
40
|
+
* settled-login cache is capped, evicting the oldest settled entries, so a
|
|
41
|
+
* rapid begin/fail loop cannot accumulate unbounded views within the TTL.
|
|
42
|
+
*/
|
|
43
|
+
const MAX_LOGIN_ATTEMPTS = 128;
|
|
44
|
+
/** Same display-name cap the api-key aliases use (core.ts MAX_ALIAS_LENGTH). */
|
|
45
|
+
const MAX_ALIAS = 80;
|
|
46
|
+
/**
|
|
47
|
+
* How long a pending projection-removal marker may absorb its deletion
|
|
48
|
+
* echo. Bounded well above any realistic echo delay (the seam's own
|
|
49
|
+
* watcher debounce dwarfs it): the old microtask sweep raced the echo
|
|
50
|
+
* handler's awaited re-read and could clear the marker too early; the
|
|
51
|
+
* race came from unbounded immediacy, not slow echoes.
|
|
52
|
+
*/
|
|
53
|
+
const INTENTIONAL_REMOVAL_TTL_MS = 5_000;
|
|
54
|
+
/** The credential-record scope pi-ai stores its per-provider grants under (llm-pi-ai/src/auth.ts:29). */
|
|
55
|
+
export const OAUTH_GRANT_RECORD_SCOPE = 'llm-pi-ai';
|
|
56
|
+
/**
|
|
57
|
+
* Upper bound on one stream's wait for projection-ownership hydration
|
|
58
|
+
* (round-9 should-fix). Hydration reads one `readRecord` per OAuth provider
|
|
59
|
+
* serially and the seam offers no timeout or cancellation, so a single
|
|
60
|
+
* stalled read can keep that promise pending indefinitely. The wait is
|
|
61
|
+
* therefore bounded: 500 ms is generous for a healthy local credential-store
|
|
62
|
+
* read while keeping a stalled read from stalling the request (and every
|
|
63
|
+
* managed provider behind it) for longer than half a second. The hydration
|
|
64
|
+
* itself is never cancelled — a late result still lands through the
|
|
65
|
+
* generation guard in {@link KeysManagerService.hydrateProjectionOwnership}.
|
|
66
|
+
*/
|
|
67
|
+
const HYDRATION_WAIT_MS = 500;
|
|
68
|
+
/**
|
|
69
|
+
* Stable failure code for a harvest whose state flush rejected AND whose
|
|
70
|
+
* credential-side rollback then failed to write.
|
|
71
|
+
*
|
|
72
|
+
* The state flush is the durability boundary `run` enforces, and the rollback
|
|
73
|
+
* that follows it deletes/restores the credential records the rolled-back row
|
|
74
|
+
* referenced. When those compensating writes fail too, the attempt's reason
|
|
75
|
+
* starts with `<code>: ` and keeps the original flush error verbatim after it,
|
|
76
|
+
* so an upstream caller can branch on {@link isFlushRollbackCompensationFailure}
|
|
77
|
+
* instead of matching prose. A flush failure whose rollback completed keeps its
|
|
78
|
+
* original error unchanged — this code is the only added spelling.
|
|
79
|
+
*/
|
|
80
|
+
export const FLUSH_ROLLBACK_COMPENSATION_FAILED = 'FLUSH_ROLLBACK_COMPENSATION_FAILED';
|
|
81
|
+
/** Whether an attempt error carries {@link FLUSH_ROLLBACK_COMPENSATION_FAILED}. */
|
|
82
|
+
export function isFlushRollbackCompensationFailure(error) {
|
|
83
|
+
return typeof error === 'string' && error.startsWith(`${FLUSH_ROLLBACK_COMPENSATION_FAILED}:`);
|
|
84
|
+
}
|
|
85
|
+
/** Whether an opaque grant payload is the OAuth credential pi-ai owns. */
|
|
86
|
+
function isOAuthGrantPayload(value) {
|
|
87
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value))
|
|
88
|
+
return false;
|
|
89
|
+
const source = value;
|
|
90
|
+
return source.type === 'oauth'
|
|
91
|
+
&& typeof source.refresh === 'string' && source.refresh.length > 0
|
|
92
|
+
&& typeof source.access === 'string';
|
|
93
|
+
}
|
|
94
|
+
/** Copy a seam prompt into its serializable view (the AbortSignal stays out). */
|
|
95
|
+
function promptView(prompt) {
|
|
96
|
+
const options = Array.isArray(prompt.options)
|
|
97
|
+
? prompt.options
|
|
98
|
+
.filter((entry) => entry !== null && typeof entry === 'object'
|
|
99
|
+
&& typeof entry.id === 'string'
|
|
100
|
+
&& typeof entry.label === 'string')
|
|
101
|
+
: undefined;
|
|
102
|
+
const kind = prompt.kind === 'secret' || prompt.kind === 'select' ? prompt.kind : 'text';
|
|
103
|
+
return {
|
|
104
|
+
kind,
|
|
105
|
+
message: prompt.message,
|
|
106
|
+
...(prompt.placeholder === undefined ? {} : { placeholder: prompt.placeholder }),
|
|
107
|
+
...(options === undefined || options.length === 0 ? {} : { options }),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
/** Copy a seam notice into its serializable view. */
|
|
111
|
+
function noticeView(notice) {
|
|
112
|
+
return {
|
|
113
|
+
message: notice.message,
|
|
114
|
+
...(notice.url === undefined ? {} : { url: notice.url }),
|
|
115
|
+
...(notice.code === undefined ? {} : { code: notice.code }),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
/** Strip the host-side handles from one attempt. */
|
|
119
|
+
function attemptView(attempt) {
|
|
120
|
+
return {
|
|
121
|
+
id: attempt.id,
|
|
122
|
+
provider: attempt.provider,
|
|
123
|
+
...(attempt.alias === undefined ? {} : { alias: attempt.alias }),
|
|
124
|
+
status: attempt.status,
|
|
125
|
+
notices: attempt.notices.map(notice => ({ ...notice })),
|
|
126
|
+
...(attempt.prompt === undefined ? {} : { prompt: { ...attempt.prompt, ...(attempt.prompt.options === undefined ? {} : { options: attempt.prompt.options.map(option => ({ ...option })) }) } }),
|
|
127
|
+
...(attempt.error === undefined ? {} : { error: attempt.error }),
|
|
128
|
+
...(attempt.accountId === undefined ? {} : { accountId: attempt.accountId }),
|
|
129
|
+
createdAt: attempt.createdAt,
|
|
130
|
+
updatedAt: attempt.updatedAt,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
function endpoint(method, parameters = []) {
|
|
134
|
+
return {
|
|
135
|
+
id: `dsh-keys-manager#${NAMESPACE}/${method}`,
|
|
136
|
+
service: SERVICE,
|
|
137
|
+
namespace: NAMESPACE,
|
|
138
|
+
method,
|
|
139
|
+
invocation: { kind: 'direct' },
|
|
140
|
+
parameters: parameters.map(field => ({
|
|
141
|
+
name: field,
|
|
142
|
+
wire: field,
|
|
143
|
+
source: 'json',
|
|
144
|
+
codec: { mode: 'src-json' },
|
|
145
|
+
})),
|
|
146
|
+
result: { mode: 'src-json' },
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
const INVOCATIONS = [
|
|
150
|
+
endpoint('getState'),
|
|
151
|
+
endpoint('adoptProvider', ['provider']),
|
|
152
|
+
endpoint('addKey', ['provider', 'alias', 'value']),
|
|
153
|
+
endpoint('updateKey', ['provider', 'keyId', 'patch']),
|
|
154
|
+
endpoint('deleteKey', ['provider', 'keyId']),
|
|
155
|
+
endpoint('releaseProvider', ['provider']),
|
|
156
|
+
endpoint('addExtraProvider', ['extraId', 'alias', 'value']),
|
|
157
|
+
endpoint('setSettings', ['patch']),
|
|
158
|
+
// OAuth account management (T2 service methods). The login dialog polls
|
|
159
|
+
// accountAttempt for attempt-state transitions: the host forwards only a
|
|
160
|
+
// fixed credentials-event allowlist to the browser, so the plugin has no
|
|
161
|
+
// own event channel a client could subscribe to.
|
|
162
|
+
// targetAccountId (round-3 finding 6): the row a re-authentication
|
|
163
|
+
// replaces, so a metadata-poor rotation re-binds in place instead of
|
|
164
|
+
// duplicating the account.
|
|
165
|
+
endpoint('beginAccountLogin', ['provider', 'alias', 'targetAccountId']),
|
|
166
|
+
endpoint('answerAccountPrompt', ['attemptId', 'value']),
|
|
167
|
+
endpoint('accountAttempt', ['attemptId']),
|
|
168
|
+
endpoint('listAccountAttempts'),
|
|
169
|
+
endpoint('activateAccount', ['provider', 'accountId']),
|
|
170
|
+
endpoint('removeAccount', ['provider', 'accountId']),
|
|
171
|
+
];
|
|
172
|
+
/** Read a nested value by path. */
|
|
173
|
+
function getPath(value, path) {
|
|
174
|
+
let current = value;
|
|
175
|
+
for (const part of path) {
|
|
176
|
+
if (current === null || typeof current !== 'object')
|
|
177
|
+
return undefined;
|
|
178
|
+
current = current[part];
|
|
179
|
+
}
|
|
180
|
+
return current;
|
|
181
|
+
}
|
|
182
|
+
function asProfile(value) {
|
|
183
|
+
return value !== null && typeof value === 'object' ? value : undefined;
|
|
184
|
+
}
|
|
185
|
+
/** Stable broker credential reference for one provider route. */
|
|
186
|
+
function brokerRefFor(provider) {
|
|
187
|
+
const stem = provider.toUpperCase().replace(/[^A-Z0-9]+/g, '_').replace(/^_+|_+$/g, '').slice(0, 34);
|
|
188
|
+
const hash = createHash('sha256').update(provider).digest('hex').slice(0, 6).toUpperCase();
|
|
189
|
+
return `${BROKER_PREFIX}${stem || 'PROVIDER'}_${hash}`;
|
|
190
|
+
}
|
|
191
|
+
/** Reference the Models page would have derived for a provider route. */
|
|
192
|
+
function derivedKeyRef(provider) {
|
|
193
|
+
const stem = provider.toUpperCase().replace(/[^A-Z0-9]+/g, '_').replace(/^_+|_+$/g, '');
|
|
194
|
+
return `${stem || 'PROVIDER'}_API_KEY`;
|
|
195
|
+
}
|
|
196
|
+
/** One fresh key credential reference beneath a provider broker. */
|
|
197
|
+
function newKeyRef(provider) {
|
|
198
|
+
return `${brokerRefFor(provider)}_K_${randomBytes(6).toString('hex').toUpperCase()}`;
|
|
199
|
+
}
|
|
200
|
+
/** One fresh non-secret key id. */
|
|
201
|
+
function newKeyId() {
|
|
202
|
+
return `k_${randomBytes(6).toString('hex')}`;
|
|
203
|
+
}
|
|
204
|
+
/** Validate a credential-reference grammar. */
|
|
205
|
+
function isCredentialRefName(value) {
|
|
206
|
+
return typeof value === 'string' && /^[A-Za-z_][A-Za-z0-9_]*$/.test(value);
|
|
207
|
+
}
|
|
208
|
+
/** Normalize one model profile row for the pi-ai route. */
|
|
209
|
+
function normalizeModel(model) {
|
|
210
|
+
if (model === null || typeof model !== 'object')
|
|
211
|
+
return undefined;
|
|
212
|
+
const source = model;
|
|
213
|
+
if (typeof source.id !== 'string' || source.id.length === 0)
|
|
214
|
+
return undefined;
|
|
215
|
+
const next = { id: source.id };
|
|
216
|
+
if (typeof source.name === 'string' && source.name.length > 0)
|
|
217
|
+
next.name = source.name;
|
|
218
|
+
if (typeof source.contextWindow === 'number' && source.contextWindow > 0)
|
|
219
|
+
next.contextWindow = Math.round(source.contextWindow);
|
|
220
|
+
if (typeof source.maxTokens === 'number' && source.maxTokens > 0)
|
|
221
|
+
next.maxTokens = Math.round(source.maxTokens);
|
|
222
|
+
const input = Array.isArray(source.input)
|
|
223
|
+
? source.input
|
|
224
|
+
: Array.isArray(source.inputModalities)
|
|
225
|
+
? source.inputModalities
|
|
226
|
+
: [];
|
|
227
|
+
const modalities = input.filter((value) => value === 'text' || value === 'image');
|
|
228
|
+
if (modalities.length > 0)
|
|
229
|
+
next.input = [...new Set(modalities)];
|
|
230
|
+
if (source.reasoningEfforts !== null && typeof source.reasoningEfforts === 'object') {
|
|
231
|
+
next.reasoningEfforts = source.reasoningEfforts;
|
|
232
|
+
}
|
|
233
|
+
if (source.compat !== null && typeof source.compat === 'object') {
|
|
234
|
+
next.compat = source.compat;
|
|
235
|
+
}
|
|
236
|
+
return next;
|
|
237
|
+
}
|
|
238
|
+
/** Normalize an array of model rows, dropping unusable entries. */
|
|
239
|
+
function normalizeModels(raw) {
|
|
240
|
+
if (!Array.isArray(raw))
|
|
241
|
+
return [];
|
|
242
|
+
return raw.map(normalizeModel).filter((model) => model !== undefined);
|
|
243
|
+
}
|
|
244
|
+
/** Normalize one discovered model into the profile model shape. */
|
|
245
|
+
function discoveredModel(model) {
|
|
246
|
+
return normalizeModel({
|
|
247
|
+
id: model.id,
|
|
248
|
+
name: model.name,
|
|
249
|
+
contextWindow: model.contextWindow,
|
|
250
|
+
maxTokens: model.maxTokens,
|
|
251
|
+
inputModalities: model.inputModalities,
|
|
252
|
+
});
|
|
253
|
+
}
|
|
254
|
+
/** Public, secret-free key row. */
|
|
255
|
+
function publicKey(key, now, projectedAccountId) {
|
|
256
|
+
return {
|
|
257
|
+
id: key.id,
|
|
258
|
+
alias: key.alias,
|
|
259
|
+
status: statusOfKey(key, now),
|
|
260
|
+
enabled: key.enabled !== false,
|
|
261
|
+
createdAt: key.createdAt,
|
|
262
|
+
lastUsedAt: key.lastUsedAt,
|
|
263
|
+
cooldownUntil: key.cooldownUntil,
|
|
264
|
+
failureLevel: key.failureLevel,
|
|
265
|
+
consecutiveFailures: key.consecutiveFailures,
|
|
266
|
+
totalUses: key.totalUses,
|
|
267
|
+
totalFailures: key.totalFailures,
|
|
268
|
+
importedFrom: key.importedFrom ?? null,
|
|
269
|
+
valueMissing: key.valueMissing === true,
|
|
270
|
+
kind: key.kind ?? 'api-key',
|
|
271
|
+
// Identity facts only; the grant reference stays host-side (T1 shapes).
|
|
272
|
+
...(key.kind === 'oauth' && key.oauth !== null
|
|
273
|
+
? {
|
|
274
|
+
oauth: {
|
|
275
|
+
provider: key.oauth.provider,
|
|
276
|
+
...(key.oauth.email === undefined ? {} : { email: key.oauth.email }),
|
|
277
|
+
...(key.oauth.accountId === undefined ? {} : { accountId: key.oauth.accountId }),
|
|
278
|
+
...(key.oauth.expiresAt === undefined ? {} : { expiresAt: key.oauth.expiresAt }),
|
|
279
|
+
},
|
|
280
|
+
// Step 6: surface which account row IS the live projection.
|
|
281
|
+
...(projectedAccountId !== undefined && projectedAccountId !== null && projectedAccountId === key.id
|
|
282
|
+
? { projected: true }
|
|
283
|
+
: {}),
|
|
284
|
+
}
|
|
285
|
+
: {}),
|
|
286
|
+
};
|
|
287
|
+
}
|
|
288
|
+
/** Env names that are a single API-key credential, not an OAuth/federation token. */
|
|
289
|
+
function isKeyishEnv(name) {
|
|
290
|
+
if (!/(?:API_KEY|_TOKEN)$/.test(name))
|
|
291
|
+
return false;
|
|
292
|
+
if (/OAUTH/.test(name))
|
|
293
|
+
return false;
|
|
294
|
+
if (/AUTH_TOKEN|FEDERATION|ORGANIZATION|IDENTITY_TOKEN_FILE|SERVICE_ACCOUNT|WORKSPACE_ID/.test(name))
|
|
295
|
+
return false;
|
|
296
|
+
return true;
|
|
297
|
+
}
|
|
298
|
+
/**
|
|
299
|
+
* DSH catalog routes that are explicitly outside the single-key pool. This
|
|
300
|
+
* is a KEY-POOL verdict only (capabilityOf): an OAuth-capable route listed
|
|
301
|
+
* here stays adoptable for account management. Keeping it as a named table
|
|
302
|
+
* (rather than only deriving it from env names) makes a reviewed decision
|
|
303
|
+
* visible and stable when a provider adds another auth path.
|
|
304
|
+
*/
|
|
305
|
+
const UNSUPPORTED_KEY_POOL_PROVIDERS = Object.freeze({
|
|
306
|
+
'amazon-bedrock': 'Uses an AWS credential chain (profile, IAM keys, container/IRSA tokens), not a single API key',
|
|
307
|
+
'cloudflare-ai-gateway': 'Needs Cloudflare account and gateway IDs in addition to the API key',
|
|
308
|
+
'cloudflare-workers-ai': 'Needs a Cloudflare account ID in addition to the API key',
|
|
309
|
+
'openai-codex': 'Uses ChatGPT OAuth sign-in only; no API-key credential',
|
|
310
|
+
});
|
|
311
|
+
/**
|
|
312
|
+
* Why an OAuth-capable catalog route cannot be managed as an account here.
|
|
313
|
+
*
|
|
314
|
+
* C6-2: `nativeAuth.oauth` is a catalog fact — the provider supports OAuth
|
|
315
|
+
* upstream. Actually signing in needs a pi-ai flow registered with this
|
|
316
|
+
* Harness build's authorization seam, which extension-backed routes may not
|
|
317
|
+
* have (they are explicitly deferred from this increment). Offering "Add
|
|
318
|
+
* account" without a flow would promise a sign-in that cannot start.
|
|
319
|
+
*/
|
|
320
|
+
const NO_OAUTH_FLOW_REASON = 'This provider has no OAuth sign-in flow registered in this Harness build';
|
|
321
|
+
/**
|
|
322
|
+
* Split one route's management capability by mode. The
|
|
323
|
+
* {@link UNSUPPORTED_KEY_POOL_PROVIDERS} table keeps its meaning: it is a
|
|
324
|
+
* verdict about API keys, so an OAuth-capable route listed there remains
|
|
325
|
+
* adoptable for accounts while its key entry stays blocked.
|
|
326
|
+
*
|
|
327
|
+
* `oauthFlow` is the live answer from {@link KeysManagerService.oauthFlowAvailable}
|
|
328
|
+
* — C6-2 requires an actually-beginable flow, not the catalog flag alone.
|
|
329
|
+
*/
|
|
330
|
+
function capabilityOf(entry, profile, oauthFlow) {
|
|
331
|
+
if (entry.error !== undefined)
|
|
332
|
+
return { keys: false, accounts: false, reason: entry.error };
|
|
333
|
+
const accounts = entry.nativeAuth?.oauth === true && oauthFlow;
|
|
334
|
+
const unsupportedReason = UNSUPPORTED_KEY_POOL_PROVIDERS[entry.provider];
|
|
335
|
+
if (unsupportedReason !== undefined) {
|
|
336
|
+
// The table's reason describes the key verdict; when the only possible
|
|
337
|
+
// capability (accounts) is unavailable too, the flow gap is the truthful
|
|
338
|
+
// explanation a user needs.
|
|
339
|
+
const reason = entry.nativeAuth?.oauth === true && !oauthFlow ? NO_OAUTH_FLOW_REASON : unsupportedReason;
|
|
340
|
+
return { keys: false, accounts, reason };
|
|
341
|
+
}
|
|
342
|
+
if (entry.provider === 'deepseek-official')
|
|
343
|
+
return { keys: true, accounts };
|
|
344
|
+
if (profile !== undefined && typeof profile.apiKeyEnv === 'string' && profile.apiKeyEnv.length > 0)
|
|
345
|
+
return { keys: true, accounts };
|
|
346
|
+
const auth = entry.nativeAuth;
|
|
347
|
+
if (auth !== undefined) {
|
|
348
|
+
// A dual-auth route (key env + OAuth, e.g. anthropic) keeps BOTH
|
|
349
|
+
// capabilities; the oauth-only verdict below is for routes with no
|
|
350
|
+
// key credential at all.
|
|
351
|
+
if (auth.env.some(isKeyishEnv))
|
|
352
|
+
return { keys: true, accounts };
|
|
353
|
+
if (auth.oauth) {
|
|
354
|
+
return oauthFlow
|
|
355
|
+
? { keys: false, accounts: true, reason: 'This provider signs in with an OAuth account; no API key is used' }
|
|
356
|
+
: { keys: false, accounts: false, reason: NO_OAUTH_FLOW_REASON };
|
|
357
|
+
}
|
|
358
|
+
if (auth.env.length > 0)
|
|
359
|
+
return { keys: false, accounts: false, reason: 'This provider uses native cloud credentials, not one API key' };
|
|
360
|
+
return { keys: false, accounts: false, reason: 'This provider declares no API-key authentication' };
|
|
361
|
+
}
|
|
362
|
+
return { keys: true, accounts };
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Whether the route can be adopted for OAuth account management (MF1).
|
|
366
|
+
*
|
|
367
|
+
* C6-2: `nativeAuth.oauth` alone is not enough. Account support requires a
|
|
368
|
+
* flow the authorization seam can begin, resolved from the live flow list on
|
|
369
|
+
* every call (see {@link KeysManagerService.oauthFlowAvailable}): the seam
|
|
370
|
+
* may mount after this plugin, so availability is never cached at
|
|
371
|
+
* construction. While the flow list is empty (seam not mounted yet) support
|
|
372
|
+
* is false; the next state build re-evaluates once flows are enumerable.
|
|
373
|
+
*/
|
|
374
|
+
function accountSupportOf(entry, oauthFlow) {
|
|
375
|
+
if (entry === undefined)
|
|
376
|
+
return { supported: false, reason: 'This provider route is not registered' };
|
|
377
|
+
if (entry.error !== undefined)
|
|
378
|
+
return { supported: false, reason: entry.error };
|
|
379
|
+
if (entry.nativeAuth?.oauth !== true)
|
|
380
|
+
return { supported: false, reason: 'This provider offers no OAuth account sign-in' };
|
|
381
|
+
if (!oauthFlow)
|
|
382
|
+
return { supported: false, reason: NO_OAUTH_FLOW_REASON };
|
|
383
|
+
return { supported: true };
|
|
384
|
+
}
|
|
385
|
+
/**
|
|
386
|
+
* The account row whose grant payload matches one live projection payload:
|
|
387
|
+
* the refresh digest names the row first; on a miss (rotated token) the
|
|
388
|
+
* payload's identity metadata re-associates.
|
|
389
|
+
*
|
|
390
|
+
* The exact `accountId` lookup runs BEFORE the email fallback (round-8
|
|
391
|
+
* finding I-000107): the fallback is only for rows that never carried an
|
|
392
|
+
* accountId, and evaluating it in array order let an earlier email-only row
|
|
393
|
+
* claim a payload that names a later row's accountId — corrupting both the
|
|
394
|
+
* owner attribution and the cleanup target.
|
|
395
|
+
*/
|
|
396
|
+
function matchOwningAccount(provider, accounts, payload) {
|
|
397
|
+
const hash = createHash('sha256').update(payload.refresh).digest('hex').slice(0, 12);
|
|
398
|
+
const projectedRef = accountRef(provider, hash);
|
|
399
|
+
const byRef = accounts.find(candidate => candidate.oauth.grantRef === projectedRef);
|
|
400
|
+
if (byRef !== undefined)
|
|
401
|
+
return byRef;
|
|
402
|
+
if (payload.accountId !== undefined) {
|
|
403
|
+
const byAccountId = accounts.find(candidate => candidate.oauth.accountId === payload.accountId);
|
|
404
|
+
if (byAccountId !== undefined)
|
|
405
|
+
return byAccountId;
|
|
406
|
+
}
|
|
407
|
+
if (payload.email === undefined)
|
|
408
|
+
return undefined;
|
|
409
|
+
return accounts.find(candidate => candidate.oauth.email === payload.email && candidate.oauth.accountId === undefined);
|
|
410
|
+
}
|
|
411
|
+
export class KeysManagerService {
|
|
412
|
+
ctx;
|
|
413
|
+
store;
|
|
414
|
+
als = new AsyncLocalStorage();
|
|
415
|
+
attempts = new Map();
|
|
416
|
+
stepAttempts = new Map();
|
|
417
|
+
streamAttempts = new Map();
|
|
418
|
+
linkedAttempts = new WeakMap();
|
|
419
|
+
retryBindings = new Map();
|
|
420
|
+
requestSteps = new Map();
|
|
421
|
+
attemptSequence = 0;
|
|
422
|
+
switches = new Map();
|
|
423
|
+
warnings = [];
|
|
424
|
+
settings;
|
|
425
|
+
rawCredentials;
|
|
426
|
+
credentialOriginals = new WeakMap();
|
|
427
|
+
credentialWrappers = new WeakMap();
|
|
428
|
+
brokerOwners = new Map();
|
|
429
|
+
bindingCache = new Map();
|
|
430
|
+
lastBindingPruneAt = 0;
|
|
431
|
+
selectionQueues = new Map();
|
|
432
|
+
queue = Promise.resolve();
|
|
433
|
+
closing = false;
|
|
434
|
+
/** Interactive sign-in attempts, by attempt id. */
|
|
435
|
+
loginAttempts = new Map();
|
|
436
|
+
/** The provider → attempt id index for the single-attempt rule. */
|
|
437
|
+
loginsInFlight = new Map();
|
|
438
|
+
loginSequence = 0;
|
|
439
|
+
/** The account row whose grant payload currently IS the projection, per provider. */
|
|
440
|
+
activeAccounts = new Map();
|
|
441
|
+
/**
|
|
442
|
+
* Account rows committed to removal (C6-1/I1 removal-aware oracle), marked
|
|
443
|
+
* by {@link removeOAuthGrant} before its first awaited delete. A multi-row
|
|
444
|
+
* release deletes every row's credential first and the provider record
|
|
445
|
+
* last, and it never splices its rows, so while it runs the provider record
|
|
446
|
+
* and the row object both still look live even though the row's grant is
|
|
447
|
+
* already gone. Object identity alone therefore cannot tell "still serving"
|
|
448
|
+
* from "mid-removal"; this mark can. A successful removal leaves it set —
|
|
449
|
+
* the row it names is either spliced out (removeAccount/deleteKey) or left
|
|
450
|
+
* flagged `valueMissing` (release) — but a removal whose awaited steps
|
|
451
|
+
* THROW must clear it: the row is then still live and still managed, and a
|
|
452
|
+
* stale mark would silently skip its rotation sync until a restart. It
|
|
453
|
+
* marks the live object, so a row object recreated under the same id is
|
|
454
|
+
* unaffected.
|
|
455
|
+
*/
|
|
456
|
+
retiringAccounts = new WeakSet();
|
|
457
|
+
/**
|
|
458
|
+
* Per-provider projection generation — the write counter for
|
|
459
|
+
* {@link activeAccounts} claims (round-3 finding 3) and the attribution
|
|
460
|
+
* identity of a projected grant (round-6 finding SF1). It is bumped exactly
|
|
461
|
+
* when a grant becomes the provider's projection (a manager-side
|
|
462
|
+
* re-projection or the seam's committed grant at the settlement broadcast),
|
|
463
|
+
* never at a later bookkeeping step. A hydration result whose generation no
|
|
464
|
+
* longer matches is discarded instead of overwriting a newer projection
|
|
465
|
+
* with a stale read.
|
|
466
|
+
*/
|
|
467
|
+
ownershipGenerations = new Map();
|
|
468
|
+
/**
|
|
469
|
+
* Per-account-row projection generation in effect when the row's CURRENT
|
|
470
|
+
* cooldown was earned (round-6 finding SF1). `applyFailureCooldown` stamps
|
|
471
|
+
* it at failure time, so a harvest can tell the previous grant's window
|
|
472
|
+
* from one the freshly projected grant earned while the harvest was still
|
|
473
|
+
* queued. A row with no entry carries a window that predates any observed
|
|
474
|
+
* attribution (one loaded from disk, for example); the next projection
|
|
475
|
+
* treats it as the older generation's.
|
|
476
|
+
*/
|
|
477
|
+
cooldownGenerations = new Map();
|
|
478
|
+
/**
|
|
479
|
+
* Per-provider chain of pending grant reconciliations (round-9 finding
|
|
480
|
+
* P1): two refresh echoes for the same route can arrive before the first
|
|
481
|
+
* write-back settles, and the bodies run read → write → commit. Arrival
|
|
482
|
+
* order is preserved so the older observation can never commit after the
|
|
483
|
+
* newer one and leave the row on a stale generation.
|
|
484
|
+
*
|
|
485
|
+
* Round-8 finding I-000106: activation joins the same chain. The only
|
|
486
|
+
* writer of `row.oauth.grantRef` is {@link reconcileGrantRecord}, which runs
|
|
487
|
+
* outside the mutation queue, so activation's record read and its projection
|
|
488
|
+
* write must be mutually exclusive with that writer — otherwise a rotation
|
|
489
|
+
* landing between them leaves the projection on a generation the row no
|
|
490
|
+
* longer names.
|
|
491
|
+
*/
|
|
492
|
+
grantReconciliations = new Map();
|
|
493
|
+
/**
|
|
494
|
+
* Providers whose projection deletion this manager initiated, mapped to
|
|
495
|
+
* the moment it was initiated. The `record-updated` echo of our own
|
|
496
|
+
* demotion consumes the marker (one-shot) and stays silent while the
|
|
497
|
+
* marker is inside its TTL window; past the window the marker is expired
|
|
498
|
+
* and a removal warns as genuinely out-of-band again.
|
|
499
|
+
*/
|
|
500
|
+
intentionalProjectionRemovals = new Map();
|
|
501
|
+
/**
|
|
502
|
+
* In-flight projection demotions by provider (round-5 finding C1): the
|
|
503
|
+
* oauth failure path cools the account and demotes asynchronously, and the
|
|
504
|
+
* failover retry must not re-enter the stream while the projection still
|
|
505
|
+
* resolves — pi-ai prefers the stored grant, so the key pool would never be
|
|
506
|
+
* reached. The request-error listener awaits this entry before deciding.
|
|
507
|
+
*/
|
|
508
|
+
projectionDemotions = new Map();
|
|
509
|
+
/**
|
|
510
|
+
* MF3 load-time hydration result (round-8 finding I-000108). The
|
|
511
|
+
* constructor starts the ownership read, but a restarted process can serve
|
|
512
|
+
* a stream before it settles. Until this provider's read lands,
|
|
513
|
+
* {@link activeAccounts} may not know the owner, so `wrapStream` would
|
|
514
|
+
* select and mutate a key (cursor, `lastUsedAt`, sticky binding) even though
|
|
515
|
+
* pi-ai serves the persisted grant, and an OAuth failure would be classified
|
|
516
|
+
* as externally authenticated instead of cooling the account.
|
|
517
|
+
*
|
|
518
|
+
* The promise never rejects (hydration catches and warns per provider), but
|
|
519
|
+
* it can stay pending forever. Requests therefore never subscribe to it
|
|
520
|
+
* (round-10 should-fix SF2: a per-call `then`/`await` would be retained on a
|
|
521
|
+
* promise that never settles); they wait per provider through
|
|
522
|
+
* {@link hydratedProviders} and {@link hydrationWaiters} instead.
|
|
523
|
+
*/
|
|
524
|
+
projectionHydration = Promise.resolve();
|
|
525
|
+
/**
|
|
526
|
+
* Upper bound on one stream's wait for its provider's hydration read
|
|
527
|
+
* (round-9 should-fix). The hydration read has no timeout of its own, so
|
|
528
|
+
* without a bound one stalled `readRecord` would gate every managed stream.
|
|
529
|
+
* Injectable so tests can exercise the bound quickly.
|
|
530
|
+
*/
|
|
531
|
+
hydrationWaitMs = HYDRATION_WAIT_MS;
|
|
532
|
+
/**
|
|
533
|
+
* Providers whose hydration read has landed — owner found or not
|
|
534
|
+
* (round-10 should-fix SF1). Hydration walks the providers serially and
|
|
535
|
+
* writes each owner as it goes, so one stalled read must not make an
|
|
536
|
+
* already-read provider look unknown: a provider whose own read landed is
|
|
537
|
+
* ready immediately, and the bounded wait's "unknown" fallback is confined
|
|
538
|
+
* to the requested provider's own unresolved read.
|
|
539
|
+
*/
|
|
540
|
+
hydratedProviders = new Set();
|
|
541
|
+
/**
|
|
542
|
+
* One resolver set per provider still being waited on (round-10 should-fix
|
|
543
|
+
* SF2). A wait registers exactly one entry here and removes it when the wait
|
|
544
|
+
* ends — on timeout or on the late landing — so a read that never settles
|
|
545
|
+
* cannot retain anything per request. {@link noteProviderHydrated} is the
|
|
546
|
+
* only notification channel; the shared hydration promise is never
|
|
547
|
+
* subscribed to.
|
|
548
|
+
*/
|
|
549
|
+
hydrationWaiters = new Map();
|
|
550
|
+
/**
|
|
551
|
+
* True once the load-time hydration read is over (round-10 SF1). From then
|
|
552
|
+
* on no provider's read can land, so attribution is known for every provider
|
|
553
|
+
* — including one adopted after the read, whose OAuth rows did not exist
|
|
554
|
+
* when the projection was looked for.
|
|
555
|
+
*/
|
|
556
|
+
hydrationSettled = false;
|
|
557
|
+
/** Providers whose bounded-wait warning already fired (warn once). */
|
|
558
|
+
hydrationWaitWarned = new Set();
|
|
559
|
+
/** Channels whose lost-subscription warning already fired (warn once). */
|
|
560
|
+
subscriptionLostWarnings = new Set();
|
|
561
|
+
/** Disposer for the `authorization/settled` subscription. */
|
|
562
|
+
settledDisposer;
|
|
563
|
+
/** Disposer for the `credentials/record-updated` subscription. */
|
|
564
|
+
recordUpdatedDisposer;
|
|
565
|
+
typertRemote;
|
|
566
|
+
constructor(ctx, store) {
|
|
567
|
+
this.ctx = ctx;
|
|
568
|
+
this.store = store;
|
|
569
|
+
this.typertRemote = Object.freeze({ service: this, serviceKey: SERVICE, namespace: NAMESPACE });
|
|
570
|
+
this.rebuildBrokerMap();
|
|
571
|
+
this.projectionHydration = this.hydrateProjectionOwnership();
|
|
572
|
+
this.settledDisposer = this.watchAuthorizationSettled();
|
|
573
|
+
this.recordUpdatedDisposer = this.watchGrantRecordUpdated();
|
|
574
|
+
}
|
|
575
|
+
/**
|
|
576
|
+
* MF3 load-time hydration: a projection outlives this process — pi-ai
|
|
577
|
+
* wrote `llm-pi-ai/<provider>` during a previous run and it survived in
|
|
578
|
+
* the credential store. Match each persisted payload back to an account
|
|
579
|
+
* row (the exact rule the rotation reconciler applies, via
|
|
580
|
+
* {@link matchOwningAccount}) so removeAccount / deleteManagedCredentials
|
|
581
|
+
* demote the projection and `onGrantRecordUpdated` can mark valueMissing
|
|
582
|
+
* after a restart. An orphan projection — no matching row, e.g. a foreign
|
|
583
|
+
* or stale grant — is left untouched: the manager never adopts a grant it
|
|
584
|
+
* does not own, and out-of-band updates keep surfacing the native-
|
|
585
|
+
* management warning.
|
|
586
|
+
*
|
|
587
|
+
* Returns its own completion (round-8 finding I-000108), and the returned
|
|
588
|
+
* promise is already caught, so a failed hydration degrades to "no owner"
|
|
589
|
+
* instead of wedging every request. Request attribution does not subscribe
|
|
590
|
+
* to that promise (round-10 should-fix SF2 — it may stay pending forever);
|
|
591
|
+
* each request waits on its own provider's landing instead.
|
|
592
|
+
*/
|
|
593
|
+
hydrateProjectionOwnership() {
|
|
594
|
+
return (async () => {
|
|
595
|
+
try {
|
|
596
|
+
const records = this.credentialRecords();
|
|
597
|
+
if (records === undefined) {
|
|
598
|
+
this.warnSubscriptionLost('credentials record access');
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
for (const providerRecord of Object.values(this.store.providers)) {
|
|
602
|
+
const provider = providerRecord.provider;
|
|
603
|
+
const accounts = providerRecord.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
604
|
+
if (accounts.length === 0)
|
|
605
|
+
continue;
|
|
606
|
+
try {
|
|
607
|
+
const grantKey = `${OAUTH_GRANT_RECORD_SCOPE}/${provider}`;
|
|
608
|
+
// Generation tag (round-3 finding 3): the read below is awaited, and
|
|
609
|
+
// an activation or login landing while it is in flight must win —
|
|
610
|
+
// the hydrated write only lands while the generation still matches.
|
|
611
|
+
const generation = this.ownershipGenerations.get(provider) ?? 0;
|
|
612
|
+
let record;
|
|
613
|
+
try {
|
|
614
|
+
record = await records.readRecord(grantKey);
|
|
615
|
+
}
|
|
616
|
+
catch (error) {
|
|
617
|
+
this.warn(`projection ownership hydration could not read ${grantKey}: ${error instanceof Error ? error.message : String(error)}`);
|
|
618
|
+
continue;
|
|
619
|
+
}
|
|
620
|
+
if (record === undefined || record.kind !== 'grant' || !isOAuthGrantPayload(record.payload))
|
|
621
|
+
continue;
|
|
622
|
+
const owner = matchOwningAccount(provider, accounts, record.payload);
|
|
623
|
+
// A newer activation/login claimed the owner while this read was in
|
|
624
|
+
// flight: the stale read is discarded, never written over it.
|
|
625
|
+
if ((this.ownershipGenerations.get(provider) ?? 0) !== generation)
|
|
626
|
+
continue;
|
|
627
|
+
// A concurrent removal may have dropped the row while this read was
|
|
628
|
+
// in flight: never resurrect an owner entry for a row that is gone.
|
|
629
|
+
if (owner !== undefined && providerRecord.keys.some(candidate => candidate.id === owner.id)) {
|
|
630
|
+
this.activeAccounts.set(provider, owner.id);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
finally {
|
|
634
|
+
// This provider's own read is over (owner or not): its requests are
|
|
635
|
+
// attributable now and must not wait behind another provider's
|
|
636
|
+
// stalled read (round-10 should-fix SF1).
|
|
637
|
+
this.noteProviderHydrated(provider);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
finally {
|
|
642
|
+
// The read is over (completed or abandoned): from now on no result can
|
|
643
|
+
// land, so every provider is ready — including one whose OAuth rows
|
|
644
|
+
// appear later, which this read could not have projected.
|
|
645
|
+
this.hydrationSettled = true;
|
|
646
|
+
this.settleHydrationWaits();
|
|
647
|
+
}
|
|
648
|
+
})().catch(error => {
|
|
649
|
+
this.warn(`projection ownership hydration failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
/**
|
|
653
|
+
* Record that one provider's hydration read is over — owner or not (SF1) —
|
|
654
|
+
* and wake the requests waiting on that provider alone (SF2).
|
|
655
|
+
*/
|
|
656
|
+
noteProviderHydrated(provider) {
|
|
657
|
+
this.hydratedProviders.add(provider);
|
|
658
|
+
const waiters = this.hydrationWaiters.get(provider);
|
|
659
|
+
if (waiters === undefined)
|
|
660
|
+
return;
|
|
661
|
+
this.hydrationWaiters.delete(provider);
|
|
662
|
+
for (const wake of [...waiters])
|
|
663
|
+
wake();
|
|
664
|
+
}
|
|
665
|
+
/** Release every provider's readiness wait when hydration cannot proceed. */
|
|
666
|
+
settleHydrationWaits() {
|
|
667
|
+
for (const provider of Object.keys(this.store.providers))
|
|
668
|
+
this.noteProviderHydrated(provider);
|
|
669
|
+
}
|
|
670
|
+
/**
|
|
671
|
+
* Wait for one provider's own hydration read to land, at most `ms`, and
|
|
672
|
+
* report whether it landed.
|
|
673
|
+
*
|
|
674
|
+
* Registers exactly one waiter entry for this wait and removes it when the
|
|
675
|
+
* wait ends either way (round-10 should-fix SF2): nothing is attached to the
|
|
676
|
+
* shared hydration promise, so a read that never settles cannot retain one
|
|
677
|
+
* reaction per request. A late landing still wakes the wait through
|
|
678
|
+
* {@link noteProviderHydrated}, and the generation guard in
|
|
679
|
+
* {@link hydrateProjectionOwnership} still decides whether its result lands
|
|
680
|
+
* (I-000108).
|
|
681
|
+
*/
|
|
682
|
+
waitForProviderHydration(provider, ms) {
|
|
683
|
+
if (this.hydrationSettled || this.hydratedProviders.has(provider))
|
|
684
|
+
return Promise.resolve(true);
|
|
685
|
+
return new Promise(resolve => {
|
|
686
|
+
let waiters = this.hydrationWaiters.get(provider);
|
|
687
|
+
if (waiters === undefined) {
|
|
688
|
+
waiters = new Set();
|
|
689
|
+
this.hydrationWaiters.set(provider, waiters);
|
|
690
|
+
}
|
|
691
|
+
const live = waiters;
|
|
692
|
+
let timer;
|
|
693
|
+
const wake = (landed) => {
|
|
694
|
+
if (timer !== undefined) {
|
|
695
|
+
clearTimeout(timer);
|
|
696
|
+
timer = undefined;
|
|
697
|
+
}
|
|
698
|
+
live.delete(onLanded);
|
|
699
|
+
if (live.size === 0 && this.hydrationWaiters.get(provider) === live)
|
|
700
|
+
this.hydrationWaiters.delete(provider);
|
|
701
|
+
resolve(landed);
|
|
702
|
+
};
|
|
703
|
+
const onLanded = () => {
|
|
704
|
+
wake(true);
|
|
705
|
+
};
|
|
706
|
+
timer = setTimeout(() => {
|
|
707
|
+
wake(false);
|
|
708
|
+
}, ms);
|
|
709
|
+
live.add(onLanded);
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Live per-provider waiter entries. A documented internal counter for the
|
|
714
|
+
* round-10 SF2 invariant: a wait that ended must leave no entry behind, so
|
|
715
|
+
* this never grows with the number of timed-out requests.
|
|
716
|
+
*/
|
|
717
|
+
hydrationWaiterCount() {
|
|
718
|
+
let total = 0;
|
|
719
|
+
for (const waiters of this.hydrationWaiters.values())
|
|
720
|
+
total += waiters.size;
|
|
721
|
+
return total;
|
|
722
|
+
}
|
|
723
|
+
/**
|
|
724
|
+
* Wait (bounded) until projection-ownership hydration can attribute this
|
|
725
|
+
* request, and report whether it did.
|
|
726
|
+
*
|
|
727
|
+
* A provider without OAuth rows can never own a projection, so it never
|
|
728
|
+
* waits — a stalled read for some other provider must not delay it
|
|
729
|
+
* (round-9 should-fix: this used to gate every managed provider, API-key-
|
|
730
|
+
* only ones included).
|
|
731
|
+
*
|
|
732
|
+
* Returns false when the bound elapsed first. The caller then treats
|
|
733
|
+
* attribution as unknown (no owner) and keeps key state untouched: a
|
|
734
|
+
* projection may still be serving, and selecting/mutating a key here would
|
|
735
|
+
* repeat the I-000108 corruption. The hydration promise is not cancelled,
|
|
736
|
+
* so a late result still lands under the generation guard and later
|
|
737
|
+
* requests are attributed normally.
|
|
738
|
+
*
|
|
739
|
+
* Readiness is decided PER PROVIDER (round-10 should-fix SF1). Hydration
|
|
740
|
+
* writes owners progressively, so racing the one global promise would
|
|
741
|
+
* discard an owner this provider already confirmed just because another
|
|
742
|
+
* provider's read stalled. A provider whose own read has landed — with an
|
|
743
|
+
* owner or with none — is ready without waiting, and the "unknown" fallback
|
|
744
|
+
* is confined to the requested provider's own unresolved read.
|
|
745
|
+
*/
|
|
746
|
+
async awaitProjectionOwnership(providerId) {
|
|
747
|
+
const provider = this.store.provider(providerId);
|
|
748
|
+
if (provider === undefined)
|
|
749
|
+
return true;
|
|
750
|
+
const hasOauthRows = provider.keys.some(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
751
|
+
if (!hasOauthRows)
|
|
752
|
+
return true;
|
|
753
|
+
// This provider's own read already landed (or a newer claim already owns
|
|
754
|
+
// it, or the whole read is over): nothing can arrive later, so no wait.
|
|
755
|
+
if (this.hydrationSettled || this.hydratedProviders.has(providerId) || this.activeAccounts.has(providerId))
|
|
756
|
+
return true;
|
|
757
|
+
const landed = await this.waitForProviderHydration(providerId, this.hydrationWaitMs);
|
|
758
|
+
if (landed)
|
|
759
|
+
return true;
|
|
760
|
+
if (!this.hydrationWaitWarned.has(providerId)) {
|
|
761
|
+
this.hydrationWaitWarned.add(providerId);
|
|
762
|
+
this.warn(`projection ownership hydration did not settle within ${this.hydrationWaitMs}ms for ${providerId}; requests proceed without an attributed account until it does`);
|
|
763
|
+
}
|
|
764
|
+
return false;
|
|
765
|
+
}
|
|
766
|
+
/**
|
|
767
|
+
* Record that a newly committed grant became the provider's projection and
|
|
768
|
+
* return that projection's generation (round-6 finding SF1).
|
|
769
|
+
*
|
|
770
|
+
* The bump happens at the moment the projection is observable — pi-ai's
|
|
771
|
+
* committed grant at the settlement broadcast, or a manager-side
|
|
772
|
+
* re-projection (activation, rotation reconcile, harvest claim) — so every
|
|
773
|
+
* failure stamped afterwards belongs to that projection. It must never be
|
|
774
|
+
* deferred to a later bookkeeping step: the harvest waits behind the
|
|
775
|
+
* mutation queue and the projection read while the new grant already
|
|
776
|
+
* serves, and a failure landing there must not be mistaken for the
|
|
777
|
+
* previous grant's.
|
|
778
|
+
*/
|
|
779
|
+
noteProjectionCommitted(provider) {
|
|
780
|
+
const generation = (this.ownershipGenerations.get(provider) ?? 0) + 1;
|
|
781
|
+
this.ownershipGenerations.set(provider, generation);
|
|
782
|
+
return generation;
|
|
783
|
+
}
|
|
784
|
+
/**
|
|
785
|
+
* Claim the projection owner for one account row, bumping the provider's
|
|
786
|
+
* ownership generation so an in-flight hydration read of an older state
|
|
787
|
+
* is discarded instead of overwriting the claim (round-3 finding 3).
|
|
788
|
+
*/
|
|
789
|
+
claimProjectionOwner(provider, accountId) {
|
|
790
|
+
this.activeAccounts.set(provider, accountId);
|
|
791
|
+
this.noteProjectionCommitted(provider);
|
|
792
|
+
}
|
|
793
|
+
/**
|
|
794
|
+
* Log without ever echoing secret material. */
|
|
795
|
+
warn(message) {
|
|
796
|
+
try {
|
|
797
|
+
this.ctx.logger.warn(`keys-manager: ${message}`);
|
|
798
|
+
}
|
|
799
|
+
catch {
|
|
800
|
+
// Logging must never break request handling.
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
/** Rebuild the broker-reference → provider index. */
|
|
804
|
+
rebuildBrokerMap() {
|
|
805
|
+
this.bindingCache.clear();
|
|
806
|
+
this.brokerOwners = new Map();
|
|
807
|
+
for (const provider of Object.values(this.store.providers)) {
|
|
808
|
+
if (provider.brokerRef.length > 0)
|
|
809
|
+
this.brokerOwners.set(provider.brokerRef, provider.provider);
|
|
810
|
+
}
|
|
811
|
+
for (const [key, pin] of this.retryBindings) {
|
|
812
|
+
const providerId = key.split('\u0000')[1];
|
|
813
|
+
const provider = providerId === undefined ? undefined : this.store.provider(providerId);
|
|
814
|
+
if (provider === undefined || !provider.keys.some(candidate => candidate.id === pin.keyId)) {
|
|
815
|
+
this.retryBindings.delete(key);
|
|
816
|
+
}
|
|
817
|
+
}
|
|
818
|
+
}
|
|
819
|
+
/** Serialize mutations, rejecting submissions that arrive during shutdown. */
|
|
820
|
+
run(operation) {
|
|
821
|
+
if (this.closing)
|
|
822
|
+
return Promise.reject(new Error('Keys Manager is shutting down'));
|
|
823
|
+
const guarded = async () => {
|
|
824
|
+
if (this.store.readOnly) {
|
|
825
|
+
throw new Error('The state file could not be read; refusing to write over it. Restart after fixing file access.');
|
|
826
|
+
}
|
|
827
|
+
const result = await operation();
|
|
828
|
+
// A mutation is not successful until its state document is durable.
|
|
829
|
+
await this.store.flush();
|
|
830
|
+
return result;
|
|
831
|
+
};
|
|
832
|
+
const next = this.queue.then(guarded, guarded);
|
|
833
|
+
this.queue = next.catch(() => { });
|
|
834
|
+
return next;
|
|
835
|
+
}
|
|
836
|
+
/**
|
|
837
|
+
* Periodically drop expired transient state: retry pins after `PIN_TTL_MS`,
|
|
838
|
+
* the remaining attempt/step records after `ATTEMPT_TTL_MS`.
|
|
839
|
+
*/
|
|
840
|
+
pruneTransient(now = Date.now()) {
|
|
841
|
+
this.pruneAttempts(now);
|
|
842
|
+
this.pruneLoginAttempts(now);
|
|
843
|
+
}
|
|
844
|
+
/** Stop accepting mutations, drain the queue, then persist the final snapshot. */
|
|
845
|
+
async close() {
|
|
846
|
+
this.closing = true;
|
|
847
|
+
this.settledDisposer?.();
|
|
848
|
+
this.settledDisposer = undefined;
|
|
849
|
+
this.recordUpdatedDisposer?.();
|
|
850
|
+
this.recordUpdatedDisposer = undefined;
|
|
851
|
+
for (const attempt of this.loginAttempts.values()) {
|
|
852
|
+
attempt.controller.abort();
|
|
853
|
+
this.dropPrompt(attempt, new Error('Keys Manager is shutting down'));
|
|
854
|
+
}
|
|
855
|
+
try {
|
|
856
|
+
await this.queue;
|
|
857
|
+
}
|
|
858
|
+
catch {
|
|
859
|
+
// The mutation queue already reports failures to its caller.
|
|
860
|
+
}
|
|
861
|
+
// Round-9 P1: the per-provider reconciliation chains are independent of
|
|
862
|
+
// the mutation queue, so drain them before the store closes — a queued
|
|
863
|
+
// echo must not write a grant record into a manager that is shutting down.
|
|
864
|
+
await Promise.allSettled([...this.grantReconciliations.values()]);
|
|
865
|
+
await this.store.close();
|
|
866
|
+
}
|
|
867
|
+
/** Push a user-facing warning that rides the next state response. */
|
|
868
|
+
pushWarning(message) {
|
|
869
|
+
this.warnings.push(message);
|
|
870
|
+
if (this.warnings.length > 4)
|
|
871
|
+
this.warnings.shift();
|
|
872
|
+
}
|
|
873
|
+
takeWarnings() {
|
|
874
|
+
const warnings = this.warnings;
|
|
875
|
+
this.warnings = [];
|
|
876
|
+
return warnings;
|
|
877
|
+
}
|
|
878
|
+
/** Current settings service, if the composition mounted one. */
|
|
879
|
+
settingsService() {
|
|
880
|
+
if (this.settings !== undefined)
|
|
881
|
+
return this.settings;
|
|
882
|
+
return this.ctx.get('settings');
|
|
883
|
+
}
|
|
884
|
+
/** Current raw credential provider. */
|
|
885
|
+
credentialsRaw() {
|
|
886
|
+
const current = this.ctx.get('credentials');
|
|
887
|
+
if (current === undefined)
|
|
888
|
+
return undefined;
|
|
889
|
+
return current[ORIGINAL] ?? current;
|
|
890
|
+
}
|
|
891
|
+
/**
|
|
892
|
+
* The credential store's record half, reached through the raw provider:
|
|
893
|
+
* the broker wrappers only ever replace the reference methods, so the
|
|
894
|
+
* record methods of the mounted credential provider are always intact on
|
|
895
|
+
* the raw instance. `llm-pi-ai` writes its grants through exactly these
|
|
896
|
+
* methods, so the manager's activation projection must use them too.
|
|
897
|
+
*/
|
|
898
|
+
credentialRecords() {
|
|
899
|
+
const raw = this.credentialsRaw();
|
|
900
|
+
if (raw === undefined)
|
|
901
|
+
return undefined;
|
|
902
|
+
if (typeof raw.readRecord !== 'function' || typeof raw.modifyRecord !== 'function')
|
|
903
|
+
return undefined;
|
|
904
|
+
return raw;
|
|
905
|
+
}
|
|
906
|
+
/**
|
|
907
|
+
* Resolve the OAuth authorization flow one provider offers.
|
|
908
|
+
*
|
|
909
|
+
* The flow key format is pinned by `llm-pi-ai/login.ts:149-171` +
|
|
910
|
+
* `auth.ts:29,36-38`: `recordKeyFor(providerId)` = `llm-pi-ai/<providerId>`,
|
|
911
|
+
* with the oauth method offered first (`login.ts:40-47`). A provider with
|
|
912
|
+
* no such flow (not OAuth-capable, or not installed) resolves to
|
|
913
|
+
* undefined, which callers must treat as "sign-in unavailable".
|
|
914
|
+
*
|
|
915
|
+
* Availability is read from the live seam on every call, never cached:
|
|
916
|
+
* the authorization service may mount after this plugin, so a route whose
|
|
917
|
+
* flow is not enumerable yet stays account-unsupported until the flow list
|
|
918
|
+
* exists (the client re-reads state after that ordering settles).
|
|
919
|
+
*/
|
|
920
|
+
oauthFlowFor(provider) {
|
|
921
|
+
const flows = this.ctx.authorization?.list() ?? [];
|
|
922
|
+
const flow = flows.find(entry => entry.key === `llm-pi-ai/${provider}`);
|
|
923
|
+
if (flow === undefined)
|
|
924
|
+
return undefined;
|
|
925
|
+
if (!flow.methods.some(method => method.id === 'oauth'))
|
|
926
|
+
return undefined;
|
|
927
|
+
return { key: flow.key, method: 'oauth' };
|
|
928
|
+
}
|
|
929
|
+
/**
|
|
930
|
+
* C6-2: can this route's OAuth sign-in actually begin right now? Account
|
|
931
|
+
* capability is gated on this instead of the catalog's `nativeAuth.oauth`
|
|
932
|
+
* flag alone, so a route without a registered pi-ai flow never offers an
|
|
933
|
+
* Add account that would fail. The check is lazy — see {@link oauthFlowFor}.
|
|
934
|
+
*/
|
|
935
|
+
oauthFlowAvailable(provider) {
|
|
936
|
+
return this.oauthFlowFor(provider) !== undefined;
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Warn once per channel: a lost subscription silently disables a whole
|
|
940
|
+
* feature (harvest never runs; rotations never sync), so the degradation
|
|
941
|
+
* must be observable instead of indistinguishable from success.
|
|
942
|
+
*/
|
|
943
|
+
warnSubscriptionLost(channel, error) {
|
|
944
|
+
if (this.subscriptionLostWarnings.has(channel))
|
|
945
|
+
return;
|
|
946
|
+
this.subscriptionLostWarnings.add(channel);
|
|
947
|
+
const detail = error === undefined ? '' : ` (${error instanceof Error ? error.message : String(error)})`;
|
|
948
|
+
this.warn(`the ${channel} subscription is unavailable${detail}; OAuth sign-in sync is degraded`);
|
|
949
|
+
}
|
|
950
|
+
/**
|
|
951
|
+
* Subscribe to the seam's settled broadcast (authorization/src/index.ts:57,
|
|
952
|
+
* :337-351). A settled attempt for a key this service is driving harvests
|
|
953
|
+
* the committed grant; everything else is ignored — another surface's
|
|
954
|
+
* sign-in is that surface's business, and the grant it leaves behind is
|
|
955
|
+
* reconciled through `credentials/record-updated` instead.
|
|
956
|
+
*/
|
|
957
|
+
watchAuthorizationSettled() {
|
|
958
|
+
const on = this.ctx.on;
|
|
959
|
+
if (typeof on !== 'function') {
|
|
960
|
+
this.warnSubscriptionLost('authorization/settled');
|
|
961
|
+
return undefined;
|
|
962
|
+
}
|
|
963
|
+
try {
|
|
964
|
+
return on.call(this.ctx, 'authorization/settled', (key, settlement) => {
|
|
965
|
+
if (typeof key !== 'string')
|
|
966
|
+
return;
|
|
967
|
+
this.onAuthorizationSettled(key, typeof settlement === 'string' ? settlement : '');
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
catch (error) {
|
|
971
|
+
this.warnSubscriptionLost('authorization/settled', error);
|
|
972
|
+
return undefined;
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
/** One settled broadcast: finalize the matching attempt, harvest on success. */
|
|
976
|
+
onAuthorizationSettled(key, settlement) {
|
|
977
|
+
if (!key.startsWith(`${OAUTH_GRANT_RECORD_SCOPE}/`))
|
|
978
|
+
return;
|
|
979
|
+
const provider = key.slice(OAUTH_GRANT_RECORD_SCOPE.length + 1);
|
|
980
|
+
const attemptId = this.loginsInFlight.get(provider);
|
|
981
|
+
if (attemptId === undefined)
|
|
982
|
+
return;
|
|
983
|
+
const attempt = this.loginAttempts.get(attemptId);
|
|
984
|
+
if (attempt === undefined || attempt.settledAt !== undefined)
|
|
985
|
+
return;
|
|
986
|
+
if (settlement === 'authorized') {
|
|
987
|
+
// F2 provenance: capture the projection payload at the settlement
|
|
988
|
+
// broadcast — the point where the seam reports THIS attempt's commit —
|
|
989
|
+
// instead of reading whatever is live when the queued harvest finally
|
|
990
|
+
// runs. The capture starts here, outside the mutation queue, so a
|
|
991
|
+
// release parked ahead of the harvest cannot let a foreign writer
|
|
992
|
+
// become this attempt's provenance.
|
|
993
|
+
//
|
|
994
|
+
// R6-SF1: the committed grant is already the provider's projection at
|
|
995
|
+
// this broadcast, so this is also the generation boundary for cooldown
|
|
996
|
+
// attribution. Everything the harvest does afterwards (queue and
|
|
997
|
+
// projection-read waits included) runs against this generation.
|
|
998
|
+
const projectionGeneration = this.noteProjectionCommitted(provider);
|
|
999
|
+
void this.harvestLogin(attempt, this.captureSettledProjection(attempt), projectionGeneration);
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
if (settlement === 'cancelled')
|
|
1003
|
+
this.finishLoginAttempt(attempt, 'cancelled');
|
|
1004
|
+
else
|
|
1005
|
+
this.failLoginAttempt(attempt, new Error(`the sign-in attempt failed (${settlement || 'unknown outcome'})`));
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Start one interactive OAuth sign-in. Returns immediately with the
|
|
1009
|
+
* attempt view; the attempt walks `pending → (prompt ⇄ pending) →
|
|
1010
|
+
* ready | failed | cancelled`, and every transition is observable through
|
|
1011
|
+
* {@link getAccountAttempt} / {@link listAccountAttempts}.
|
|
1012
|
+
*/
|
|
1013
|
+
async beginAccountLogin(provider, alias, targetAccountId) {
|
|
1014
|
+
if (typeof provider !== 'string' || provider.length === 0)
|
|
1015
|
+
throw new TypeError('provider is required');
|
|
1016
|
+
const flow = this.oauthFlowFor(provider);
|
|
1017
|
+
if (flow === undefined)
|
|
1018
|
+
throw new Error(`Provider "${provider}" offers no OAuth sign-in flow`);
|
|
1019
|
+
const providerRecord = this.store.provider(provider);
|
|
1020
|
+
if (providerRecord === undefined)
|
|
1021
|
+
throw new Error(`Provider "${provider}" is not managed; adopt it before signing in`);
|
|
1022
|
+
// Validate the re-authentication target up front (round-3 finding 6):
|
|
1023
|
+
// the harvest binds the new grant to this row, so it must already name
|
|
1024
|
+
// a live OAuth account of this provider. It runs BEFORE the pending
|
|
1025
|
+
// resume (round-4 finding 3): a resume must never smuggle an older
|
|
1026
|
+
// attempt's target past the request's own validation, so an unknown or
|
|
1027
|
+
// foreign id is refused even while another sign-in is pending.
|
|
1028
|
+
let cleanTarget;
|
|
1029
|
+
if (typeof targetAccountId === 'string' && targetAccountId.length > 0) {
|
|
1030
|
+
const target = providerRecord.keys.find(candidate => candidate.id === targetAccountId);
|
|
1031
|
+
if (target === undefined)
|
|
1032
|
+
throw new Error(`Account "${targetAccountId}" not found on ${provider}`);
|
|
1033
|
+
if (target.kind !== 'oauth' || target.oauth === null)
|
|
1034
|
+
throw new Error(`Key "${targetAccountId}" is not an OAuth account`);
|
|
1035
|
+
cleanTarget = targetAccountId;
|
|
1036
|
+
}
|
|
1037
|
+
if (this.loginsInFlight.has(provider)) {
|
|
1038
|
+
// F3 (I-000082): a pending attempt IS the flow — reopening the dialog
|
|
1039
|
+
// resumes it (same id, same notices, same parked prompt) instead of
|
|
1040
|
+
// minting a competing second flow the seam's single-flight slot would
|
|
1041
|
+
// have to refuse. Terminal attempts never reach here: finishing an
|
|
1042
|
+
// attempt clears its provider slot, so the next begin starts fresh.
|
|
1043
|
+
const attemptId = this.loginsInFlight.get(provider);
|
|
1044
|
+
const inFlight = attemptId === undefined ? undefined : this.loginAttempts.get(attemptId);
|
|
1045
|
+
if (inFlight !== undefined && inFlight.settledAt === undefined) {
|
|
1046
|
+
// Round-4 finding 3: reuse is sound only when the resumed attempt
|
|
1047
|
+
// signs in the SAME target the caller asked for (or both name no
|
|
1048
|
+
// target). Silently resuming another row's flow would bind the new
|
|
1049
|
+
// grant to the older attempt's target.
|
|
1050
|
+
if (inFlight.targetAccountId === cleanTarget)
|
|
1051
|
+
return attemptView(inFlight);
|
|
1052
|
+
throw new Error(inFlight.targetAccountId === undefined
|
|
1053
|
+
? `A sign-in for ${provider} is already pending without a target account; finish or cancel it before signing in to a specific account`
|
|
1054
|
+
: `A sign-in for ${provider} is already pending for a different account; finish or cancel it before signing in to ${cleanTarget ?? 'this provider'}`);
|
|
1055
|
+
}
|
|
1056
|
+
// Stale slot (should not happen — finishing always clears it): drop it
|
|
1057
|
+
// and start a fresh flow rather than wedging the provider.
|
|
1058
|
+
this.loginsInFlight.delete(provider);
|
|
1059
|
+
}
|
|
1060
|
+
const cleanAlias = typeof alias === 'string' ? alias.trim().slice(0, MAX_ALIAS) : undefined;
|
|
1061
|
+
const now = Date.now();
|
|
1062
|
+
const attempt = {
|
|
1063
|
+
id: `la_${(++this.loginSequence).toString(36)}_${randomBytes(4).toString('hex')}`,
|
|
1064
|
+
provider,
|
|
1065
|
+
...(cleanAlias !== undefined && cleanAlias.length > 0 ? { alias: cleanAlias } : {}),
|
|
1066
|
+
...(cleanTarget !== undefined ? { targetAccountId: cleanTarget } : {}),
|
|
1067
|
+
status: 'pending',
|
|
1068
|
+
notices: [],
|
|
1069
|
+
createdAt: now,
|
|
1070
|
+
updatedAt: now,
|
|
1071
|
+
controller: new AbortController(),
|
|
1072
|
+
};
|
|
1073
|
+
this.loginAttempts.set(attempt.id, attempt);
|
|
1074
|
+
this.capLoginAttempts();
|
|
1075
|
+
this.loginsInFlight.set(provider, attempt.id);
|
|
1076
|
+
const records = this.credentialRecords();
|
|
1077
|
+
if (records === undefined) {
|
|
1078
|
+
this.loginAttempts.delete(attempt.id);
|
|
1079
|
+
this.loginsInFlight.delete(provider);
|
|
1080
|
+
throw new Error('The credential store does not support records; OAuth sign-in is unavailable');
|
|
1081
|
+
}
|
|
1082
|
+
const interaction = {
|
|
1083
|
+
notify: (notice) => {
|
|
1084
|
+
attempt.notices.push(noticeView(notice));
|
|
1085
|
+
if (attempt.notices.length > 32)
|
|
1086
|
+
attempt.notices.shift();
|
|
1087
|
+
attempt.updatedAt = Date.now();
|
|
1088
|
+
},
|
|
1089
|
+
prompt: (prompt) => new Promise((resolve, reject) => {
|
|
1090
|
+
if (attempt.settledAt !== undefined) {
|
|
1091
|
+
reject(new Error('This sign-in attempt has already ended'));
|
|
1092
|
+
return;
|
|
1093
|
+
}
|
|
1094
|
+
if (prompt.signal?.aborted === true) {
|
|
1095
|
+
reject(new Error('The sign-in prompt was withdrawn'));
|
|
1096
|
+
return;
|
|
1097
|
+
}
|
|
1098
|
+
attempt.status = 'prompt';
|
|
1099
|
+
attempt.updatedAt = Date.now();
|
|
1100
|
+
attempt.prompt = promptView(prompt);
|
|
1101
|
+
attempt.resolvePrompt = resolve;
|
|
1102
|
+
attempt.rejectPrompt = reject;
|
|
1103
|
+
// Cancellation propagates to the awaited prompt: a seam-side
|
|
1104
|
+
// withdrawal of this signal releases a flow parked here instead
|
|
1105
|
+
// of leaving it pending forever.
|
|
1106
|
+
prompt.signal?.addEventListener('abort', () => reject(new Error('The sign-in prompt was withdrawn')), { once: true });
|
|
1107
|
+
}),
|
|
1108
|
+
};
|
|
1109
|
+
const beginPromise = this.ctx.authorization.begin({
|
|
1110
|
+
key: flow.key,
|
|
1111
|
+
method: flow.method,
|
|
1112
|
+
interaction,
|
|
1113
|
+
signal: attempt.controller.signal,
|
|
1114
|
+
});
|
|
1115
|
+
void beginPromise.then(outcome => {
|
|
1116
|
+
// The settled broadcast always lands first (the seam fires it in the
|
|
1117
|
+
// attempt's finally, authorization/src/index.ts:316-322). An
|
|
1118
|
+
// authorized outcome therefore means harvestLogin owns the terminal
|
|
1119
|
+
// transition (SF1): publishing ready the moment begin resolves would
|
|
1120
|
+
// close the dialog with "Signed in." while the grant capture is
|
|
1121
|
+
// still in flight — or failing. A cancelled outcome still finalizes
|
|
1122
|
+
// attempts that no settlement reached.
|
|
1123
|
+
if (outcome.status !== 'authorized')
|
|
1124
|
+
this.finishLoginAttempt(attempt, 'cancelled');
|
|
1125
|
+
}, error => {
|
|
1126
|
+
this.failLoginAttempt(attempt, error);
|
|
1127
|
+
});
|
|
1128
|
+
return attemptView(attempt);
|
|
1129
|
+
}
|
|
1130
|
+
/** Resolve the attempt's parked prompt with the human's answer. */
|
|
1131
|
+
async answerAccountPrompt(attemptId, value) {
|
|
1132
|
+
const attempt = this.loginAttempts.get(attemptId);
|
|
1133
|
+
if (attempt === undefined)
|
|
1134
|
+
throw new Error('Unknown sign-in attempt');
|
|
1135
|
+
if (attempt.settledAt !== undefined)
|
|
1136
|
+
throw new Error('This sign-in attempt has already ended');
|
|
1137
|
+
if (attempt.status !== 'prompt' || attempt.resolvePrompt === undefined)
|
|
1138
|
+
throw new Error('This attempt has no pending prompt');
|
|
1139
|
+
if (typeof value !== 'string')
|
|
1140
|
+
throw new TypeError('An answer value is required');
|
|
1141
|
+
const resolve = attempt.resolvePrompt;
|
|
1142
|
+
this.dropPrompt(attempt);
|
|
1143
|
+
resolve(value);
|
|
1144
|
+
}
|
|
1145
|
+
/** One attempt's serializable view, or undefined when unknown. */
|
|
1146
|
+
getAccountAttempt(attemptId) {
|
|
1147
|
+
const attempt = this.loginAttempts.get(attemptId);
|
|
1148
|
+
return attempt === undefined ? undefined : attemptView(attempt);
|
|
1149
|
+
}
|
|
1150
|
+
/** Remote: poll one attempt's serializable state (T3 dialog channel). */
|
|
1151
|
+
accountAttempt(attemptId) {
|
|
1152
|
+
return this.getAccountAttempt(attemptId);
|
|
1153
|
+
}
|
|
1154
|
+
/** Every attempt, newest first, for T3 surfaces. */
|
|
1155
|
+
listAccountAttempts() {
|
|
1156
|
+
return [...this.loginAttempts.values()].sort((a, b) => b.createdAt - a.createdAt).map(attemptView);
|
|
1157
|
+
}
|
|
1158
|
+
/**
|
|
1159
|
+
* Detach the parked prompt. With a reason the flow's awaited promise is
|
|
1160
|
+
* rejected too — a parked prompt dropped un-rejected would leave the
|
|
1161
|
+
* seam's flow (and the provider's single-flight slot) wedged until its
|
|
1162
|
+
* own timeout, so every attempt-ending path passes one.
|
|
1163
|
+
*/
|
|
1164
|
+
dropPrompt(attempt, reason) {
|
|
1165
|
+
const reject = attempt.rejectPrompt;
|
|
1166
|
+
attempt.resolvePrompt = undefined;
|
|
1167
|
+
attempt.rejectPrompt = undefined;
|
|
1168
|
+
attempt.prompt = undefined;
|
|
1169
|
+
if (attempt.status === 'prompt')
|
|
1170
|
+
attempt.status = 'pending';
|
|
1171
|
+
attempt.updatedAt = Date.now();
|
|
1172
|
+
if (reject !== undefined && reason !== undefined)
|
|
1173
|
+
reject(reason);
|
|
1174
|
+
}
|
|
1175
|
+
/** Finalize an attempt with a terminal status exactly once. */
|
|
1176
|
+
finishLoginAttempt(attempt, status, error) {
|
|
1177
|
+
if (attempt.settledAt !== undefined)
|
|
1178
|
+
return;
|
|
1179
|
+
attempt.settledAt = Date.now();
|
|
1180
|
+
attempt.status = status;
|
|
1181
|
+
if (error !== undefined) {
|
|
1182
|
+
attempt.error = error instanceof Error ? error.message : String(error);
|
|
1183
|
+
}
|
|
1184
|
+
this.dropPrompt(attempt, new Error(status === 'failed'
|
|
1185
|
+
? attempt.error ?? 'The sign-in attempt failed'
|
|
1186
|
+
: 'The sign-in attempt was cancelled'));
|
|
1187
|
+
if (this.loginsInFlight.get(attempt.provider) === attempt.id)
|
|
1188
|
+
this.loginsInFlight.delete(attempt.provider);
|
|
1189
|
+
attempt.controller.abort();
|
|
1190
|
+
attempt.updatedAt = attempt.settledAt;
|
|
1191
|
+
}
|
|
1192
|
+
/** Finalize an attempt as failed with the given reason. */
|
|
1193
|
+
failLoginAttempt(attempt, error) {
|
|
1194
|
+
this.finishLoginAttempt(attempt, 'failed', error);
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Turn a committed sign-in into pool state: persist the grant pi-ai wrote
|
|
1198
|
+
* to `llm-pi-ai/<provider>` according to the settlement-time capture,
|
|
1199
|
+
* under the plugin-scope account ref, and upsert the `kind: 'oauth'` row.
|
|
1200
|
+
* Any failure fails the attempt loudly — a login whose grant could not be
|
|
1201
|
+
* captured must not look like success.
|
|
1202
|
+
*
|
|
1203
|
+
* `committed` is the F2 provenance promise started by
|
|
1204
|
+
* {@link captureSettledProjection}: the projection payload observed at the
|
|
1205
|
+
* settlement broadcast, never a later live read. The harvest persists
|
|
1206
|
+
* exactly that payload and compensation may only revert exactly that
|
|
1207
|
+
* payload, so a foreign grant cannot become this attempt's authority just
|
|
1208
|
+
* because it landed before the queued mutation ran.
|
|
1209
|
+
*
|
|
1210
|
+
* `projectionGeneration` is the generation {@link noteProjectionCommitted}
|
|
1211
|
+
* recorded at that same broadcast (R6-SF1): it names the projection whose
|
|
1212
|
+
* failures own a cooldown, so the stale-window reset below clears only a
|
|
1213
|
+
* window attributed to an OLDER projection.
|
|
1214
|
+
*/
|
|
1215
|
+
async harvestLogin(attempt, committed, projectionGeneration) {
|
|
1216
|
+
// A revoke (provider release) may have finalized this attempt before
|
|
1217
|
+
// the seam's authorized settlement arrives: never resurrect rows for a
|
|
1218
|
+
// settled login (round-2 review, step 4).
|
|
1219
|
+
if (attempt.settledAt !== undefined)
|
|
1220
|
+
return;
|
|
1221
|
+
// Records this login's grant identity for the failure-path
|
|
1222
|
+
// compensation below: `wroteRef` says the plugin-scope copy exists, and
|
|
1223
|
+
// the projection is reverted only while it still carries exactly this
|
|
1224
|
+
// login's committed payload. The row/owner facts are only needed when
|
|
1225
|
+
// the state flush rejects after the mutation resolved.
|
|
1226
|
+
let harvest;
|
|
1227
|
+
try {
|
|
1228
|
+
await this.run(async () => {
|
|
1229
|
+
const records = this.credentialRecords();
|
|
1230
|
+
if (records === undefined)
|
|
1231
|
+
throw new Error('The credential store does not support records');
|
|
1232
|
+
const grantKey = `${OAUTH_GRANT_RECORD_SCOPE}/${attempt.provider}`;
|
|
1233
|
+
const payload = await committed;
|
|
1234
|
+
if (payload === undefined) {
|
|
1235
|
+
throw new Error(`No pi-ai OAuth credential was committed to ${grantKey}`);
|
|
1236
|
+
}
|
|
1237
|
+
// The refresh token is the stable per-grant secret; its digest —
|
|
1238
|
+
// never the token — names the account's plugin-scope record.
|
|
1239
|
+
const hash = createHash('sha256').update(payload.refresh).digest('hex').slice(0, 12);
|
|
1240
|
+
const grantRef = accountRef(attempt.provider, hash);
|
|
1241
|
+
// Capture the compensation facts BEFORE the managed-state check
|
|
1242
|
+
// (round-3 finding 2): release may win the race — it settles this
|
|
1243
|
+
// attempt and removes the provider while the harvest mutation is
|
|
1244
|
+
// queued behind it — and when the check below throws, the
|
|
1245
|
+
// compensation must still hold this login's payload so the pi-ai
|
|
1246
|
+
// projection it already committed can be reverted.
|
|
1247
|
+
const facts = { payload, grantRef, wroteRef: false };
|
|
1248
|
+
harvest = facts;
|
|
1249
|
+
// Managed state and key capacity are checked BEFORE the
|
|
1250
|
+
// plugin-scope write (round-2 review, step 4): a login whose row
|
|
1251
|
+
// cannot exist must not strand the persisted grant.
|
|
1252
|
+
const providerRecord = this.store.provider(attempt.provider);
|
|
1253
|
+
if (providerRecord === undefined)
|
|
1254
|
+
throw new Error(`Provider "${attempt.provider}" is no longer managed`);
|
|
1255
|
+
// Round-9 finding I-000110: reuse the ordered ownership matcher the
|
|
1256
|
+
// rotation reconciler and hydration apply. The previous single `find`
|
|
1257
|
+
// ORing grantRef / accountId / email evaluated the email fallback in
|
|
1258
|
+
// ARRAY order, so an earlier email-only row adopted a later account's
|
|
1259
|
+
// grant when both carried the same address; the harvest then deleted
|
|
1260
|
+
// that earlier row's prior record as "superseded" — another account's
|
|
1261
|
+
// credential. Ref first, then the exact accountId, then the guarded
|
|
1262
|
+
// email fallback.
|
|
1263
|
+
const oauthRows = providerRecord.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
1264
|
+
const existing = matchOwningAccount(attempt.provider, oauthRows, payload)
|
|
1265
|
+
// Re-authentication target binding (round-3 finding 6): when the
|
|
1266
|
+
// payload matches nothing — a rotated token with no identity
|
|
1267
|
+
// metadata — the validated target row from beginAccountLogin
|
|
1268
|
+
// receives the grant instead of minting a duplicate row.
|
|
1269
|
+
?? (attempt.targetAccountId !== undefined
|
|
1270
|
+
? providerRecord.keys.find(candidate => candidate.id === attempt.targetAccountId && candidate.kind === 'oauth' && candidate.oauth !== null)
|
|
1271
|
+
: undefined);
|
|
1272
|
+
if (existing === undefined && providerRecord.keys.length >= KEY_LIMIT) {
|
|
1273
|
+
throw new Error(`A provider may have at most ${KEY_LIMIT} keys`);
|
|
1274
|
+
}
|
|
1275
|
+
// Snapshot the record this write is about to overwrite (same-ref
|
|
1276
|
+
// re-authentication): a rejected state flush rolls the write back,
|
|
1277
|
+
// and the restored row must keep the payload it referenced.
|
|
1278
|
+
if (existing !== undefined && existing.oauth !== null && existing.oauth.grantRef === grantRef) {
|
|
1279
|
+
facts.previousRef = await records.readRecord(grantRef);
|
|
1280
|
+
}
|
|
1281
|
+
// R5-SF3: the cooldown reset below clears only the window the OLD
|
|
1282
|
+
// grant earned. The seam broadcasts the committed grant as soon as
|
|
1283
|
+
// pi-ai writes it, so the freshly committed grant can already be
|
|
1284
|
+
// serving requests while this write is awaited; a failure landing in
|
|
1285
|
+
// that window belongs to the new generation and must keep cooling.
|
|
1286
|
+
//
|
|
1287
|
+
// R6-SF1: the boundary is the projection generation, not the instant
|
|
1288
|
+
// this plugin-copy write starts. That instant sits too late — the
|
|
1289
|
+
// harvest also waited behind the mutation queue and the projection
|
|
1290
|
+
// read, and a failure of the already-projected grant landing there
|
|
1291
|
+
// used to look older than the write. Attribution is stamped onto the
|
|
1292
|
+
// row's window at failure time instead: only a window belonging to a
|
|
1293
|
+
// generation older than the one projected here is the previous
|
|
1294
|
+
// grant's. Timestamps play no part, so a same-millisecond recovery no
|
|
1295
|
+
// longer stalls.
|
|
1296
|
+
//
|
|
1297
|
+
// Codex P2 (PR #17): the settled guard at the top of this method
|
|
1298
|
+
// cannot see a settlement that lands AFTER this callback was queued
|
|
1299
|
+
// but BEFORE it runs. A release queued ahead does exactly that — the
|
|
1300
|
+
// settlement broadcast arrives while the release body is still
|
|
1301
|
+
// parked, so the pre-queue check passes — and when the release then
|
|
1302
|
+
// fails its own preflight (unreadable settings) the provider stays
|
|
1303
|
+
// managed, so this callback still runs against it. The attempt the
|
|
1304
|
+
// user sees is failed while the harvest would write the grant and
|
|
1305
|
+
// the account row. Revalidate settlement immediately before the
|
|
1306
|
+
// first credential write: a settled attempt must never grow a grant
|
|
1307
|
+
// copy or a row. Nothing of this harvest is written yet, so the
|
|
1308
|
+
// abort needs no compensation, and the release already published the
|
|
1309
|
+
// terminal it owns.
|
|
1310
|
+
if (attempt.settledAt !== undefined)
|
|
1311
|
+
return;
|
|
1312
|
+
await records.modifyRecord(grantRef, () => Promise.resolve({ kind: 'grant', payload: structuredClone(payload) }));
|
|
1313
|
+
facts.wroteRef = true;
|
|
1314
|
+
const now = Date.now();
|
|
1315
|
+
// The id of the row this mutation appends, when it appends one.
|
|
1316
|
+
let addedId;
|
|
1317
|
+
if (existing !== undefined) {
|
|
1318
|
+
// Replacing the credential removes the superseded plugin-scope
|
|
1319
|
+
// record: a changed refresh token hashes to a new ref, and the
|
|
1320
|
+
// old payload (old refresh token included) must not strand as a
|
|
1321
|
+
// secret-bearing orphan. Cleanup is best-effort — the fresh grant
|
|
1322
|
+
// is already persisted, so a locked store must not fail an
|
|
1323
|
+
// otherwise successful login; the warning keeps it observable.
|
|
1324
|
+
if (existing.oauth !== null && existing.oauth.grantRef !== grantRef) {
|
|
1325
|
+
try {
|
|
1326
|
+
// Snapshot before deleting: a rejected state flush restores
|
|
1327
|
+
// this record, because the rolled-back row still references
|
|
1328
|
+
// it. Refusing the delete when the read fails keeps the
|
|
1329
|
+
// superseded record reversible instead of dangling.
|
|
1330
|
+
const superseded = await records.readRecord(existing.oauth.grantRef);
|
|
1331
|
+
if (superseded !== undefined)
|
|
1332
|
+
facts.superseded = { ref: existing.oauth.grantRef, record: superseded };
|
|
1333
|
+
await records.deleteRecord(existing.oauth.grantRef);
|
|
1334
|
+
}
|
|
1335
|
+
catch (error) {
|
|
1336
|
+
this.warn(`cleanup of the superseded grant record failed for ${attempt.provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
// Snapshot the fields this upsert replaces, so a flush rejection
|
|
1340
|
+
// can restore the row exactly as it was before the harvest.
|
|
1341
|
+
facts.rowUndo = {
|
|
1342
|
+
kind: 'updated',
|
|
1343
|
+
id: existing.id,
|
|
1344
|
+
snapshot: {
|
|
1345
|
+
oauth: existing.oauth === null ? null : structuredClone(existing.oauth),
|
|
1346
|
+
valueMissing: existing.valueMissing,
|
|
1347
|
+
cooldownUntil: existing.cooldownUntil,
|
|
1348
|
+
},
|
|
1349
|
+
};
|
|
1350
|
+
existing.oauth = {
|
|
1351
|
+
...existing.oauth,
|
|
1352
|
+
provider: attempt.provider,
|
|
1353
|
+
grantRef,
|
|
1354
|
+
...(payload.email === undefined ? {} : { email: payload.email }),
|
|
1355
|
+
...(payload.accountId === undefined ? {} : { accountId: payload.accountId }),
|
|
1356
|
+
...(typeof payload.expires === 'number' && Number.isFinite(payload.expires) ? { expiresAt: payload.expires } : {}),
|
|
1357
|
+
};
|
|
1358
|
+
// A successful re-authentication recovers the account (round-3
|
|
1359
|
+
// finding 5): the missing flag a demotion set must go, or the now
|
|
1360
|
+
// live projection keeps rendering as missing and stays out of the
|
|
1361
|
+
// healthy counts. Same reset activateAccount applies.
|
|
1362
|
+
existing.valueMissing = false;
|
|
1363
|
+
// Round-5 finding C3: a demoted-and-cooling row kept its stale
|
|
1364
|
+
// cooldown timer, so the freshly projected grant still rendered as
|
|
1365
|
+
// cooling and stayed out of the healthy counts until that timer
|
|
1366
|
+
// expired. The window ends with the new grant.
|
|
1367
|
+
// Round-5 finding M2: clearing the window also resets escalation.
|
|
1368
|
+
// `applyFailureCooldown` derives the next level only from the
|
|
1369
|
+
// previous `cooldownUntil` (core.ts:168-174), so a zeroed window
|
|
1370
|
+
// starts the next failure at level 0 / the base cooldown; the
|
|
1371
|
+
// recovery grace window does not apply to a zeroed cooldown. The
|
|
1372
|
+
// leftover `failureLevel`/`consecutiveFailures` fields do not
|
|
1373
|
+
// escalate anything.
|
|
1374
|
+
// Round-5 finding SF3: only a cooldown the OLD grant earned may be
|
|
1375
|
+
// cleared. R6-SF1: attribution is the projection generation stamped
|
|
1376
|
+
// on the window at failure time, not a timestamp taken here — a
|
|
1377
|
+
// failure of the freshly projected grant landing while this harvest
|
|
1378
|
+
// waited (queue, projection read) keeps its window, and an older
|
|
1379
|
+
// window clears even when its stamp shares this millisecond.
|
|
1380
|
+
const earnedGeneration = this.cooldownGenerations.get(existing.id);
|
|
1381
|
+
if (earnedGeneration === undefined || earnedGeneration < projectionGeneration) {
|
|
1382
|
+
existing.cooldownUntil = 0;
|
|
1383
|
+
this.cooldownGenerations.delete(existing.id);
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
else {
|
|
1387
|
+
const rowId = newKeyId();
|
|
1388
|
+
addedId = rowId;
|
|
1389
|
+
// Record the appended row before the push resolves, so the
|
|
1390
|
+
// failure handler can remove exactly this row.
|
|
1391
|
+
facts.rowUndo = { kind: 'added', id: rowId };
|
|
1392
|
+
providerRecord.keys.push({
|
|
1393
|
+
id: rowId,
|
|
1394
|
+
alias: attempt.alias ?? payload.email ?? providerRecord.displayName,
|
|
1395
|
+
// Deliberately not a credential-ref-grammar name: account rows
|
|
1396
|
+
// carry no broker-resolvable literal, and an unparseable ref keeps
|
|
1397
|
+
// the api-key probe paths (probeKey/findVerifiedAlternate) from
|
|
1398
|
+
// misreading the grant as a missing API key.
|
|
1399
|
+
ref: grantRef,
|
|
1400
|
+
kind: 'oauth',
|
|
1401
|
+
enabled: true,
|
|
1402
|
+
createdAt: now,
|
|
1403
|
+
lastUsedAt: 0,
|
|
1404
|
+
cooldownUntil: 0,
|
|
1405
|
+
lastFailureAt: 0,
|
|
1406
|
+
lastFailureCode: '',
|
|
1407
|
+
failureLevel: 0,
|
|
1408
|
+
consecutiveFailures: 0,
|
|
1409
|
+
totalUses: 0,
|
|
1410
|
+
totalFailures: 0,
|
|
1411
|
+
valueMissing: false,
|
|
1412
|
+
importedFrom: null,
|
|
1413
|
+
oauth: {
|
|
1414
|
+
provider: attempt.provider,
|
|
1415
|
+
grantRef,
|
|
1416
|
+
...(payload.email === undefined ? {} : { email: payload.email }),
|
|
1417
|
+
...(payload.accountId === undefined ? {} : { accountId: payload.accountId }),
|
|
1418
|
+
...(typeof payload.expires === 'number' && Number.isFinite(payload.expires) ? { expiresAt: payload.expires } : {}),
|
|
1419
|
+
},
|
|
1420
|
+
});
|
|
1421
|
+
}
|
|
1422
|
+
this.store.touch();
|
|
1423
|
+
attempt.accountId = existing !== undefined ? existing.id : addedId;
|
|
1424
|
+
// Remember who owned the projection before this harvest claims it:
|
|
1425
|
+
// a flush rejection restores that entry instead of leaving the
|
|
1426
|
+
// rolled-back row as the live owner.
|
|
1427
|
+
facts.ownerUndo = this.activeAccounts.has(attempt.provider)
|
|
1428
|
+
? { had: true, owner: this.activeAccounts.get(attempt.provider) }
|
|
1429
|
+
: { had: false };
|
|
1430
|
+
// The grant pi-ai just committed IS the projection; the fresh row
|
|
1431
|
+
// owns it until activation or fallback says otherwise.
|
|
1432
|
+
if (attempt.accountId !== undefined)
|
|
1433
|
+
this.claimProjectionOwner(attempt.provider, attempt.accountId);
|
|
1434
|
+
// No terminal here: `run` flushes the state document after this
|
|
1435
|
+
// callback resolves, and the ready terminal publishes below — only
|
|
1436
|
+
// once persistence has actually resolved (round-2 review, step 3).
|
|
1437
|
+
});
|
|
1438
|
+
this.finishLoginAttempt(attempt, 'ready');
|
|
1439
|
+
}
|
|
1440
|
+
catch (error) {
|
|
1441
|
+
// A rejected state flush lands here AFTER the mutation resolved, so
|
|
1442
|
+
// `attempt.accountId` is already set even though `run` never made the
|
|
1443
|
+
// document durable. Undo the in-memory upsert synchronously — before
|
|
1444
|
+
// any queued mutation or retry can serialize a row whose grant the
|
|
1445
|
+
// credential rollback then deletes — and roll the credential writes
|
|
1446
|
+
// back. Only a failure BEFORE the upsert takes the pre-existing
|
|
1447
|
+
// compensation path.
|
|
1448
|
+
const upserted = attempt.accountId !== undefined;
|
|
1449
|
+
if (upserted)
|
|
1450
|
+
this.undoHarvestRow(attempt, harvest);
|
|
1451
|
+
// Whether THIS failure handler settles the attempt (a release may have
|
|
1452
|
+
// finalized it first): only an attempt this handler failed may have its
|
|
1453
|
+
// reason enriched with the rollback code below.
|
|
1454
|
+
const settledHere = attempt.settledAt === undefined;
|
|
1455
|
+
this.failLoginAttempt(attempt, error);
|
|
1456
|
+
this.warn(`oauth sign-in harvest failed for ${attempt.provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1457
|
+
// Round-2 review step 4: a harvest whose row was never created is
|
|
1458
|
+
// compensated by deleting both written records — the plugin-scope
|
|
1459
|
+
// grant copy (when its write resolved), and the llm-pi-ai projection
|
|
1460
|
+
// only while it still carries THIS login's payload. A harvest that
|
|
1461
|
+
// DID create its row is rolled back the same way, plus the credential
|
|
1462
|
+
// writes its restored row depends on, because the rejected flush
|
|
1463
|
+
// left that upsert undurable.
|
|
1464
|
+
if (upserted) {
|
|
1465
|
+
const rollback = await this.compensateFailedFlush(attempt, harvest);
|
|
1466
|
+
// A rollback that could not write leaves a credential surface the
|
|
1467
|
+
// rolled-back row no longer owns. Tag the terminal reason with the
|
|
1468
|
+
// stable code (keeping the original flush cause after it) so an
|
|
1469
|
+
// upstream caller can branch instead of matching prose.
|
|
1470
|
+
if (rollback.failed && settledHere) {
|
|
1471
|
+
const cause = attempt.error ?? 'the state flush failed';
|
|
1472
|
+
attempt.error = `${FLUSH_ROLLBACK_COMPENSATION_FAILED}: ${cause} (credential rollback incomplete: ${rollback.failures.join('; ')})`;
|
|
1473
|
+
attempt.updatedAt = Date.now();
|
|
1474
|
+
this.warn(`${FLUSH_ROLLBACK_COMPENSATION_FAILED} for ${attempt.provider}: the state flush failed and the credential rollback could not be completed`);
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
else {
|
|
1478
|
+
await this.compensateFailedHarvest(attempt, harvest);
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
/**
|
|
1483
|
+
* Capture this attempt's committed projection payload (F2 provenance).
|
|
1484
|
+
*
|
|
1485
|
+
* The seam reports `authorized` only after the flow's commit is observed
|
|
1486
|
+
* and still present, so a read STARTED at that broadcast observes the
|
|
1487
|
+
* record this attempt wrote. The read is deliberately not queued on
|
|
1488
|
+
* `run`: a release parked ahead of the harvest must not delay provenance
|
|
1489
|
+
* long enough for a foreign writer to become this attempt's authority.
|
|
1490
|
+
*
|
|
1491
|
+
* The seam offers no writer identity or record version, so this capture is
|
|
1492
|
+
* the closest provenance the manager can obtain: `undefined` means the
|
|
1493
|
+
* committed payload is unknown, and compensation then refuses to touch the
|
|
1494
|
+
* projection instead of guessing. See I-000090 for the seam capability
|
|
1495
|
+
* (compare-and-delete / versioned records) that would make it exact.
|
|
1496
|
+
*/
|
|
1497
|
+
captureSettledProjection(attempt) {
|
|
1498
|
+
const records = this.credentialRecords();
|
|
1499
|
+
if (records === undefined)
|
|
1500
|
+
return Promise.resolve(undefined);
|
|
1501
|
+
const projectionKey = `${OAUTH_GRANT_RECORD_SCOPE}/${attempt.provider}`;
|
|
1502
|
+
return records.readRecord(projectionKey).then(record => record !== undefined && record.kind === 'grant' && isOAuthGrantPayload(record.payload)
|
|
1503
|
+
? structuredClone(record.payload)
|
|
1504
|
+
: undefined, error => {
|
|
1505
|
+
this.warn(`could not capture the committed grant for ${attempt.provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1506
|
+
return undefined;
|
|
1507
|
+
});
|
|
1508
|
+
}
|
|
1509
|
+
/**
|
|
1510
|
+
* Undo the in-memory half of a harvest whose state document was never made
|
|
1511
|
+
* durable (a rejected `store.flush()` after the mutation resolved).
|
|
1512
|
+
*
|
|
1513
|
+
* Runs synchronously from the failure handler so no queued mutation,
|
|
1514
|
+
* debounced write, or retry can observe — or persist — the rolled-back row
|
|
1515
|
+
* between the rejected flush and this undo. The credential rollback that
|
|
1516
|
+
* follows ({@link compensateFailedFlush}) deletes the grant the row
|
|
1517
|
+
* referenced, so leaving the upsert in memory would make the next
|
|
1518
|
+
* successful flush point at a deleted record.
|
|
1519
|
+
*/
|
|
1520
|
+
undoHarvestRow(attempt, harvest) {
|
|
1521
|
+
attempt.accountId = undefined;
|
|
1522
|
+
const providerRecord = this.store.provider(attempt.provider);
|
|
1523
|
+
if (providerRecord !== undefined && harvest?.rowUndo !== undefined) {
|
|
1524
|
+
const undo = harvest.rowUndo;
|
|
1525
|
+
if (undo.kind === 'added') {
|
|
1526
|
+
const index = providerRecord.keys.findIndex(candidate => candidate.id === undo.id);
|
|
1527
|
+
// Remove exactly the appended row: identity, never position.
|
|
1528
|
+
if (index >= 0)
|
|
1529
|
+
providerRecord.keys.splice(index, 1);
|
|
1530
|
+
}
|
|
1531
|
+
else {
|
|
1532
|
+
const row = providerRecord.keys.find(candidate => candidate.id === undo.id);
|
|
1533
|
+
if (row !== undefined) {
|
|
1534
|
+
row.oauth = undo.snapshot.oauth;
|
|
1535
|
+
row.valueMissing = undo.snapshot.valueMissing;
|
|
1536
|
+
row.cooldownUntil = undo.snapshot.cooldownUntil;
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
if (harvest?.ownerUndo !== undefined) {
|
|
1541
|
+
if (harvest.ownerUndo.had && harvest.ownerUndo.owner !== undefined) {
|
|
1542
|
+
this.activeAccounts.set(attempt.provider, harvest.ownerUndo.owner);
|
|
1543
|
+
}
|
|
1544
|
+
else {
|
|
1545
|
+
this.activeAccounts.delete(attempt.provider);
|
|
1546
|
+
}
|
|
1547
|
+
// Same generation guard the claim bumped: a hydration read started
|
|
1548
|
+
// before/during the harvest must not re-apply the claim just undone.
|
|
1549
|
+
this.ownershipGenerations.set(attempt.provider, (this.ownershipGenerations.get(attempt.provider) ?? 0) + 1);
|
|
1550
|
+
}
|
|
1551
|
+
this.store.touch();
|
|
1552
|
+
}
|
|
1553
|
+
/**
|
|
1554
|
+
* Undo the credential writes of a harvest whose state flush rejected.
|
|
1555
|
+
*
|
|
1556
|
+
* Differs from {@link compensateFailedHarvest} in one respect: the row
|
|
1557
|
+
* upsert already happened, so the plugin-scope record cannot simply be
|
|
1558
|
+
* deleted. A same-ref re-authentication overwrote the record the restored
|
|
1559
|
+
* row references, and a changed-ref re-authentication deleted the
|
|
1560
|
+
* superseded generation before the flush failed — both are reverted here,
|
|
1561
|
+
* each write guarded by the exact payload this login committed so a
|
|
1562
|
+
* foreign writer's record is never clobbered. The `llm-pi-ai` projection is
|
|
1563
|
+
* then demoted exactly like any failed harvest (ownership-checked, never a
|
|
1564
|
+
* foreign grant), so no committed credential outlives the rolled-back row
|
|
1565
|
+
* without a pool owner.
|
|
1566
|
+
*
|
|
1567
|
+
* Resolves with the write failures the rollback hit, if any: a refusal to
|
|
1568
|
+
* touch the surface when provenance is unknown is a deliberate skip, not a
|
|
1569
|
+
* failure, but a rejected compensating write means the credential surface
|
|
1570
|
+
* may still hold records the rolled-back row no longer owns. The caller
|
|
1571
|
+
* tags the attempt with {@link FLUSH_ROLLBACK_COMPENSATION_FAILED} so that
|
|
1572
|
+
* state is distinguishable from a clean rollback (each failure is also
|
|
1573
|
+
* warned about here, message by message).
|
|
1574
|
+
*/
|
|
1575
|
+
async compensateFailedFlush(attempt, harvest) {
|
|
1576
|
+
const failures = [];
|
|
1577
|
+
const recordFailure = (prefix, error) => {
|
|
1578
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
1579
|
+
failures.push(`${prefix}: ${detail}`);
|
|
1580
|
+
this.warn(`${prefix}: ${detail}`);
|
|
1581
|
+
};
|
|
1582
|
+
const records = this.credentialRecords();
|
|
1583
|
+
if (records === undefined)
|
|
1584
|
+
return { failed: false, failures };
|
|
1585
|
+
if (harvest === undefined) {
|
|
1586
|
+
this.warn(`flush-failure compensation refused for ${attempt.provider}: this attempt's committed grant payload is unknown, so the credential surface was left untouched`);
|
|
1587
|
+
return { failed: false, failures };
|
|
1588
|
+
}
|
|
1589
|
+
const provenance = JSON.stringify(harvest.payload);
|
|
1590
|
+
const stillThisPayload = (record) => record !== undefined && record.kind === 'grant'
|
|
1591
|
+
&& isOAuthGrantPayload(record.payload) && JSON.stringify(record.payload) === provenance;
|
|
1592
|
+
try {
|
|
1593
|
+
if (harvest.previousRef !== undefined) {
|
|
1594
|
+
const previous = harvest.previousRef;
|
|
1595
|
+
// Restore the overwritten same-ref record only while the live record
|
|
1596
|
+
// still holds this harvest's payload; the replacement runs inside
|
|
1597
|
+
// the seam's per-key lock.
|
|
1598
|
+
await records.modifyRecord(harvest.grantRef, async (current) => stillThisPayload(current) ? structuredClone(previous) : undefined);
|
|
1599
|
+
}
|
|
1600
|
+
else if (harvest.wroteRef) {
|
|
1601
|
+
let owns = false;
|
|
1602
|
+
await records.modifyRecord(harvest.grantRef, async (current) => {
|
|
1603
|
+
owns = stillThisPayload(current);
|
|
1604
|
+
return undefined;
|
|
1605
|
+
});
|
|
1606
|
+
if (owns)
|
|
1607
|
+
await records.deleteRecord(harvest.grantRef);
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
catch (error) {
|
|
1611
|
+
recordFailure(`flush-failure compensation could not restore ${harvest.grantRef} for ${attempt.provider}`, error);
|
|
1612
|
+
}
|
|
1613
|
+
if (harvest.superseded !== undefined) {
|
|
1614
|
+
const superseded = harvest.superseded;
|
|
1615
|
+
try {
|
|
1616
|
+
// Write the superseded generation back only while its ref is still
|
|
1617
|
+
// free: a native writer that reused the ref owns the record now.
|
|
1618
|
+
await records.modifyRecord(superseded.ref, async (current) => current === undefined ? structuredClone(superseded.record) : undefined);
|
|
1619
|
+
}
|
|
1620
|
+
catch (error) {
|
|
1621
|
+
recordFailure(`flush-failure compensation could not restore ${superseded.ref} for ${attempt.provider}`, error);
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
const projection = await this.revertHarvestProjection(attempt, harvest);
|
|
1625
|
+
if (!projection.ok)
|
|
1626
|
+
failures.push(projection.failure);
|
|
1627
|
+
return { failed: failures.length > 0, failures };
|
|
1628
|
+
}
|
|
1629
|
+
/**
|
|
1630
|
+
* Roll a failed harvest back to a clean credential surface: the
|
|
1631
|
+
* plugin-scope grant copy is deleted when it was written; the
|
|
1632
|
+
* `llm-pi-ai` projection is deleted only while it still carries EXACTLY
|
|
1633
|
+
* this login's settlement-time committed payload, so compensation can
|
|
1634
|
+
* never delete a grant another account owns.
|
|
1635
|
+
*
|
|
1636
|
+
* Provenance unknown (the settlement-time capture found no usable record)
|
|
1637
|
+
* refuses both deletions with a warning: without the committed payload
|
|
1638
|
+
* there is no proof of ownership, and deleting "whatever is live" is the
|
|
1639
|
+
* round-4 defect (I-000083).
|
|
1640
|
+
*/
|
|
1641
|
+
async compensateFailedHarvest(attempt, harvest) {
|
|
1642
|
+
const records = this.credentialRecords();
|
|
1643
|
+
if (records === undefined)
|
|
1644
|
+
return;
|
|
1645
|
+
if (harvest === undefined) {
|
|
1646
|
+
this.warn(`harvest compensation refused for ${attempt.provider}: this attempt's committed grant payload is unknown, so the projection was left untouched`);
|
|
1647
|
+
return;
|
|
1648
|
+
}
|
|
1649
|
+
if (harvest.wroteRef) {
|
|
1650
|
+
try {
|
|
1651
|
+
await records.deleteRecord(harvest.grantRef);
|
|
1652
|
+
}
|
|
1653
|
+
catch (error) {
|
|
1654
|
+
this.warn(`harvest compensation could not delete ${harvest.grantRef}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
await this.revertHarvestProjection(attempt, harvest);
|
|
1658
|
+
}
|
|
1659
|
+
/**
|
|
1660
|
+
* Delete the `llm-pi-ai` projection only while it still carries exactly
|
|
1661
|
+
* this login's settlement-time committed payload. Shared by the
|
|
1662
|
+
* failed-harvest compensation and the flush-failure rollback.
|
|
1663
|
+
*
|
|
1664
|
+
* Resolves `ok: false` only when the compare/delete writes themselves
|
|
1665
|
+
* rejected; an ownership mismatch is the guard doing its job (the live
|
|
1666
|
+
* record belongs to someone else) and counts as a clean skip.
|
|
1667
|
+
*/
|
|
1668
|
+
async revertHarvestProjection(attempt, harvest) {
|
|
1669
|
+
const records = this.credentialRecords();
|
|
1670
|
+
if (records === undefined)
|
|
1671
|
+
return { ok: true };
|
|
1672
|
+
try {
|
|
1673
|
+
const projectionKey = `${OAUTH_GRANT_RECORD_SCOPE}/${attempt.provider}`;
|
|
1674
|
+
// Same atomic decide-and-delete as the demotion path (round-3
|
|
1675
|
+
// finding 1): the compare runs inside `modifyRecord`'s per-key lock
|
|
1676
|
+
// against the FULL settlement-time payload (not merely its refresh
|
|
1677
|
+
// token), so a native writer that replaced the projection — even by a
|
|
1678
|
+
// payload that reused our refresh token — is never matched, and the
|
|
1679
|
+
// delete runs only on a locked-in exact match (no delete-inside-modify
|
|
1680
|
+
// primitive exists on the seam; see decideAndDeleteProjection and
|
|
1681
|
+
// I-000090).
|
|
1682
|
+
const provenance = JSON.stringify(harvest.payload);
|
|
1683
|
+
let owns = false;
|
|
1684
|
+
let decidedAgainst;
|
|
1685
|
+
await records.modifyRecord(projectionKey, async (current) => {
|
|
1686
|
+
owns = current !== undefined && current.kind === 'grant'
|
|
1687
|
+
&& isOAuthGrantPayload(current.payload) && JSON.stringify(current.payload) === provenance;
|
|
1688
|
+
// F2 pre-delete re-read snapshot, identical to the demotion path.
|
|
1689
|
+
if (owns && current !== undefined)
|
|
1690
|
+
decidedAgainst = provenance;
|
|
1691
|
+
return undefined;
|
|
1692
|
+
});
|
|
1693
|
+
if (owns && await this.projectionStillMatches(records, projectionKey, decidedAgainst)) {
|
|
1694
|
+
await records.deleteRecord(projectionKey);
|
|
1695
|
+
}
|
|
1696
|
+
else if (!owns) {
|
|
1697
|
+
this.warn(`harvest compensation kept the projection for ${attempt.provider}: the live grant no longer matches this attempt's committed payload`);
|
|
1698
|
+
}
|
|
1699
|
+
return { ok: true };
|
|
1700
|
+
}
|
|
1701
|
+
catch (error) {
|
|
1702
|
+
const failure = `harvest compensation could not revert the projection for ${attempt.provider}: ${error instanceof Error ? error.message : String(error)}`;
|
|
1703
|
+
this.warn(failure);
|
|
1704
|
+
return { ok: false, failure };
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
/** Expire wedged sign-ins and drop long-settled attempt records. */
|
|
1708
|
+
pruneLoginAttempts(now) {
|
|
1709
|
+
for (const [id, attempt] of this.loginAttempts) {
|
|
1710
|
+
if (attempt.settledAt === undefined) {
|
|
1711
|
+
if (now - attempt.updatedAt <= LOGIN_ATTEMPT_TTL_MS)
|
|
1712
|
+
continue;
|
|
1713
|
+
attempt.controller.abort();
|
|
1714
|
+
this.failLoginAttempt(attempt, new Error('The sign-in attempt timed out'));
|
|
1715
|
+
continue;
|
|
1716
|
+
}
|
|
1717
|
+
if (now - attempt.settledAt > ATTEMPT_TTL_MS)
|
|
1718
|
+
this.loginAttempts.delete(id);
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
/** Keep the attempt cache bounded: evict the oldest settled entries first. */
|
|
1722
|
+
capLoginAttempts() {
|
|
1723
|
+
while (this.loginAttempts.size > MAX_LOGIN_ATTEMPTS) {
|
|
1724
|
+
let oldestId;
|
|
1725
|
+
let oldestSettledAt = Number.POSITIVE_INFINITY;
|
|
1726
|
+
for (const [id, attempt] of this.loginAttempts) {
|
|
1727
|
+
// In-flight attempts are never eviction candidates.
|
|
1728
|
+
if (attempt.settledAt === undefined)
|
|
1729
|
+
continue;
|
|
1730
|
+
if (attempt.settledAt < oldestSettledAt) {
|
|
1731
|
+
oldestSettledAt = attempt.settledAt;
|
|
1732
|
+
oldestId = id;
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
if (oldestId === undefined)
|
|
1736
|
+
break;
|
|
1737
|
+
this.loginAttempts.delete(oldestId);
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
/**
|
|
1741
|
+
* Subscribe to the credential seam's post-commit record broadcast
|
|
1742
|
+
* (credentials/src/index.ts:271-277; the event carries the key only, so
|
|
1743
|
+
* the handler re-reads the record for its payload).
|
|
1744
|
+
*/
|
|
1745
|
+
watchGrantRecordUpdated() {
|
|
1746
|
+
const on = this.ctx.on;
|
|
1747
|
+
if (typeof on !== 'function') {
|
|
1748
|
+
this.warnSubscriptionLost('credentials/record-updated');
|
|
1749
|
+
return undefined;
|
|
1750
|
+
}
|
|
1751
|
+
try {
|
|
1752
|
+
return on.call(this.ctx, 'credentials/record-updated', (key) => {
|
|
1753
|
+
if (typeof key === 'string')
|
|
1754
|
+
this.onGrantRecordUpdated(key);
|
|
1755
|
+
});
|
|
1756
|
+
}
|
|
1757
|
+
catch (error) {
|
|
1758
|
+
this.warnSubscriptionLost('credentials/record-updated', error);
|
|
1759
|
+
return undefined;
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
/**
|
|
1763
|
+
* Reconcile one `llm-pi-ai/<provider>` record change with the pool.
|
|
1764
|
+
*
|
|
1765
|
+
* The manager writes this record only through {@link activateAccount}
|
|
1766
|
+
* (D1); pi-ai rewrites it on every lazy refresh and owns every login. A
|
|
1767
|
+
* change whose payload belongs to the active account is a rotation and
|
|
1768
|
+
* syncs back into the owning row; anything else the manager did not write
|
|
1769
|
+
* is native management of the route and surfaces as a warning instead of
|
|
1770
|
+
* being adopted silently.
|
|
1771
|
+
*
|
|
1772
|
+
* C6-1: reconciliation writes a plugin-scope record for the rotated
|
|
1773
|
+
* generation and then mutates the owning row. removeAccount /
|
|
1774
|
+
* releaseProvider mutate the same row but cannot be serialized against
|
|
1775
|
+
* that write — the credential seam offers only a per-record lock
|
|
1776
|
+
* (`modifyRecord`), never a transaction across the record write and the
|
|
1777
|
+
* pool row. So {@link reconcileGrantRecord} revalidates the row immediately
|
|
1778
|
+
* before AND after the awaited write and compensates by deleting the record
|
|
1779
|
+
* it just wrote. An orphan record must never outlive a removed account.
|
|
1780
|
+
*
|
|
1781
|
+
* Round-9 finding P1: two echoes for the same provider can
|
|
1782
|
+
* arrive before the first reconciliation commits. Their write-backs then
|
|
1783
|
+
* race and an out-of-order completion could move the row back onto an
|
|
1784
|
+
* older generation, stranding the newer record outside manager state. The
|
|
1785
|
+
* bodies are therefore chained per provider: one observation's
|
|
1786
|
+
* read → write → commit window is never interleaved with another's, and
|
|
1787
|
+
* {@link reconcileGrantRecord} additionally re-checks that its generation
|
|
1788
|
+
* is still current before committing.
|
|
1789
|
+
*/
|
|
1790
|
+
onGrantRecordUpdated(key) {
|
|
1791
|
+
if (!key.startsWith(`${OAUTH_GRANT_RECORD_SCOPE}/`))
|
|
1792
|
+
return;
|
|
1793
|
+
const provider = key.slice(OAUTH_GRANT_RECORD_SCOPE.length + 1);
|
|
1794
|
+
if (this.store.provider(provider) === undefined)
|
|
1795
|
+
return;
|
|
1796
|
+
// Fire-and-forget: the seam event is synchronous and carries no
|
|
1797
|
+
// completion channel, so failures are logged like any async work. The
|
|
1798
|
+
// body is appended to the provider's chain, preserving arrival order
|
|
1799
|
+
// (round-9 P1); the chain itself never rejects, so a failed
|
|
1800
|
+
// reconciliation does not stop the next echo.
|
|
1801
|
+
const next = this.scheduleGrantReconciliation(provider, () => this.reconcileGrantRecord(provider, key));
|
|
1802
|
+
void next.catch(error => {
|
|
1803
|
+
this.warn(`grant reconciliation failed for ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1806
|
+
/**
|
|
1807
|
+
* Append one body to the provider's per-route grant chain and return it.
|
|
1808
|
+
*
|
|
1809
|
+
* The chain is the single serialization point for every operation that can
|
|
1810
|
+
* repoint `row.oauth.grantRef` or reproject the route: the echo reconciler
|
|
1811
|
+
* (round-9 P1 ordering) and, since round-8 finding I-000106, activation.
|
|
1812
|
+
* The stored tail never rejects, so one failed body cannot stop the chain;
|
|
1813
|
+
* callers that must observe the failure await the returned promise (the
|
|
1814
|
+
* fire-and-forget echo caller attaches its own catch).
|
|
1815
|
+
*/
|
|
1816
|
+
scheduleGrantReconciliation(provider, body) {
|
|
1817
|
+
const previous = this.grantReconciliations.get(provider) ?? Promise.resolve();
|
|
1818
|
+
const next = previous.then(body, body);
|
|
1819
|
+
const tail = next.then(() => undefined, () => undefined);
|
|
1820
|
+
this.grantReconciliations.set(provider, tail);
|
|
1821
|
+
void tail.then(() => {
|
|
1822
|
+
if (this.grantReconciliations.get(provider) === tail)
|
|
1823
|
+
this.grantReconciliations.delete(provider);
|
|
1824
|
+
});
|
|
1825
|
+
return next;
|
|
1826
|
+
}
|
|
1827
|
+
/**
|
|
1828
|
+
* The body of {@link onGrantRecordUpdated}. Pool state is read live, and
|
|
1829
|
+
* every awaited step around the write is guarded by revalidation: a
|
|
1830
|
+
* removal is allowed to complete while the write is in flight.
|
|
1831
|
+
*/
|
|
1832
|
+
async reconcileGrantRecord(provider, key) {
|
|
1833
|
+
const records = this.credentialRecords();
|
|
1834
|
+
if (records === undefined)
|
|
1835
|
+
return;
|
|
1836
|
+
const providerRecord = this.store.provider(provider);
|
|
1837
|
+
if (providerRecord === undefined)
|
|
1838
|
+
return;
|
|
1839
|
+
const accounts = providerRecord.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
1840
|
+
if (accounts.length === 0)
|
|
1841
|
+
return;
|
|
1842
|
+
const activeId = this.activeAccounts.get(provider);
|
|
1843
|
+
const active = activeId === undefined
|
|
1844
|
+
? undefined
|
|
1845
|
+
: accounts.find(candidate => candidate.id === activeId);
|
|
1846
|
+
// F1 (I-000081): the owner claim below must not overwrite a claim an
|
|
1847
|
+
// activation or login lands while this awaited re-read is in flight —
|
|
1848
|
+
// the same generation guard load-time hydration applies.
|
|
1849
|
+
const generation = this.ownershipGenerations.get(provider) ?? 0;
|
|
1850
|
+
const record = await records.readRecord(key);
|
|
1851
|
+
if (record === undefined) {
|
|
1852
|
+
// This manager's own demotion marked the removal just before
|
|
1853
|
+
// deleting; while the marker is inside its TTL window the echo is
|
|
1854
|
+
// ours — consume the one-shot marker and stay silent. An expired
|
|
1855
|
+
// marker no longer masks anything (and is dropped for hygiene).
|
|
1856
|
+
const markedAt = this.intentionalProjectionRemovals.get(provider);
|
|
1857
|
+
if (markedAt !== undefined && Date.now() - markedAt <= INTENTIONAL_REMOVAL_TTL_MS) {
|
|
1858
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
1859
|
+
return;
|
|
1860
|
+
}
|
|
1861
|
+
if (markedAt !== undefined)
|
|
1862
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
1863
|
+
if (active !== undefined) {
|
|
1864
|
+
// Not our deletion: someone removed the projection, so the route
|
|
1865
|
+
// silently serves the key pool while the account row looks
|
|
1866
|
+
// healthy. Mark, warn, and drop the stale owner entry (round-2
|
|
1867
|
+
// review, step 1): nothing owns the projection anymore.
|
|
1868
|
+
active.valueMissing = true;
|
|
1869
|
+
this.activeAccounts.delete(provider);
|
|
1870
|
+
this.store.touch();
|
|
1871
|
+
this.pushWarning(`The ${provider} OAuth grant was removed outside Keys Manager; the route now resolves through its API-key pool.`);
|
|
1872
|
+
}
|
|
1873
|
+
return;
|
|
1874
|
+
}
|
|
1875
|
+
if (record.kind === 'api-key') {
|
|
1876
|
+
// Native api-key management took the route over: the manager no
|
|
1877
|
+
// longer owns any projection here.
|
|
1878
|
+
this.activeAccounts.delete(provider);
|
|
1879
|
+
this.pushWarning(`The ${provider} route now holds a natively managed API-key credential; the Keys Manager account pool is not serving it.`);
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1882
|
+
if (!isOAuthGrantPayload(record.payload)) {
|
|
1883
|
+
// An unrecognized payload is not ours to interpret: drop the stale
|
|
1884
|
+
// owner entry so failure attribution and demotion never fire on it.
|
|
1885
|
+
this.activeAccounts.delete(provider);
|
|
1886
|
+
this.pushWarning(`The ${provider} OAuth record holds an unrecognized payload; Keys Manager left it untouched.`);
|
|
1887
|
+
return;
|
|
1888
|
+
}
|
|
1889
|
+
const payload = record.payload;
|
|
1890
|
+
const hash = createHash('sha256').update(payload.refresh).digest('hex').slice(0, 12);
|
|
1891
|
+
const rotatedRef = accountRef(provider, hash);
|
|
1892
|
+
// Owner resolution runs through the shared matcher (refresh digest
|
|
1893
|
+
// first, then identity metadata) — the same rule load-time hydration
|
|
1894
|
+
// applies, so a restarted process and the live reconciler agree.
|
|
1895
|
+
const owner = matchOwningAccount(provider, accounts, payload);
|
|
1896
|
+
if (owner === undefined) {
|
|
1897
|
+
// A foreign account's grant serves the route: no pool row owns the
|
|
1898
|
+
// projection, so the previous owner entry must not survive it.
|
|
1899
|
+
this.activeAccounts.delete(provider);
|
|
1900
|
+
this.pushWarning(`An OAuth account for ${provider} is managed natively outside Keys Manager and is not part of the account pool.`);
|
|
1901
|
+
return;
|
|
1902
|
+
}
|
|
1903
|
+
// C6-1 pre-write revalidation: removeAccount / releaseProvider run on
|
|
1904
|
+
// the mutation queue and delete the row (and its grant record) while
|
|
1905
|
+
// this reconciliation is still resolving its owner. Writing the rotated
|
|
1906
|
+
// generation for an account removed in that window would recreate a
|
|
1907
|
+
// secret-bearing record nothing owns, so the write is skipped entirely.
|
|
1908
|
+
// R7-SF3: this proof is adoption-oriented, so a merely demoted row
|
|
1909
|
+
// (`valueMissing`) still qualifies — a projection that reappears for it
|
|
1910
|
+
// is re-adopted below. Only a removal in progress is refused.
|
|
1911
|
+
if (!this.accountRowIsManaged(provider, owner))
|
|
1912
|
+
return;
|
|
1913
|
+
// F1 (I-000081): the matched grant IS the live projection, so this row
|
|
1914
|
+
// owns it from now on — even when no in-process login or activation
|
|
1915
|
+
// ran (the post-restart echo-rotation case: a hydrated row, no owner
|
|
1916
|
+
// entry). Claiming here is what makes request attribution
|
|
1917
|
+
// (oauthOwnerFor), cooling, and demotion act on the serving account.
|
|
1918
|
+
// The guard above keeps unrecognized and foreign payloads on the
|
|
1919
|
+
// warn-and-clear path; ownership is never adopted for them. A newer
|
|
1920
|
+
// activation/login that claimed while the read was in flight wins:
|
|
1921
|
+
// the stale echo claim is discarded exactly like a stale hydration
|
|
1922
|
+
// read (the rotation sync below still runs, but a commit that a newer
|
|
1923
|
+
// claim has superseded is rejected — round-9 P1).
|
|
1924
|
+
let claimedGeneration;
|
|
1925
|
+
if ((this.ownershipGenerations.get(provider) ?? 0) === generation) {
|
|
1926
|
+
this.claimProjectionOwner(provider, owner.id);
|
|
1927
|
+
claimedGeneration = this.ownershipGenerations.get(provider);
|
|
1928
|
+
// R7-SF3: the live projection proves this row's identity again, so a
|
|
1929
|
+
// demoted row is re-adopted — restore its owner claim (above) and end
|
|
1930
|
+
// the demoted state. `valueMissing` means "not serving but still
|
|
1931
|
+
// managed", not "removal in progress": a native sign-in or pi-ai
|
|
1932
|
+
// write that re-projects the grant recovers the account without a
|
|
1933
|
+
// manual re-activation. The flag is persisted with the rest of the
|
|
1934
|
+
// state document.
|
|
1935
|
+
if (owner.valueMissing === true) {
|
|
1936
|
+
owner.valueMissing = false;
|
|
1937
|
+
// R14-SF3: a demotion is earned by a failure, so the row usually
|
|
1938
|
+
// still carries the cooldown `markFailure` wrote. Re-adoption is the
|
|
1939
|
+
// same manual-recovery equivalence `activateAccount` and re-auth
|
|
1940
|
+
// implement: the live projection is serving again, so the old
|
|
1941
|
+
// window ends with it — otherwise the resurrected row renders
|
|
1942
|
+
// "cooling" and stays out of the healthy counts until the timer
|
|
1943
|
+
// expires. Only this proven-adoption branch clears; a row whose
|
|
1944
|
+
// projection did not reappear keeps its window.
|
|
1945
|
+
owner.cooldownUntil = 0;
|
|
1946
|
+
this.store.touch();
|
|
1947
|
+
}
|
|
1948
|
+
}
|
|
1949
|
+
// Rotation sync: the rotated payload is authoritative (the plan's
|
|
1950
|
+
// single-writer contract — "sync them back to the owning plugin-scope
|
|
1951
|
+
// record on record-updated"), so write it back to the row's grant
|
|
1952
|
+
// record instead of trusting that refresh tokens never rotate. The
|
|
1953
|
+
// plugin-scope record is written FIRST; the row's grantRef commits
|
|
1954
|
+
// only after that write resolves, so a failed write leaves the row
|
|
1955
|
+
// pointing at the old, still-present record and Remove keeps working
|
|
1956
|
+
// against the live grant (round-2 review, step 5). This write cannot
|
|
1957
|
+
// re-enter this handler: it is keys-manager/-scope, and the handler
|
|
1958
|
+
// above only reconciles llm-pi-ai/-scope keys.
|
|
1959
|
+
const previousRef = owner.oauth.grantRef;
|
|
1960
|
+
await records.modifyRecord(rotatedRef, () => Promise.resolve({ kind: 'grant', payload: structuredClone(payload) }));
|
|
1961
|
+
// C6-1 post-write revalidation: a removal that landed while the write
|
|
1962
|
+
// was in flight already deleted its own record, so the generation just
|
|
1963
|
+
// written must be compensated or it outlives the account. The removed
|
|
1964
|
+
// row object is detached, so the mutation below would be invisible
|
|
1965
|
+
// anyway; it is skipped, and a stale owner claim is withdrawn.
|
|
1966
|
+
if (!this.accountRowStillCurrent(provider, owner)) {
|
|
1967
|
+
if (this.activeAccounts.get(provider) === owner.id)
|
|
1968
|
+
this.activeAccounts.delete(provider);
|
|
1969
|
+
try {
|
|
1970
|
+
await records.deleteRecord(rotatedRef);
|
|
1971
|
+
}
|
|
1972
|
+
catch (error) {
|
|
1973
|
+
this.warn(`cancelled rotation record cleanup failed for ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1974
|
+
}
|
|
1975
|
+
return;
|
|
1976
|
+
}
|
|
1977
|
+
// Round-9 P1 commit guard: the awaited write gave a newer echo, an
|
|
1978
|
+
// activation, or a login time to supersede the generation this body
|
|
1979
|
+
// observed. Committing then would point the row back at an older
|
|
1980
|
+
// generation and leave the newer generation's record outside manager
|
|
1981
|
+
// state, so the superseded write-back is removed instead. Only a
|
|
1982
|
+
// distinct record is deleted: in the same-ref case the write landed on
|
|
1983
|
+
// the row's own current record and must stay.
|
|
1984
|
+
const stillCurrent = await this.rotationStillCurrent(records, provider, key, rotatedRef, claimedGeneration);
|
|
1985
|
+
if (!stillCurrent) {
|
|
1986
|
+
if (rotatedRef !== previousRef) {
|
|
1987
|
+
// R5-MF1: the staleness probe is awaited, and a settled harvest can
|
|
1988
|
+
// adopt this very generation into a live row while it is in flight
|
|
1989
|
+
// (same account, same committed refresh token). Removing the record
|
|
1990
|
+
// then would delete the row's own grant — keep anything a live row
|
|
1991
|
+
// still serves.
|
|
1992
|
+
await this.discardRotationRecord(records, provider, rotatedRef, 'cleanup of the superseded rotation record failed for');
|
|
1993
|
+
}
|
|
1994
|
+
return;
|
|
1995
|
+
}
|
|
1996
|
+
// R5-SF1: the staleness probe above is itself awaited, so the post-write
|
|
1997
|
+
// revalidation is already stale by the time the commit runs. A removal
|
|
1998
|
+
// that completes inside that probe is only visible through the
|
|
1999
|
+
// removal-aware check, so it runs again immediately before the commit:
|
|
2000
|
+
// committing would write grantRef onto a detached row and leave the record
|
|
2001
|
+
// just written orphaned. Compensation mirrors the post-write branch.
|
|
2002
|
+
if (!this.accountRowStillCurrent(provider, owner)) {
|
|
2003
|
+
if (this.activeAccounts.get(provider) === owner.id)
|
|
2004
|
+
this.activeAccounts.delete(provider);
|
|
2005
|
+
if (rotatedRef !== previousRef) {
|
|
2006
|
+
await this.discardRotationRecord(records, provider, rotatedRef, 'cancelled rotation record cleanup failed for');
|
|
2007
|
+
}
|
|
2008
|
+
return;
|
|
2009
|
+
}
|
|
2010
|
+
// SF2: once the new payload persists and the row points at it, delete
|
|
2011
|
+
// the superseded generation's plugin-scope record — harvest parity. A
|
|
2012
|
+
// rotated refresh token must not strand its secret-bearing record
|
|
2013
|
+
// outside manager state; cleanup is best-effort with a warning,
|
|
2014
|
+
// exactly like the harvest-time cleanup.
|
|
2015
|
+
if (previousRef !== rotatedRef) {
|
|
2016
|
+
owner.oauth = { ...owner.oauth, grantRef: rotatedRef };
|
|
2017
|
+
try {
|
|
2018
|
+
await records.deleteRecord(previousRef);
|
|
2019
|
+
}
|
|
2020
|
+
catch (error) {
|
|
2021
|
+
this.warn(`cleanup of the superseded grant record failed for ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
// Refresh-derived identity fields are informational and update in place.
|
|
2025
|
+
owner.oauth = {
|
|
2026
|
+
...owner.oauth,
|
|
2027
|
+
...(payload.email === undefined ? {} : { email: payload.email }),
|
|
2028
|
+
...(payload.accountId === undefined ? {} : { accountId: payload.accountId }),
|
|
2029
|
+
...(typeof payload.expires === 'number' && Number.isFinite(payload.expires) ? { expiresAt: payload.expires } : {}),
|
|
2030
|
+
};
|
|
2031
|
+
this.store.touch();
|
|
2032
|
+
}
|
|
2033
|
+
/**
|
|
2034
|
+
* C6-1 revalidation: the row captured before an awaited credential write
|
|
2035
|
+
* must still be the provider's live row — same object identity, still an
|
|
2036
|
+
* OAuth account. A remove/release landing in between detaches the row
|
|
2037
|
+
* object; object identity is what tells "this account is still there"
|
|
2038
|
+
* apart from "some same-id row was recreated".
|
|
2039
|
+
*
|
|
2040
|
+
* C6-1/I1: a removal that is still IN PROGRESS is not visible through
|
|
2041
|
+
* identity alone. A multi-row release deletes each row's grant first and
|
|
2042
|
+
* the provider record last, and it never splices its rows, so after row X's
|
|
2043
|
+
* grant is deleted the provider record and row X both still look live while
|
|
2044
|
+
* the release awaits later rows. Syncing a rotation into that window would
|
|
2045
|
+
* commit `grantRef = rotatedRef` on a row nothing will serve, and the
|
|
2046
|
+
* release's final provider deletion leaves that record orphaned. The
|
|
2047
|
+
* removal-aware check is {@link retiringAccounts}, set before the removal's
|
|
2048
|
+
* first awaited delete.
|
|
2049
|
+
*
|
|
2050
|
+
* R7-SF3: `valueMissing` is NOT a removal signal — it only says the row is
|
|
2051
|
+
* currently not serving. A demoted row is still managed, and a projection
|
|
2052
|
+
* that reappears for it must be re-adopted (the caller proves the
|
|
2053
|
+
* projection belongs to this row before claiming it). This guard therefore
|
|
2054
|
+
* keeps the credential-write semantics strict: writing to a demoted row is
|
|
2055
|
+
* pointless until the adoption path has cleared the flag.
|
|
2056
|
+
*/
|
|
2057
|
+
accountRowStillCurrent(provider, owner) {
|
|
2058
|
+
if (owner.valueMissing === true)
|
|
2059
|
+
return false;
|
|
2060
|
+
return this.accountRowIsManaged(provider, owner);
|
|
2061
|
+
}
|
|
2062
|
+
/**
|
|
2063
|
+
* R7-SF3 adoption proof: the row is live and still managed, whether or not
|
|
2064
|
+
* it is currently serving. "Retiring" (removal in progress) refuses
|
|
2065
|
+
* everything; "demoted" (`valueMissing`) refuses nothing here — the caller
|
|
2066
|
+
* has already proven that the live projection belongs to this exact row
|
|
2067
|
+
* ({@link matchOwningAccount}) and that no newer projection superseded the
|
|
2068
|
+
* observation (generation guard).
|
|
2069
|
+
*/
|
|
2070
|
+
accountRowIsManaged(provider, owner) {
|
|
2071
|
+
if (this.retiringAccounts.has(owner))
|
|
2072
|
+
return false;
|
|
2073
|
+
const providerRecord = this.store.provider(provider);
|
|
2074
|
+
if (providerRecord === undefined)
|
|
2075
|
+
return false;
|
|
2076
|
+
if (owner.kind !== 'oauth' || owner.oauth === null)
|
|
2077
|
+
return false;
|
|
2078
|
+
return providerRecord.keys.some(candidate => candidate === owner);
|
|
2079
|
+
}
|
|
2080
|
+
/**
|
|
2081
|
+
* Round-5 review (R5-MF1): whether a live account row of `provider` still
|
|
2082
|
+
* names `ref` as its grant. The settled harvest can adopt the generation a
|
|
2083
|
+
* reconciliation just wrote while that reconciliation is still awaiting its
|
|
2084
|
+
* staleness probe — the row's `grantRef` then IS the record the cleanup
|
|
2085
|
+
* would delete, so deleting it destroys the account's own live grant.
|
|
2086
|
+
* A row already marked for removal does not count: its removal deletes the
|
|
2087
|
+
* refs it names, so keeping the record would strand it instead.
|
|
2088
|
+
*/
|
|
2089
|
+
grantRefServed(provider, ref) {
|
|
2090
|
+
const providerRecord = this.store.provider(provider);
|
|
2091
|
+
if (providerRecord === undefined)
|
|
2092
|
+
return false;
|
|
2093
|
+
return providerRecord.keys.some(candidate => candidate.kind === 'oauth'
|
|
2094
|
+
&& candidate.oauth !== null
|
|
2095
|
+
&& candidate.oauth.grantRef === ref
|
|
2096
|
+
&& !this.retiringAccounts.has(candidate));
|
|
2097
|
+
}
|
|
2098
|
+
/**
|
|
2099
|
+
* Delete a rotation record a reconciliation wrote but must not keep, unless
|
|
2100
|
+
* a live row already serves it (R5-MF1). Cleanup is best-effort with a
|
|
2101
|
+
* warning, exactly like the harvest-time superseded-record cleanup.
|
|
2102
|
+
*/
|
|
2103
|
+
async discardRotationRecord(records, provider, ref, failure) {
|
|
2104
|
+
if (this.grantRefServed(provider, ref))
|
|
2105
|
+
return;
|
|
2106
|
+
try {
|
|
2107
|
+
await records.deleteRecord(ref);
|
|
2108
|
+
}
|
|
2109
|
+
catch (error) {
|
|
2110
|
+
this.warn(`${failure} ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
/**
|
|
2114
|
+
* Round-9 P1 revalidation: the generation a reconciliation observed is
|
|
2115
|
+
* still the one the route holds, and the ownership claim this body made (if
|
|
2116
|
+
* any) is still the newest. Two independent writes can supersede it while
|
|
2117
|
+
* the plugin-scope write-back is in flight: pi-ai refreshing again (the
|
|
2118
|
+
* projection's refresh digest — the ref's identity — moves on) and an
|
|
2119
|
+
* activation or login claiming the projection (the ownership generation
|
|
2120
|
+
* moves). Committing the older ref in either case would move the row
|
|
2121
|
+
* backwards and strand the newer record outside manager state.
|
|
2122
|
+
*
|
|
2123
|
+
* A failed re-read cannot prove staleness, so it must not block the sync:
|
|
2124
|
+
* the commit then proceeds exactly as it did before this guard existed.
|
|
2125
|
+
*/
|
|
2126
|
+
async rotationStillCurrent(records, provider, key, rotatedRef, claimedGeneration) {
|
|
2127
|
+
if (claimedGeneration !== undefined
|
|
2128
|
+
&& (this.ownershipGenerations.get(provider) ?? 0) !== claimedGeneration)
|
|
2129
|
+
return false;
|
|
2130
|
+
try {
|
|
2131
|
+
const current = await records.readRecord(key);
|
|
2132
|
+
if (current === undefined || current.kind !== 'grant' || !isOAuthGrantPayload(current.payload))
|
|
2133
|
+
return false;
|
|
2134
|
+
const hash = createHash('sha256').update(current.payload.refresh).digest('hex').slice(0, 12);
|
|
2135
|
+
return accountRef(provider, hash) === rotatedRef;
|
|
2136
|
+
}
|
|
2137
|
+
catch (error) {
|
|
2138
|
+
this.warn(`rotation staleness check could not re-read ${key}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2139
|
+
return true;
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2142
|
+
/**
|
|
2143
|
+
* Make one account's grant payload the route's stored credential
|
|
2144
|
+
* (D1 activation projection).
|
|
2145
|
+
*
|
|
2146
|
+
* Round-8 finding I-000106 (must-fix): the decide snapshot used to be read
|
|
2147
|
+
* before the seam's lock, and `reconcileGrantRecord` runs OUTSIDE the
|
|
2148
|
+
* mutation queue — so a rotation echo could repoint `row.oauth.grantRef` and
|
|
2149
|
+
* delete the record being projected between the read and the write, leaving
|
|
2150
|
+
* the route on an older generation than the row names. The whole
|
|
2151
|
+
* read → project → claim section therefore runs on the provider's grant
|
|
2152
|
+
* chain ({@link scheduleGrantReconciliation}) — the same chain the
|
|
2153
|
+
* reconciler bodies use. A rotation committed before this body is observed
|
|
2154
|
+
* in `row.oauth.grantRef`; one that arrives while the body runs is queued
|
|
2155
|
+
* behind it and reconciles the generation this body projected. Either way
|
|
2156
|
+
* the projection holds exactly the generation the row names.
|
|
2157
|
+
*
|
|
2158
|
+
* The seam's per-key lock still serializes the read-decide-replace steps of
|
|
2159
|
+
* `modifyRecord` (credentials/src/index.ts:236-250) against other writers.
|
|
2160
|
+
* The account whose payload lands becomes the pool's active account and is
|
|
2161
|
+
* cascade-selectable again.
|
|
2162
|
+
*/
|
|
2163
|
+
activateAccount(provider, accountId) {
|
|
2164
|
+
return this.run(async () => {
|
|
2165
|
+
const providerRecord = this.store.provider(provider);
|
|
2166
|
+
if (providerRecord === undefined)
|
|
2167
|
+
throw new Error(`Provider "${provider}" is not managed`);
|
|
2168
|
+
const row = providerRecord.keys.find(candidate => candidate.id === accountId);
|
|
2169
|
+
if (row === undefined)
|
|
2170
|
+
throw new Error(`Account "${accountId}" not found on ${provider}`);
|
|
2171
|
+
if (row.kind !== 'oauth' || row.oauth === null)
|
|
2172
|
+
throw new Error(`Key "${accountId}" is not an OAuth account`);
|
|
2173
|
+
const records = this.credentialRecords();
|
|
2174
|
+
if (records === undefined)
|
|
2175
|
+
throw new Error('The credential store does not support records');
|
|
2176
|
+
await this.scheduleGrantReconciliation(provider, async () => {
|
|
2177
|
+
// Re-read the generation under the chain: a rotation that committed
|
|
2178
|
+
// before this body claimed its slot is already reflected here.
|
|
2179
|
+
const grantRef = row.oauth.grantRef;
|
|
2180
|
+
const stored = await records.readRecord(grantRef);
|
|
2181
|
+
if (stored === undefined || stored.kind !== 'grant' || !isOAuthGrantPayload(stored.payload)) {
|
|
2182
|
+
throw new Error(`The grant payload at ${grantRef} is missing or unusable; sign in to this account again`);
|
|
2183
|
+
}
|
|
2184
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
2185
|
+
await records.modifyRecord(`${OAUTH_GRANT_RECORD_SCOPE}/${provider}`, () => Promise.resolve({ kind: 'grant', payload: structuredClone(stored.payload) }));
|
|
2186
|
+
this.claimProjectionOwner(provider, row.id);
|
|
2187
|
+
});
|
|
2188
|
+
row.valueMissing = false;
|
|
2189
|
+
// Round-5 finding M1: manual recovery ends the cooling window too, the
|
|
2190
|
+
// same reset a successful re-authentication applies. A demoted account
|
|
2191
|
+
// is re-activated while its stale cooldown still sits on the row, which
|
|
2192
|
+
// would render the freshly projected grant as cooling and keep it out
|
|
2193
|
+
// of the healthy counts until the timer expires. Clearing the window
|
|
2194
|
+
// also resets escalation: `applyFailureCooldown` derives the next level
|
|
2195
|
+
// from `cooldownUntil` only (core.ts:168-174).
|
|
2196
|
+
row.cooldownUntil = 0;
|
|
2197
|
+
this.store.touch();
|
|
2198
|
+
// Remote methods hand back the refreshed state view (addKey/deleteKey
|
|
2199
|
+
// precedent), so the client updates its snapshot in one round trip.
|
|
2200
|
+
return this.buildState();
|
|
2201
|
+
});
|
|
2202
|
+
}
|
|
2203
|
+
/**
|
|
2204
|
+
* Demote the cascade to the key pool by deleting the projection record:
|
|
2205
|
+
* pi-ai's per-request resolution prefers the stored credential and only
|
|
2206
|
+
* falls through to the broker `apiKeyEnv` when none is stored
|
|
2207
|
+
* (research, resolve.js:26-55).
|
|
2208
|
+
*
|
|
2209
|
+
* Recovery is deliberately manual: the cooled account stays demoted until
|
|
2210
|
+
* the user re-activates it or signs in again. An automatic re-activation
|
|
2211
|
+
* on cooldown expiry would replay the failing grant every request; the
|
|
2212
|
+
* user (who can see whether the account recovered upstream) is the
|
|
2213
|
+
* arbiter. `fallbackToKeys` marks every account row value-missing so the
|
|
2214
|
+
* live arbitration — projection presence plus probe fail-skip over a
|
|
2215
|
+
* kind-agnostic cursor ring — keeps serving only real candidates.
|
|
2216
|
+
*
|
|
2217
|
+
* `accountId` names the row whose failure triggers this demotion (R7-MF).
|
|
2218
|
+
* It is resolved when the queued body runs, so the demotion deletes only a
|
|
2219
|
+
* projection that still belongs to THAT row: activating another account
|
|
2220
|
+
* while the failing request is in flight must not lose the new account's
|
|
2221
|
+
* healthy projection.
|
|
2222
|
+
*
|
|
2223
|
+
* R14-M3: the targetless form (omitted `accountId`) is TEST-ONLY. It
|
|
2224
|
+
* resolves the provider's active account when the queued body runs — the
|
|
2225
|
+
* same execution-time race R7-MF removed from the failure path — so it is
|
|
2226
|
+
* kept only because the tests drive the demotion without a failure event.
|
|
2227
|
+
* Production callers must name the row; the failure path always passes
|
|
2228
|
+
* `key.id`.
|
|
2229
|
+
*/
|
|
2230
|
+
fallbackToKeys(provider, accountId) {
|
|
2231
|
+
return this.run(() => this.fallbackToKeysUnqueued(provider, accountId));
|
|
2232
|
+
}
|
|
2233
|
+
/**
|
|
2234
|
+
* Atomic decide-and-delete of the `llm-pi-ai/<provider>` projection for one
|
|
2235
|
+
* row (round-3 finding 1): the payload compare runs INSIDE the credential
|
|
2236
|
+
* seam's per-key lock — `modifyRecord` hands `mutate` the record as it
|
|
2237
|
+
* stands while the write is exclusive, so a native sign-in that already
|
|
2238
|
+
* replaced the projection is visible here and the delete is skipped. The
|
|
2239
|
+
* seam exposes no delete-inside-modify primitive: a `mutate` may only
|
|
2240
|
+
* return a replacement or `undefined` (leave untouched), and calling
|
|
2241
|
+
* `deleteRecord` inside the mutate would deadlock the provider's own
|
|
2242
|
+
* serialized operation queue (credentials-local `enqueue`). The closest
|
|
2243
|
+
* equivalent — the exact primitives this method uses — are a decide-only
|
|
2244
|
+
* modify (returns `undefined`, no write) and a second pre-delete re-read
|
|
2245
|
+
* against the decided-against snapshot ({@link projectionStillMatches},
|
|
2246
|
+
* F2 step-1 hardening): a native writer landing between the decide and
|
|
2247
|
+
* the delete is therefore visible before the delete runs, and its grant
|
|
2248
|
+
* survives. The residual window is the sub-microtask gap between the
|
|
2249
|
+
* re-read releasing the lock and the delete's own queue entry; a native
|
|
2250
|
+
* writer landing there is still deleted, because closing that gap needs a
|
|
2251
|
+
* compare-and-delete primitive the seam does not expose (store issue
|
|
2252
|
+
* I-000090). `tests/host.test.ts` ("a native grant landing after the final
|
|
2253
|
+
* comparison is still lost") pins that exact outcome so the seam fix has a
|
|
2254
|
+
* test to flip.
|
|
2255
|
+
*
|
|
2256
|
+
* This is the only delete authority: a natively replaced grant — another
|
|
2257
|
+
* account signed in, a native api-key record, an unrecognized payload —
|
|
2258
|
+
* never matches and must survive the demotion of a stale row, and the
|
|
2259
|
+
* async hydration window must not decide ownership (round-2 review,
|
|
2260
|
+
* steps 1–2).
|
|
2261
|
+
*/
|
|
2262
|
+
async decideAndDeleteProjection(provider, row) {
|
|
2263
|
+
if (row.kind !== 'oauth' || row.oauth === null)
|
|
2264
|
+
return false;
|
|
2265
|
+
const records = this.credentialRecords();
|
|
2266
|
+
if (records === undefined)
|
|
2267
|
+
return false;
|
|
2268
|
+
const accounts = this.store.provider(provider)?.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null) ?? [];
|
|
2269
|
+
const projectionKey = `${OAUTH_GRANT_RECORD_SCOPE}/${provider}`;
|
|
2270
|
+
let owns = false;
|
|
2271
|
+
let decidedAgainst;
|
|
2272
|
+
await records.modifyRecord(projectionKey, async (current) => {
|
|
2273
|
+
owns = current !== undefined && current.kind === 'grant'
|
|
2274
|
+
&& isOAuthGrantPayload(current.payload)
|
|
2275
|
+
&& matchOwningAccount(provider, accounts, current.payload)?.id === row.id;
|
|
2276
|
+
// The decided-against payload snapshot (in-memory JSON, never logged
|
|
2277
|
+
// or persisted): the pre-delete re-read below skips the removal when
|
|
2278
|
+
// the live record no longer holds exactly this content.
|
|
2279
|
+
if (owns && current !== undefined)
|
|
2280
|
+
decidedAgainst = JSON.stringify(current.payload);
|
|
2281
|
+
// Decide only: the removal goes through deleteRecord below, outside
|
|
2282
|
+
// the mutate (see the primitive note above).
|
|
2283
|
+
return undefined;
|
|
2284
|
+
});
|
|
2285
|
+
if (!owns)
|
|
2286
|
+
return false;
|
|
2287
|
+
// F2 (step-1 removal window): re-read the live record under the seam's
|
|
2288
|
+
// per-key lock immediately before the delete and skip when a native
|
|
2289
|
+
// grant landed between the decide and the delete. Residual window,
|
|
2290
|
+
// documented honestly: the seam exposes no delete-inside-modify
|
|
2291
|
+
// primitive (a mutate may only return a replacement or undefined, and
|
|
2292
|
+
// deleteRecord inside the mutate would deadlock the provider's
|
|
2293
|
+
// serialized operation queue), so a native writer landing between THIS
|
|
2294
|
+
// re-read releasing the lock and the delete's own queue entry can still
|
|
2295
|
+
// lose its grant. That writer is the next writer's grant, and the
|
|
2296
|
+
// ownership oracle re-identifies it on the record-updated echo.
|
|
2297
|
+
if (!await this.projectionStillMatches(records, projectionKey, decidedAgainst))
|
|
2298
|
+
return false;
|
|
2299
|
+
// Mark BEFORE the delete: the deletion's echo handler reads this
|
|
2300
|
+
// marker across its own awaited record re-read, whenever that
|
|
2301
|
+
// continuation resumes. The TTL window — not scheduling order —
|
|
2302
|
+
// decides how long the marker stays trustworthy.
|
|
2303
|
+
const markedAt = Date.now();
|
|
2304
|
+
this.intentionalProjectionRemovals.set(provider, markedAt);
|
|
2305
|
+
try {
|
|
2306
|
+
await records.deleteRecord(projectionKey);
|
|
2307
|
+
}
|
|
2308
|
+
catch (error) {
|
|
2309
|
+
// R7-SF2: a delete that threw removed nothing, so no own-removal echo
|
|
2310
|
+
// is coming. Revoke this call's marker before rethrowing — left in
|
|
2311
|
+
// place it would consume a genuine out-of-band removal as the
|
|
2312
|
+
// manager's own for the whole TTL, hiding the warning, the missing
|
|
2313
|
+
// mark, and the stale owner cleanup. The identity check keeps a newer
|
|
2314
|
+
// marker (there is none on the serialized mutation queue) untouched.
|
|
2315
|
+
// R14-SF2: seam assumption — the throwing delete may have removed the
|
|
2316
|
+
// record before throwing; the later echo is then read as external
|
|
2317
|
+
// (warning + stale-owner drop), the conservative direction over
|
|
2318
|
+
// masking a genuine out-of-band removal.
|
|
2319
|
+
if (this.intentionalProjectionRemovals.get(provider) === markedAt) {
|
|
2320
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
2321
|
+
}
|
|
2322
|
+
throw error;
|
|
2323
|
+
}
|
|
2324
|
+
return true;
|
|
2325
|
+
}
|
|
2326
|
+
/**
|
|
2327
|
+
* F2 pre-delete re-read, shared by the demotion and harvest-compensation
|
|
2328
|
+
* decide-then-delete pairs: `true` while the live projection still holds
|
|
2329
|
+
* exactly the decided-against payload. Runs inside `modifyRecord`'s
|
|
2330
|
+
* per-key lock (the same primitive as the decide), so a native writer
|
|
2331
|
+
* that already replaced the record is visible here and the delete is
|
|
2332
|
+
* skipped instead of destroying the foreign grant. The lock is released
|
|
2333
|
+
* before the delete, so a writer landing in that final gap is still lost:
|
|
2334
|
+
* closing it needs the seam's compare-and-delete (store issue I-000090).
|
|
2335
|
+
*/
|
|
2336
|
+
async projectionStillMatches(records, projectionKey, decidedAgainst) {
|
|
2337
|
+
if (decidedAgainst === undefined)
|
|
2338
|
+
return false;
|
|
2339
|
+
let liveMatches = false;
|
|
2340
|
+
await records.modifyRecord(projectionKey, async (current) => {
|
|
2341
|
+
liveMatches = current !== undefined && current.kind === 'grant'
|
|
2342
|
+
&& JSON.stringify(current.payload) === decidedAgainst;
|
|
2343
|
+
return undefined;
|
|
2344
|
+
});
|
|
2345
|
+
return liveMatches;
|
|
2346
|
+
}
|
|
2347
|
+
/**
|
|
2348
|
+
* The fallback body; callers own the mutation queue. `target` narrows the
|
|
2349
|
+
* ownership check to one explicit row: removal call sites pass the removed
|
|
2350
|
+
* row object, the failure path passes the failing row's id (R7-MF) — an
|
|
2351
|
+
* id is re-resolved here because the failure's demotion is queued and the
|
|
2352
|
+
* row object may be detached by the time it runs. Without a target the
|
|
2353
|
+
* provider's active account decides.
|
|
2354
|
+
*/
|
|
2355
|
+
async fallbackToKeysUnqueued(provider, target) {
|
|
2356
|
+
const providerRecord = this.store.provider(provider);
|
|
2357
|
+
if (providerRecord === undefined)
|
|
2358
|
+
return;
|
|
2359
|
+
const activeId = this.activeAccounts.get(provider);
|
|
2360
|
+
const resolveOAuthRow = (candidate) => candidate !== undefined && candidate.kind === 'oauth' && candidate.oauth !== null ? candidate : undefined;
|
|
2361
|
+
const active = resolveOAuthRow(typeof target === 'string'
|
|
2362
|
+
? providerRecord.keys.find(candidate => candidate.id === target)
|
|
2363
|
+
: target ?? (activeId === undefined
|
|
2364
|
+
? undefined
|
|
2365
|
+
: providerRecord.keys.find(candidate => candidate.id === activeId)));
|
|
2366
|
+
// Ownership-verified delete: demote the projection only while it still
|
|
2367
|
+
// belongs to this row. The compare runs inside the seam's per-key lock
|
|
2368
|
+
// and the delete follows it (decideAndDeleteProjection, round-3
|
|
2369
|
+
// finding 1) — both inside this mutation.
|
|
2370
|
+
const ownsProjection = active !== undefined && await this.decideAndDeleteProjection(provider, active);
|
|
2371
|
+
if (ownsProjection) {
|
|
2372
|
+
// Full cascade demotion: no pool row serves the route anymore, so
|
|
2373
|
+
// every account row is demoted until re-activation. The owner entry
|
|
2374
|
+
// must go too (round-3 finding 4): oauthOwnerFor otherwise keeps
|
|
2375
|
+
// naming this row, so the client renders "Serving provider" next to
|
|
2376
|
+
// the missing pill and leaves "Set active" disabled — the documented
|
|
2377
|
+
// re-activation path unreachable until a restart or a new sign-in.
|
|
2378
|
+
this.activeAccounts.delete(provider);
|
|
2379
|
+
for (const row of providerRecord.keys) {
|
|
2380
|
+
if (row.kind === 'oauth')
|
|
2381
|
+
row.valueMissing = true;
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
else if (active !== undefined && activeId === active.id) {
|
|
2385
|
+
// The projection is not ours anymore (native replacement): keep the
|
|
2386
|
+
// stale owner entry from routing attribution or demotion at this row.
|
|
2387
|
+
this.activeAccounts.delete(provider);
|
|
2388
|
+
}
|
|
2389
|
+
this.store.touch();
|
|
2390
|
+
}
|
|
2391
|
+
/**
|
|
2392
|
+
* Drop one OAuth account's plugin-scope grant payload and — only while
|
|
2393
|
+
* this row still owns the live projection — demote the cascade to the key
|
|
2394
|
+
* pool first, so the route falls through to the API-key pool instead of
|
|
2395
|
+
* serving a grant whose owning row is gone. The demotion re-reads
|
|
2396
|
+
* `llm-pi-ai/<provider>` inside this mutation and runs it through the
|
|
2397
|
+
* ownership oracle ({@link matchOwningAccount}); a natively replaced
|
|
2398
|
+
* projection (another account signed in, a native api-key record) is left
|
|
2399
|
+
* untouched. Non-oauth rows are a no-op; every OAuth-removal call site
|
|
2400
|
+
* goes through this helper.
|
|
2401
|
+
*/
|
|
2402
|
+
async removeOAuthGrant(provider, row) {
|
|
2403
|
+
if (row.kind !== 'oauth' || row.oauth === null)
|
|
2404
|
+
return;
|
|
2405
|
+
// C6-1/I1: mark the row BEFORE the first awaited delete. A release
|
|
2406
|
+
// deletes every row's credential before it deletes the provider record
|
|
2407
|
+
// and never splices the rows, so from the first delete onward the row
|
|
2408
|
+
// object is no longer a truthful "still serving" signal; the reconciler
|
|
2409
|
+
// consults this mark instead (see accountRowStillCurrent).
|
|
2410
|
+
this.retiringAccounts.add(row);
|
|
2411
|
+
try {
|
|
2412
|
+
await this.fallbackToKeysUnqueued(provider, row);
|
|
2413
|
+
const records = this.credentialRecords();
|
|
2414
|
+
if (records !== undefined)
|
|
2415
|
+
await records.deleteRecord(row.oauth.grantRef);
|
|
2416
|
+
}
|
|
2417
|
+
catch (error) {
|
|
2418
|
+
// Round-7 minor: a throwing awaited step aborts the removal — the
|
|
2419
|
+
// callers rethrow before splicing the row or releasing the provider —
|
|
2420
|
+
// so the row is still live and still managed. Clear the mark or the
|
|
2421
|
+
// row would stay permanently retired and its rotation echoes would be
|
|
2422
|
+
// skipped until a restart. A retried removal re-marks before its own
|
|
2423
|
+
// first delete, and a successful removal never reaches this path.
|
|
2424
|
+
this.retiringAccounts.delete(row);
|
|
2425
|
+
throw error;
|
|
2426
|
+
}
|
|
2427
|
+
}
|
|
2428
|
+
/**
|
|
2429
|
+
* Drop one account: its plugin-scope grant payload, its row, and — when
|
|
2430
|
+
* it was the active projection — the projection itself, so the route
|
|
2431
|
+
* falls back to the key pool.
|
|
2432
|
+
*/
|
|
2433
|
+
removeAccount(provider, accountId) {
|
|
2434
|
+
return this.run(async () => {
|
|
2435
|
+
const providerRecord = this.store.provider(provider);
|
|
2436
|
+
if (providerRecord === undefined)
|
|
2437
|
+
throw new Error(`Provider "${provider}" is not managed`);
|
|
2438
|
+
const index = providerRecord.keys.findIndex(candidate => candidate.id === accountId);
|
|
2439
|
+
if (index < 0)
|
|
2440
|
+
throw new Error(`Account "${accountId}" not found on ${provider}`);
|
|
2441
|
+
const row = providerRecord.keys[index];
|
|
2442
|
+
if (row.kind !== 'oauth' || row.oauth === null)
|
|
2443
|
+
throw new Error(`Key "${accountId}" is not an OAuth account; use deleteKey`);
|
|
2444
|
+
await this.removeOAuthGrant(provider, row);
|
|
2445
|
+
providerRecord.keys.splice(index, 1);
|
|
2446
|
+
this.store.clearBindings((_key, binding) => binding.keyId === row.id);
|
|
2447
|
+
this.store.touch();
|
|
2448
|
+
return this.buildState();
|
|
2449
|
+
});
|
|
2450
|
+
}
|
|
2451
|
+
/** Install the broker layer on one credential provider instance. */
|
|
2452
|
+
attachCredentials(credentials) {
|
|
2453
|
+
const raw = credentials[ORIGINAL] ?? credentials;
|
|
2454
|
+
if (this.credentialWrappers.has(raw))
|
|
2455
|
+
return;
|
|
2456
|
+
const original = {
|
|
2457
|
+
resolve: raw.resolve,
|
|
2458
|
+
describe: raw.describe,
|
|
2459
|
+
set: raw.set,
|
|
2460
|
+
unset: raw.unset,
|
|
2461
|
+
};
|
|
2462
|
+
const service = this;
|
|
2463
|
+
const wrappers = {
|
|
2464
|
+
resolve: async (ref) => {
|
|
2465
|
+
const owner = service.brokerOwners.get(ref);
|
|
2466
|
+
if (owner === undefined)
|
|
2467
|
+
return original.resolve.call(raw, ref);
|
|
2468
|
+
const active = service.als.getStore();
|
|
2469
|
+
if (active !== undefined && active.provider === owner) {
|
|
2470
|
+
if (typeof active.keyValue === 'string' && active.keyValue.length > 0) {
|
|
2471
|
+
active.keyUsed = true;
|
|
2472
|
+
return { value: active.keyValue, source: 'keys-manager' };
|
|
2473
|
+
}
|
|
2474
|
+
return undefined;
|
|
2475
|
+
}
|
|
2476
|
+
// Listing models has no request scope; answer from any stored key
|
|
2477
|
+
// without moving the session cursor or changing health.
|
|
2478
|
+
const provider = service.store.provider(owner);
|
|
2479
|
+
if (provider !== undefined) {
|
|
2480
|
+
const now = Date.now();
|
|
2481
|
+
for (const key of provider.keys) {
|
|
2482
|
+
if (key.ref.length === 0 || !isSelectable(key, now))
|
|
2483
|
+
continue;
|
|
2484
|
+
const hit = await original.resolve.call(raw, key.ref);
|
|
2485
|
+
if (hit !== undefined && typeof hit.value === 'string' && hit.value.length > 0) {
|
|
2486
|
+
return { ...hit, source: 'keys-manager' };
|
|
2487
|
+
}
|
|
2488
|
+
}
|
|
2489
|
+
}
|
|
2490
|
+
return undefined;
|
|
2491
|
+
},
|
|
2492
|
+
describe: async (ref) => {
|
|
2493
|
+
const owner = service.brokerOwners.get(ref);
|
|
2494
|
+
if (owner === undefined)
|
|
2495
|
+
return original.describe.call(raw, ref);
|
|
2496
|
+
const provider = service.store.provider(owner);
|
|
2497
|
+
const configured = provider !== undefined && provider.keys.length > 0;
|
|
2498
|
+
return {
|
|
2499
|
+
configured,
|
|
2500
|
+
...(configured ? { source: 'keys-manager' } : {}),
|
|
2501
|
+
writable: false,
|
|
2502
|
+
};
|
|
2503
|
+
},
|
|
2504
|
+
set: async (ref, value) => {
|
|
2505
|
+
if (service.brokerOwners.has(ref)) {
|
|
2506
|
+
throw new Error('This provider key is managed by Keys Manager; edit it on the plugin detail page');
|
|
2507
|
+
}
|
|
2508
|
+
return original.set.call(raw, ref, value);
|
|
2509
|
+
},
|
|
2510
|
+
unset: async (ref) => {
|
|
2511
|
+
if (service.brokerOwners.has(ref)) {
|
|
2512
|
+
throw new Error('This provider key is managed by Keys Manager; edit it on the plugin detail page');
|
|
2513
|
+
}
|
|
2514
|
+
return original.unset.call(raw, ref);
|
|
2515
|
+
},
|
|
2516
|
+
};
|
|
2517
|
+
raw.resolve = wrappers.resolve;
|
|
2518
|
+
raw.describe = wrappers.describe;
|
|
2519
|
+
raw.set = wrappers.set;
|
|
2520
|
+
raw.unset = wrappers.unset;
|
|
2521
|
+
this.credentialOriginals.set(raw, original);
|
|
2522
|
+
this.credentialWrappers.set(raw, wrappers);
|
|
2523
|
+
if (this.rawCredentials === undefined)
|
|
2524
|
+
this.rawCredentials = raw;
|
|
2525
|
+
}
|
|
2526
|
+
/** Restore the credential provider methods when our fiber unloads. */
|
|
2527
|
+
detachCredentials(credentials) {
|
|
2528
|
+
const raw = credentials[ORIGINAL] ?? credentials;
|
|
2529
|
+
const wrappers = this.credentialWrappers.get(raw);
|
|
2530
|
+
const original = this.credentialOriginals.get(raw);
|
|
2531
|
+
if (wrappers === undefined || original === undefined)
|
|
2532
|
+
return;
|
|
2533
|
+
if (raw.resolve === wrappers.resolve)
|
|
2534
|
+
raw.resolve = original.resolve;
|
|
2535
|
+
if (raw.describe === wrappers.describe)
|
|
2536
|
+
raw.describe = original.describe;
|
|
2537
|
+
if (raw.set === wrappers.set)
|
|
2538
|
+
raw.set = original.set;
|
|
2539
|
+
if (raw.unset === wrappers.unset)
|
|
2540
|
+
raw.unset = original.unset;
|
|
2541
|
+
this.credentialWrappers.delete(raw);
|
|
2542
|
+
this.credentialOriginals.delete(raw);
|
|
2543
|
+
if (this.rawCredentials === raw)
|
|
2544
|
+
this.rawCredentials = undefined;
|
|
2545
|
+
}
|
|
2546
|
+
/** Resolve a credential through the unwrapped provider. */
|
|
2547
|
+
async resolveOriginal(ref) {
|
|
2548
|
+
if (!isCredentialRefName(ref))
|
|
2549
|
+
return undefined;
|
|
2550
|
+
const raw = this.credentialsRaw();
|
|
2551
|
+
if (raw === undefined)
|
|
2552
|
+
return undefined;
|
|
2553
|
+
const original = this.credentialOriginals.get(raw);
|
|
2554
|
+
const resolve = original?.resolve ?? raw.resolve;
|
|
2555
|
+
try {
|
|
2556
|
+
return await resolve.call(raw, ref);
|
|
2557
|
+
}
|
|
2558
|
+
catch {
|
|
2559
|
+
return undefined;
|
|
2560
|
+
}
|
|
2561
|
+
}
|
|
2562
|
+
/**
|
|
2563
|
+
* Probe one credential reference while distinguishing absence from a
|
|
2564
|
+
* transient credential-store failure. Selection must never persist a
|
|
2565
|
+
* permanent `valueMissing` marker for an error.
|
|
2566
|
+
*/
|
|
2567
|
+
async probeKey(ref) {
|
|
2568
|
+
if (!isCredentialRefName(ref))
|
|
2569
|
+
return { missing: false, failed: true };
|
|
2570
|
+
const raw = this.credentialsRaw();
|
|
2571
|
+
if (raw === undefined)
|
|
2572
|
+
return { missing: false, failed: true };
|
|
2573
|
+
const original = this.credentialOriginals.get(raw);
|
|
2574
|
+
const resolve = original?.resolve ?? raw.resolve;
|
|
2575
|
+
try {
|
|
2576
|
+
const hit = await resolve.call(raw, ref);
|
|
2577
|
+
if (hit === undefined || typeof hit.value !== 'string' || hit.value.length === 0) {
|
|
2578
|
+
return { missing: true, failed: false };
|
|
2579
|
+
}
|
|
2580
|
+
return { hit, missing: false, failed: false };
|
|
2581
|
+
}
|
|
2582
|
+
catch {
|
|
2583
|
+
return { missing: false, failed: true };
|
|
2584
|
+
}
|
|
2585
|
+
}
|
|
2586
|
+
/** Write one key literal through the unwrapped provider. */
|
|
2587
|
+
async setOriginal(ref, value) {
|
|
2588
|
+
const raw = this.credentialsRaw();
|
|
2589
|
+
if (raw === undefined)
|
|
2590
|
+
throw new Error('The credential store is unavailable');
|
|
2591
|
+
const original = this.credentialOriginals.get(raw);
|
|
2592
|
+
const set = original?.set ?? raw.set;
|
|
2593
|
+
await set.call(raw, ref, value);
|
|
2594
|
+
}
|
|
2595
|
+
/**
|
|
2596
|
+
* Remove one key literal through the unwrapped provider. A rejected unset is
|
|
2597
|
+
* only ignored when the credential is confirmed absent; a transient or
|
|
2598
|
+
* permission failure must keep the metadata so a later retry can finish.
|
|
2599
|
+
*/
|
|
2600
|
+
async unsetOriginal(ref) {
|
|
2601
|
+
if (!isCredentialRefName(ref))
|
|
2602
|
+
return;
|
|
2603
|
+
const raw = this.credentialsRaw();
|
|
2604
|
+
if (raw === undefined)
|
|
2605
|
+
throw new Error('The credential store is unavailable');
|
|
2606
|
+
const original = this.credentialOriginals.get(raw);
|
|
2607
|
+
const unset = original?.unset ?? raw.unset;
|
|
2608
|
+
try {
|
|
2609
|
+
await unset.call(raw, ref);
|
|
2610
|
+
}
|
|
2611
|
+
catch (error) {
|
|
2612
|
+
const probe = await this.probeKey(ref);
|
|
2613
|
+
if (probe.missing)
|
|
2614
|
+
return;
|
|
2615
|
+
throw new Error(`Could not remove credential ${ref}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2616
|
+
}
|
|
2617
|
+
}
|
|
2618
|
+
/** All settings descriptors, with an explicit unreadable state. */
|
|
2619
|
+
describeSettings() {
|
|
2620
|
+
const settings = this.settingsService();
|
|
2621
|
+
if (settings === undefined)
|
|
2622
|
+
return { writable: false, namespaces: [], readable: false };
|
|
2623
|
+
try {
|
|
2624
|
+
return { writable: settings.writable !== false, namespaces: settings.describe({ redactSecrets: true }), readable: true };
|
|
2625
|
+
}
|
|
2626
|
+
catch (error) {
|
|
2627
|
+
this.warn(`settings describe failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
2628
|
+
return { writable: false, namespaces: [], readable: false };
|
|
2629
|
+
}
|
|
2630
|
+
}
|
|
2631
|
+
/** Find one live settings descriptor by entry id. */
|
|
2632
|
+
namespace(descriptor, ns) {
|
|
2633
|
+
return descriptor.namespaces.find(entry => entry.ns === ns);
|
|
2634
|
+
}
|
|
2635
|
+
/** Apply one path-addressed settings write through the live revision. */
|
|
2636
|
+
async mutateSettings(ns, ops) {
|
|
2637
|
+
const settings = this.settingsService();
|
|
2638
|
+
if (settings === undefined)
|
|
2639
|
+
throw new Error('Settings are unavailable in this composition');
|
|
2640
|
+
if (settings.writable === false)
|
|
2641
|
+
throw new Error('The active profile is read-only');
|
|
2642
|
+
let descriptor;
|
|
2643
|
+
try {
|
|
2644
|
+
descriptor = settings.describe({ redactSecrets: true }).find(entry => entry.ns === ns);
|
|
2645
|
+
}
|
|
2646
|
+
catch (error) {
|
|
2647
|
+
throw new Error(`Cannot read settings: ${error instanceof Error ? error.message : String(error)}`);
|
|
2648
|
+
}
|
|
2649
|
+
if (descriptor === undefined)
|
|
2650
|
+
throw new Error(`Settings namespace "${ns}" is not mounted`);
|
|
2651
|
+
await settings.mutate(ns, ops, descriptor.revision);
|
|
2652
|
+
}
|
|
2653
|
+
/** Prune expired bindings at most once a minute, without a restart. */
|
|
2654
|
+
pruneBindings(now) {
|
|
2655
|
+
if (now - this.lastBindingPruneAt < 60_000)
|
|
2656
|
+
return;
|
|
2657
|
+
this.lastBindingPruneAt = now;
|
|
2658
|
+
this.store.pruneBindings(now);
|
|
2659
|
+
this.pruneAttempts(now);
|
|
2660
|
+
}
|
|
2661
|
+
/** Build the complete UI state. */
|
|
2662
|
+
buildState() {
|
|
2663
|
+
const now = Date.now();
|
|
2664
|
+
this.pruneBindings(now);
|
|
2665
|
+
const descriptor = this.describeSettings();
|
|
2666
|
+
const directory = this.ctx.llm.listConfigurableProviders();
|
|
2667
|
+
const active = new Set(this.ctx.llm.listProviders().map(provider => provider.id));
|
|
2668
|
+
const views = [];
|
|
2669
|
+
const seen = new Set();
|
|
2670
|
+
for (const entry of directory) {
|
|
2671
|
+
if (entry.provider === 'deepseek-account' || entry.settingsNs === 'deepseek-account' || entry.provider.startsWith('deepseek-account'))
|
|
2672
|
+
continue;
|
|
2673
|
+
const namespace = this.namespace(descriptor, entry.settingsNs);
|
|
2674
|
+
const profile = namespace === undefined ? undefined : asProfile(getPath(namespace.value, entry.settingsPath));
|
|
2675
|
+
const managed = this.store.provider(entry.provider);
|
|
2676
|
+
const extra = EXTRA_PROVIDER_MAP.get(entry.provider);
|
|
2677
|
+
const oauthFlow = this.oauthFlowAvailable(entry.provider);
|
|
2678
|
+
const caps = capabilityOf(entry, profile, oauthFlow);
|
|
2679
|
+
const support = { supported: caps.keys, ...(caps.reason === undefined ? {} : { reason: caps.reason }) };
|
|
2680
|
+
const accountSupport = accountSupportOf(entry, oauthFlow);
|
|
2681
|
+
const apiKeyEnv = profile !== undefined && typeof profile.apiKeyEnv === 'string' ? profile.apiKeyEnv : null;
|
|
2682
|
+
const binding = managed === undefined ? undefined : this.bindingState(managed, descriptor);
|
|
2683
|
+
const bindingHealthy = binding === 'healthy' ? true : binding === 'mismatch' ? false : undefined;
|
|
2684
|
+
const projectedAccountId = managed === undefined ? null : this.oauthOwnerFor(managed.provider);
|
|
2685
|
+
const keys = managed !== undefined ? managed.keys.map(key => publicKey(key, now, projectedAccountId)) : [];
|
|
2686
|
+
const warning = binding === 'mismatch'
|
|
2687
|
+
? 'The provider profile no longer points at Keys Manager. Re-adopt it after releasing it, or repair the provider in Models.'
|
|
2688
|
+
: undefined;
|
|
2689
|
+
const models = profile !== undefined ? normalizeModels(profile.models) : extra?.models ?? managed?.models ?? [];
|
|
2690
|
+
views.push({
|
|
2691
|
+
provider: entry.provider,
|
|
2692
|
+
displayName: extra?.name ?? entry.displayName ?? entry.provider,
|
|
2693
|
+
source: extra !== undefined ? 'extra' : 'dsh',
|
|
2694
|
+
origin: extra !== undefined ? 'plugin' : entry.declared === true ? 'custom' : 'dsh',
|
|
2695
|
+
managed: managed !== undefined,
|
|
2696
|
+
createdProfile: managed?.createdProfile ?? false,
|
|
2697
|
+
configured: active.has(entry.provider) || profile !== undefined,
|
|
2698
|
+
active: active.has(entry.provider),
|
|
2699
|
+
supported: support.supported,
|
|
2700
|
+
...(support.reason === undefined ? {} : { reason: support.reason }),
|
|
2701
|
+
accountSupported: accountSupport.supported,
|
|
2702
|
+
settingsNs: entry.settingsNs,
|
|
2703
|
+
settingsPath: [...entry.settingsPath],
|
|
2704
|
+
apiKeyEnv,
|
|
2705
|
+
brokerRef: managed?.brokerRef ?? null,
|
|
2706
|
+
...(bindingHealthy === undefined ? {} : { bindingHealthy }),
|
|
2707
|
+
extra: extra !== undefined,
|
|
2708
|
+
extraId: extra?.id ?? managed?.extraId ?? null,
|
|
2709
|
+
api: profile?.api ?? extra?.api ?? managed?.api ?? null,
|
|
2710
|
+
baseURL: profile?.baseURL ?? extra?.baseURL ?? entry.defaultBaseURL ?? managed?.baseURL ?? null,
|
|
2711
|
+
env: [...(extra?.env ?? entry.nativeAuth?.env ?? [])],
|
|
2712
|
+
defaultBaseURL: entry.defaultBaseURL ?? extra?.baseURL ?? null,
|
|
2713
|
+
nativeAuth: entry.nativeAuth === undefined ? null : { env: [...entry.nativeAuth.env], oauth: entry.nativeAuth.oauth },
|
|
2714
|
+
models,
|
|
2715
|
+
keyCount: keys.length,
|
|
2716
|
+
healthyKeyCount: keys.filter(key => key.status === 'active').length,
|
|
2717
|
+
status: managed !== undefined ? statusOfProvider(managed.keys, now) : 'unmanaged',
|
|
2718
|
+
keys,
|
|
2719
|
+
...(warning === undefined ? {} : { warning }),
|
|
2720
|
+
});
|
|
2721
|
+
seen.add(entry.provider);
|
|
2722
|
+
}
|
|
2723
|
+
for (const extra of EXTRA_PROVIDERS) {
|
|
2724
|
+
if (seen.has(extra.id))
|
|
2725
|
+
continue;
|
|
2726
|
+
const managed = this.store.provider(extra.id);
|
|
2727
|
+
const binding = managed === undefined ? undefined : this.bindingState(managed, descriptor);
|
|
2728
|
+
const projectedAccountId = managed === undefined ? null : this.oauthOwnerFor(managed.provider);
|
|
2729
|
+
const keys = managed !== undefined ? managed.keys.map(key => publicKey(key, now, projectedAccountId)) : [];
|
|
2730
|
+
views.push({
|
|
2731
|
+
provider: extra.id,
|
|
2732
|
+
displayName: extra.name,
|
|
2733
|
+
source: 'extra',
|
|
2734
|
+
origin: 'plugin',
|
|
2735
|
+
managed: managed !== undefined,
|
|
2736
|
+
createdProfile: managed?.createdProfile ?? false,
|
|
2737
|
+
configured: managed !== undefined,
|
|
2738
|
+
active: false,
|
|
2739
|
+
supported: true,
|
|
2740
|
+
// Extra providers are key-pool routes; they offer no OAuth flow.
|
|
2741
|
+
accountSupported: false,
|
|
2742
|
+
settingsNs: 'llm-pi-ai',
|
|
2743
|
+
settingsPath: ['providers', extra.id],
|
|
2744
|
+
apiKeyEnv: managed?.brokerRef ?? null,
|
|
2745
|
+
brokerRef: managed?.brokerRef ?? null,
|
|
2746
|
+
...(binding === 'healthy' ? { bindingHealthy: true } : binding === 'mismatch' ? { bindingHealthy: false } : {}),
|
|
2747
|
+
extra: true,
|
|
2748
|
+
extraId: extra.id,
|
|
2749
|
+
api: extra.api,
|
|
2750
|
+
baseURL: extra.baseURL,
|
|
2751
|
+
env: [...extra.env],
|
|
2752
|
+
defaultBaseURL: extra.baseURL,
|
|
2753
|
+
nativeAuth: null,
|
|
2754
|
+
models: extra.models,
|
|
2755
|
+
keyCount: keys.length,
|
|
2756
|
+
healthyKeyCount: keys.filter(key => key.status === 'active').length,
|
|
2757
|
+
status: managed !== undefined ? statusOfProvider(managed.keys, now) : 'unmanaged',
|
|
2758
|
+
keys,
|
|
2759
|
+
...(managed === undefined ? {} : { warning: 'This extra provider is managed but its pi-ai route is not registered. Check the provider profile in Models.' }),
|
|
2760
|
+
});
|
|
2761
|
+
seen.add(extra.id);
|
|
2762
|
+
}
|
|
2763
|
+
for (const managed of Object.values(this.store.providers)) {
|
|
2764
|
+
if (seen.has(managed.provider))
|
|
2765
|
+
continue;
|
|
2766
|
+
const projectedAccountId = this.oauthOwnerFor(managed.provider);
|
|
2767
|
+
const keys = managed.keys.map(key => publicKey(key, now, projectedAccountId));
|
|
2768
|
+
const binding = this.bindingState(managed, descriptor);
|
|
2769
|
+
views.push({
|
|
2770
|
+
provider: managed.provider,
|
|
2771
|
+
displayName: managed.displayName,
|
|
2772
|
+
source: managed.source,
|
|
2773
|
+
origin: managed.source === 'extra' ? 'plugin' : 'custom',
|
|
2774
|
+
managed: true,
|
|
2775
|
+
createdProfile: managed.createdProfile,
|
|
2776
|
+
configured: false,
|
|
2777
|
+
active: false,
|
|
2778
|
+
supported: true,
|
|
2779
|
+
accountSupported: false,
|
|
2780
|
+
settingsNs: managed.settingsNs,
|
|
2781
|
+
settingsPath: [...managed.settingsPath],
|
|
2782
|
+
apiKeyEnv: managed.brokerRef,
|
|
2783
|
+
brokerRef: managed.brokerRef,
|
|
2784
|
+
...(binding === 'healthy' ? { bindingHealthy: true } : binding === 'mismatch' ? { bindingHealthy: false } : {}),
|
|
2785
|
+
extra: managed.source === 'extra',
|
|
2786
|
+
extraId: managed.extraId,
|
|
2787
|
+
api: managed.api,
|
|
2788
|
+
baseURL: managed.baseURL,
|
|
2789
|
+
env: [],
|
|
2790
|
+
defaultBaseURL: managed.baseURL,
|
|
2791
|
+
nativeAuth: null,
|
|
2792
|
+
models: managed.models,
|
|
2793
|
+
keyCount: keys.length,
|
|
2794
|
+
healthyKeyCount: keys.filter(key => key.status === 'active').length,
|
|
2795
|
+
status: statusOfProvider(managed.keys, now),
|
|
2796
|
+
keys,
|
|
2797
|
+
warning: 'This provider route is not currently registered. Check its profile in Models.',
|
|
2798
|
+
});
|
|
2799
|
+
seen.add(managed.provider);
|
|
2800
|
+
}
|
|
2801
|
+
views.sort((a, b) => {
|
|
2802
|
+
if (a.managed !== b.managed)
|
|
2803
|
+
return a.managed ? -1 : 1;
|
|
2804
|
+
if (a.configured !== b.configured)
|
|
2805
|
+
return a.configured ? -1 : 1;
|
|
2806
|
+
if (a.source !== b.source)
|
|
2807
|
+
return a.source === 'extra' ? 1 : -1;
|
|
2808
|
+
return a.displayName.localeCompare(b.displayName) || a.provider.localeCompare(b.provider);
|
|
2809
|
+
});
|
|
2810
|
+
const allKeys = views.flatMap(view => view.keys);
|
|
2811
|
+
const warnings = this.takeWarnings();
|
|
2812
|
+
if (this.store.loadError !== undefined)
|
|
2813
|
+
warnings.unshift(`State file could not be read; started fresh (${this.store.loadError})`);
|
|
2814
|
+
if (descriptor.writable === false)
|
|
2815
|
+
warnings.push('The active profile is read-only; key changes cannot be saved.');
|
|
2816
|
+
return {
|
|
2817
|
+
now,
|
|
2818
|
+
writable: descriptor.writable && !this.store.readOnly,
|
|
2819
|
+
settings: { ...normalizeSettings(this.store.settings) },
|
|
2820
|
+
providers: views,
|
|
2821
|
+
managedCount: views.filter(view => view.managed).length,
|
|
2822
|
+
keyCount: allKeys.length,
|
|
2823
|
+
healthyKeyCount: allKeys.filter(key => key.status === 'active').length,
|
|
2824
|
+
coolingKeyCount: allKeys.filter(key => key.status === 'cooling').length,
|
|
2825
|
+
pausedKeyCount: allKeys.filter(key => key.status === 'paused').length,
|
|
2826
|
+
bindingsCount: Object.keys(this.store.state.sessionBindings).length,
|
|
2827
|
+
warnings,
|
|
2828
|
+
};
|
|
2829
|
+
}
|
|
2830
|
+
/** Remote: complete manager state. */
|
|
2831
|
+
getState() {
|
|
2832
|
+
return this.buildState();
|
|
2833
|
+
}
|
|
2834
|
+
/** Find one directory entry by route id. */
|
|
2835
|
+
directoryEntry(provider) {
|
|
2836
|
+
return this.ctx.llm.listConfigurableProviders().find(entry => entry.provider === provider);
|
|
2837
|
+
}
|
|
2838
|
+
/**
|
|
2839
|
+
* Classify the live binding. An unreadable settings snapshot is `unknown`,
|
|
2840
|
+
* not a confirmed mismatch: transient settings failures must not disable
|
|
2841
|
+
* key-pool recovery for a provider that is still managed.
|
|
2842
|
+
*/
|
|
2843
|
+
bindingState(record, descriptor = this.describeSettings()) {
|
|
2844
|
+
if (!descriptor.readable) {
|
|
2845
|
+
// Never re-enable a previously confirmed mismatch from an unreadable
|
|
2846
|
+
// snapshot; without evidence, leave the binding unknown.
|
|
2847
|
+
return this.bindingCache.get(record.provider) ?? 'unknown';
|
|
2848
|
+
}
|
|
2849
|
+
const namespace = this.namespace(descriptor, record.settingsNs);
|
|
2850
|
+
const profile = namespace === undefined ? undefined : asProfile(getPath(namespace.value, record.settingsPath));
|
|
2851
|
+
const state = profile !== undefined && profile.apiKeyEnv === record.brokerRef ? 'healthy' : 'mismatch';
|
|
2852
|
+
this.bindingCache.set(record.provider, state);
|
|
2853
|
+
return state;
|
|
2854
|
+
}
|
|
2855
|
+
/** Candidate credential refs from which an existing key can be imported. */
|
|
2856
|
+
importRefs(entry, profile) {
|
|
2857
|
+
const refs = [];
|
|
2858
|
+
if (profile !== undefined && typeof profile.apiKeyEnv === 'string' && profile.apiKeyEnv.length > 0)
|
|
2859
|
+
refs.push(profile.apiKeyEnv);
|
|
2860
|
+
for (const name of entry.nativeAuth?.env ?? []) {
|
|
2861
|
+
if (isKeyishEnv(name))
|
|
2862
|
+
refs.push(name);
|
|
2863
|
+
}
|
|
2864
|
+
if (entry.provider === 'deepseek-official')
|
|
2865
|
+
refs.push('DEEPSEEK_API_KEY');
|
|
2866
|
+
refs.push(derivedKeyRef(entry.provider));
|
|
2867
|
+
return [...new Set(refs.filter(ref => isCredentialRefName(ref) && !ref.startsWith(BROKER_PREFIX)))];
|
|
2868
|
+
}
|
|
2869
|
+
/** Create and store one key record without adding it to a provider. */
|
|
2870
|
+
async createKeyRecord(provider, alias, value, importedFrom = null) {
|
|
2871
|
+
const keyRef = newKeyRef(provider);
|
|
2872
|
+
await this.setOriginal(keyRef, value);
|
|
2873
|
+
return {
|
|
2874
|
+
id: newKeyId(),
|
|
2875
|
+
alias,
|
|
2876
|
+
ref: keyRef,
|
|
2877
|
+
kind: 'api-key',
|
|
2878
|
+
enabled: true,
|
|
2879
|
+
createdAt: Date.now(),
|
|
2880
|
+
lastUsedAt: 0,
|
|
2881
|
+
cooldownUntil: 0,
|
|
2882
|
+
lastFailureAt: 0,
|
|
2883
|
+
lastFailureCode: '',
|
|
2884
|
+
failureLevel: 0,
|
|
2885
|
+
consecutiveFailures: 0,
|
|
2886
|
+
totalUses: 0,
|
|
2887
|
+
totalFailures: 0,
|
|
2888
|
+
valueMissing: false,
|
|
2889
|
+
importedFrom,
|
|
2890
|
+
oauth: null,
|
|
2891
|
+
};
|
|
2892
|
+
}
|
|
2893
|
+
/** Find the first existing key value worth importing. */
|
|
2894
|
+
async findImport(entry, profile) {
|
|
2895
|
+
for (const ref of this.importRefs(entry, profile)) {
|
|
2896
|
+
const probe = await this.probeKey(ref);
|
|
2897
|
+
if (probe.hit !== undefined)
|
|
2898
|
+
return { ref, value: probe.hit.value };
|
|
2899
|
+
if (probe.failed) {
|
|
2900
|
+
throw new Error(`Could not read credential ${ref}; the credential store is unavailable. Try adoption again after it recovers.`);
|
|
2901
|
+
}
|
|
2902
|
+
}
|
|
2903
|
+
return undefined;
|
|
2904
|
+
}
|
|
2905
|
+
/** Remote: take over one existing provider route. */
|
|
2906
|
+
adoptProvider(provider) {
|
|
2907
|
+
return this.run(async () => {
|
|
2908
|
+
if (typeof provider !== 'string' || provider.length === 0)
|
|
2909
|
+
throw new Error('provider is required');
|
|
2910
|
+
if (this.store.provider(provider) !== undefined)
|
|
2911
|
+
return this.buildState();
|
|
2912
|
+
const entry = this.directoryEntry(provider);
|
|
2913
|
+
const extra = EXTRA_PROVIDER_MAP.get(provider);
|
|
2914
|
+
if (entry === undefined) {
|
|
2915
|
+
if (extra !== undefined)
|
|
2916
|
+
throw new Error('This extra provider needs a key; add it from the extra-provider section');
|
|
2917
|
+
throw new Error(`Provider "${provider}" is not registered by any adapter`);
|
|
2918
|
+
}
|
|
2919
|
+
const descriptor = this.describeSettings();
|
|
2920
|
+
const namespace = this.namespace(descriptor, entry.settingsNs);
|
|
2921
|
+
if (namespace === undefined)
|
|
2922
|
+
throw new Error(`Settings namespace "${entry.settingsNs}" is not mounted`);
|
|
2923
|
+
const profile = asProfile(getPath(namespace.value, entry.settingsPath));
|
|
2924
|
+
// MF1: adoption is reachable through EITHER capability. An OAuth-only
|
|
2925
|
+
// route adopts with zero keys (account-only management); a key-only
|
|
2926
|
+
// route adopts as before; the key-pool table alone no longer blocks
|
|
2927
|
+
// an adoptable account route.
|
|
2928
|
+
const oauthFlow = this.oauthFlowAvailable(entry.provider);
|
|
2929
|
+
const caps = capabilityOf(entry, profile, oauthFlow);
|
|
2930
|
+
const accountSupport = accountSupportOf(entry, oauthFlow);
|
|
2931
|
+
if (!caps.keys && !accountSupport.supported) {
|
|
2932
|
+
throw new Error(caps.reason ?? accountSupport.reason ?? 'This provider cannot be managed');
|
|
2933
|
+
}
|
|
2934
|
+
if (entry.declared === true && profile === undefined) {
|
|
2935
|
+
throw new Error('Add this hand-declared provider in Models first, then adopt it here');
|
|
2936
|
+
}
|
|
2937
|
+
const brokerRef = brokerRefFor(provider);
|
|
2938
|
+
const collision = [...this.brokerOwners.keys()].find(ref => ref === brokerRef);
|
|
2939
|
+
if (collision !== undefined)
|
|
2940
|
+
throw new Error(`Broker reference collision for "${provider}"`);
|
|
2941
|
+
const originalApiKeyEnv = profile !== undefined && typeof profile.apiKeyEnv === 'string' ? profile.apiKeyEnv : null;
|
|
2942
|
+
const path = [...entry.settingsPath, 'apiKeyEnv'];
|
|
2943
|
+
const hit = await this.findImport(entry, profile);
|
|
2944
|
+
let imported;
|
|
2945
|
+
if (hit !== undefined)
|
|
2946
|
+
imported = await this.createKeyRecord(provider, 'Imported', hit.value, hit.ref);
|
|
2947
|
+
try {
|
|
2948
|
+
await this.mutateSettings(entry.settingsNs, [{ op: 'set', path, value: brokerRef }]);
|
|
2949
|
+
}
|
|
2950
|
+
catch (error) {
|
|
2951
|
+
if (imported !== undefined) {
|
|
2952
|
+
try {
|
|
2953
|
+
await this.unsetOriginal(imported.ref);
|
|
2954
|
+
}
|
|
2955
|
+
catch (cleanupError) {
|
|
2956
|
+
this.warn(`adoption cleanup left credential ${imported.ref}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`);
|
|
2957
|
+
}
|
|
2958
|
+
}
|
|
2959
|
+
throw error;
|
|
2960
|
+
}
|
|
2961
|
+
const record = {
|
|
2962
|
+
provider,
|
|
2963
|
+
displayName: extra?.name ?? entry.displayName ?? provider,
|
|
2964
|
+
source: extra !== undefined ? 'extra' : 'dsh',
|
|
2965
|
+
settingsNs: entry.settingsNs,
|
|
2966
|
+
settingsPath: [...entry.settingsPath],
|
|
2967
|
+
adapter: entry.settingsNs.includes('deepseek') || provider === 'deepseek-official' ? 'deepseek' : 'pi-ai',
|
|
2968
|
+
brokerRef,
|
|
2969
|
+
originalApiKeyEnv,
|
|
2970
|
+
createdProfile: false,
|
|
2971
|
+
hadProfile: profile !== undefined,
|
|
2972
|
+
extraId: extra?.id ?? null,
|
|
2973
|
+
api: profile?.api ?? extra?.api ?? null,
|
|
2974
|
+
baseURL: profile?.baseURL ?? extra?.baseURL ?? entry.defaultBaseURL ?? null,
|
|
2975
|
+
models: profile !== undefined ? normalizeModels(profile.models) : extra?.models ?? [],
|
|
2976
|
+
adoptedAt: Date.now(),
|
|
2977
|
+
cursor: 0,
|
|
2978
|
+
keys: imported !== undefined ? [imported] : [],
|
|
2979
|
+
};
|
|
2980
|
+
this.store.providers[provider] = record;
|
|
2981
|
+
this.rebuildBrokerMap();
|
|
2982
|
+
if (hit !== undefined)
|
|
2983
|
+
this.pushWarning(`Imported the existing credential ${hit.ref}.`);
|
|
2984
|
+
this.store.touch();
|
|
2985
|
+
return this.buildState();
|
|
2986
|
+
});
|
|
2987
|
+
}
|
|
2988
|
+
/** Discover models for a new extra provider, falling back to the seed catalog. */
|
|
2989
|
+
async discoverModels(extra, value) {
|
|
2990
|
+
let discovered = [];
|
|
2991
|
+
let failure;
|
|
2992
|
+
try {
|
|
2993
|
+
discovered = await this.ctx.llm.discoverModels('llm-pi-ai', { provider: extra.id, baseURL: extra.baseURL, api: extra.api, apiKey: value }, AbortSignal.timeout(20_000));
|
|
2994
|
+
}
|
|
2995
|
+
catch (error) {
|
|
2996
|
+
failure = error instanceof Error ? error.message : String(error);
|
|
2997
|
+
}
|
|
2998
|
+
const models = [];
|
|
2999
|
+
const seen = new Set();
|
|
3000
|
+
for (const model of discovered) {
|
|
3001
|
+
const normalized = discoveredModel(model);
|
|
3002
|
+
if (normalized === undefined || seen.has(normalized.id))
|
|
3003
|
+
continue;
|
|
3004
|
+
seen.add(normalized.id);
|
|
3005
|
+
models.push(normalized);
|
|
3006
|
+
if (models.length >= 60)
|
|
3007
|
+
break;
|
|
3008
|
+
}
|
|
3009
|
+
if (models.length > 0)
|
|
3010
|
+
return { models, failure };
|
|
3011
|
+
return { models: normalizeModels(extra.models), failure };
|
|
3012
|
+
}
|
|
3013
|
+
/** Remote: install and adopt one extra built-in provider. */
|
|
3014
|
+
addExtraProvider(extraId, alias, value) {
|
|
3015
|
+
return this.run(async () => {
|
|
3016
|
+
if (typeof extraId !== 'string' || extraId.length === 0)
|
|
3017
|
+
throw new Error('extraId is required');
|
|
3018
|
+
const input = validateKeyInput(alias, value);
|
|
3019
|
+
const extra = EXTRA_PROVIDER_MAP.get(extraId);
|
|
3020
|
+
if (extra === undefined)
|
|
3021
|
+
throw new Error(`Unknown extra provider "${extraId}"`);
|
|
3022
|
+
if (this.store.provider(extraId) !== undefined) {
|
|
3023
|
+
this.pushWarning('Provider is already managed; the key was added instead.');
|
|
3024
|
+
await this.addKeyLocked(extraId, input.alias, input.value);
|
|
3025
|
+
this.store.touch();
|
|
3026
|
+
return this.buildState();
|
|
3027
|
+
}
|
|
3028
|
+
const descriptor = this.describeSettings();
|
|
3029
|
+
const namespace = this.namespace(descriptor, 'llm-pi-ai');
|
|
3030
|
+
if (namespace === undefined)
|
|
3031
|
+
throw new Error('The llm-pi-ai settings namespace is not mounted');
|
|
3032
|
+
const existing = asProfile(getPath(namespace.value, ['providers', extraId]));
|
|
3033
|
+
if (existing !== undefined) {
|
|
3034
|
+
this.pushWarning('A stored profile already exists for this provider; adopting it without replacing its models.');
|
|
3035
|
+
const record = {
|
|
3036
|
+
provider: extraId,
|
|
3037
|
+
displayName: extra.name,
|
|
3038
|
+
source: 'extra',
|
|
3039
|
+
settingsNs: 'llm-pi-ai',
|
|
3040
|
+
settingsPath: ['providers', extraId],
|
|
3041
|
+
adapter: 'pi-ai',
|
|
3042
|
+
brokerRef: brokerRefFor(extraId),
|
|
3043
|
+
originalApiKeyEnv: typeof existing.apiKeyEnv === 'string' ? existing.apiKeyEnv : null,
|
|
3044
|
+
createdProfile: false,
|
|
3045
|
+
hadProfile: true,
|
|
3046
|
+
extraId,
|
|
3047
|
+
api: existing.api ?? extra.api,
|
|
3048
|
+
baseURL: existing.baseURL ?? extra.baseURL,
|
|
3049
|
+
models: normalizeModels(existing.models).length > 0 ? normalizeModels(existing.models) : normalizeModels(extra.models),
|
|
3050
|
+
adoptedAt: Date.now(),
|
|
3051
|
+
cursor: 0,
|
|
3052
|
+
keys: [],
|
|
3053
|
+
};
|
|
3054
|
+
const key = await this.createKeyRecord(extraId, input.alias, input.value);
|
|
3055
|
+
try {
|
|
3056
|
+
await this.mutateSettings('llm-pi-ai', [{ op: 'set', path: ['providers', extraId, 'apiKeyEnv'], value: record.brokerRef }]);
|
|
3057
|
+
}
|
|
3058
|
+
catch (error) {
|
|
3059
|
+
try {
|
|
3060
|
+
await this.unsetOriginal(key.ref);
|
|
3061
|
+
}
|
|
3062
|
+
catch (cleanupError) {
|
|
3063
|
+
this.warn(`extra-provider cleanup left credential ${key.ref}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`);
|
|
3064
|
+
}
|
|
3065
|
+
throw error;
|
|
3066
|
+
}
|
|
3067
|
+
record.keys.push(key);
|
|
3068
|
+
this.store.providers[extraId] = record;
|
|
3069
|
+
this.rebuildBrokerMap();
|
|
3070
|
+
this.store.touch();
|
|
3071
|
+
return this.buildState();
|
|
3072
|
+
}
|
|
3073
|
+
const { models, failure } = await this.discoverModels(extra, input.value);
|
|
3074
|
+
if (models.length === 0)
|
|
3075
|
+
throw new Error('No models were discovered and the seed catalog is empty');
|
|
3076
|
+
if (failure !== undefined)
|
|
3077
|
+
this.pushWarning(`Model discovery failed; installed ${models.length} seed models instead: ${failure}`);
|
|
3078
|
+
const brokerRef = brokerRefFor(extraId);
|
|
3079
|
+
const profile = {
|
|
3080
|
+
displayName: extra.name,
|
|
3081
|
+
api: extra.api,
|
|
3082
|
+
baseURL: extra.baseURL,
|
|
3083
|
+
apiKeyEnv: brokerRef,
|
|
3084
|
+
models,
|
|
3085
|
+
};
|
|
3086
|
+
const key = await this.createKeyRecord(extraId, input.alias, input.value);
|
|
3087
|
+
try {
|
|
3088
|
+
await this.mutateSettings('llm-pi-ai', [{ op: 'set', path: ['providers', extraId], value: profile }]);
|
|
3089
|
+
}
|
|
3090
|
+
catch (error) {
|
|
3091
|
+
try {
|
|
3092
|
+
await this.unsetOriginal(key.ref);
|
|
3093
|
+
}
|
|
3094
|
+
catch (cleanupError) {
|
|
3095
|
+
this.warn(`extra-provider cleanup left credential ${key.ref}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`);
|
|
3096
|
+
}
|
|
3097
|
+
throw error;
|
|
3098
|
+
}
|
|
3099
|
+
const record = {
|
|
3100
|
+
provider: extraId,
|
|
3101
|
+
displayName: extra.name,
|
|
3102
|
+
source: 'extra',
|
|
3103
|
+
settingsNs: 'llm-pi-ai',
|
|
3104
|
+
settingsPath: ['providers', extraId],
|
|
3105
|
+
adapter: 'pi-ai',
|
|
3106
|
+
brokerRef,
|
|
3107
|
+
originalApiKeyEnv: null,
|
|
3108
|
+
createdProfile: true,
|
|
3109
|
+
hadProfile: false,
|
|
3110
|
+
extraId,
|
|
3111
|
+
api: extra.api,
|
|
3112
|
+
baseURL: extra.baseURL,
|
|
3113
|
+
models,
|
|
3114
|
+
adoptedAt: Date.now(),
|
|
3115
|
+
cursor: 0,
|
|
3116
|
+
keys: [key],
|
|
3117
|
+
};
|
|
3118
|
+
this.store.providers[extraId] = record;
|
|
3119
|
+
this.rebuildBrokerMap();
|
|
3120
|
+
this.store.touch();
|
|
3121
|
+
return this.buildState();
|
|
3122
|
+
});
|
|
3123
|
+
}
|
|
3124
|
+
/** Add one key; caller already owns the mutation queue. */
|
|
3125
|
+
async addKeyLocked(provider, alias, value) {
|
|
3126
|
+
const record = this.store.provider(provider);
|
|
3127
|
+
if (record === undefined)
|
|
3128
|
+
throw new Error(`Provider "${provider}" is not managed`);
|
|
3129
|
+
// MF1: the key-pool restriction lives on the key entry path. A route
|
|
3130
|
+
// adopted for accounts only (no key-pool credential) refuses new keys.
|
|
3131
|
+
const entry = this.directoryEntry(provider);
|
|
3132
|
+
if (entry !== undefined) {
|
|
3133
|
+
const namespace = this.namespace(this.describeSettings(), entry.settingsNs);
|
|
3134
|
+
const profile = namespace === undefined ? undefined : asProfile(getPath(namespace.value, entry.settingsPath));
|
|
3135
|
+
const caps = capabilityOf(entry, profile, this.oauthFlowAvailable(entry.provider));
|
|
3136
|
+
if (!caps.keys)
|
|
3137
|
+
throw new Error(caps.reason ?? `Provider "${provider}" does not accept API keys`);
|
|
3138
|
+
}
|
|
3139
|
+
const input = validateKeyInput(alias, value);
|
|
3140
|
+
if (record.keys.length >= KEY_LIMIT)
|
|
3141
|
+
throw new Error(`A provider may have at most ${KEY_LIMIT} keys`);
|
|
3142
|
+
const key = await this.createKeyRecord(provider, input.alias, input.value);
|
|
3143
|
+
record.keys.push(key);
|
|
3144
|
+
}
|
|
3145
|
+
/** Remote: add one key to a managed provider. */
|
|
3146
|
+
addKey(provider, alias, value) {
|
|
3147
|
+
return this.run(async () => {
|
|
3148
|
+
await this.addKeyLocked(provider, alias, value);
|
|
3149
|
+
this.store.touch();
|
|
3150
|
+
return this.buildState();
|
|
3151
|
+
});
|
|
3152
|
+
}
|
|
3153
|
+
/** Remote: rename or pause/resume one key. */
|
|
3154
|
+
updateKey(provider, keyId, patch) {
|
|
3155
|
+
return this.run(async () => {
|
|
3156
|
+
const record = this.store.provider(provider);
|
|
3157
|
+
if (record === undefined)
|
|
3158
|
+
throw new Error(`Provider "${provider}" is not managed`);
|
|
3159
|
+
const key = record.keys.find(candidate => candidate.id === keyId);
|
|
3160
|
+
if (key === undefined)
|
|
3161
|
+
throw new Error('Key not found');
|
|
3162
|
+
const clean = validateKeyPatch(patch);
|
|
3163
|
+
if (clean.alias !== undefined)
|
|
3164
|
+
key.alias = clean.alias;
|
|
3165
|
+
if (clean.enabled !== undefined)
|
|
3166
|
+
key.enabled = clean.enabled;
|
|
3167
|
+
this.store.touch();
|
|
3168
|
+
return this.buildState();
|
|
3169
|
+
});
|
|
3170
|
+
}
|
|
3171
|
+
/** Remote: delete one key, releasing the provider when it was the last one. */
|
|
3172
|
+
deleteKey(provider, keyId) {
|
|
3173
|
+
return this.run(async () => {
|
|
3174
|
+
const record = this.store.provider(provider);
|
|
3175
|
+
if (record === undefined)
|
|
3176
|
+
throw new Error(`Provider "${provider}" is not managed`);
|
|
3177
|
+
const index = record.keys.findIndex(candidate => candidate.id === keyId);
|
|
3178
|
+
if (index < 0)
|
|
3179
|
+
throw new Error('Key not found');
|
|
3180
|
+
if (record.keys.length === 1) {
|
|
3181
|
+
await this.releaseLocked(provider);
|
|
3182
|
+
return this.buildState();
|
|
3183
|
+
}
|
|
3184
|
+
const key = record.keys[index];
|
|
3185
|
+
// An OAuth account leaves the pool like removeAccount: drop its
|
|
3186
|
+
// plugin-scope payload and, when it was the projection, fall back to
|
|
3187
|
+
// the key pool. API keys remove their broker literal.
|
|
3188
|
+
if (key.kind === 'oauth' && key.oauth !== null) {
|
|
3189
|
+
await this.removeOAuthGrant(provider, key);
|
|
3190
|
+
}
|
|
3191
|
+
else {
|
|
3192
|
+
// Unset first: if the credential backend refuses, keep the metadata so a
|
|
3193
|
+
// later retry can remove both together.
|
|
3194
|
+
await this.unsetOriginal(key.ref);
|
|
3195
|
+
}
|
|
3196
|
+
record.keys.splice(index, 1);
|
|
3197
|
+
this.store.clearBindings((_key, binding) => binding.keyId === key.id);
|
|
3198
|
+
this.store.touch();
|
|
3199
|
+
return this.buildState();
|
|
3200
|
+
});
|
|
3201
|
+
}
|
|
3202
|
+
/** Read one live profile for release-time cleanup decisions. */
|
|
3203
|
+
profileAt(ns, path) {
|
|
3204
|
+
const descriptor = this.describeSettings();
|
|
3205
|
+
const namespace = this.namespace(descriptor, ns);
|
|
3206
|
+
return namespace === undefined ? undefined : asProfile(getPath(namespace.value, path));
|
|
3207
|
+
}
|
|
3208
|
+
/** Restore the exact live profile captured before release mutated it. */
|
|
3209
|
+
async restoreProfileSnapshot(record, snapshot) {
|
|
3210
|
+
if (record.settingsPath.length === 0) {
|
|
3211
|
+
const apiKeyEnv = snapshot !== undefined && typeof snapshot.apiKeyEnv === 'string' ? snapshot.apiKeyEnv : record.brokerRef;
|
|
3212
|
+
await this.mutateSettings(record.settingsNs, [{ op: 'set', path: ['apiKeyEnv'], value: apiKeyEnv }]);
|
|
3213
|
+
return;
|
|
3214
|
+
}
|
|
3215
|
+
await this.mutateSettings(record.settingsNs, [
|
|
3216
|
+
snapshot === undefined
|
|
3217
|
+
? { op: 'unset', path: record.settingsPath }
|
|
3218
|
+
: { op: 'set', path: record.settingsPath, value: snapshot },
|
|
3219
|
+
]);
|
|
3220
|
+
}
|
|
3221
|
+
/** Remove every managed credential, recording progress before a retry. */
|
|
3222
|
+
async deleteManagedCredentials(record) {
|
|
3223
|
+
for (const key of record.keys) {
|
|
3224
|
+
if (key.kind === 'oauth' && key.oauth !== null) {
|
|
3225
|
+
// Same cleanup as removeAccount/deleteKey: when this row owns the
|
|
3226
|
+
// activation projection, demote it first so the released route
|
|
3227
|
+
// resolves through the restored apiKeyEnv instead of a grant whose
|
|
3228
|
+
// manager is gone (pi-ai resolution prefers a stored credential).
|
|
3229
|
+
await this.removeOAuthGrant(record.provider, key);
|
|
3230
|
+
}
|
|
3231
|
+
else {
|
|
3232
|
+
await this.unsetOriginal(key.ref);
|
|
3233
|
+
}
|
|
3234
|
+
key.valueMissing = true;
|
|
3235
|
+
this.store.touch();
|
|
3236
|
+
}
|
|
3237
|
+
this.activeAccounts.delete(record.provider);
|
|
3238
|
+
}
|
|
3239
|
+
/** Restore the provider profile and drop management; caller owns the queue. */
|
|
3240
|
+
async releaseLocked(provider) {
|
|
3241
|
+
const record = this.store.provider(provider);
|
|
3242
|
+
if (record === undefined)
|
|
3243
|
+
return;
|
|
3244
|
+
// Round-2 review step 4: releasing a provider revokes its in-flight
|
|
3245
|
+
// sign-in attempts. A login whose seam flow is aborted can no longer
|
|
3246
|
+
// harvest rows for a provider that is no longer managed (the harvest
|
|
3247
|
+
// itself also refuses settled attempts).
|
|
3248
|
+
for (const attempt of this.loginAttempts.values()) {
|
|
3249
|
+
if (attempt.provider !== provider || attempt.settledAt !== undefined)
|
|
3250
|
+
continue;
|
|
3251
|
+
this.failLoginAttempt(attempt, new Error(`Provider "${provider}" was released during sign-in`));
|
|
3252
|
+
}
|
|
3253
|
+
const descriptor = this.describeSettings();
|
|
3254
|
+
if (!descriptor.readable) {
|
|
3255
|
+
throw new Error('Settings are currently unreadable; refusing to release a provider whose binding cannot be inspected.');
|
|
3256
|
+
}
|
|
3257
|
+
const namespace = this.namespace(descriptor, record.settingsNs);
|
|
3258
|
+
if (namespace === undefined)
|
|
3259
|
+
throw new Error(`Settings namespace "${record.settingsNs}" is not mounted`);
|
|
3260
|
+
const binding = this.bindingState(record, descriptor);
|
|
3261
|
+
if (binding === 'unknown') {
|
|
3262
|
+
throw new Error('Settings are currently unreadable; refusing to release a provider whose binding cannot be inspected.');
|
|
3263
|
+
}
|
|
3264
|
+
// The user may have repaired or replaced the provider in Models. In that
|
|
3265
|
+
// case release only drops manager metadata and credentials; it must not
|
|
3266
|
+
// overwrite the live profile.
|
|
3267
|
+
if (binding === 'mismatch') {
|
|
3268
|
+
await this.deleteManagedCredentials(record);
|
|
3269
|
+
delete this.store.providers[provider];
|
|
3270
|
+
this.store.clearBindings(key => key.endsWith(`\u0000${provider}`));
|
|
3271
|
+
this.rebuildBrokerMap();
|
|
3272
|
+
this.store.touch();
|
|
3273
|
+
return;
|
|
3274
|
+
}
|
|
3275
|
+
const liveProfile = asProfile(getPath(namespace.value, record.settingsPath));
|
|
3276
|
+
const snapshot = liveProfile === undefined ? undefined : structuredClone(liveProfile);
|
|
3277
|
+
let profileMutated = false;
|
|
3278
|
+
try {
|
|
3279
|
+
if (record.createdProfile && record.extraId !== null && record.settingsNs === 'llm-pi-ai') {
|
|
3280
|
+
await this.mutateSettings(record.settingsNs, [{ op: 'unset', path: ['providers', record.extraId] }]);
|
|
3281
|
+
profileMutated = true;
|
|
3282
|
+
}
|
|
3283
|
+
else {
|
|
3284
|
+
// Native and pre-existing extra routes own only apiKeyEnv. Leave every
|
|
3285
|
+
// other profile field (baseURL, models, hand edits) exactly as it is.
|
|
3286
|
+
const path = [...record.settingsPath, 'apiKeyEnv'];
|
|
3287
|
+
await this.mutateSettings(record.settingsNs, [
|
|
3288
|
+
record.originalApiKeyEnv !== null
|
|
3289
|
+
? { op: 'set', path, value: record.originalApiKeyEnv }
|
|
3290
|
+
: { op: 'unset', path },
|
|
3291
|
+
]);
|
|
3292
|
+
profileMutated = true;
|
|
3293
|
+
// Remove an auto-created profile only when it is still empty after
|
|
3294
|
+
// clearing apiKeyEnv; a profile with later user edits stays.
|
|
3295
|
+
if (!record.hadProfile && record.settingsPath.length > 0) {
|
|
3296
|
+
const current = this.profileAt(record.settingsNs, record.settingsPath);
|
|
3297
|
+
if (current !== undefined && Object.keys(current).length === 0) {
|
|
3298
|
+
await this.mutateSettings(record.settingsNs, [{ op: 'unset', path: record.settingsPath }]);
|
|
3299
|
+
}
|
|
3300
|
+
}
|
|
3301
|
+
}
|
|
3302
|
+
// The profile mutation came first so a read-only profile fails before
|
|
3303
|
+
// any managed credential is deleted.
|
|
3304
|
+
await this.deleteManagedCredentials(record);
|
|
3305
|
+
}
|
|
3306
|
+
catch (error) {
|
|
3307
|
+
if (profileMutated) {
|
|
3308
|
+
try {
|
|
3309
|
+
await this.restoreProfileSnapshot(record, snapshot);
|
|
3310
|
+
}
|
|
3311
|
+
catch (restoreError) {
|
|
3312
|
+
this.warn(`release compensation failed for ${provider}: ${restoreError instanceof Error ? restoreError.message : String(restoreError)}`);
|
|
3313
|
+
}
|
|
3314
|
+
}
|
|
3315
|
+
throw error;
|
|
3316
|
+
}
|
|
3317
|
+
delete this.store.providers[provider];
|
|
3318
|
+
this.store.clearBindings(key => key.endsWith(`\u0000${provider}`));
|
|
3319
|
+
this.rebuildBrokerMap();
|
|
3320
|
+
this.store.touch();
|
|
3321
|
+
}
|
|
3322
|
+
/** Remote: stop managing a provider and restore its original key reference. */
|
|
3323
|
+
releaseProvider(provider) {
|
|
3324
|
+
return this.run(async () => {
|
|
3325
|
+
if (this.store.provider(provider) === undefined)
|
|
3326
|
+
throw new Error(`Provider "${provider}" is not managed`);
|
|
3327
|
+
await this.releaseLocked(provider);
|
|
3328
|
+
return this.buildState();
|
|
3329
|
+
});
|
|
3330
|
+
}
|
|
3331
|
+
/** Remote: update manager policy. */
|
|
3332
|
+
setSettings(patch) {
|
|
3333
|
+
return this.run(async () => {
|
|
3334
|
+
if (patch === null || typeof patch !== 'object' || Array.isArray(patch))
|
|
3335
|
+
throw new Error('patch is required');
|
|
3336
|
+
const next = patchSettings(this.store.settings, patch);
|
|
3337
|
+
this.store.state.settings = next;
|
|
3338
|
+
this.store.touch();
|
|
3339
|
+
return this.buildState();
|
|
3340
|
+
});
|
|
3341
|
+
}
|
|
3342
|
+
/**
|
|
3343
|
+
* Serialize the first selection for a sticky session/provider pair so two
|
|
3344
|
+
* concurrent requests cannot advance the cursor and bind different keys.
|
|
3345
|
+
*/
|
|
3346
|
+
async selectKey(providerId, sessionId, step) {
|
|
3347
|
+
if (!this.store.settings.stickySession || sessionId === undefined) {
|
|
3348
|
+
return this.doSelectKey(providerId, sessionId, step);
|
|
3349
|
+
}
|
|
3350
|
+
const queueKey = `${sessionId}\u0000${providerId}`;
|
|
3351
|
+
const previous = this.selectionQueues.get(queueKey) ?? Promise.resolve();
|
|
3352
|
+
const next = previous.then(() => this.doSelectKey(providerId, sessionId, step), () => this.doSelectKey(providerId, sessionId, step));
|
|
3353
|
+
const tail = next.then(() => undefined, () => undefined);
|
|
3354
|
+
this.selectionQueues.set(queueKey, tail);
|
|
3355
|
+
try {
|
|
3356
|
+
return await next;
|
|
3357
|
+
}
|
|
3358
|
+
finally {
|
|
3359
|
+
if (this.selectionQueues.get(queueKey) === tail)
|
|
3360
|
+
this.selectionQueues.delete(queueKey);
|
|
3361
|
+
}
|
|
3362
|
+
}
|
|
3363
|
+
/** Pin the key a retry must use, scoped to the failing step. */
|
|
3364
|
+
pinRetry(providerId, sessionId, turn, step, keyId) {
|
|
3365
|
+
if (sessionId === undefined)
|
|
3366
|
+
return;
|
|
3367
|
+
this.retryBindings.set(`${bindingKey(sessionId, providerId)}\u0000${turn}\u0000${step}`, { keyId, at: Date.now() });
|
|
3368
|
+
}
|
|
3369
|
+
/** Record the turn/step the agent loop is about to request for one provider. */
|
|
3370
|
+
noteRequestStep(payload, providerId) {
|
|
3371
|
+
const raw = payload.agent?.session?.id;
|
|
3372
|
+
if (raw === undefined || providerId.length === 0)
|
|
3373
|
+
return;
|
|
3374
|
+
// Unmanaged providers never consult this map; do not let them accumulate.
|
|
3375
|
+
if (this.store.provider(providerId) === undefined)
|
|
3376
|
+
return;
|
|
3377
|
+
const sessionId = String(raw);
|
|
3378
|
+
this.requestSteps.set(`${sessionId}\u0000${providerId}`, { turn: payload.turn, step: payload.step, at: Date.now() });
|
|
3379
|
+
}
|
|
3380
|
+
/** Drop one step's retry pin once its attempt settles. */
|
|
3381
|
+
clearRetry(providerId, sessionId, turn, step) {
|
|
3382
|
+
if (sessionId === undefined)
|
|
3383
|
+
return;
|
|
3384
|
+
this.retryBindings.delete(`${bindingKey(sessionId, providerId)}\u0000${turn}\u0000${step}`);
|
|
3385
|
+
}
|
|
3386
|
+
/** Selection body; callers reach it through {@link selectKey}. */
|
|
3387
|
+
async doSelectKey(providerId, sessionId, step) {
|
|
3388
|
+
const provider = this.store.provider(providerId);
|
|
3389
|
+
if (provider === undefined)
|
|
3390
|
+
return { key: undefined, value: undefined };
|
|
3391
|
+
const now = Date.now();
|
|
3392
|
+
this.pruneBindings(now);
|
|
3393
|
+
if (sessionId !== undefined && step !== undefined) {
|
|
3394
|
+
const pinKey = `${bindingKey(sessionId, providerId)}\u0000${step.turn}\u0000${step.step}`;
|
|
3395
|
+
const pinned = this.retryBindings.get(pinKey);
|
|
3396
|
+
this.retryBindings.delete(pinKey);
|
|
3397
|
+
if (pinned !== undefined && now - pinned.at <= PIN_TTL_MS) {
|
|
3398
|
+
const candidate = provider.keys.find(entry => entry.id === pinned.keyId);
|
|
3399
|
+
if (isSelectable(candidate, now)) {
|
|
3400
|
+
const probe = await this.probeKey(candidate.ref);
|
|
3401
|
+
if (probe.hit !== undefined) {
|
|
3402
|
+
candidate.valueMissing = false;
|
|
3403
|
+
markKeyUsed(candidate, now);
|
|
3404
|
+
this.store.touch();
|
|
3405
|
+
if (this.store.settings.stickySession)
|
|
3406
|
+
this.store.bind(sessionId, providerId, candidate.id, now);
|
|
3407
|
+
return { key: candidate, value: probe.hit.value };
|
|
3408
|
+
}
|
|
3409
|
+
if (probe.missing) {
|
|
3410
|
+
candidate.valueMissing = true;
|
|
3411
|
+
this.store.touch();
|
|
3412
|
+
}
|
|
3413
|
+
}
|
|
3414
|
+
}
|
|
3415
|
+
}
|
|
3416
|
+
let key;
|
|
3417
|
+
if (sessionId !== undefined && this.store.settings.stickySession) {
|
|
3418
|
+
const binding = this.store.binding(sessionId, providerId);
|
|
3419
|
+
if (binding !== undefined) {
|
|
3420
|
+
const bound = provider.keys.find(candidate => candidate.id === binding.keyId);
|
|
3421
|
+
if (isSelectable(bound, now))
|
|
3422
|
+
key = bound;
|
|
3423
|
+
}
|
|
3424
|
+
}
|
|
3425
|
+
if (key === undefined)
|
|
3426
|
+
key = pickRoundRobin(provider, now);
|
|
3427
|
+
const tryCandidate = async (candidate) => {
|
|
3428
|
+
if (candidate === undefined)
|
|
3429
|
+
return undefined;
|
|
3430
|
+
const probe = await this.probeKey(candidate.ref);
|
|
3431
|
+
if (probe.hit !== undefined) {
|
|
3432
|
+
candidate.valueMissing = false;
|
|
3433
|
+
markKeyUsed(candidate, now);
|
|
3434
|
+
this.store.touch();
|
|
3435
|
+
if (sessionId !== undefined && this.store.settings.stickySession) {
|
|
3436
|
+
this.store.bind(sessionId, providerId, candidate.id, now);
|
|
3437
|
+
}
|
|
3438
|
+
return { key: candidate, value: probe.hit.value };
|
|
3439
|
+
}
|
|
3440
|
+
if (probe.missing) {
|
|
3441
|
+
candidate.valueMissing = true;
|
|
3442
|
+
this.store.touch();
|
|
3443
|
+
}
|
|
3444
|
+
return undefined;
|
|
3445
|
+
};
|
|
3446
|
+
const tried = new Set();
|
|
3447
|
+
while (key !== undefined && !tried.has(key.id)) {
|
|
3448
|
+
tried.add(key.id);
|
|
3449
|
+
const selected = await tryCandidate(key);
|
|
3450
|
+
if (selected !== undefined)
|
|
3451
|
+
return selected;
|
|
3452
|
+
key = pickRoundRobin(provider, now, tried);
|
|
3453
|
+
}
|
|
3454
|
+
// A previously missing key may have been restored in the credential store.
|
|
3455
|
+
// Re-probe those candidates when no healthy key remains; a transient read
|
|
3456
|
+
// failure is never written back as missing.
|
|
3457
|
+
for (const candidate of provider.keys) {
|
|
3458
|
+
if (candidate.valueMissing !== true || candidate.enabled === false || Number(candidate.cooldownUntil) > now)
|
|
3459
|
+
continue;
|
|
3460
|
+
const selected = await tryCandidate(candidate);
|
|
3461
|
+
if (selected !== undefined)
|
|
3462
|
+
return selected;
|
|
3463
|
+
}
|
|
3464
|
+
return { key: undefined, value: undefined };
|
|
3465
|
+
}
|
|
3466
|
+
/** Record a terminal failure without deciding policy yet. */
|
|
3467
|
+
recordFailure(providerId, sessionId, keyId, failure, usedManagedKey = false) {
|
|
3468
|
+
if (sessionId === undefined)
|
|
3469
|
+
return;
|
|
3470
|
+
const key = bindingKey(sessionId, providerId);
|
|
3471
|
+
const previous = this.attempts.get(key);
|
|
3472
|
+
const effectiveKeyId = keyId ?? previous?.keyId;
|
|
3473
|
+
this.attempts.set(key, {
|
|
3474
|
+
keyId: effectiveKeyId,
|
|
3475
|
+
retries: previous !== undefined && previous.keyId === effectiveKeyId ? previous.retries : 0,
|
|
3476
|
+
failure,
|
|
3477
|
+
at: Date.now(),
|
|
3478
|
+
turn: previous?.turn,
|
|
3479
|
+
step: previous?.step,
|
|
3480
|
+
usedManagedKey,
|
|
3481
|
+
});
|
|
3482
|
+
}
|
|
3483
|
+
/** Record a successful terminal finish. */
|
|
3484
|
+
recordSuccess(providerId, sessionId, keyId, step) {
|
|
3485
|
+
const provider = this.store.provider(providerId);
|
|
3486
|
+
if (provider !== undefined) {
|
|
3487
|
+
const key = provider.keys.find(candidate => candidate.id === keyId);
|
|
3488
|
+
if (key !== undefined)
|
|
3489
|
+
markKeySuccess(key, Date.now());
|
|
3490
|
+
this.store.touch();
|
|
3491
|
+
}
|
|
3492
|
+
if (sessionId !== undefined) {
|
|
3493
|
+
this.attempts.delete(bindingKey(sessionId, providerId));
|
|
3494
|
+
if (step !== undefined)
|
|
3495
|
+
this.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3496
|
+
// Do not clear step-scoped switch/attempt records here: another step of
|
|
3497
|
+
// the same session may still be in flight. Step records expire by TTL or
|
|
3498
|
+
// are cleared when their own request-error path terminates.
|
|
3499
|
+
}
|
|
3500
|
+
}
|
|
3501
|
+
/**
|
|
3502
|
+
* Codex P2: record a projection-served success on the owning account row —
|
|
3503
|
+
* `lastUsedAt`/`totalUses` move and the failure streak resets, symmetric to
|
|
3504
|
+
* the MF2 failure attribution.
|
|
3505
|
+
*/
|
|
3506
|
+
recordProjectionSuccess(providerId, accountId) {
|
|
3507
|
+
const provider = this.store.provider(providerId);
|
|
3508
|
+
if (provider === undefined)
|
|
3509
|
+
return;
|
|
3510
|
+
const row = provider.keys.find(candidate => candidate.id === accountId);
|
|
3511
|
+
if (row !== undefined)
|
|
3512
|
+
markKeySuccess(row, Date.now());
|
|
3513
|
+
this.store.touch();
|
|
3514
|
+
}
|
|
3515
|
+
/** Mark one key cooling, preserving escalation across recovery windows. */
|
|
3516
|
+
markFailure(providerId, keyId, failure) {
|
|
3517
|
+
const provider = this.store.provider(providerId);
|
|
3518
|
+
if (provider === undefined || keyId === undefined)
|
|
3519
|
+
return;
|
|
3520
|
+
const key = provider.keys.find(candidate => candidate.id === keyId);
|
|
3521
|
+
if (key === undefined)
|
|
3522
|
+
return;
|
|
3523
|
+
const now = Date.now();
|
|
3524
|
+
// R6-SF1/R7-SF1: the projection generation is read at failure time and
|
|
3525
|
+
// stamped on the row's window. It must be stamped on EVERY observed
|
|
3526
|
+
// failure, including one the dedupe below skips: a failure of a freshly
|
|
3527
|
+
// projected grant that lands inside an existing window still belongs to
|
|
3528
|
+
// the new projection, and leaving the old stamp would let a later
|
|
3529
|
+
// harvest clear a window the new projection earned.
|
|
3530
|
+
const generation = this.ownershipGenerations.get(providerId) ?? 0;
|
|
3531
|
+
// D1/D2 pool arbitration: when the OAuth grant itself failed (the pinned
|
|
3532
|
+
// pi-ai `ModelsError("oauth", …)` code), the account cools down AND the
|
|
3533
|
+
// cascade demotes to the key pool by deleting the projection record.
|
|
3534
|
+
const oauthDemotion = key.kind === 'oauth' && classifyOAuthFailure(failure) === 'oauth';
|
|
3535
|
+
if (key.cooldownUntil > now && now - key.lastFailureAt < 10_000) {
|
|
3536
|
+
// Dedupe skips only the cooldown EXTENSION; the attribution stamp
|
|
3537
|
+
// always follows the live projection.
|
|
3538
|
+
this.cooldownGenerations.set(key.id, generation);
|
|
3539
|
+
// R14-M1: the demotion is not part of the extension this dedupe
|
|
3540
|
+
// suppresses. The first attempt may have thrown on its delete (or a
|
|
3541
|
+
// native write re-projected the grant inside the window), leaving the
|
|
3542
|
+
// route on a grant whose row already cooled while C1's await pointed
|
|
3543
|
+
// at that settled attempt. Re-running the idempotent demotion repairs
|
|
3544
|
+
// exactly that; skipping it made C1's awaited demotion pointless for
|
|
3545
|
+
// every deduped failure.
|
|
3546
|
+
if (oauthDemotion)
|
|
3547
|
+
this.demoteProjection(providerId, key.id);
|
|
3548
|
+
return;
|
|
3549
|
+
}
|
|
3550
|
+
const result = applyFailureCooldown(key, this.store.settings, now, failure);
|
|
3551
|
+
// R6-SF1: stamp which projection earned this window. The generation is
|
|
3552
|
+
// read at failure time — never derived later from a cooldown timestamp —
|
|
3553
|
+
// so a harvest can clear only a window an older projection earned.
|
|
3554
|
+
this.cooldownGenerations.set(key.id, generation);
|
|
3555
|
+
if (oauthDemotion) {
|
|
3556
|
+
this.warn(`oauth account ${key.id} on ${providerId} failed (${failure.code}); demoting to the key pool for ${Math.round(result.cooldownMs / 1000)}s`);
|
|
3557
|
+
// Persist the applied cooldown on this path too: the deferred
|
|
3558
|
+
// fallbackToKeys touch never runs when its deleteRecord throws, and
|
|
3559
|
+
// this touch is what schedules the debounced write regardless.
|
|
3560
|
+
this.store.touch();
|
|
3561
|
+
// Round-5 finding C1: publish the demotion so the request-error handler
|
|
3562
|
+
// can wait for the projection to actually be gone before its retry
|
|
3563
|
+
// re-enters the stream (otherwise pi-ai serves the stored grant again).
|
|
3564
|
+
// R7-MF: carry THIS row's identity through the queued demotion. The
|
|
3565
|
+
// failure belongs to the account that served the request, not to
|
|
3566
|
+
// whichever account is active when the queue drains — a "Set active"
|
|
3567
|
+
// landing in between must keep its healthy projection.
|
|
3568
|
+
this.demoteProjection(providerId, key.id);
|
|
3569
|
+
return;
|
|
3570
|
+
}
|
|
3571
|
+
this.warn(`key ${key.id} on ${providerId} cooled for ${Math.round(result.cooldownMs / 1000)}s`);
|
|
3572
|
+
this.store.touch();
|
|
3573
|
+
}
|
|
3574
|
+
/**
|
|
3575
|
+
* Queue one projection demotion for a failed account and publish its
|
|
3576
|
+
* promise so the request-error handler can wait for the projection to be
|
|
3577
|
+
* gone (round-5 finding C1). `keyId` is the failing row (R7-MF), so the
|
|
3578
|
+
* queued body deletes only a projection that still belongs to it. R14-M1:
|
|
3579
|
+
* both the cooldown-extension branch and the dedupe branch start the
|
|
3580
|
+
* demotion; it is idempotent when the projection is already gone.
|
|
3581
|
+
*/
|
|
3582
|
+
demoteProjection(providerId, keyId) {
|
|
3583
|
+
const demotion = this.fallbackToKeys(providerId, keyId).catch(error => {
|
|
3584
|
+
this.warn(`key-pool fallback failed for ${providerId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
3585
|
+
});
|
|
3586
|
+
this.projectionDemotions.set(providerId, demotion);
|
|
3587
|
+
void demotion.then(() => {
|
|
3588
|
+
if (this.projectionDemotions.get(providerId) === demotion)
|
|
3589
|
+
this.projectionDemotions.delete(providerId);
|
|
3590
|
+
});
|
|
3591
|
+
}
|
|
3592
|
+
/**
|
|
3593
|
+
* Wait for any in-flight projection demotion on `providerId` (round-5
|
|
3594
|
+
* finding C1). Resolves immediately when nothing is pending, and always
|
|
3595
|
+
* resolves — the recorded promise already absorbed its own error.
|
|
3596
|
+
*/
|
|
3597
|
+
settleProjectionDemotion(providerId) {
|
|
3598
|
+
const pending = this.projectionDemotions.get(providerId);
|
|
3599
|
+
return pending === undefined ? Promise.resolve() : pending;
|
|
3600
|
+
}
|
|
3601
|
+
/**
|
|
3602
|
+
* Whether the key pool can serve a request right now (round-5 finding C1):
|
|
3603
|
+
* a selectable non-account row other than `excludeKeyId`. Account rows are
|
|
3604
|
+
* excluded because their reference is not broker-resolvable — only a
|
|
3605
|
+
* projected grant serves them.
|
|
3606
|
+
*/
|
|
3607
|
+
hasFailoverCandidate(providerId, excludeKeyId) {
|
|
3608
|
+
const provider = this.store.provider(providerId);
|
|
3609
|
+
if (provider === undefined)
|
|
3610
|
+
return false;
|
|
3611
|
+
const now = Date.now();
|
|
3612
|
+
return provider.keys.some(key => key.id !== excludeKeyId && key.kind !== 'oauth' && isSelectable(key, now));
|
|
3613
|
+
}
|
|
3614
|
+
/** Prune stale attempt records. */
|
|
3615
|
+
pruneAttempts(now) {
|
|
3616
|
+
if (this.attempts.size >= 128) {
|
|
3617
|
+
for (const [key, attempt] of this.attempts) {
|
|
3618
|
+
if (now - attempt.at <= ATTEMPT_TTL_MS)
|
|
3619
|
+
continue;
|
|
3620
|
+
const stepKey = attempt.turn !== undefined && attempt.step !== undefined
|
|
3621
|
+
? `${key}\u0000${attempt.turn}\u0000${attempt.step}`
|
|
3622
|
+
: undefined;
|
|
3623
|
+
// A step-scoped record or a recent switch means the step still owns a
|
|
3624
|
+
// budget; do not drop the in-flight attribution record first.
|
|
3625
|
+
if (stepKey !== undefined && (this.stepAttempts.has(stepKey) || (this.switches.get(stepKey)?.at ?? 0) > now - ATTEMPT_TTL_MS))
|
|
3626
|
+
continue;
|
|
3627
|
+
this.attempts.delete(key);
|
|
3628
|
+
}
|
|
3629
|
+
}
|
|
3630
|
+
const liveSteps = new Set();
|
|
3631
|
+
const maxRetention = 24 * 60 * 60 * 1000;
|
|
3632
|
+
for (const [key, attempt] of this.stepAttempts) {
|
|
3633
|
+
const age = now - attempt.at;
|
|
3634
|
+
const hasSwitch = this.switches.has(key);
|
|
3635
|
+
if (age <= ATTEMPT_TTL_MS || (hasSwitch && age <= maxRetention)) {
|
|
3636
|
+
liveSteps.add(key);
|
|
3637
|
+
continue;
|
|
3638
|
+
}
|
|
3639
|
+
this.stepAttempts.delete(key);
|
|
3640
|
+
}
|
|
3641
|
+
for (const [key, entry] of this.switches) {
|
|
3642
|
+
if (liveSteps.has(key))
|
|
3643
|
+
continue;
|
|
3644
|
+
if (now - entry.at <= ATTEMPT_TTL_MS)
|
|
3645
|
+
continue;
|
|
3646
|
+
this.switches.delete(key);
|
|
3647
|
+
}
|
|
3648
|
+
for (const [key, attempts] of this.streamAttempts) {
|
|
3649
|
+
const kept = attempts.filter(entry => now - entry.at <= ATTEMPT_TTL_MS);
|
|
3650
|
+
if (kept.length === 0)
|
|
3651
|
+
this.streamAttempts.delete(key);
|
|
3652
|
+
else
|
|
3653
|
+
this.streamAttempts.set(key, kept);
|
|
3654
|
+
}
|
|
3655
|
+
for (const [key, pin] of this.retryBindings) {
|
|
3656
|
+
if (now - pin.at > PIN_TTL_MS)
|
|
3657
|
+
this.retryBindings.delete(key);
|
|
3658
|
+
}
|
|
3659
|
+
for (const [key, step] of this.requestSteps) {
|
|
3660
|
+
if (now - step.at > ATTEMPT_TTL_MS)
|
|
3661
|
+
this.requestSteps.delete(key);
|
|
3662
|
+
}
|
|
3663
|
+
}
|
|
3664
|
+
/** Waterfall listener: bind a key to the request and observe its terminal chunk. */
|
|
3665
|
+
streamListener(options, next) {
|
|
3666
|
+
const providerId = options.provider;
|
|
3667
|
+
const sessionId = options.sessionId === undefined ? undefined : String(options.sessionId);
|
|
3668
|
+
// `options.provider` is the route the agent/request waterfall finally
|
|
3669
|
+
// resolved; noteRequestStep keys its record by the same
|
|
3670
|
+
// `${sessionId}\u0000${providerId}` pair, so the two must stay in sync or
|
|
3671
|
+
// the retry pin below will not be found.
|
|
3672
|
+
const current = sessionId === undefined || options.purpose !== undefined
|
|
3673
|
+
? undefined
|
|
3674
|
+
: this.requestSteps.get(`${sessionId}\u0000${providerId}`);
|
|
3675
|
+
const step = current !== undefined && Date.now() - current.at <= ATTEMPT_TTL_MS
|
|
3676
|
+
? { turn: current.turn, step: current.step }
|
|
3677
|
+
: undefined;
|
|
3678
|
+
const record = this.store.provider(providerId);
|
|
3679
|
+
if (record === undefined) {
|
|
3680
|
+
if (sessionId !== undefined && step !== undefined)
|
|
3681
|
+
this.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3682
|
+
return next();
|
|
3683
|
+
}
|
|
3684
|
+
const binding = this.bindingState(record);
|
|
3685
|
+
if (binding === 'mismatch') {
|
|
3686
|
+
if (sessionId !== undefined && step !== undefined)
|
|
3687
|
+
this.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3688
|
+
return next();
|
|
3689
|
+
}
|
|
3690
|
+
return this.wrapStream(options, next, providerId, sessionId, step);
|
|
3691
|
+
}
|
|
3692
|
+
/**
|
|
3693
|
+
* The account row whose grant payload currently IS the live projection,
|
|
3694
|
+
* verified against the provider's rows (MF2/MF3 read side). A stale map
|
|
3695
|
+
* entry (row removed) resolves to null.
|
|
3696
|
+
*/
|
|
3697
|
+
oauthOwnerFor(provider) {
|
|
3698
|
+
const accountId = this.activeAccounts.get(provider);
|
|
3699
|
+
if (accountId === undefined)
|
|
3700
|
+
return null;
|
|
3701
|
+
const row = this.store.provider(provider)?.keys.find(candidate => candidate.id === accountId);
|
|
3702
|
+
return row !== undefined && row.kind === 'oauth' && row.oauth !== null ? row.id : null;
|
|
3703
|
+
}
|
|
3704
|
+
/** Wrap one managed stream so credential resolution sees this key. */
|
|
3705
|
+
wrapStream(_options, next, providerId, sessionId, step) {
|
|
3706
|
+
const service = this;
|
|
3707
|
+
let iterator;
|
|
3708
|
+
let store;
|
|
3709
|
+
let started = false;
|
|
3710
|
+
let keyId;
|
|
3711
|
+
return {
|
|
3712
|
+
[Symbol.asyncIterator]() {
|
|
3713
|
+
return {
|
|
3714
|
+
async next() {
|
|
3715
|
+
if (!started) {
|
|
3716
|
+
started = true;
|
|
3717
|
+
// Round-8 finding I-000108: a restarted process serves the
|
|
3718
|
+
// persisted projection before the ownership hydration settles.
|
|
3719
|
+
// Attribution must wait for it — otherwise this request selects
|
|
3720
|
+
// and mutates a key cursor even though pi-ai serves the stored
|
|
3721
|
+
// grant, and an OAuth failure is classified as externally
|
|
3722
|
+
// authenticated. Round-9 should-fix: the wait is bounded and is
|
|
3723
|
+
// skipped entirely for providers that hold no OAuth rows, so a
|
|
3724
|
+
// stalled credential read can neither wedge this request nor
|
|
3725
|
+
// delay an API-key-only provider.
|
|
3726
|
+
const hydrated = await service.awaitProjectionOwnership(providerId);
|
|
3727
|
+
// Round-5 finding C4: while a projection exists, pi-ai serves
|
|
3728
|
+
// the stored grant and never consults the broker, so selecting
|
|
3729
|
+
// a key here would only mutate unused key state (cursor,
|
|
3730
|
+
// lastUsedAt, sticky binding) on every OAuth-backed request.
|
|
3731
|
+
// Skip selection entirely; the request is attributed to the
|
|
3732
|
+
// serving account row below. The retry after a demotion runs
|
|
3733
|
+
// with no projection, so it selects from the key pool.
|
|
3734
|
+
//
|
|
3735
|
+
// When the bounded wait elapsed, attribution is unknown: no
|
|
3736
|
+
// owner, and no key selection either, because a projection may
|
|
3737
|
+
// still be serving (the hydration result lands later under its
|
|
3738
|
+
// generation guard).
|
|
3739
|
+
const oauthAccountId = hydrated ? service.oauthOwnerFor(providerId) : null;
|
|
3740
|
+
let selection = { key: undefined, value: undefined };
|
|
3741
|
+
if (hydrated && oauthAccountId === null) {
|
|
3742
|
+
try {
|
|
3743
|
+
selection = await service.selectKey(providerId, sessionId, step);
|
|
3744
|
+
}
|
|
3745
|
+
catch (error) {
|
|
3746
|
+
service.warn(`key selection failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3747
|
+
}
|
|
3748
|
+
}
|
|
3749
|
+
else if (hydrated && sessionId !== undefined && step !== undefined) {
|
|
3750
|
+
// A projection-served attempt cannot consume a step retry pin;
|
|
3751
|
+
// dropping it here keeps a stale pin from leaking into a later
|
|
3752
|
+
// attempt of the same step.
|
|
3753
|
+
service.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3754
|
+
}
|
|
3755
|
+
keyId = selection.key?.id;
|
|
3756
|
+
store = {
|
|
3757
|
+
provider: providerId,
|
|
3758
|
+
keyId: keyId ?? null,
|
|
3759
|
+
keyValue: selection.value ?? null,
|
|
3760
|
+
// Only the broker resolver proves that this request actually
|
|
3761
|
+
// used a managed key credential.
|
|
3762
|
+
keyUsed: false,
|
|
3763
|
+
// Captured where the broker resolve path is set up (MF2):
|
|
3764
|
+
// when a projection exists, pi-ai serves the stored grant
|
|
3765
|
+
// and this scope owns the failure attribution.
|
|
3766
|
+
oauthAccountId,
|
|
3767
|
+
};
|
|
3768
|
+
if (sessionId !== undefined && keyId !== undefined) {
|
|
3769
|
+
const key = bindingKey(sessionId, providerId);
|
|
3770
|
+
const previous = service.attempts.get(key);
|
|
3771
|
+
service.attempts.set(key, {
|
|
3772
|
+
keyId,
|
|
3773
|
+
// Preserve the retry budget across key changes. With sticky
|
|
3774
|
+
// sessions off, round-robin picks a different key each request;
|
|
3775
|
+
// resetting here would make the retry budget unbounded.
|
|
3776
|
+
retries: previous?.retries ?? 0,
|
|
3777
|
+
failure: undefined,
|
|
3778
|
+
at: Date.now(),
|
|
3779
|
+
turn: previous?.turn,
|
|
3780
|
+
step: previous?.step,
|
|
3781
|
+
usedManagedKey: false,
|
|
3782
|
+
});
|
|
3783
|
+
}
|
|
3784
|
+
iterator = await service.als.run(store, () => next()[Symbol.asyncIterator]());
|
|
3785
|
+
}
|
|
3786
|
+
if (iterator === undefined)
|
|
3787
|
+
throw new Error('managed stream has no iterator');
|
|
3788
|
+
const result = await service.als.run(store, () => iterator.next());
|
|
3789
|
+
if (result.done === true)
|
|
3790
|
+
return result;
|
|
3791
|
+
const chunk = result.value;
|
|
3792
|
+
if (chunk.type === 'finish' && chunk.reason !== undefined) {
|
|
3793
|
+
const reason = chunk.reason;
|
|
3794
|
+
const used = store?.keyUsed === true;
|
|
3795
|
+
// Captured where the broker resolve path is set up: when a
|
|
3796
|
+
// projection exists, pi-ai serves the stored grant, so both
|
|
3797
|
+
// terminal kinds attribute to the owning account row.
|
|
3798
|
+
const oauthAccountId = store?.oauthAccountId ?? null;
|
|
3799
|
+
if (reason.kind === 'error') {
|
|
3800
|
+
// MF2 attribution: with an active projection, pi-ai served the
|
|
3801
|
+
// stored grant, so an oauth-coded failure belongs to the
|
|
3802
|
+
// account row — cool it and demote the projection exactly as
|
|
3803
|
+
// markFailure's oauth branch does, instead of delegating with
|
|
3804
|
+
// no recovery evidence.
|
|
3805
|
+
const failure = reason.failure;
|
|
3806
|
+
if (oauthAccountId !== null && failure !== undefined && classifyOAuthFailure(failure) === 'oauth') {
|
|
3807
|
+
service.recordFailure(providerId, sessionId, oauthAccountId, failure, true);
|
|
3808
|
+
service.markFailure(providerId, oauthAccountId, failure);
|
|
3809
|
+
// Round-5 finding C1: the demotion alone only helps a LATER
|
|
3810
|
+
// request. Recording request-scoped evidence in the same
|
|
3811
|
+
// linked form the key paths produce lets the manager's retry
|
|
3812
|
+
// policy fail over inside THIS request. It is recorded only
|
|
3813
|
+
// when the pool holds a candidate: with none, neither the
|
|
3814
|
+
// demoted account nor a key can serve the retry, and a retry
|
|
3815
|
+
// would be a phantom loop — delegation stays the answer.
|
|
3816
|
+
if (service.hasFailoverCandidate(providerId, oauthAccountId)) {
|
|
3817
|
+
service.recordStreamAttempt(providerId, sessionId, oauthAccountId, true, failure);
|
|
3818
|
+
}
|
|
3819
|
+
}
|
|
3820
|
+
else {
|
|
3821
|
+
if (reason.failure !== undefined && shouldFailover(reason.failure.code) && service.store.settings.autoSwitch) {
|
|
3822
|
+
service.recordStreamAttempt(providerId, sessionId, keyId, used, reason.failure);
|
|
3823
|
+
}
|
|
3824
|
+
if (used && reason.failure !== undefined) {
|
|
3825
|
+
service.recordFailure(providerId, sessionId, keyId, reason.failure, true);
|
|
3826
|
+
}
|
|
3827
|
+
else if (sessionId !== undefined) {
|
|
3828
|
+
// The request failed on an external credential; do not
|
|
3829
|
+
// attribute it to a managed key.
|
|
3830
|
+
service.attempts.delete(bindingKey(sessionId, providerId));
|
|
3831
|
+
if (step !== undefined)
|
|
3832
|
+
service.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3833
|
+
}
|
|
3834
|
+
}
|
|
3835
|
+
service.pruneAttempts(Date.now());
|
|
3836
|
+
}
|
|
3837
|
+
else if (reason.kind === 'aborted') {
|
|
3838
|
+
if (sessionId !== undefined && step !== undefined)
|
|
3839
|
+
service.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3840
|
+
}
|
|
3841
|
+
else if (used) {
|
|
3842
|
+
service.recordSuccess(providerId, sessionId, keyId, step);
|
|
3843
|
+
}
|
|
3844
|
+
else if (oauthAccountId !== null) {
|
|
3845
|
+
// Codex P2: symmetric to the failure attribution — a request
|
|
3846
|
+
// served by the projected grant succeeded on the owning
|
|
3847
|
+
// account row, so its counters and failure streak update
|
|
3848
|
+
// there, not on a broker key.
|
|
3849
|
+
service.recordProjectionSuccess(providerId, oauthAccountId);
|
|
3850
|
+
if (sessionId !== undefined) {
|
|
3851
|
+
service.attempts.delete(bindingKey(sessionId, providerId));
|
|
3852
|
+
if (step !== undefined)
|
|
3853
|
+
service.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3854
|
+
}
|
|
3855
|
+
}
|
|
3856
|
+
else if (sessionId !== undefined) {
|
|
3857
|
+
service.attempts.delete(bindingKey(sessionId, providerId));
|
|
3858
|
+
if (step !== undefined)
|
|
3859
|
+
service.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3860
|
+
}
|
|
3861
|
+
}
|
|
3862
|
+
return result;
|
|
3863
|
+
},
|
|
3864
|
+
async return(value) {
|
|
3865
|
+
if (iterator?.return !== undefined && store !== undefined) {
|
|
3866
|
+
return service.als.run(store, () => iterator.return(value));
|
|
3867
|
+
}
|
|
3868
|
+
return { done: true, value: undefined };
|
|
3869
|
+
},
|
|
3870
|
+
async throw(error) {
|
|
3871
|
+
if (iterator?.throw !== undefined && store !== undefined) {
|
|
3872
|
+
return service.als.run(store, () => iterator.throw(error));
|
|
3873
|
+
}
|
|
3874
|
+
throw error;
|
|
3875
|
+
},
|
|
3876
|
+
};
|
|
3877
|
+
},
|
|
3878
|
+
};
|
|
3879
|
+
}
|
|
3880
|
+
/**
|
|
3881
|
+
* Delegate recovery to the next agent/request-error listener. Preserve the
|
|
3882
|
+
* current key only when that listener actually requests a retry; a declined
|
|
3883
|
+
* (undefined) recovery is terminal and must still cool the exhausted key.
|
|
3884
|
+
*/
|
|
3885
|
+
async delegateAndMaybeCool(payload, providerId, keyId, next) {
|
|
3886
|
+
let action;
|
|
3887
|
+
try {
|
|
3888
|
+
action = await next();
|
|
3889
|
+
}
|
|
3890
|
+
catch (error) {
|
|
3891
|
+
this.markFailure(providerId, keyId, payload.failure);
|
|
3892
|
+
throw error;
|
|
3893
|
+
}
|
|
3894
|
+
if (action === undefined)
|
|
3895
|
+
this.markFailure(providerId, keyId, payload.failure);
|
|
3896
|
+
return action;
|
|
3897
|
+
}
|
|
3898
|
+
/** Record one failed stream attempt for turn/step attribution. */
|
|
3899
|
+
recordStreamAttempt(providerId, sessionId, keyId, usedManagedKey, failure) {
|
|
3900
|
+
if (sessionId === undefined)
|
|
3901
|
+
return;
|
|
3902
|
+
const entry = {
|
|
3903
|
+
keyId,
|
|
3904
|
+
usedManagedKey,
|
|
3905
|
+
at: Date.now(),
|
|
3906
|
+
sequence: ++this.attemptSequence,
|
|
3907
|
+
claimed: false,
|
|
3908
|
+
};
|
|
3909
|
+
if (typeof failure === 'object' && failure !== null) {
|
|
3910
|
+
// The agent loop forwards this exact failure object to
|
|
3911
|
+
// agent/request-error; it is a request-scoped identity.
|
|
3912
|
+
this.linkedAttempts.set(failure, entry);
|
|
3913
|
+
return;
|
|
3914
|
+
}
|
|
3915
|
+
const key = bindingKey(sessionId, providerId);
|
|
3916
|
+
const attempts = this.streamAttempts.get(key) ?? [];
|
|
3917
|
+
attempts.push(entry);
|
|
3918
|
+
while (attempts.length > 8)
|
|
3919
|
+
attempts.shift();
|
|
3920
|
+
this.streamAttempts.set(key, attempts);
|
|
3921
|
+
}
|
|
3922
|
+
/** Consume the stream record linked to this exact failure object. */
|
|
3923
|
+
takeLinkedAttempt(failure) {
|
|
3924
|
+
if (typeof failure !== 'object' || failure === null)
|
|
3925
|
+
return undefined;
|
|
3926
|
+
const entry = this.linkedAttempts.get(failure);
|
|
3927
|
+
if (entry === undefined)
|
|
3928
|
+
return undefined;
|
|
3929
|
+
this.linkedAttempts.delete(failure);
|
|
3930
|
+
return entry;
|
|
3931
|
+
}
|
|
3932
|
+
/** Claim the next unclaimed stream attempt after a step's last sequence. */
|
|
3933
|
+
claimStreamAttempt(providerId, sessionId, afterSequence) {
|
|
3934
|
+
if (sessionId === undefined)
|
|
3935
|
+
return undefined;
|
|
3936
|
+
const attempts = this.streamAttempts.get(bindingKey(sessionId, providerId));
|
|
3937
|
+
if (attempts === undefined)
|
|
3938
|
+
return undefined;
|
|
3939
|
+
const found = attempts.find(entry => !entry.claimed && entry.sequence > afterSequence);
|
|
3940
|
+
if (found === undefined)
|
|
3941
|
+
return undefined;
|
|
3942
|
+
found.claimed = true;
|
|
3943
|
+
return found;
|
|
3944
|
+
}
|
|
3945
|
+
/**
|
|
3946
|
+
* Find a metadata-selectable alternate whose credential actually resolves.
|
|
3947
|
+
* A key whose credential was removed outside the manager must not cause the
|
|
3948
|
+
* current, working key to be cooled and the session rebound.
|
|
3949
|
+
*/
|
|
3950
|
+
async findVerifiedAlternate(record, currentKeyId, now) {
|
|
3951
|
+
const tried = new Set();
|
|
3952
|
+
for (;;) {
|
|
3953
|
+
const candidate = pickHealthyExcluding(record, currentKeyId, now);
|
|
3954
|
+
if (candidate === undefined || tried.has(candidate.id))
|
|
3955
|
+
return undefined;
|
|
3956
|
+
tried.add(candidate.id);
|
|
3957
|
+
const probe = await this.probeKey(candidate.ref);
|
|
3958
|
+
if (probe.hit !== undefined)
|
|
3959
|
+
return candidate;
|
|
3960
|
+
if (probe.missing) {
|
|
3961
|
+
candidate.valueMissing = true;
|
|
3962
|
+
this.store.touch();
|
|
3963
|
+
}
|
|
3964
|
+
}
|
|
3965
|
+
}
|
|
3966
|
+
/** Waterfall listener: bounded retry, then key switch and cool-down. */
|
|
3967
|
+
async requestErrorListener(payload, next) {
|
|
3968
|
+
// Consume a linked record on every path, including skipped recovery, so a
|
|
3969
|
+
// later request cannot claim this failure's evidence.
|
|
3970
|
+
const linked = this.takeLinkedAttempt(payload.failure);
|
|
3971
|
+
const providerId = payload.provider;
|
|
3972
|
+
const sessionId = payload.agent?.session?.id === undefined ? undefined : String(payload.agent.session.id);
|
|
3973
|
+
const dropPin = () => {
|
|
3974
|
+
if (sessionId !== undefined)
|
|
3975
|
+
this.clearRetry(providerId, sessionId, payload.turn, payload.step);
|
|
3976
|
+
};
|
|
3977
|
+
if (payload.signal?.aborted) {
|
|
3978
|
+
dropPin();
|
|
3979
|
+
return next();
|
|
3980
|
+
}
|
|
3981
|
+
const record = this.store.provider(providerId);
|
|
3982
|
+
if (record === undefined || this.bindingState(record) === 'mismatch') {
|
|
3983
|
+
dropPin();
|
|
3984
|
+
return next();
|
|
3985
|
+
}
|
|
3986
|
+
const settings = this.store.settings;
|
|
3987
|
+
if (!settings.autoSwitch) {
|
|
3988
|
+
dropPin();
|
|
3989
|
+
return next();
|
|
3990
|
+
}
|
|
3991
|
+
if (!shouldFailover(payload.failure?.code)) {
|
|
3992
|
+
dropPin();
|
|
3993
|
+
return next();
|
|
3994
|
+
}
|
|
3995
|
+
if (sessionId === undefined)
|
|
3996
|
+
return next();
|
|
3997
|
+
// Round-5 finding C1: an OAuth-grant failure cools the account and starts
|
|
3998
|
+
// an asynchronous projection demotion. The retry must not re-enter the
|
|
3999
|
+
// stream while that demotion is pending — pi-ai would serve the stored
|
|
4000
|
+
// grant again and the key-pool failover would never happen. Awaiting here
|
|
4001
|
+
// is what makes the retry attempt land on the pool within this request.
|
|
4002
|
+
await this.settleProjectionDemotion(providerId);
|
|
4003
|
+
const now = Date.now();
|
|
4004
|
+
const attemptKey = bindingKey(sessionId, providerId);
|
|
4005
|
+
const switchKey = `${attemptKey}\u0000${payload.turn}\u0000${payload.step}`;
|
|
4006
|
+
let attempt = this.stepAttempts.get(switchKey);
|
|
4007
|
+
const evidence = linked ?? this.claimStreamAttempt(providerId, sessionId, attempt?.lastSequence ?? 0);
|
|
4008
|
+
if (attempt === undefined) {
|
|
4009
|
+
// Do not fall back to the shared session record: another step may have
|
|
4010
|
+
// overwritten it. Without request-scoped evidence, delegating is safer.
|
|
4011
|
+
if (evidence === undefined || evidence.usedManagedKey !== true)
|
|
4012
|
+
return next();
|
|
4013
|
+
attempt = {
|
|
4014
|
+
keyId: evidence.keyId,
|
|
4015
|
+
retries: 0,
|
|
4016
|
+
at: now,
|
|
4017
|
+
turn: payload.turn,
|
|
4018
|
+
step: payload.step,
|
|
4019
|
+
usedManagedKey: true,
|
|
4020
|
+
lastSequence: evidence.sequence,
|
|
4021
|
+
keyRetries: {},
|
|
4022
|
+
};
|
|
4023
|
+
this.stepAttempts.set(switchKey, attempt);
|
|
4024
|
+
}
|
|
4025
|
+
else {
|
|
4026
|
+
// Every recovery needs evidence for the request that just failed; an
|
|
4027
|
+
// existing step must not reuse stale attribution or budget.
|
|
4028
|
+
if (evidence === undefined) {
|
|
4029
|
+
this.switches.delete(switchKey);
|
|
4030
|
+
this.stepAttempts.delete(switchKey);
|
|
4031
|
+
this.clearRetry(providerId, sessionId, payload.turn, payload.step);
|
|
4032
|
+
return next();
|
|
4033
|
+
}
|
|
4034
|
+
const keyChanged = attempt.keyId !== evidence.keyId;
|
|
4035
|
+
attempt.keyId = evidence.keyId;
|
|
4036
|
+
attempt.usedManagedKey = evidence.usedManagedKey;
|
|
4037
|
+
attempt.lastSequence = evidence.sequence;
|
|
4038
|
+
attempt.at = now;
|
|
4039
|
+
if (keyChanged) {
|
|
4040
|
+
attempt.keyRetries ??= {};
|
|
4041
|
+
attempt.retries = evidence.keyId === undefined ? 0 : attempt.keyRetries[evidence.keyId] ?? 0;
|
|
4042
|
+
}
|
|
4043
|
+
}
|
|
4044
|
+
if (attempt.usedManagedKey !== true) {
|
|
4045
|
+
this.switches.delete(switchKey);
|
|
4046
|
+
this.stepAttempts.delete(switchKey);
|
|
4047
|
+
this.clearRetry(providerId, sessionId, payload.turn, payload.step);
|
|
4048
|
+
return next();
|
|
4049
|
+
}
|
|
4050
|
+
const currentKeyId = attempt.keyId;
|
|
4051
|
+
if (currentKeyId === undefined) {
|
|
4052
|
+
this.clearRetry(providerId, sessionId, payload.turn, payload.step);
|
|
4053
|
+
return next();
|
|
4054
|
+
}
|
|
4055
|
+
const current = record.keys.find(key => key.id === currentKeyId);
|
|
4056
|
+
if (current === undefined)
|
|
4057
|
+
return next();
|
|
4058
|
+
const switchCount = this.switches.get(switchKey)?.count ?? 0;
|
|
4059
|
+
if (attempt.retries < settings.retryCount) {
|
|
4060
|
+
attempt.retries += 1;
|
|
4061
|
+
attempt.keyRetries ??= {};
|
|
4062
|
+
if (attempt.keyId !== undefined)
|
|
4063
|
+
attempt.keyRetries[attempt.keyId] = attempt.retries;
|
|
4064
|
+
this.pinRetry(providerId, sessionId, payload.turn, payload.step, currentKeyId);
|
|
4065
|
+
this.warn(`${providerId} key ${currentKeyId} failed (${payload.failure?.code ?? 'error'}); retry ${attempt.retries}/${settings.retryCount}`);
|
|
4066
|
+
return { kind: 'retry' };
|
|
4067
|
+
}
|
|
4068
|
+
if (switchCount >= Math.min(MAX_SWITCHES, Math.max(1, record.keys.length))) {
|
|
4069
|
+
this.warn(`${providerId} exhausted key switches for this step; delegating recovery`);
|
|
4070
|
+
const action = await this.delegateAndMaybeCool(payload, providerId, currentKeyId, next);
|
|
4071
|
+
if (action === undefined) {
|
|
4072
|
+
this.switches.delete(switchKey);
|
|
4073
|
+
this.stepAttempts.delete(switchKey);
|
|
4074
|
+
this.clearRetry(providerId, sessionId, payload.turn, payload.step);
|
|
4075
|
+
}
|
|
4076
|
+
else {
|
|
4077
|
+
this.pinRetry(providerId, sessionId, payload.turn, payload.step, currentKeyId);
|
|
4078
|
+
}
|
|
4079
|
+
return action;
|
|
4080
|
+
}
|
|
4081
|
+
const candidate = await this.findVerifiedAlternate(record, currentKeyId, now);
|
|
4082
|
+
if (candidate === undefined) {
|
|
4083
|
+
this.warn(`${providerId} has no alternate with a resolvable credential; preserving the current key via delegation`);
|
|
4084
|
+
const action = await this.delegateAndMaybeCool(payload, providerId, currentKeyId, next);
|
|
4085
|
+
if (action === undefined) {
|
|
4086
|
+
this.switches.delete(switchKey);
|
|
4087
|
+
this.stepAttempts.delete(switchKey);
|
|
4088
|
+
this.clearRetry(providerId, sessionId, payload.turn, payload.step);
|
|
4089
|
+
}
|
|
4090
|
+
else {
|
|
4091
|
+
this.pinRetry(providerId, sessionId, payload.turn, payload.step, currentKeyId);
|
|
4092
|
+
}
|
|
4093
|
+
return action;
|
|
4094
|
+
}
|
|
4095
|
+
// Keep the current key usable until an alternate is actually chosen. If
|
|
4096
|
+
// the delegated retry re-enters llm/stream, selectKey must still find a
|
|
4097
|
+
// credential; cooling it before this point would strand that retry.
|
|
4098
|
+
this.markFailure(providerId, currentKeyId, payload.failure);
|
|
4099
|
+
this.store.bind(sessionId, providerId, candidate.id, now);
|
|
4100
|
+
attempt.keyId = candidate.id;
|
|
4101
|
+
attempt.keyRetries ??= {};
|
|
4102
|
+
attempt.keyRetries[candidate.id] ??= 0;
|
|
4103
|
+
attempt.retries = attempt.keyRetries[candidate.id];
|
|
4104
|
+
attempt.at = now;
|
|
4105
|
+
this.pinRetry(providerId, sessionId, payload.turn, payload.step, candidate.id);
|
|
4106
|
+
this.switches.set(switchKey, { count: switchCount + 1, at: now });
|
|
4107
|
+
this.store.touch();
|
|
4108
|
+
this.warn(`${providerId} switched to key ${candidate.id} after ${settings.retryCount} retries`);
|
|
4109
|
+
return { kind: 'retry' };
|
|
4110
|
+
}
|
|
4111
|
+
}
|
|
4112
|
+
/** Host entry. */
|
|
4113
|
+
export function apply(ctx) {
|
|
4114
|
+
const file = statePath();
|
|
4115
|
+
ensureStateFile(file);
|
|
4116
|
+
const store = new KeysManagerStore(file, { warn: message => ctx.logger.warn(message) });
|
|
4117
|
+
const service = new KeysManagerService(ctx, store);
|
|
4118
|
+
ctx.effect(() => () => service.close(), 'keys-manager: persist state');
|
|
4119
|
+
ctx.inject(['credentials'], child => {
|
|
4120
|
+
const credentials = child.credentials;
|
|
4121
|
+
service.attachCredentials(credentials);
|
|
4122
|
+
return () => service.detachCredentials(credentials);
|
|
4123
|
+
});
|
|
4124
|
+
ctx.inject(['settings'], child => {
|
|
4125
|
+
service.settings = child.settings;
|
|
4126
|
+
return () => {
|
|
4127
|
+
if (service.settings === child.settings)
|
|
4128
|
+
service.settings = undefined;
|
|
4129
|
+
};
|
|
4130
|
+
});
|
|
4131
|
+
const on = ctx.on;
|
|
4132
|
+
on('llm/stream', (options, next) => service.streamListener(options, next));
|
|
4133
|
+
const pruneTimer = setInterval(() => service.pruneTransient(), 60_000);
|
|
4134
|
+
pruneTimer.unref?.();
|
|
4135
|
+
ctx.effect(() => () => {
|
|
4136
|
+
clearInterval(pruneTimer);
|
|
4137
|
+
}, 'keys-manager: prune transient state');
|
|
4138
|
+
on('agent/request', async (payload, next) => {
|
|
4139
|
+
const config = await next();
|
|
4140
|
+
const provider = config !== null && typeof config === 'object' && typeof config.provider === 'string'
|
|
4141
|
+
? config.provider
|
|
4142
|
+
: '';
|
|
4143
|
+
service.noteRequestStep(payload, provider);
|
|
4144
|
+
return config;
|
|
4145
|
+
}, { prepend: true });
|
|
4146
|
+
on('agent/request-error', (payload, next) => service.requestErrorListener(payload, next), { prepend: true });
|
|
4147
|
+
ctx.provide(SERVICE, service);
|
|
4148
|
+
ctx.effect(() => ctx.typert.register({
|
|
4149
|
+
package: 'dsh-keys-manager',
|
|
4150
|
+
face: 'host',
|
|
4151
|
+
schemas: [],
|
|
4152
|
+
model: { services: [], events: [], objects: [] },
|
|
4153
|
+
invocations: INVOCATIONS,
|
|
4154
|
+
}), 'keys-manager: typert registration');
|
|
4155
|
+
ctx.logger.info(`keys-manager: ${Object.keys(store.providers).length} managed provider(s), ${file}`);
|
|
4156
|
+
}
|
|
4157
|
+
//# sourceMappingURL=index.js.map
|