dsh-keys-manager 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -8
- package/README.zh-CN.md +16 -5
- package/lib/client/styles.d.ts +1 -1
- package/lib/client/types.d.ts +47 -4
- package/lib/client.js +549 -37
- package/lib/core.d.ts +44 -0
- package/lib/core.js +44 -0
- package/lib/core.js.map +1 -1
- package/lib/domain.d.ts +79 -4
- package/lib/host-types.d.ts +63 -1
- package/lib/index.d.ts +681 -2
- package/lib/index.js +2423 -48
- package/lib/index.js.map +1 -1
- package/lib/store.js +44 -10
- package/lib/store.js.map +1 -1
- package/locale/en.json +20 -1
- package/locale/zh.json +20 -1
- package/package.json +1 -1
package/lib/index.js
CHANGED
|
@@ -15,7 +15,7 @@
|
|
|
15
15
|
*/
|
|
16
16
|
import { AsyncLocalStorage } from 'node:async_hooks';
|
|
17
17
|
import { createHash, randomBytes } from 'node:crypto';
|
|
18
|
-
import { KEY_LIMIT, applyFailureCooldown, bindingKey, isSelectable, markKeySuccess, markKeyUsed, normalizeSettings, patchSettings, pickHealthyExcluding, pickRoundRobin, shouldFailover, statusOfKey, statusOfProvider, validateKeyInput, validateKeyPatch, } from './core.js';
|
|
18
|
+
import { KEY_LIMIT, accountRef, applyFailureCooldown, bindingKey, classifyOAuthFailure, isSelectable, markKeySuccess, markKeyUsed, normalizeSettings, patchSettings, pickHealthyExcluding, pickRoundRobin, shouldFailover, statusOfKey, statusOfProvider, validateKeyInput, validateKeyPatch, } from './core.js';
|
|
19
19
|
import { EXTRA_PROVIDER_MAP, EXTRA_PROVIDERS } from './providers.js';
|
|
20
20
|
import { KeysManagerStore, ensureStateFile, statePath } from './store.js';
|
|
21
21
|
export const name = 'keys-manager';
|
|
@@ -29,6 +29,107 @@ const MAX_SWITCHES = 6;
|
|
|
29
29
|
const ATTEMPT_TTL_MS = 60 * 60 * 1000;
|
|
30
30
|
/** A retry pin only lives for this long; an expired pin is treated as absent. */
|
|
31
31
|
const PIN_TTL_MS = 60_000;
|
|
32
|
+
/**
|
|
33
|
+
* An interactive sign-in that has not settled within this window is failed
|
|
34
|
+
* and withdrawn: real flows finish in minutes, and a wedged attempt would
|
|
35
|
+
* otherwise hold the provider's single authorization slot forever.
|
|
36
|
+
*/
|
|
37
|
+
const LOGIN_ATTEMPT_TTL_MS = 10 * 60 * 1000;
|
|
38
|
+
/**
|
|
39
|
+
* Same shape of bound the api-key attempt cache uses (pruneAttempts): the
|
|
40
|
+
* settled-login cache is capped, evicting the oldest settled entries, so a
|
|
41
|
+
* rapid begin/fail loop cannot accumulate unbounded views within the TTL.
|
|
42
|
+
*/
|
|
43
|
+
const MAX_LOGIN_ATTEMPTS = 128;
|
|
44
|
+
/** Same display-name cap the api-key aliases use (core.ts MAX_ALIAS_LENGTH). */
|
|
45
|
+
const MAX_ALIAS = 80;
|
|
46
|
+
/**
|
|
47
|
+
* How long a pending projection-removal marker may absorb its deletion
|
|
48
|
+
* echo. Bounded well above any realistic echo delay (the seam's own
|
|
49
|
+
* watcher debounce dwarfs it): the old microtask sweep raced the echo
|
|
50
|
+
* handler's awaited re-read and could clear the marker too early; the
|
|
51
|
+
* race came from unbounded immediacy, not slow echoes.
|
|
52
|
+
*/
|
|
53
|
+
const INTENTIONAL_REMOVAL_TTL_MS = 5_000;
|
|
54
|
+
/** The credential-record scope pi-ai stores its per-provider grants under (llm-pi-ai/src/auth.ts:29). */
|
|
55
|
+
export const OAUTH_GRANT_RECORD_SCOPE = 'llm-pi-ai';
|
|
56
|
+
/**
|
|
57
|
+
* Upper bound on one stream's wait for projection-ownership hydration
|
|
58
|
+
* (round-9 should-fix). Hydration reads one `readRecord` per OAuth provider
|
|
59
|
+
* serially and the seam offers no timeout or cancellation, so a single
|
|
60
|
+
* stalled read can keep that promise pending indefinitely. The wait is
|
|
61
|
+
* therefore bounded: 500 ms is generous for a healthy local credential-store
|
|
62
|
+
* read while keeping a stalled read from stalling the request (and every
|
|
63
|
+
* managed provider behind it) for longer than half a second. The hydration
|
|
64
|
+
* itself is never cancelled — a late result still lands through the
|
|
65
|
+
* generation guard in {@link KeysManagerService.hydrateProjectionOwnership}.
|
|
66
|
+
*/
|
|
67
|
+
const HYDRATION_WAIT_MS = 500;
|
|
68
|
+
/**
|
|
69
|
+
* Stable failure code for a harvest whose state flush rejected AND whose
|
|
70
|
+
* credential-side rollback then failed to write.
|
|
71
|
+
*
|
|
72
|
+
* The state flush is the durability boundary `run` enforces, and the rollback
|
|
73
|
+
* that follows it deletes/restores the credential records the rolled-back row
|
|
74
|
+
* referenced. When those compensating writes fail too, the attempt's reason
|
|
75
|
+
* starts with `<code>: ` and keeps the original flush error verbatim after it,
|
|
76
|
+
* so an upstream caller can branch on {@link isFlushRollbackCompensationFailure}
|
|
77
|
+
* instead of matching prose. A flush failure whose rollback completed keeps its
|
|
78
|
+
* original error unchanged — this code is the only added spelling.
|
|
79
|
+
*/
|
|
80
|
+
export const FLUSH_ROLLBACK_COMPENSATION_FAILED = 'FLUSH_ROLLBACK_COMPENSATION_FAILED';
|
|
81
|
+
/** Whether an attempt error carries {@link FLUSH_ROLLBACK_COMPENSATION_FAILED}. */
|
|
82
|
+
export function isFlushRollbackCompensationFailure(error) {
|
|
83
|
+
return typeof error === 'string' && error.startsWith(`${FLUSH_ROLLBACK_COMPENSATION_FAILED}:`);
|
|
84
|
+
}
|
|
85
|
+
/** Whether an opaque grant payload is the OAuth credential pi-ai owns. */
|
|
86
|
+
function isOAuthGrantPayload(value) {
|
|
87
|
+
if (value === null || typeof value !== 'object' || Array.isArray(value))
|
|
88
|
+
return false;
|
|
89
|
+
const source = value;
|
|
90
|
+
return source.type === 'oauth'
|
|
91
|
+
&& typeof source.refresh === 'string' && source.refresh.length > 0
|
|
92
|
+
&& typeof source.access === 'string';
|
|
93
|
+
}
|
|
94
|
+
/** Copy a seam prompt into its serializable view (the AbortSignal stays out). */
|
|
95
|
+
function promptView(prompt) {
|
|
96
|
+
const options = Array.isArray(prompt.options)
|
|
97
|
+
? prompt.options
|
|
98
|
+
.filter((entry) => entry !== null && typeof entry === 'object'
|
|
99
|
+
&& typeof entry.id === 'string'
|
|
100
|
+
&& typeof entry.label === 'string')
|
|
101
|
+
: undefined;
|
|
102
|
+
const kind = prompt.kind === 'secret' || prompt.kind === 'select' ? prompt.kind : 'text';
|
|
103
|
+
return {
|
|
104
|
+
kind,
|
|
105
|
+
message: prompt.message,
|
|
106
|
+
...(prompt.placeholder === undefined ? {} : { placeholder: prompt.placeholder }),
|
|
107
|
+
...(options === undefined || options.length === 0 ? {} : { options }),
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
/** Copy a seam notice into its serializable view. */
|
|
111
|
+
function noticeView(notice) {
|
|
112
|
+
return {
|
|
113
|
+
message: notice.message,
|
|
114
|
+
...(notice.url === undefined ? {} : { url: notice.url }),
|
|
115
|
+
...(notice.code === undefined ? {} : { code: notice.code }),
|
|
116
|
+
};
|
|
117
|
+
}
|
|
118
|
+
/** Strip the host-side handles from one attempt. */
|
|
119
|
+
function attemptView(attempt) {
|
|
120
|
+
return {
|
|
121
|
+
id: attempt.id,
|
|
122
|
+
provider: attempt.provider,
|
|
123
|
+
...(attempt.alias === undefined ? {} : { alias: attempt.alias }),
|
|
124
|
+
status: attempt.status,
|
|
125
|
+
notices: attempt.notices.map(notice => ({ ...notice })),
|
|
126
|
+
...(attempt.prompt === undefined ? {} : { prompt: { ...attempt.prompt, ...(attempt.prompt.options === undefined ? {} : { options: attempt.prompt.options.map(option => ({ ...option })) }) } }),
|
|
127
|
+
...(attempt.error === undefined ? {} : { error: attempt.error }),
|
|
128
|
+
...(attempt.accountId === undefined ? {} : { accountId: attempt.accountId }),
|
|
129
|
+
createdAt: attempt.createdAt,
|
|
130
|
+
updatedAt: attempt.updatedAt,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
32
133
|
function endpoint(method, parameters = []) {
|
|
33
134
|
return {
|
|
34
135
|
id: `dsh-keys-manager#${NAMESPACE}/${method}`,
|
|
@@ -54,6 +155,19 @@ const INVOCATIONS = [
|
|
|
54
155
|
endpoint('releaseProvider', ['provider']),
|
|
55
156
|
endpoint('addExtraProvider', ['extraId', 'alias', 'value']),
|
|
56
157
|
endpoint('setSettings', ['patch']),
|
|
158
|
+
// OAuth account management (T2 service methods). The login dialog polls
|
|
159
|
+
// accountAttempt for attempt-state transitions: the host forwards only a
|
|
160
|
+
// fixed credentials-event allowlist to the browser, so the plugin has no
|
|
161
|
+
// own event channel a client could subscribe to.
|
|
162
|
+
// targetAccountId (round-3 finding 6): the row a re-authentication
|
|
163
|
+
// replaces, so a metadata-poor rotation re-binds in place instead of
|
|
164
|
+
// duplicating the account.
|
|
165
|
+
endpoint('beginAccountLogin', ['provider', 'alias', 'targetAccountId']),
|
|
166
|
+
endpoint('answerAccountPrompt', ['attemptId', 'value']),
|
|
167
|
+
endpoint('accountAttempt', ['attemptId']),
|
|
168
|
+
endpoint('listAccountAttempts'),
|
|
169
|
+
endpoint('activateAccount', ['provider', 'accountId']),
|
|
170
|
+
endpoint('removeAccount', ['provider', 'accountId']),
|
|
57
171
|
];
|
|
58
172
|
/** Read a nested value by path. */
|
|
59
173
|
function getPath(value, path) {
|
|
@@ -138,7 +252,7 @@ function discoveredModel(model) {
|
|
|
138
252
|
});
|
|
139
253
|
}
|
|
140
254
|
/** Public, secret-free key row. */
|
|
141
|
-
function publicKey(key, now) {
|
|
255
|
+
function publicKey(key, now, projectedAccountId) {
|
|
142
256
|
return {
|
|
143
257
|
id: key.id,
|
|
144
258
|
alias: key.alias,
|
|
@@ -154,6 +268,21 @@ function publicKey(key, now) {
|
|
|
154
268
|
importedFrom: key.importedFrom ?? null,
|
|
155
269
|
valueMissing: key.valueMissing === true,
|
|
156
270
|
kind: key.kind ?? 'api-key',
|
|
271
|
+
// Identity facts only; the grant reference stays host-side (T1 shapes).
|
|
272
|
+
...(key.kind === 'oauth' && key.oauth !== null
|
|
273
|
+
? {
|
|
274
|
+
oauth: {
|
|
275
|
+
provider: key.oauth.provider,
|
|
276
|
+
...(key.oauth.email === undefined ? {} : { email: key.oauth.email }),
|
|
277
|
+
...(key.oauth.accountId === undefined ? {} : { accountId: key.oauth.accountId }),
|
|
278
|
+
...(key.oauth.expiresAt === undefined ? {} : { expiresAt: key.oauth.expiresAt }),
|
|
279
|
+
},
|
|
280
|
+
// Step 6: surface which account row IS the live projection.
|
|
281
|
+
...(projectedAccountId !== undefined && projectedAccountId !== null && projectedAccountId === key.id
|
|
282
|
+
? { projected: true }
|
|
283
|
+
: {}),
|
|
284
|
+
}
|
|
285
|
+
: {}),
|
|
157
286
|
};
|
|
158
287
|
}
|
|
159
288
|
/** Env names that are a single API-key credential, not an OAuth/federation token. */
|
|
@@ -167,9 +296,11 @@ function isKeyishEnv(name) {
|
|
|
167
296
|
return true;
|
|
168
297
|
}
|
|
169
298
|
/**
|
|
170
|
-
* DSH catalog routes that are explicitly outside the single-key pool.
|
|
171
|
-
*
|
|
172
|
-
*
|
|
299
|
+
* DSH catalog routes that are explicitly outside the single-key pool. This
|
|
300
|
+
* is a KEY-POOL verdict only (capabilityOf): an OAuth-capable route listed
|
|
301
|
+
* here stays adoptable for account management. Keeping it as a named table
|
|
302
|
+
* (rather than only deriving it from env names) makes a reviewed decision
|
|
303
|
+
* visible and stable when a provider adds another auth path.
|
|
173
304
|
*/
|
|
174
305
|
const UNSUPPORTED_KEY_POOL_PROVIDERS = Object.freeze({
|
|
175
306
|
'amazon-bedrock': 'Uses an AWS credential chain (profile, IAM keys, container/IRSA tokens), not a single API key',
|
|
@@ -177,31 +308,106 @@ const UNSUPPORTED_KEY_POOL_PROVIDERS = Object.freeze({
|
|
|
177
308
|
'cloudflare-workers-ai': 'Needs a Cloudflare account ID in addition to the API key',
|
|
178
309
|
'openai-codex': 'Uses ChatGPT OAuth sign-in only; no API-key credential',
|
|
179
310
|
});
|
|
180
|
-
/**
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
311
|
+
/**
|
|
312
|
+
* Why an OAuth-capable catalog route cannot be managed as an account here.
|
|
313
|
+
*
|
|
314
|
+
* C6-2: `nativeAuth.oauth` is a catalog fact — the provider supports OAuth
|
|
315
|
+
* upstream. Actually signing in needs a pi-ai flow registered with this
|
|
316
|
+
* Harness build's authorization seam, which extension-backed routes may not
|
|
317
|
+
* have (they are explicitly deferred from this increment). Offering "Add
|
|
318
|
+
* account" without a flow would promise a sign-in that cannot start.
|
|
319
|
+
*/
|
|
320
|
+
const NO_OAUTH_FLOW_REASON = 'This provider has no OAuth sign-in flow registered in this Harness build';
|
|
321
|
+
/**
|
|
322
|
+
* Split one route's management capability by mode. The
|
|
323
|
+
* {@link UNSUPPORTED_KEY_POOL_PROVIDERS} table keeps its meaning: it is a
|
|
324
|
+
* verdict about API keys, so an OAuth-capable route listed there remains
|
|
325
|
+
* adoptable for accounts while its key entry stays blocked.
|
|
326
|
+
*
|
|
327
|
+
* `oauthFlow` is the live answer from {@link KeysManagerService.oauthFlowAvailable}
|
|
328
|
+
* — C6-2 requires an actually-beginable flow, not the catalog flag alone.
|
|
329
|
+
*/
|
|
330
|
+
function capabilityOf(entry, profile, oauthFlow) {
|
|
184
331
|
if (entry.error !== undefined)
|
|
185
|
-
return {
|
|
332
|
+
return { keys: false, accounts: false, reason: entry.error };
|
|
333
|
+
const accounts = entry.nativeAuth?.oauth === true && oauthFlow;
|
|
186
334
|
const unsupportedReason = UNSUPPORTED_KEY_POOL_PROVIDERS[entry.provider];
|
|
187
|
-
if (unsupportedReason !== undefined)
|
|
188
|
-
|
|
335
|
+
if (unsupportedReason !== undefined) {
|
|
336
|
+
// The table's reason describes the key verdict; when the only possible
|
|
337
|
+
// capability (accounts) is unavailable too, the flow gap is the truthful
|
|
338
|
+
// explanation a user needs.
|
|
339
|
+
const reason = entry.nativeAuth?.oauth === true && !oauthFlow ? NO_OAUTH_FLOW_REASON : unsupportedReason;
|
|
340
|
+
return { keys: false, accounts, reason };
|
|
341
|
+
}
|
|
189
342
|
if (entry.provider === 'deepseek-official')
|
|
190
|
-
return {
|
|
343
|
+
return { keys: true, accounts };
|
|
191
344
|
if (profile !== undefined && typeof profile.apiKeyEnv === 'string' && profile.apiKeyEnv.length > 0)
|
|
192
|
-
return {
|
|
345
|
+
return { keys: true, accounts };
|
|
193
346
|
const auth = entry.nativeAuth;
|
|
194
347
|
if (auth !== undefined) {
|
|
348
|
+
// A dual-auth route (key env + OAuth, e.g. anthropic) keeps BOTH
|
|
349
|
+
// capabilities; the oauth-only verdict below is for routes with no
|
|
350
|
+
// key credential at all.
|
|
195
351
|
if (auth.env.some(isKeyishEnv))
|
|
196
|
-
return {
|
|
197
|
-
if (auth.oauth)
|
|
198
|
-
return
|
|
352
|
+
return { keys: true, accounts };
|
|
353
|
+
if (auth.oauth) {
|
|
354
|
+
return oauthFlow
|
|
355
|
+
? { keys: false, accounts: true, reason: 'This provider signs in with an OAuth account; no API key is used' }
|
|
356
|
+
: { keys: false, accounts: false, reason: NO_OAUTH_FLOW_REASON };
|
|
357
|
+
}
|
|
199
358
|
if (auth.env.length > 0)
|
|
200
|
-
return {
|
|
201
|
-
return {
|
|
359
|
+
return { keys: false, accounts: false, reason: 'This provider uses native cloud credentials, not one API key' };
|
|
360
|
+
return { keys: false, accounts: false, reason: 'This provider declares no API-key authentication' };
|
|
202
361
|
}
|
|
362
|
+
return { keys: true, accounts };
|
|
363
|
+
}
|
|
364
|
+
/**
|
|
365
|
+
* Whether the route can be adopted for OAuth account management (MF1).
|
|
366
|
+
*
|
|
367
|
+
* C6-2: `nativeAuth.oauth` alone is not enough. Account support requires a
|
|
368
|
+
* flow the authorization seam can begin, resolved from the live flow list on
|
|
369
|
+
* every call (see {@link KeysManagerService.oauthFlowAvailable}): the seam
|
|
370
|
+
* may mount after this plugin, so availability is never cached at
|
|
371
|
+
* construction. While the flow list is empty (seam not mounted yet) support
|
|
372
|
+
* is false; the next state build re-evaluates once flows are enumerable.
|
|
373
|
+
*/
|
|
374
|
+
function accountSupportOf(entry, oauthFlow) {
|
|
375
|
+
if (entry === undefined)
|
|
376
|
+
return { supported: false, reason: 'This provider route is not registered' };
|
|
377
|
+
if (entry.error !== undefined)
|
|
378
|
+
return { supported: false, reason: entry.error };
|
|
379
|
+
if (entry.nativeAuth?.oauth !== true)
|
|
380
|
+
return { supported: false, reason: 'This provider offers no OAuth account sign-in' };
|
|
381
|
+
if (!oauthFlow)
|
|
382
|
+
return { supported: false, reason: NO_OAUTH_FLOW_REASON };
|
|
203
383
|
return { supported: true };
|
|
204
384
|
}
|
|
385
|
+
/**
|
|
386
|
+
* The account row whose grant payload matches one live projection payload:
|
|
387
|
+
* the refresh digest names the row first; on a miss (rotated token) the
|
|
388
|
+
* payload's identity metadata re-associates.
|
|
389
|
+
*
|
|
390
|
+
* The exact `accountId` lookup runs BEFORE the email fallback (round-8
|
|
391
|
+
* finding I-000107): the fallback is only for rows that never carried an
|
|
392
|
+
* accountId, and evaluating it in array order let an earlier email-only row
|
|
393
|
+
* claim a payload that names a later row's accountId — corrupting both the
|
|
394
|
+
* owner attribution and the cleanup target.
|
|
395
|
+
*/
|
|
396
|
+
function matchOwningAccount(provider, accounts, payload) {
|
|
397
|
+
const hash = createHash('sha256').update(payload.refresh).digest('hex').slice(0, 12);
|
|
398
|
+
const projectedRef = accountRef(provider, hash);
|
|
399
|
+
const byRef = accounts.find(candidate => candidate.oauth.grantRef === projectedRef);
|
|
400
|
+
if (byRef !== undefined)
|
|
401
|
+
return byRef;
|
|
402
|
+
if (payload.accountId !== undefined) {
|
|
403
|
+
const byAccountId = accounts.find(candidate => candidate.oauth.accountId === payload.accountId);
|
|
404
|
+
if (byAccountId !== undefined)
|
|
405
|
+
return byAccountId;
|
|
406
|
+
}
|
|
407
|
+
if (payload.email === undefined)
|
|
408
|
+
return undefined;
|
|
409
|
+
return accounts.find(candidate => candidate.oauth.email === payload.email && candidate.oauth.accountId === undefined);
|
|
410
|
+
}
|
|
205
411
|
export class KeysManagerService {
|
|
206
412
|
ctx;
|
|
207
413
|
store;
|
|
@@ -225,14 +431,367 @@ export class KeysManagerService {
|
|
|
225
431
|
selectionQueues = new Map();
|
|
226
432
|
queue = Promise.resolve();
|
|
227
433
|
closing = false;
|
|
434
|
+
/** Interactive sign-in attempts, by attempt id. */
|
|
435
|
+
loginAttempts = new Map();
|
|
436
|
+
/** The provider → attempt id index for the single-attempt rule. */
|
|
437
|
+
loginsInFlight = new Map();
|
|
438
|
+
loginSequence = 0;
|
|
439
|
+
/** The account row whose grant payload currently IS the projection, per provider. */
|
|
440
|
+
activeAccounts = new Map();
|
|
441
|
+
/**
|
|
442
|
+
* Account rows committed to removal (C6-1/I1 removal-aware oracle), marked
|
|
443
|
+
* by {@link removeOAuthGrant} before its first awaited delete. A multi-row
|
|
444
|
+
* release deletes every row's credential first and the provider record
|
|
445
|
+
* last, and it never splices its rows, so while it runs the provider record
|
|
446
|
+
* and the row object both still look live even though the row's grant is
|
|
447
|
+
* already gone. Object identity alone therefore cannot tell "still serving"
|
|
448
|
+
* from "mid-removal"; this mark can. A successful removal leaves it set —
|
|
449
|
+
* the row it names is either spliced out (removeAccount/deleteKey) or left
|
|
450
|
+
* flagged `valueMissing` (release) — but a removal whose awaited steps
|
|
451
|
+
* THROW must clear it: the row is then still live and still managed, and a
|
|
452
|
+
* stale mark would silently skip its rotation sync until a restart. It
|
|
453
|
+
* marks the live object, so a row object recreated under the same id is
|
|
454
|
+
* unaffected.
|
|
455
|
+
*/
|
|
456
|
+
retiringAccounts = new WeakSet();
|
|
457
|
+
/**
|
|
458
|
+
* Per-provider projection generation — the write counter for
|
|
459
|
+
* {@link activeAccounts} claims (round-3 finding 3) and the attribution
|
|
460
|
+
* identity of a projected grant (round-6 finding SF1). It is bumped exactly
|
|
461
|
+
* when a grant becomes the provider's projection (a manager-side
|
|
462
|
+
* re-projection or the seam's committed grant at the settlement broadcast),
|
|
463
|
+
* never at a later bookkeeping step. A hydration result whose generation no
|
|
464
|
+
* longer matches is discarded instead of overwriting a newer projection
|
|
465
|
+
* with a stale read.
|
|
466
|
+
*/
|
|
467
|
+
ownershipGenerations = new Map();
|
|
468
|
+
/**
|
|
469
|
+
* Per-account-row projection generation in effect when the row's CURRENT
|
|
470
|
+
* cooldown was earned (round-6 finding SF1). `applyFailureCooldown` stamps
|
|
471
|
+
* it at failure time, so a harvest can tell the previous grant's window
|
|
472
|
+
* from one the freshly projected grant earned while the harvest was still
|
|
473
|
+
* queued. A row with no entry carries a window that predates any observed
|
|
474
|
+
* attribution (one loaded from disk, for example); the next projection
|
|
475
|
+
* treats it as the older generation's.
|
|
476
|
+
*/
|
|
477
|
+
cooldownGenerations = new Map();
|
|
478
|
+
/**
|
|
479
|
+
* Per-provider chain of pending grant reconciliations (round-9 finding
|
|
480
|
+
* P1): two refresh echoes for the same route can arrive before the first
|
|
481
|
+
* write-back settles, and the bodies run read → write → commit. Arrival
|
|
482
|
+
* order is preserved so the older observation can never commit after the
|
|
483
|
+
* newer one and leave the row on a stale generation.
|
|
484
|
+
*
|
|
485
|
+
* Round-8 finding I-000106: activation joins the same chain. The only
|
|
486
|
+
* writer of `row.oauth.grantRef` is {@link reconcileGrantRecord}, which runs
|
|
487
|
+
* outside the mutation queue, so activation's record read and its projection
|
|
488
|
+
* write must be mutually exclusive with that writer — otherwise a rotation
|
|
489
|
+
* landing between them leaves the projection on a generation the row no
|
|
490
|
+
* longer names.
|
|
491
|
+
*/
|
|
492
|
+
grantReconciliations = new Map();
|
|
493
|
+
/**
|
|
494
|
+
* Providers whose projection deletion this manager initiated, mapped to
|
|
495
|
+
* the moment it was initiated. The `record-updated` echo of our own
|
|
496
|
+
* demotion consumes the marker (one-shot) and stays silent while the
|
|
497
|
+
* marker is inside its TTL window; past the window the marker is expired
|
|
498
|
+
* and a removal warns as genuinely out-of-band again.
|
|
499
|
+
*/
|
|
500
|
+
intentionalProjectionRemovals = new Map();
|
|
501
|
+
/**
|
|
502
|
+
* In-flight projection demotions by provider (round-5 finding C1): the
|
|
503
|
+
* oauth failure path cools the account and demotes asynchronously, and the
|
|
504
|
+
* failover retry must not re-enter the stream while the projection still
|
|
505
|
+
* resolves — pi-ai prefers the stored grant, so the key pool would never be
|
|
506
|
+
* reached. The request-error listener awaits this entry before deciding.
|
|
507
|
+
*/
|
|
508
|
+
projectionDemotions = new Map();
|
|
509
|
+
/**
|
|
510
|
+
* MF3 load-time hydration result (round-8 finding I-000108). The
|
|
511
|
+
* constructor starts the ownership read, but a restarted process can serve
|
|
512
|
+
* a stream before it settles. Until this provider's read lands,
|
|
513
|
+
* {@link activeAccounts} may not know the owner, so `wrapStream` would
|
|
514
|
+
* select and mutate a key (cursor, `lastUsedAt`, sticky binding) even though
|
|
515
|
+
* pi-ai serves the persisted grant, and an OAuth failure would be classified
|
|
516
|
+
* as externally authenticated instead of cooling the account.
|
|
517
|
+
*
|
|
518
|
+
* The promise never rejects (hydration catches and warns per provider), but
|
|
519
|
+
* it can stay pending forever. Requests therefore never subscribe to it
|
|
520
|
+
* (round-10 should-fix SF2: a per-call `then`/`await` would be retained on a
|
|
521
|
+
* promise that never settles); they wait per provider through
|
|
522
|
+
* {@link hydratedProviders} and {@link hydrationWaiters} instead.
|
|
523
|
+
*/
|
|
524
|
+
projectionHydration = Promise.resolve();
|
|
525
|
+
/**
|
|
526
|
+
* Upper bound on one stream's wait for its provider's hydration read
|
|
527
|
+
* (round-9 should-fix). The hydration read has no timeout of its own, so
|
|
528
|
+
* without a bound one stalled `readRecord` would gate every managed stream.
|
|
529
|
+
* Injectable so tests can exercise the bound quickly.
|
|
530
|
+
*/
|
|
531
|
+
hydrationWaitMs = HYDRATION_WAIT_MS;
|
|
532
|
+
/**
|
|
533
|
+
* Providers whose hydration read has landed — owner found or not
|
|
534
|
+
* (round-10 should-fix SF1). Hydration walks the providers serially and
|
|
535
|
+
* writes each owner as it goes, so one stalled read must not make an
|
|
536
|
+
* already-read provider look unknown: a provider whose own read landed is
|
|
537
|
+
* ready immediately, and the bounded wait's "unknown" fallback is confined
|
|
538
|
+
* to the requested provider's own unresolved read.
|
|
539
|
+
*/
|
|
540
|
+
hydratedProviders = new Set();
|
|
541
|
+
/**
|
|
542
|
+
* One resolver set per provider still being waited on (round-10 should-fix
|
|
543
|
+
* SF2). A wait registers exactly one entry here and removes it when the wait
|
|
544
|
+
* ends — on timeout or on the late landing — so a read that never settles
|
|
545
|
+
* cannot retain anything per request. {@link noteProviderHydrated} is the
|
|
546
|
+
* only notification channel; the shared hydration promise is never
|
|
547
|
+
* subscribed to.
|
|
548
|
+
*/
|
|
549
|
+
hydrationWaiters = new Map();
|
|
550
|
+
/**
|
|
551
|
+
* True once the load-time hydration read is over (round-10 SF1). From then
|
|
552
|
+
* on no provider's read can land, so attribution is known for every provider
|
|
553
|
+
* — including one adopted after the read, whose OAuth rows did not exist
|
|
554
|
+
* when the projection was looked for.
|
|
555
|
+
*/
|
|
556
|
+
hydrationSettled = false;
|
|
557
|
+
/** Providers whose bounded-wait warning already fired (warn once). */
|
|
558
|
+
hydrationWaitWarned = new Set();
|
|
559
|
+
/** Channels whose lost-subscription warning already fired (warn once). */
|
|
560
|
+
subscriptionLostWarnings = new Set();
|
|
561
|
+
/** Disposer for the `authorization/settled` subscription. */
|
|
562
|
+
settledDisposer;
|
|
563
|
+
/** Disposer for the `credentials/record-updated` subscription. */
|
|
564
|
+
recordUpdatedDisposer;
|
|
228
565
|
typertRemote;
|
|
229
566
|
constructor(ctx, store) {
|
|
230
567
|
this.ctx = ctx;
|
|
231
568
|
this.store = store;
|
|
232
569
|
this.typertRemote = Object.freeze({ service: this, serviceKey: SERVICE, namespace: NAMESPACE });
|
|
233
570
|
this.rebuildBrokerMap();
|
|
571
|
+
this.projectionHydration = this.hydrateProjectionOwnership();
|
|
572
|
+
this.settledDisposer = this.watchAuthorizationSettled();
|
|
573
|
+
this.recordUpdatedDisposer = this.watchGrantRecordUpdated();
|
|
574
|
+
}
|
|
575
|
+
/**
|
|
576
|
+
* MF3 load-time hydration: a projection outlives this process — pi-ai
|
|
577
|
+
* wrote `llm-pi-ai/<provider>` during a previous run and it survived in
|
|
578
|
+
* the credential store. Match each persisted payload back to an account
|
|
579
|
+
* row (the exact rule the rotation reconciler applies, via
|
|
580
|
+
* {@link matchOwningAccount}) so removeAccount / deleteManagedCredentials
|
|
581
|
+
* demote the projection and `onGrantRecordUpdated` can mark valueMissing
|
|
582
|
+
* after a restart. An orphan projection — no matching row, e.g. a foreign
|
|
583
|
+
* or stale grant — is left untouched: the manager never adopts a grant it
|
|
584
|
+
* does not own, and out-of-band updates keep surfacing the native-
|
|
585
|
+
* management warning.
|
|
586
|
+
*
|
|
587
|
+
* Returns its own completion (round-8 finding I-000108), and the returned
|
|
588
|
+
* promise is already caught, so a failed hydration degrades to "no owner"
|
|
589
|
+
* instead of wedging every request. Request attribution does not subscribe
|
|
590
|
+
* to that promise (round-10 should-fix SF2 — it may stay pending forever);
|
|
591
|
+
* each request waits on its own provider's landing instead.
|
|
592
|
+
*/
|
|
593
|
+
hydrateProjectionOwnership() {
|
|
594
|
+
return (async () => {
|
|
595
|
+
try {
|
|
596
|
+
const records = this.credentialRecords();
|
|
597
|
+
if (records === undefined) {
|
|
598
|
+
this.warnSubscriptionLost('credentials record access');
|
|
599
|
+
return;
|
|
600
|
+
}
|
|
601
|
+
for (const providerRecord of Object.values(this.store.providers)) {
|
|
602
|
+
const provider = providerRecord.provider;
|
|
603
|
+
const accounts = providerRecord.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
604
|
+
if (accounts.length === 0)
|
|
605
|
+
continue;
|
|
606
|
+
try {
|
|
607
|
+
const grantKey = `${OAUTH_GRANT_RECORD_SCOPE}/${provider}`;
|
|
608
|
+
// Generation tag (round-3 finding 3): the read below is awaited, and
|
|
609
|
+
// an activation or login landing while it is in flight must win —
|
|
610
|
+
// the hydrated write only lands while the generation still matches.
|
|
611
|
+
const generation = this.ownershipGenerations.get(provider) ?? 0;
|
|
612
|
+
let record;
|
|
613
|
+
try {
|
|
614
|
+
record = await records.readRecord(grantKey);
|
|
615
|
+
}
|
|
616
|
+
catch (error) {
|
|
617
|
+
this.warn(`projection ownership hydration could not read ${grantKey}: ${error instanceof Error ? error.message : String(error)}`);
|
|
618
|
+
continue;
|
|
619
|
+
}
|
|
620
|
+
if (record === undefined || record.kind !== 'grant' || !isOAuthGrantPayload(record.payload))
|
|
621
|
+
continue;
|
|
622
|
+
const owner = matchOwningAccount(provider, accounts, record.payload);
|
|
623
|
+
// A newer activation/login claimed the owner while this read was in
|
|
624
|
+
// flight: the stale read is discarded, never written over it.
|
|
625
|
+
if ((this.ownershipGenerations.get(provider) ?? 0) !== generation)
|
|
626
|
+
continue;
|
|
627
|
+
// A concurrent removal may have dropped the row while this read was
|
|
628
|
+
// in flight: never resurrect an owner entry for a row that is gone.
|
|
629
|
+
if (owner !== undefined && providerRecord.keys.some(candidate => candidate.id === owner.id)) {
|
|
630
|
+
this.activeAccounts.set(provider, owner.id);
|
|
631
|
+
}
|
|
632
|
+
}
|
|
633
|
+
finally {
|
|
634
|
+
// This provider's own read is over (owner or not): its requests are
|
|
635
|
+
// attributable now and must not wait behind another provider's
|
|
636
|
+
// stalled read (round-10 should-fix SF1).
|
|
637
|
+
this.noteProviderHydrated(provider);
|
|
638
|
+
}
|
|
639
|
+
}
|
|
640
|
+
}
|
|
641
|
+
finally {
|
|
642
|
+
// The read is over (completed or abandoned): from now on no result can
|
|
643
|
+
// land, so every provider is ready — including one whose OAuth rows
|
|
644
|
+
// appear later, which this read could not have projected.
|
|
645
|
+
this.hydrationSettled = true;
|
|
646
|
+
this.settleHydrationWaits();
|
|
647
|
+
}
|
|
648
|
+
})().catch(error => {
|
|
649
|
+
this.warn(`projection ownership hydration failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
650
|
+
});
|
|
651
|
+
}
|
|
652
|
+
/**
|
|
653
|
+
* Record that one provider's hydration read is over — owner or not (SF1) —
|
|
654
|
+
* and wake the requests waiting on that provider alone (SF2).
|
|
655
|
+
*/
|
|
656
|
+
noteProviderHydrated(provider) {
|
|
657
|
+
this.hydratedProviders.add(provider);
|
|
658
|
+
const waiters = this.hydrationWaiters.get(provider);
|
|
659
|
+
if (waiters === undefined)
|
|
660
|
+
return;
|
|
661
|
+
this.hydrationWaiters.delete(provider);
|
|
662
|
+
for (const wake of [...waiters])
|
|
663
|
+
wake();
|
|
234
664
|
}
|
|
235
|
-
/**
|
|
665
|
+
/** Release every provider's readiness wait when hydration cannot proceed. */
|
|
666
|
+
settleHydrationWaits() {
|
|
667
|
+
for (const provider of Object.keys(this.store.providers))
|
|
668
|
+
this.noteProviderHydrated(provider);
|
|
669
|
+
}
|
|
670
|
+
/**
|
|
671
|
+
* Wait for one provider's own hydration read to land, at most `ms`, and
|
|
672
|
+
* report whether it landed.
|
|
673
|
+
*
|
|
674
|
+
* Registers exactly one waiter entry for this wait and removes it when the
|
|
675
|
+
* wait ends either way (round-10 should-fix SF2): nothing is attached to the
|
|
676
|
+
* shared hydration promise, so a read that never settles cannot retain one
|
|
677
|
+
* reaction per request. A late landing still wakes the wait through
|
|
678
|
+
* {@link noteProviderHydrated}, and the generation guard in
|
|
679
|
+
* {@link hydrateProjectionOwnership} still decides whether its result lands
|
|
680
|
+
* (I-000108).
|
|
681
|
+
*/
|
|
682
|
+
waitForProviderHydration(provider, ms) {
|
|
683
|
+
if (this.hydrationSettled || this.hydratedProviders.has(provider))
|
|
684
|
+
return Promise.resolve(true);
|
|
685
|
+
return new Promise(resolve => {
|
|
686
|
+
let waiters = this.hydrationWaiters.get(provider);
|
|
687
|
+
if (waiters === undefined) {
|
|
688
|
+
waiters = new Set();
|
|
689
|
+
this.hydrationWaiters.set(provider, waiters);
|
|
690
|
+
}
|
|
691
|
+
const live = waiters;
|
|
692
|
+
let timer;
|
|
693
|
+
const wake = (landed) => {
|
|
694
|
+
if (timer !== undefined) {
|
|
695
|
+
clearTimeout(timer);
|
|
696
|
+
timer = undefined;
|
|
697
|
+
}
|
|
698
|
+
live.delete(onLanded);
|
|
699
|
+
if (live.size === 0 && this.hydrationWaiters.get(provider) === live)
|
|
700
|
+
this.hydrationWaiters.delete(provider);
|
|
701
|
+
resolve(landed);
|
|
702
|
+
};
|
|
703
|
+
const onLanded = () => {
|
|
704
|
+
wake(true);
|
|
705
|
+
};
|
|
706
|
+
timer = setTimeout(() => {
|
|
707
|
+
wake(false);
|
|
708
|
+
}, ms);
|
|
709
|
+
live.add(onLanded);
|
|
710
|
+
});
|
|
711
|
+
}
|
|
712
|
+
/**
|
|
713
|
+
* Live per-provider waiter entries. A documented internal counter for the
|
|
714
|
+
* round-10 SF2 invariant: a wait that ended must leave no entry behind, so
|
|
715
|
+
* this never grows with the number of timed-out requests.
|
|
716
|
+
*/
|
|
717
|
+
hydrationWaiterCount() {
|
|
718
|
+
let total = 0;
|
|
719
|
+
for (const waiters of this.hydrationWaiters.values())
|
|
720
|
+
total += waiters.size;
|
|
721
|
+
return total;
|
|
722
|
+
}
|
|
723
|
+
/**
|
|
724
|
+
* Wait (bounded) until projection-ownership hydration can attribute this
|
|
725
|
+
* request, and report whether it did.
|
|
726
|
+
*
|
|
727
|
+
* A provider without OAuth rows can never own a projection, so it never
|
|
728
|
+
* waits — a stalled read for some other provider must not delay it
|
|
729
|
+
* (round-9 should-fix: this used to gate every managed provider, API-key-
|
|
730
|
+
* only ones included).
|
|
731
|
+
*
|
|
732
|
+
* Returns false when the bound elapsed first. The caller then treats
|
|
733
|
+
* attribution as unknown (no owner) and keeps key state untouched: a
|
|
734
|
+
* projection may still be serving, and selecting/mutating a key here would
|
|
735
|
+
* repeat the I-000108 corruption. The hydration promise is not cancelled,
|
|
736
|
+
* so a late result still lands under the generation guard and later
|
|
737
|
+
* requests are attributed normally.
|
|
738
|
+
*
|
|
739
|
+
* Readiness is decided PER PROVIDER (round-10 should-fix SF1). Hydration
|
|
740
|
+
* writes owners progressively, so racing the one global promise would
|
|
741
|
+
* discard an owner this provider already confirmed just because another
|
|
742
|
+
* provider's read stalled. A provider whose own read has landed — with an
|
|
743
|
+
* owner or with none — is ready without waiting, and the "unknown" fallback
|
|
744
|
+
* is confined to the requested provider's own unresolved read.
|
|
745
|
+
*/
|
|
746
|
+
async awaitProjectionOwnership(providerId) {
|
|
747
|
+
const provider = this.store.provider(providerId);
|
|
748
|
+
if (provider === undefined)
|
|
749
|
+
return true;
|
|
750
|
+
const hasOauthRows = provider.keys.some(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
751
|
+
if (!hasOauthRows)
|
|
752
|
+
return true;
|
|
753
|
+
// This provider's own read already landed (or a newer claim already owns
|
|
754
|
+
// it, or the whole read is over): nothing can arrive later, so no wait.
|
|
755
|
+
if (this.hydrationSettled || this.hydratedProviders.has(providerId) || this.activeAccounts.has(providerId))
|
|
756
|
+
return true;
|
|
757
|
+
const landed = await this.waitForProviderHydration(providerId, this.hydrationWaitMs);
|
|
758
|
+
if (landed)
|
|
759
|
+
return true;
|
|
760
|
+
if (!this.hydrationWaitWarned.has(providerId)) {
|
|
761
|
+
this.hydrationWaitWarned.add(providerId);
|
|
762
|
+
this.warn(`projection ownership hydration did not settle within ${this.hydrationWaitMs}ms for ${providerId}; requests proceed without an attributed account until it does`);
|
|
763
|
+
}
|
|
764
|
+
return false;
|
|
765
|
+
}
|
|
766
|
+
/**
|
|
767
|
+
* Record that a newly committed grant became the provider's projection and
|
|
768
|
+
* return that projection's generation (round-6 finding SF1).
|
|
769
|
+
*
|
|
770
|
+
* The bump happens at the moment the projection is observable — pi-ai's
|
|
771
|
+
* committed grant at the settlement broadcast, or a manager-side
|
|
772
|
+
* re-projection (activation, rotation reconcile, harvest claim) — so every
|
|
773
|
+
* failure stamped afterwards belongs to that projection. It must never be
|
|
774
|
+
* deferred to a later bookkeeping step: the harvest waits behind the
|
|
775
|
+
* mutation queue and the projection read while the new grant already
|
|
776
|
+
* serves, and a failure landing there must not be mistaken for the
|
|
777
|
+
* previous grant's.
|
|
778
|
+
*/
|
|
779
|
+
noteProjectionCommitted(provider) {
|
|
780
|
+
const generation = (this.ownershipGenerations.get(provider) ?? 0) + 1;
|
|
781
|
+
this.ownershipGenerations.set(provider, generation);
|
|
782
|
+
return generation;
|
|
783
|
+
}
|
|
784
|
+
/**
|
|
785
|
+
* Claim the projection owner for one account row, bumping the provider's
|
|
786
|
+
* ownership generation so an in-flight hydration read of an older state
|
|
787
|
+
* is discarded instead of overwriting the claim (round-3 finding 3).
|
|
788
|
+
*/
|
|
789
|
+
claimProjectionOwner(provider, accountId) {
|
|
790
|
+
this.activeAccounts.set(provider, accountId);
|
|
791
|
+
this.noteProjectionCommitted(provider);
|
|
792
|
+
}
|
|
793
|
+
/**
|
|
794
|
+
* Log without ever echoing secret material. */
|
|
236
795
|
warn(message) {
|
|
237
796
|
try {
|
|
238
797
|
this.ctx.logger.warn(`keys-manager: ${message}`);
|
|
@@ -280,16 +839,29 @@ export class KeysManagerService {
|
|
|
280
839
|
*/
|
|
281
840
|
pruneTransient(now = Date.now()) {
|
|
282
841
|
this.pruneAttempts(now);
|
|
842
|
+
this.pruneLoginAttempts(now);
|
|
283
843
|
}
|
|
284
844
|
/** Stop accepting mutations, drain the queue, then persist the final snapshot. */
|
|
285
845
|
async close() {
|
|
286
846
|
this.closing = true;
|
|
847
|
+
this.settledDisposer?.();
|
|
848
|
+
this.settledDisposer = undefined;
|
|
849
|
+
this.recordUpdatedDisposer?.();
|
|
850
|
+
this.recordUpdatedDisposer = undefined;
|
|
851
|
+
for (const attempt of this.loginAttempts.values()) {
|
|
852
|
+
attempt.controller.abort();
|
|
853
|
+
this.dropPrompt(attempt, new Error('Keys Manager is shutting down'));
|
|
854
|
+
}
|
|
287
855
|
try {
|
|
288
856
|
await this.queue;
|
|
289
857
|
}
|
|
290
858
|
catch {
|
|
291
859
|
// The mutation queue already reports failures to its caller.
|
|
292
860
|
}
|
|
861
|
+
// Round-9 P1: the per-provider reconciliation chains are independent of
|
|
862
|
+
// the mutation queue, so drain them before the store closes — a queued
|
|
863
|
+
// echo must not write a grant record into a manager that is shutting down.
|
|
864
|
+
await Promise.allSettled([...this.grantReconciliations.values()]);
|
|
293
865
|
await this.store.close();
|
|
294
866
|
}
|
|
295
867
|
/** Push a user-facing warning that rides the next state response. */
|
|
@@ -316,6 +888,1566 @@ export class KeysManagerService {
|
|
|
316
888
|
return undefined;
|
|
317
889
|
return current[ORIGINAL] ?? current;
|
|
318
890
|
}
|
|
891
|
+
/**
|
|
892
|
+
* The credential store's record half, reached through the raw provider:
|
|
893
|
+
* the broker wrappers only ever replace the reference methods, so the
|
|
894
|
+
* record methods of the mounted credential provider are always intact on
|
|
895
|
+
* the raw instance. `llm-pi-ai` writes its grants through exactly these
|
|
896
|
+
* methods, so the manager's activation projection must use them too.
|
|
897
|
+
*/
|
|
898
|
+
credentialRecords() {
|
|
899
|
+
const raw = this.credentialsRaw();
|
|
900
|
+
if (raw === undefined)
|
|
901
|
+
return undefined;
|
|
902
|
+
if (typeof raw.readRecord !== 'function' || typeof raw.modifyRecord !== 'function')
|
|
903
|
+
return undefined;
|
|
904
|
+
return raw;
|
|
905
|
+
}
|
|
906
|
+
/**
|
|
907
|
+
* Resolve the OAuth authorization flow one provider offers.
|
|
908
|
+
*
|
|
909
|
+
* The flow key format is pinned by `llm-pi-ai/login.ts:149-171` +
|
|
910
|
+
* `auth.ts:29,36-38`: `recordKeyFor(providerId)` = `llm-pi-ai/<providerId>`,
|
|
911
|
+
* with the oauth method offered first (`login.ts:40-47`). A provider with
|
|
912
|
+
* no such flow (not OAuth-capable, or not installed) resolves to
|
|
913
|
+
* undefined, which callers must treat as "sign-in unavailable".
|
|
914
|
+
*
|
|
915
|
+
* Availability is read from the live seam on every call, never cached:
|
|
916
|
+
* the authorization service may mount after this plugin, so a route whose
|
|
917
|
+
* flow is not enumerable yet stays account-unsupported until the flow list
|
|
918
|
+
* exists (the client re-reads state after that ordering settles).
|
|
919
|
+
*/
|
|
920
|
+
oauthFlowFor(provider) {
|
|
921
|
+
const flows = this.ctx.authorization?.list() ?? [];
|
|
922
|
+
const flow = flows.find(entry => entry.key === `llm-pi-ai/${provider}`);
|
|
923
|
+
if (flow === undefined)
|
|
924
|
+
return undefined;
|
|
925
|
+
if (!flow.methods.some(method => method.id === 'oauth'))
|
|
926
|
+
return undefined;
|
|
927
|
+
return { key: flow.key, method: 'oauth' };
|
|
928
|
+
}
|
|
929
|
+
/**
|
|
930
|
+
* C6-2: can this route's OAuth sign-in actually begin right now? Account
|
|
931
|
+
* capability is gated on this instead of the catalog's `nativeAuth.oauth`
|
|
932
|
+
* flag alone, so a route without a registered pi-ai flow never offers an
|
|
933
|
+
* Add account that would fail. The check is lazy — see {@link oauthFlowFor}.
|
|
934
|
+
*/
|
|
935
|
+
oauthFlowAvailable(provider) {
|
|
936
|
+
return this.oauthFlowFor(provider) !== undefined;
|
|
937
|
+
}
|
|
938
|
+
/**
|
|
939
|
+
* Warn once per channel: a lost subscription silently disables a whole
|
|
940
|
+
* feature (harvest never runs; rotations never sync), so the degradation
|
|
941
|
+
* must be observable instead of indistinguishable from success.
|
|
942
|
+
*/
|
|
943
|
+
warnSubscriptionLost(channel, error) {
|
|
944
|
+
if (this.subscriptionLostWarnings.has(channel))
|
|
945
|
+
return;
|
|
946
|
+
this.subscriptionLostWarnings.add(channel);
|
|
947
|
+
const detail = error === undefined ? '' : ` (${error instanceof Error ? error.message : String(error)})`;
|
|
948
|
+
this.warn(`the ${channel} subscription is unavailable${detail}; OAuth sign-in sync is degraded`);
|
|
949
|
+
}
|
|
950
|
+
/**
|
|
951
|
+
* Subscribe to the seam's settled broadcast (authorization/src/index.ts:57,
|
|
952
|
+
* :337-351). A settled attempt for a key this service is driving harvests
|
|
953
|
+
* the committed grant; everything else is ignored — another surface's
|
|
954
|
+
* sign-in is that surface's business, and the grant it leaves behind is
|
|
955
|
+
* reconciled through `credentials/record-updated` instead.
|
|
956
|
+
*/
|
|
957
|
+
watchAuthorizationSettled() {
|
|
958
|
+
const on = this.ctx.on;
|
|
959
|
+
if (typeof on !== 'function') {
|
|
960
|
+
this.warnSubscriptionLost('authorization/settled');
|
|
961
|
+
return undefined;
|
|
962
|
+
}
|
|
963
|
+
try {
|
|
964
|
+
return on.call(this.ctx, 'authorization/settled', (key, settlement) => {
|
|
965
|
+
if (typeof key !== 'string')
|
|
966
|
+
return;
|
|
967
|
+
this.onAuthorizationSettled(key, typeof settlement === 'string' ? settlement : '');
|
|
968
|
+
});
|
|
969
|
+
}
|
|
970
|
+
catch (error) {
|
|
971
|
+
this.warnSubscriptionLost('authorization/settled', error);
|
|
972
|
+
return undefined;
|
|
973
|
+
}
|
|
974
|
+
}
|
|
975
|
+
/** One settled broadcast: finalize the matching attempt, harvest on success. */
|
|
976
|
+
onAuthorizationSettled(key, settlement) {
|
|
977
|
+
if (!key.startsWith(`${OAUTH_GRANT_RECORD_SCOPE}/`))
|
|
978
|
+
return;
|
|
979
|
+
const provider = key.slice(OAUTH_GRANT_RECORD_SCOPE.length + 1);
|
|
980
|
+
const attemptId = this.loginsInFlight.get(provider);
|
|
981
|
+
if (attemptId === undefined)
|
|
982
|
+
return;
|
|
983
|
+
const attempt = this.loginAttempts.get(attemptId);
|
|
984
|
+
if (attempt === undefined || attempt.settledAt !== undefined)
|
|
985
|
+
return;
|
|
986
|
+
if (settlement === 'authorized') {
|
|
987
|
+
// F2 provenance: capture the projection payload at the settlement
|
|
988
|
+
// broadcast — the point where the seam reports THIS attempt's commit —
|
|
989
|
+
// instead of reading whatever is live when the queued harvest finally
|
|
990
|
+
// runs. The capture starts here, outside the mutation queue, so a
|
|
991
|
+
// release parked ahead of the harvest cannot let a foreign writer
|
|
992
|
+
// become this attempt's provenance.
|
|
993
|
+
//
|
|
994
|
+
// R6-SF1: the committed grant is already the provider's projection at
|
|
995
|
+
// this broadcast, so this is also the generation boundary for cooldown
|
|
996
|
+
// attribution. Everything the harvest does afterwards (queue and
|
|
997
|
+
// projection-read waits included) runs against this generation.
|
|
998
|
+
const projectionGeneration = this.noteProjectionCommitted(provider);
|
|
999
|
+
void this.harvestLogin(attempt, this.captureSettledProjection(attempt), projectionGeneration);
|
|
1000
|
+
return;
|
|
1001
|
+
}
|
|
1002
|
+
if (settlement === 'cancelled')
|
|
1003
|
+
this.finishLoginAttempt(attempt, 'cancelled');
|
|
1004
|
+
else
|
|
1005
|
+
this.failLoginAttempt(attempt, new Error(`the sign-in attempt failed (${settlement || 'unknown outcome'})`));
|
|
1006
|
+
}
|
|
1007
|
+
/**
|
|
1008
|
+
* Start one interactive OAuth sign-in. Returns immediately with the
|
|
1009
|
+
* attempt view; the attempt walks `pending → (prompt ⇄ pending) →
|
|
1010
|
+
* ready | failed | cancelled`, and every transition is observable through
|
|
1011
|
+
* {@link getAccountAttempt} / {@link listAccountAttempts}.
|
|
1012
|
+
*/
|
|
1013
|
+
async beginAccountLogin(provider, alias, targetAccountId) {
|
|
1014
|
+
if (typeof provider !== 'string' || provider.length === 0)
|
|
1015
|
+
throw new TypeError('provider is required');
|
|
1016
|
+
const flow = this.oauthFlowFor(provider);
|
|
1017
|
+
if (flow === undefined)
|
|
1018
|
+
throw new Error(`Provider "${provider}" offers no OAuth sign-in flow`);
|
|
1019
|
+
const providerRecord = this.store.provider(provider);
|
|
1020
|
+
if (providerRecord === undefined)
|
|
1021
|
+
throw new Error(`Provider "${provider}" is not managed; adopt it before signing in`);
|
|
1022
|
+
// Validate the re-authentication target up front (round-3 finding 6):
|
|
1023
|
+
// the harvest binds the new grant to this row, so it must already name
|
|
1024
|
+
// a live OAuth account of this provider. It runs BEFORE the pending
|
|
1025
|
+
// resume (round-4 finding 3): a resume must never smuggle an older
|
|
1026
|
+
// attempt's target past the request's own validation, so an unknown or
|
|
1027
|
+
// foreign id is refused even while another sign-in is pending.
|
|
1028
|
+
let cleanTarget;
|
|
1029
|
+
if (typeof targetAccountId === 'string' && targetAccountId.length > 0) {
|
|
1030
|
+
const target = providerRecord.keys.find(candidate => candidate.id === targetAccountId);
|
|
1031
|
+
if (target === undefined)
|
|
1032
|
+
throw new Error(`Account "${targetAccountId}" not found on ${provider}`);
|
|
1033
|
+
if (target.kind !== 'oauth' || target.oauth === null)
|
|
1034
|
+
throw new Error(`Key "${targetAccountId}" is not an OAuth account`);
|
|
1035
|
+
cleanTarget = targetAccountId;
|
|
1036
|
+
}
|
|
1037
|
+
if (this.loginsInFlight.has(provider)) {
|
|
1038
|
+
// F3 (I-000082): a pending attempt IS the flow — reopening the dialog
|
|
1039
|
+
// resumes it (same id, same notices, same parked prompt) instead of
|
|
1040
|
+
// minting a competing second flow the seam's single-flight slot would
|
|
1041
|
+
// have to refuse. Terminal attempts never reach here: finishing an
|
|
1042
|
+
// attempt clears its provider slot, so the next begin starts fresh.
|
|
1043
|
+
const attemptId = this.loginsInFlight.get(provider);
|
|
1044
|
+
const inFlight = attemptId === undefined ? undefined : this.loginAttempts.get(attemptId);
|
|
1045
|
+
if (inFlight !== undefined && inFlight.settledAt === undefined) {
|
|
1046
|
+
// Round-4 finding 3: reuse is sound only when the resumed attempt
|
|
1047
|
+
// signs in the SAME target the caller asked for (or both name no
|
|
1048
|
+
// target). Silently resuming another row's flow would bind the new
|
|
1049
|
+
// grant to the older attempt's target.
|
|
1050
|
+
if (inFlight.targetAccountId === cleanTarget)
|
|
1051
|
+
return attemptView(inFlight);
|
|
1052
|
+
throw new Error(inFlight.targetAccountId === undefined
|
|
1053
|
+
? `A sign-in for ${provider} is already pending without a target account; finish or cancel it before signing in to a specific account`
|
|
1054
|
+
: `A sign-in for ${provider} is already pending for a different account; finish or cancel it before signing in to ${cleanTarget ?? 'this provider'}`);
|
|
1055
|
+
}
|
|
1056
|
+
// Stale slot (should not happen — finishing always clears it): drop it
|
|
1057
|
+
// and start a fresh flow rather than wedging the provider.
|
|
1058
|
+
this.loginsInFlight.delete(provider);
|
|
1059
|
+
}
|
|
1060
|
+
const cleanAlias = typeof alias === 'string' ? alias.trim().slice(0, MAX_ALIAS) : undefined;
|
|
1061
|
+
const now = Date.now();
|
|
1062
|
+
const attempt = {
|
|
1063
|
+
id: `la_${(++this.loginSequence).toString(36)}_${randomBytes(4).toString('hex')}`,
|
|
1064
|
+
provider,
|
|
1065
|
+
...(cleanAlias !== undefined && cleanAlias.length > 0 ? { alias: cleanAlias } : {}),
|
|
1066
|
+
...(cleanTarget !== undefined ? { targetAccountId: cleanTarget } : {}),
|
|
1067
|
+
status: 'pending',
|
|
1068
|
+
notices: [],
|
|
1069
|
+
createdAt: now,
|
|
1070
|
+
updatedAt: now,
|
|
1071
|
+
controller: new AbortController(),
|
|
1072
|
+
};
|
|
1073
|
+
this.loginAttempts.set(attempt.id, attempt);
|
|
1074
|
+
this.capLoginAttempts();
|
|
1075
|
+
this.loginsInFlight.set(provider, attempt.id);
|
|
1076
|
+
const records = this.credentialRecords();
|
|
1077
|
+
if (records === undefined) {
|
|
1078
|
+
this.loginAttempts.delete(attempt.id);
|
|
1079
|
+
this.loginsInFlight.delete(provider);
|
|
1080
|
+
throw new Error('The credential store does not support records; OAuth sign-in is unavailable');
|
|
1081
|
+
}
|
|
1082
|
+
const interaction = {
|
|
1083
|
+
notify: (notice) => {
|
|
1084
|
+
attempt.notices.push(noticeView(notice));
|
|
1085
|
+
if (attempt.notices.length > 32)
|
|
1086
|
+
attempt.notices.shift();
|
|
1087
|
+
attempt.updatedAt = Date.now();
|
|
1088
|
+
},
|
|
1089
|
+
prompt: (prompt) => new Promise((resolve, reject) => {
|
|
1090
|
+
if (attempt.settledAt !== undefined) {
|
|
1091
|
+
reject(new Error('This sign-in attempt has already ended'));
|
|
1092
|
+
return;
|
|
1093
|
+
}
|
|
1094
|
+
if (prompt.signal?.aborted === true) {
|
|
1095
|
+
reject(new Error('The sign-in prompt was withdrawn'));
|
|
1096
|
+
return;
|
|
1097
|
+
}
|
|
1098
|
+
attempt.status = 'prompt';
|
|
1099
|
+
attempt.updatedAt = Date.now();
|
|
1100
|
+
attempt.prompt = promptView(prompt);
|
|
1101
|
+
attempt.resolvePrompt = resolve;
|
|
1102
|
+
attempt.rejectPrompt = reject;
|
|
1103
|
+
// Cancellation propagates to the awaited prompt: a seam-side
|
|
1104
|
+
// withdrawal of this signal releases a flow parked here instead
|
|
1105
|
+
// of leaving it pending forever.
|
|
1106
|
+
prompt.signal?.addEventListener('abort', () => reject(new Error('The sign-in prompt was withdrawn')), { once: true });
|
|
1107
|
+
}),
|
|
1108
|
+
};
|
|
1109
|
+
const beginPromise = this.ctx.authorization.begin({
|
|
1110
|
+
key: flow.key,
|
|
1111
|
+
method: flow.method,
|
|
1112
|
+
interaction,
|
|
1113
|
+
signal: attempt.controller.signal,
|
|
1114
|
+
});
|
|
1115
|
+
void beginPromise.then(outcome => {
|
|
1116
|
+
// The settled broadcast always lands first (the seam fires it in the
|
|
1117
|
+
// attempt's finally, authorization/src/index.ts:316-322). An
|
|
1118
|
+
// authorized outcome therefore means harvestLogin owns the terminal
|
|
1119
|
+
// transition (SF1): publishing ready the moment begin resolves would
|
|
1120
|
+
// close the dialog with "Signed in." while the grant capture is
|
|
1121
|
+
// still in flight — or failing. A cancelled outcome still finalizes
|
|
1122
|
+
// attempts that no settlement reached.
|
|
1123
|
+
if (outcome.status !== 'authorized')
|
|
1124
|
+
this.finishLoginAttempt(attempt, 'cancelled');
|
|
1125
|
+
}, error => {
|
|
1126
|
+
this.failLoginAttempt(attempt, error);
|
|
1127
|
+
});
|
|
1128
|
+
return attemptView(attempt);
|
|
1129
|
+
}
|
|
1130
|
+
/** Resolve the attempt's parked prompt with the human's answer. */
|
|
1131
|
+
async answerAccountPrompt(attemptId, value) {
|
|
1132
|
+
const attempt = this.loginAttempts.get(attemptId);
|
|
1133
|
+
if (attempt === undefined)
|
|
1134
|
+
throw new Error('Unknown sign-in attempt');
|
|
1135
|
+
if (attempt.settledAt !== undefined)
|
|
1136
|
+
throw new Error('This sign-in attempt has already ended');
|
|
1137
|
+
if (attempt.status !== 'prompt' || attempt.resolvePrompt === undefined)
|
|
1138
|
+
throw new Error('This attempt has no pending prompt');
|
|
1139
|
+
if (typeof value !== 'string')
|
|
1140
|
+
throw new TypeError('An answer value is required');
|
|
1141
|
+
const resolve = attempt.resolvePrompt;
|
|
1142
|
+
this.dropPrompt(attempt);
|
|
1143
|
+
resolve(value);
|
|
1144
|
+
}
|
|
1145
|
+
/** One attempt's serializable view, or undefined when unknown. */
|
|
1146
|
+
getAccountAttempt(attemptId) {
|
|
1147
|
+
const attempt = this.loginAttempts.get(attemptId);
|
|
1148
|
+
return attempt === undefined ? undefined : attemptView(attempt);
|
|
1149
|
+
}
|
|
1150
|
+
/** Remote: poll one attempt's serializable state (T3 dialog channel). */
|
|
1151
|
+
accountAttempt(attemptId) {
|
|
1152
|
+
return this.getAccountAttempt(attemptId);
|
|
1153
|
+
}
|
|
1154
|
+
/** Every attempt, newest first, for T3 surfaces. */
|
|
1155
|
+
listAccountAttempts() {
|
|
1156
|
+
return [...this.loginAttempts.values()].sort((a, b) => b.createdAt - a.createdAt).map(attemptView);
|
|
1157
|
+
}
|
|
1158
|
+
/**
|
|
1159
|
+
* Detach the parked prompt. With a reason the flow's awaited promise is
|
|
1160
|
+
* rejected too — a parked prompt dropped un-rejected would leave the
|
|
1161
|
+
* seam's flow (and the provider's single-flight slot) wedged until its
|
|
1162
|
+
* own timeout, so every attempt-ending path passes one.
|
|
1163
|
+
*/
|
|
1164
|
+
dropPrompt(attempt, reason) {
|
|
1165
|
+
const reject = attempt.rejectPrompt;
|
|
1166
|
+
attempt.resolvePrompt = undefined;
|
|
1167
|
+
attempt.rejectPrompt = undefined;
|
|
1168
|
+
attempt.prompt = undefined;
|
|
1169
|
+
if (attempt.status === 'prompt')
|
|
1170
|
+
attempt.status = 'pending';
|
|
1171
|
+
attempt.updatedAt = Date.now();
|
|
1172
|
+
if (reject !== undefined && reason !== undefined)
|
|
1173
|
+
reject(reason);
|
|
1174
|
+
}
|
|
1175
|
+
/** Finalize an attempt with a terminal status exactly once. */
|
|
1176
|
+
finishLoginAttempt(attempt, status, error) {
|
|
1177
|
+
if (attempt.settledAt !== undefined)
|
|
1178
|
+
return;
|
|
1179
|
+
attempt.settledAt = Date.now();
|
|
1180
|
+
attempt.status = status;
|
|
1181
|
+
if (error !== undefined) {
|
|
1182
|
+
attempt.error = error instanceof Error ? error.message : String(error);
|
|
1183
|
+
}
|
|
1184
|
+
this.dropPrompt(attempt, new Error(status === 'failed'
|
|
1185
|
+
? attempt.error ?? 'The sign-in attempt failed'
|
|
1186
|
+
: 'The sign-in attempt was cancelled'));
|
|
1187
|
+
if (this.loginsInFlight.get(attempt.provider) === attempt.id)
|
|
1188
|
+
this.loginsInFlight.delete(attempt.provider);
|
|
1189
|
+
attempt.controller.abort();
|
|
1190
|
+
attempt.updatedAt = attempt.settledAt;
|
|
1191
|
+
}
|
|
1192
|
+
/** Finalize an attempt as failed with the given reason. */
|
|
1193
|
+
failLoginAttempt(attempt, error) {
|
|
1194
|
+
this.finishLoginAttempt(attempt, 'failed', error);
|
|
1195
|
+
}
|
|
1196
|
+
/**
|
|
1197
|
+
* Turn a committed sign-in into pool state: persist the grant pi-ai wrote
|
|
1198
|
+
* to `llm-pi-ai/<provider>` according to the settlement-time capture,
|
|
1199
|
+
* under the plugin-scope account ref, and upsert the `kind: 'oauth'` row.
|
|
1200
|
+
* Any failure fails the attempt loudly — a login whose grant could not be
|
|
1201
|
+
* captured must not look like success.
|
|
1202
|
+
*
|
|
1203
|
+
* `committed` is the F2 provenance promise started by
|
|
1204
|
+
* {@link captureSettledProjection}: the projection payload observed at the
|
|
1205
|
+
* settlement broadcast, never a later live read. The harvest persists
|
|
1206
|
+
* exactly that payload and compensation may only revert exactly that
|
|
1207
|
+
* payload, so a foreign grant cannot become this attempt's authority just
|
|
1208
|
+
* because it landed before the queued mutation ran.
|
|
1209
|
+
*
|
|
1210
|
+
* `projectionGeneration` is the generation {@link noteProjectionCommitted}
|
|
1211
|
+
* recorded at that same broadcast (R6-SF1): it names the projection whose
|
|
1212
|
+
* failures own a cooldown, so the stale-window reset below clears only a
|
|
1213
|
+
* window attributed to an OLDER projection.
|
|
1214
|
+
*/
|
|
1215
|
+
async harvestLogin(attempt, committed, projectionGeneration) {
|
|
1216
|
+
// A revoke (provider release) may have finalized this attempt before
|
|
1217
|
+
// the seam's authorized settlement arrives: never resurrect rows for a
|
|
1218
|
+
// settled login (round-2 review, step 4).
|
|
1219
|
+
if (attempt.settledAt !== undefined)
|
|
1220
|
+
return;
|
|
1221
|
+
// Records this login's grant identity for the failure-path
|
|
1222
|
+
// compensation below: `wroteRef` says the plugin-scope copy exists, and
|
|
1223
|
+
// the projection is reverted only while it still carries exactly this
|
|
1224
|
+
// login's committed payload. The row/owner facts are only needed when
|
|
1225
|
+
// the state flush rejects after the mutation resolved.
|
|
1226
|
+
let harvest;
|
|
1227
|
+
try {
|
|
1228
|
+
await this.run(async () => {
|
|
1229
|
+
const records = this.credentialRecords();
|
|
1230
|
+
if (records === undefined)
|
|
1231
|
+
throw new Error('The credential store does not support records');
|
|
1232
|
+
const grantKey = `${OAUTH_GRANT_RECORD_SCOPE}/${attempt.provider}`;
|
|
1233
|
+
const payload = await committed;
|
|
1234
|
+
if (payload === undefined) {
|
|
1235
|
+
throw new Error(`No pi-ai OAuth credential was committed to ${grantKey}`);
|
|
1236
|
+
}
|
|
1237
|
+
// The refresh token is the stable per-grant secret; its digest —
|
|
1238
|
+
// never the token — names the account's plugin-scope record.
|
|
1239
|
+
const hash = createHash('sha256').update(payload.refresh).digest('hex').slice(0, 12);
|
|
1240
|
+
const grantRef = accountRef(attempt.provider, hash);
|
|
1241
|
+
// Capture the compensation facts BEFORE the managed-state check
|
|
1242
|
+
// (round-3 finding 2): release may win the race — it settles this
|
|
1243
|
+
// attempt and removes the provider while the harvest mutation is
|
|
1244
|
+
// queued behind it — and when the check below throws, the
|
|
1245
|
+
// compensation must still hold this login's payload so the pi-ai
|
|
1246
|
+
// projection it already committed can be reverted.
|
|
1247
|
+
const facts = { payload, grantRef, wroteRef: false };
|
|
1248
|
+
harvest = facts;
|
|
1249
|
+
// Managed state and key capacity are checked BEFORE the
|
|
1250
|
+
// plugin-scope write (round-2 review, step 4): a login whose row
|
|
1251
|
+
// cannot exist must not strand the persisted grant.
|
|
1252
|
+
const providerRecord = this.store.provider(attempt.provider);
|
|
1253
|
+
if (providerRecord === undefined)
|
|
1254
|
+
throw new Error(`Provider "${attempt.provider}" is no longer managed`);
|
|
1255
|
+
// Round-9 finding I-000110: reuse the ordered ownership matcher the
|
|
1256
|
+
// rotation reconciler and hydration apply. The previous single `find`
|
|
1257
|
+
// ORing grantRef / accountId / email evaluated the email fallback in
|
|
1258
|
+
// ARRAY order, so an earlier email-only row adopted a later account's
|
|
1259
|
+
// grant when both carried the same address; the harvest then deleted
|
|
1260
|
+
// that earlier row's prior record as "superseded" — another account's
|
|
1261
|
+
// credential. Ref first, then the exact accountId, then the guarded
|
|
1262
|
+
// email fallback.
|
|
1263
|
+
const oauthRows = providerRecord.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
1264
|
+
const existing = matchOwningAccount(attempt.provider, oauthRows, payload)
|
|
1265
|
+
// Re-authentication target binding (round-3 finding 6): when the
|
|
1266
|
+
// payload matches nothing — a rotated token with no identity
|
|
1267
|
+
// metadata — the validated target row from beginAccountLogin
|
|
1268
|
+
// receives the grant instead of minting a duplicate row.
|
|
1269
|
+
?? (attempt.targetAccountId !== undefined
|
|
1270
|
+
? providerRecord.keys.find(candidate => candidate.id === attempt.targetAccountId && candidate.kind === 'oauth' && candidate.oauth !== null)
|
|
1271
|
+
: undefined);
|
|
1272
|
+
if (existing === undefined && providerRecord.keys.length >= KEY_LIMIT) {
|
|
1273
|
+
throw new Error(`A provider may have at most ${KEY_LIMIT} keys`);
|
|
1274
|
+
}
|
|
1275
|
+
// Snapshot the record this write is about to overwrite (same-ref
|
|
1276
|
+
// re-authentication): a rejected state flush rolls the write back,
|
|
1277
|
+
// and the restored row must keep the payload it referenced.
|
|
1278
|
+
if (existing !== undefined && existing.oauth !== null && existing.oauth.grantRef === grantRef) {
|
|
1279
|
+
facts.previousRef = await records.readRecord(grantRef);
|
|
1280
|
+
}
|
|
1281
|
+
// R5-SF3: the cooldown reset below clears only the window the OLD
|
|
1282
|
+
// grant earned. The seam broadcasts the committed grant as soon as
|
|
1283
|
+
// pi-ai writes it, so the freshly committed grant can already be
|
|
1284
|
+
// serving requests while this write is awaited; a failure landing in
|
|
1285
|
+
// that window belongs to the new generation and must keep cooling.
|
|
1286
|
+
//
|
|
1287
|
+
// R6-SF1: the boundary is the projection generation, not the instant
|
|
1288
|
+
// this plugin-copy write starts. That instant sits too late — the
|
|
1289
|
+
// harvest also waited behind the mutation queue and the projection
|
|
1290
|
+
// read, and a failure of the already-projected grant landing there
|
|
1291
|
+
// used to look older than the write. Attribution is stamped onto the
|
|
1292
|
+
// row's window at failure time instead: only a window belonging to a
|
|
1293
|
+
// generation older than the one projected here is the previous
|
|
1294
|
+
// grant's. Timestamps play no part, so a same-millisecond recovery no
|
|
1295
|
+
// longer stalls.
|
|
1296
|
+
//
|
|
1297
|
+
// Codex P2 (PR #17): the settled guard at the top of this method
|
|
1298
|
+
// cannot see a settlement that lands AFTER this callback was queued
|
|
1299
|
+
// but BEFORE it runs. A release queued ahead does exactly that — the
|
|
1300
|
+
// settlement broadcast arrives while the release body is still
|
|
1301
|
+
// parked, so the pre-queue check passes — and when the release then
|
|
1302
|
+
// fails its own preflight (unreadable settings) the provider stays
|
|
1303
|
+
// managed, so this callback still runs against it. The attempt the
|
|
1304
|
+
// user sees is failed while the harvest would write the grant and
|
|
1305
|
+
// the account row. Revalidate settlement immediately before the
|
|
1306
|
+
// first credential write: a settled attempt must never grow a grant
|
|
1307
|
+
// copy or a row. Nothing of this harvest is written yet, so the
|
|
1308
|
+
// abort needs no compensation, and the release already published the
|
|
1309
|
+
// terminal it owns.
|
|
1310
|
+
if (attempt.settledAt !== undefined)
|
|
1311
|
+
return;
|
|
1312
|
+
await records.modifyRecord(grantRef, () => Promise.resolve({ kind: 'grant', payload: structuredClone(payload) }));
|
|
1313
|
+
facts.wroteRef = true;
|
|
1314
|
+
const now = Date.now();
|
|
1315
|
+
// The id of the row this mutation appends, when it appends one.
|
|
1316
|
+
let addedId;
|
|
1317
|
+
if (existing !== undefined) {
|
|
1318
|
+
// Replacing the credential removes the superseded plugin-scope
|
|
1319
|
+
// record: a changed refresh token hashes to a new ref, and the
|
|
1320
|
+
// old payload (old refresh token included) must not strand as a
|
|
1321
|
+
// secret-bearing orphan. Cleanup is best-effort — the fresh grant
|
|
1322
|
+
// is already persisted, so a locked store must not fail an
|
|
1323
|
+
// otherwise successful login; the warning keeps it observable.
|
|
1324
|
+
if (existing.oauth !== null && existing.oauth.grantRef !== grantRef) {
|
|
1325
|
+
try {
|
|
1326
|
+
// Snapshot before deleting: a rejected state flush restores
|
|
1327
|
+
// this record, because the rolled-back row still references
|
|
1328
|
+
// it. Refusing the delete when the read fails keeps the
|
|
1329
|
+
// superseded record reversible instead of dangling.
|
|
1330
|
+
const superseded = await records.readRecord(existing.oauth.grantRef);
|
|
1331
|
+
if (superseded !== undefined)
|
|
1332
|
+
facts.superseded = { ref: existing.oauth.grantRef, record: superseded };
|
|
1333
|
+
await records.deleteRecord(existing.oauth.grantRef);
|
|
1334
|
+
}
|
|
1335
|
+
catch (error) {
|
|
1336
|
+
this.warn(`cleanup of the superseded grant record failed for ${attempt.provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
// Snapshot the fields this upsert replaces, so a flush rejection
|
|
1340
|
+
// can restore the row exactly as it was before the harvest.
|
|
1341
|
+
facts.rowUndo = {
|
|
1342
|
+
kind: 'updated',
|
|
1343
|
+
id: existing.id,
|
|
1344
|
+
snapshot: {
|
|
1345
|
+
oauth: existing.oauth === null ? null : structuredClone(existing.oauth),
|
|
1346
|
+
valueMissing: existing.valueMissing,
|
|
1347
|
+
cooldownUntil: existing.cooldownUntil,
|
|
1348
|
+
},
|
|
1349
|
+
};
|
|
1350
|
+
existing.oauth = {
|
|
1351
|
+
...existing.oauth,
|
|
1352
|
+
provider: attempt.provider,
|
|
1353
|
+
grantRef,
|
|
1354
|
+
...(payload.email === undefined ? {} : { email: payload.email }),
|
|
1355
|
+
...(payload.accountId === undefined ? {} : { accountId: payload.accountId }),
|
|
1356
|
+
...(typeof payload.expires === 'number' && Number.isFinite(payload.expires) ? { expiresAt: payload.expires } : {}),
|
|
1357
|
+
};
|
|
1358
|
+
// A successful re-authentication recovers the account (round-3
|
|
1359
|
+
// finding 5): the missing flag a demotion set must go, or the now
|
|
1360
|
+
// live projection keeps rendering as missing and stays out of the
|
|
1361
|
+
// healthy counts. Same reset activateAccount applies.
|
|
1362
|
+
existing.valueMissing = false;
|
|
1363
|
+
// Round-5 finding C3: a demoted-and-cooling row kept its stale
|
|
1364
|
+
// cooldown timer, so the freshly projected grant still rendered as
|
|
1365
|
+
// cooling and stayed out of the healthy counts until that timer
|
|
1366
|
+
// expired. The window ends with the new grant.
|
|
1367
|
+
// Round-5 finding M2: clearing the window also resets escalation.
|
|
1368
|
+
// `applyFailureCooldown` derives the next level only from the
|
|
1369
|
+
// previous `cooldownUntil` (core.ts:168-174), so a zeroed window
|
|
1370
|
+
// starts the next failure at level 0 / the base cooldown; the
|
|
1371
|
+
// recovery grace window does not apply to a zeroed cooldown. The
|
|
1372
|
+
// leftover `failureLevel`/`consecutiveFailures` fields do not
|
|
1373
|
+
// escalate anything.
|
|
1374
|
+
// Round-5 finding SF3: only a cooldown the OLD grant earned may be
|
|
1375
|
+
// cleared. R6-SF1: attribution is the projection generation stamped
|
|
1376
|
+
// on the window at failure time, not a timestamp taken here — a
|
|
1377
|
+
// failure of the freshly projected grant landing while this harvest
|
|
1378
|
+
// waited (queue, projection read) keeps its window, and an older
|
|
1379
|
+
// window clears even when its stamp shares this millisecond.
|
|
1380
|
+
const earnedGeneration = this.cooldownGenerations.get(existing.id);
|
|
1381
|
+
if (earnedGeneration === undefined || earnedGeneration < projectionGeneration) {
|
|
1382
|
+
existing.cooldownUntil = 0;
|
|
1383
|
+
this.cooldownGenerations.delete(existing.id);
|
|
1384
|
+
}
|
|
1385
|
+
}
|
|
1386
|
+
else {
|
|
1387
|
+
const rowId = newKeyId();
|
|
1388
|
+
addedId = rowId;
|
|
1389
|
+
// Record the appended row before the push resolves, so the
|
|
1390
|
+
// failure handler can remove exactly this row.
|
|
1391
|
+
facts.rowUndo = { kind: 'added', id: rowId };
|
|
1392
|
+
providerRecord.keys.push({
|
|
1393
|
+
id: rowId,
|
|
1394
|
+
alias: attempt.alias ?? payload.email ?? providerRecord.displayName,
|
|
1395
|
+
// Deliberately not a credential-ref-grammar name: account rows
|
|
1396
|
+
// carry no broker-resolvable literal, and an unparseable ref keeps
|
|
1397
|
+
// the api-key probe paths (probeKey/findVerifiedAlternate) from
|
|
1398
|
+
// misreading the grant as a missing API key.
|
|
1399
|
+
ref: grantRef,
|
|
1400
|
+
kind: 'oauth',
|
|
1401
|
+
enabled: true,
|
|
1402
|
+
createdAt: now,
|
|
1403
|
+
lastUsedAt: 0,
|
|
1404
|
+
cooldownUntil: 0,
|
|
1405
|
+
lastFailureAt: 0,
|
|
1406
|
+
lastFailureCode: '',
|
|
1407
|
+
failureLevel: 0,
|
|
1408
|
+
consecutiveFailures: 0,
|
|
1409
|
+
totalUses: 0,
|
|
1410
|
+
totalFailures: 0,
|
|
1411
|
+
valueMissing: false,
|
|
1412
|
+
importedFrom: null,
|
|
1413
|
+
oauth: {
|
|
1414
|
+
provider: attempt.provider,
|
|
1415
|
+
grantRef,
|
|
1416
|
+
...(payload.email === undefined ? {} : { email: payload.email }),
|
|
1417
|
+
...(payload.accountId === undefined ? {} : { accountId: payload.accountId }),
|
|
1418
|
+
...(typeof payload.expires === 'number' && Number.isFinite(payload.expires) ? { expiresAt: payload.expires } : {}),
|
|
1419
|
+
},
|
|
1420
|
+
});
|
|
1421
|
+
}
|
|
1422
|
+
this.store.touch();
|
|
1423
|
+
attempt.accountId = existing !== undefined ? existing.id : addedId;
|
|
1424
|
+
// Remember who owned the projection before this harvest claims it:
|
|
1425
|
+
// a flush rejection restores that entry instead of leaving the
|
|
1426
|
+
// rolled-back row as the live owner.
|
|
1427
|
+
facts.ownerUndo = this.activeAccounts.has(attempt.provider)
|
|
1428
|
+
? { had: true, owner: this.activeAccounts.get(attempt.provider) }
|
|
1429
|
+
: { had: false };
|
|
1430
|
+
// The grant pi-ai just committed IS the projection; the fresh row
|
|
1431
|
+
// owns it until activation or fallback says otherwise.
|
|
1432
|
+
if (attempt.accountId !== undefined)
|
|
1433
|
+
this.claimProjectionOwner(attempt.provider, attempt.accountId);
|
|
1434
|
+
// No terminal here: `run` flushes the state document after this
|
|
1435
|
+
// callback resolves, and the ready terminal publishes below — only
|
|
1436
|
+
// once persistence has actually resolved (round-2 review, step 3).
|
|
1437
|
+
});
|
|
1438
|
+
this.finishLoginAttempt(attempt, 'ready');
|
|
1439
|
+
}
|
|
1440
|
+
catch (error) {
|
|
1441
|
+
// A rejected state flush lands here AFTER the mutation resolved, so
|
|
1442
|
+
// `attempt.accountId` is already set even though `run` never made the
|
|
1443
|
+
// document durable. Undo the in-memory upsert synchronously — before
|
|
1444
|
+
// any queued mutation or retry can serialize a row whose grant the
|
|
1445
|
+
// credential rollback then deletes — and roll the credential writes
|
|
1446
|
+
// back. Only a failure BEFORE the upsert takes the pre-existing
|
|
1447
|
+
// compensation path.
|
|
1448
|
+
const upserted = attempt.accountId !== undefined;
|
|
1449
|
+
if (upserted)
|
|
1450
|
+
this.undoHarvestRow(attempt, harvest);
|
|
1451
|
+
// Whether THIS failure handler settles the attempt (a release may have
|
|
1452
|
+
// finalized it first): only an attempt this handler failed may have its
|
|
1453
|
+
// reason enriched with the rollback code below.
|
|
1454
|
+
const settledHere = attempt.settledAt === undefined;
|
|
1455
|
+
this.failLoginAttempt(attempt, error);
|
|
1456
|
+
this.warn(`oauth sign-in harvest failed for ${attempt.provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1457
|
+
// Round-2 review step 4: a harvest whose row was never created is
|
|
1458
|
+
// compensated by deleting both written records — the plugin-scope
|
|
1459
|
+
// grant copy (when its write resolved), and the llm-pi-ai projection
|
|
1460
|
+
// only while it still carries THIS login's payload. A harvest that
|
|
1461
|
+
// DID create its row is rolled back the same way, plus the credential
|
|
1462
|
+
// writes its restored row depends on, because the rejected flush
|
|
1463
|
+
// left that upsert undurable.
|
|
1464
|
+
if (upserted) {
|
|
1465
|
+
const rollback = await this.compensateFailedFlush(attempt, harvest);
|
|
1466
|
+
// A rollback that could not write leaves a credential surface the
|
|
1467
|
+
// rolled-back row no longer owns. Tag the terminal reason with the
|
|
1468
|
+
// stable code (keeping the original flush cause after it) so an
|
|
1469
|
+
// upstream caller can branch instead of matching prose.
|
|
1470
|
+
if (rollback.failed && settledHere) {
|
|
1471
|
+
const cause = attempt.error ?? 'the state flush failed';
|
|
1472
|
+
attempt.error = `${FLUSH_ROLLBACK_COMPENSATION_FAILED}: ${cause} (credential rollback incomplete: ${rollback.failures.join('; ')})`;
|
|
1473
|
+
attempt.updatedAt = Date.now();
|
|
1474
|
+
this.warn(`${FLUSH_ROLLBACK_COMPENSATION_FAILED} for ${attempt.provider}: the state flush failed and the credential rollback could not be completed`);
|
|
1475
|
+
}
|
|
1476
|
+
}
|
|
1477
|
+
else {
|
|
1478
|
+
await this.compensateFailedHarvest(attempt, harvest);
|
|
1479
|
+
}
|
|
1480
|
+
}
|
|
1481
|
+
}
|
|
1482
|
+
/**
|
|
1483
|
+
* Capture this attempt's committed projection payload (F2 provenance).
|
|
1484
|
+
*
|
|
1485
|
+
* The seam reports `authorized` only after the flow's commit is observed
|
|
1486
|
+
* and still present, so a read STARTED at that broadcast observes the
|
|
1487
|
+
* record this attempt wrote. The read is deliberately not queued on
|
|
1488
|
+
* `run`: a release parked ahead of the harvest must not delay provenance
|
|
1489
|
+
* long enough for a foreign writer to become this attempt's authority.
|
|
1490
|
+
*
|
|
1491
|
+
* The seam offers no writer identity or record version, so this capture is
|
|
1492
|
+
* the closest provenance the manager can obtain: `undefined` means the
|
|
1493
|
+
* committed payload is unknown, and compensation then refuses to touch the
|
|
1494
|
+
* projection instead of guessing. See I-000090 for the seam capability
|
|
1495
|
+
* (compare-and-delete / versioned records) that would make it exact.
|
|
1496
|
+
*/
|
|
1497
|
+
captureSettledProjection(attempt) {
|
|
1498
|
+
const records = this.credentialRecords();
|
|
1499
|
+
if (records === undefined)
|
|
1500
|
+
return Promise.resolve(undefined);
|
|
1501
|
+
const projectionKey = `${OAUTH_GRANT_RECORD_SCOPE}/${attempt.provider}`;
|
|
1502
|
+
return records.readRecord(projectionKey).then(record => record !== undefined && record.kind === 'grant' && isOAuthGrantPayload(record.payload)
|
|
1503
|
+
? structuredClone(record.payload)
|
|
1504
|
+
: undefined, error => {
|
|
1505
|
+
this.warn(`could not capture the committed grant for ${attempt.provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1506
|
+
return undefined;
|
|
1507
|
+
});
|
|
1508
|
+
}
|
|
1509
|
+
/**
|
|
1510
|
+
* Undo the in-memory half of a harvest whose state document was never made
|
|
1511
|
+
* durable (a rejected `store.flush()` after the mutation resolved).
|
|
1512
|
+
*
|
|
1513
|
+
* Runs synchronously from the failure handler so no queued mutation,
|
|
1514
|
+
* debounced write, or retry can observe — or persist — the rolled-back row
|
|
1515
|
+
* between the rejected flush and this undo. The credential rollback that
|
|
1516
|
+
* follows ({@link compensateFailedFlush}) deletes the grant the row
|
|
1517
|
+
* referenced, so leaving the upsert in memory would make the next
|
|
1518
|
+
* successful flush point at a deleted record.
|
|
1519
|
+
*/
|
|
1520
|
+
undoHarvestRow(attempt, harvest) {
|
|
1521
|
+
attempt.accountId = undefined;
|
|
1522
|
+
const providerRecord = this.store.provider(attempt.provider);
|
|
1523
|
+
if (providerRecord !== undefined && harvest?.rowUndo !== undefined) {
|
|
1524
|
+
const undo = harvest.rowUndo;
|
|
1525
|
+
if (undo.kind === 'added') {
|
|
1526
|
+
const index = providerRecord.keys.findIndex(candidate => candidate.id === undo.id);
|
|
1527
|
+
// Remove exactly the appended row: identity, never position.
|
|
1528
|
+
if (index >= 0)
|
|
1529
|
+
providerRecord.keys.splice(index, 1);
|
|
1530
|
+
}
|
|
1531
|
+
else {
|
|
1532
|
+
const row = providerRecord.keys.find(candidate => candidate.id === undo.id);
|
|
1533
|
+
if (row !== undefined) {
|
|
1534
|
+
row.oauth = undo.snapshot.oauth;
|
|
1535
|
+
row.valueMissing = undo.snapshot.valueMissing;
|
|
1536
|
+
row.cooldownUntil = undo.snapshot.cooldownUntil;
|
|
1537
|
+
}
|
|
1538
|
+
}
|
|
1539
|
+
}
|
|
1540
|
+
if (harvest?.ownerUndo !== undefined) {
|
|
1541
|
+
if (harvest.ownerUndo.had && harvest.ownerUndo.owner !== undefined) {
|
|
1542
|
+
this.activeAccounts.set(attempt.provider, harvest.ownerUndo.owner);
|
|
1543
|
+
}
|
|
1544
|
+
else {
|
|
1545
|
+
this.activeAccounts.delete(attempt.provider);
|
|
1546
|
+
}
|
|
1547
|
+
// Same generation guard the claim bumped: a hydration read started
|
|
1548
|
+
// before/during the harvest must not re-apply the claim just undone.
|
|
1549
|
+
this.ownershipGenerations.set(attempt.provider, (this.ownershipGenerations.get(attempt.provider) ?? 0) + 1);
|
|
1550
|
+
}
|
|
1551
|
+
this.store.touch();
|
|
1552
|
+
}
|
|
1553
|
+
/**
|
|
1554
|
+
* Undo the credential writes of a harvest whose state flush rejected.
|
|
1555
|
+
*
|
|
1556
|
+
* Differs from {@link compensateFailedHarvest} in one respect: the row
|
|
1557
|
+
* upsert already happened, so the plugin-scope record cannot simply be
|
|
1558
|
+
* deleted. A same-ref re-authentication overwrote the record the restored
|
|
1559
|
+
* row references, and a changed-ref re-authentication deleted the
|
|
1560
|
+
* superseded generation before the flush failed — both are reverted here,
|
|
1561
|
+
* each write guarded by the exact payload this login committed so a
|
|
1562
|
+
* foreign writer's record is never clobbered. The `llm-pi-ai` projection is
|
|
1563
|
+
* then demoted exactly like any failed harvest (ownership-checked, never a
|
|
1564
|
+
* foreign grant), so no committed credential outlives the rolled-back row
|
|
1565
|
+
* without a pool owner.
|
|
1566
|
+
*
|
|
1567
|
+
* Resolves with the write failures the rollback hit, if any: a refusal to
|
|
1568
|
+
* touch the surface when provenance is unknown is a deliberate skip, not a
|
|
1569
|
+
* failure, but a rejected compensating write means the credential surface
|
|
1570
|
+
* may still hold records the rolled-back row no longer owns. The caller
|
|
1571
|
+
* tags the attempt with {@link FLUSH_ROLLBACK_COMPENSATION_FAILED} so that
|
|
1572
|
+
* state is distinguishable from a clean rollback (each failure is also
|
|
1573
|
+
* warned about here, message by message).
|
|
1574
|
+
*/
|
|
1575
|
+
async compensateFailedFlush(attempt, harvest) {
|
|
1576
|
+
const failures = [];
|
|
1577
|
+
const recordFailure = (prefix, error) => {
|
|
1578
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
1579
|
+
failures.push(`${prefix}: ${detail}`);
|
|
1580
|
+
this.warn(`${prefix}: ${detail}`);
|
|
1581
|
+
};
|
|
1582
|
+
const records = this.credentialRecords();
|
|
1583
|
+
if (records === undefined)
|
|
1584
|
+
return { failed: false, failures };
|
|
1585
|
+
if (harvest === undefined) {
|
|
1586
|
+
this.warn(`flush-failure compensation refused for ${attempt.provider}: this attempt's committed grant payload is unknown, so the credential surface was left untouched`);
|
|
1587
|
+
return { failed: false, failures };
|
|
1588
|
+
}
|
|
1589
|
+
const provenance = JSON.stringify(harvest.payload);
|
|
1590
|
+
const stillThisPayload = (record) => record !== undefined && record.kind === 'grant'
|
|
1591
|
+
&& isOAuthGrantPayload(record.payload) && JSON.stringify(record.payload) === provenance;
|
|
1592
|
+
try {
|
|
1593
|
+
if (harvest.previousRef !== undefined) {
|
|
1594
|
+
const previous = harvest.previousRef;
|
|
1595
|
+
// Restore the overwritten same-ref record only while the live record
|
|
1596
|
+
// still holds this harvest's payload; the replacement runs inside
|
|
1597
|
+
// the seam's per-key lock.
|
|
1598
|
+
await records.modifyRecord(harvest.grantRef, async (current) => stillThisPayload(current) ? structuredClone(previous) : undefined);
|
|
1599
|
+
}
|
|
1600
|
+
else if (harvest.wroteRef) {
|
|
1601
|
+
let owns = false;
|
|
1602
|
+
await records.modifyRecord(harvest.grantRef, async (current) => {
|
|
1603
|
+
owns = stillThisPayload(current);
|
|
1604
|
+
return undefined;
|
|
1605
|
+
});
|
|
1606
|
+
if (owns)
|
|
1607
|
+
await records.deleteRecord(harvest.grantRef);
|
|
1608
|
+
}
|
|
1609
|
+
}
|
|
1610
|
+
catch (error) {
|
|
1611
|
+
recordFailure(`flush-failure compensation could not restore ${harvest.grantRef} for ${attempt.provider}`, error);
|
|
1612
|
+
}
|
|
1613
|
+
if (harvest.superseded !== undefined) {
|
|
1614
|
+
const superseded = harvest.superseded;
|
|
1615
|
+
try {
|
|
1616
|
+
// Write the superseded generation back only while its ref is still
|
|
1617
|
+
// free: a native writer that reused the ref owns the record now.
|
|
1618
|
+
await records.modifyRecord(superseded.ref, async (current) => current === undefined ? structuredClone(superseded.record) : undefined);
|
|
1619
|
+
}
|
|
1620
|
+
catch (error) {
|
|
1621
|
+
recordFailure(`flush-failure compensation could not restore ${superseded.ref} for ${attempt.provider}`, error);
|
|
1622
|
+
}
|
|
1623
|
+
}
|
|
1624
|
+
const projection = await this.revertHarvestProjection(attempt, harvest);
|
|
1625
|
+
if (!projection.ok)
|
|
1626
|
+
failures.push(projection.failure);
|
|
1627
|
+
return { failed: failures.length > 0, failures };
|
|
1628
|
+
}
|
|
1629
|
+
/**
|
|
1630
|
+
* Roll a failed harvest back to a clean credential surface: the
|
|
1631
|
+
* plugin-scope grant copy is deleted when it was written; the
|
|
1632
|
+
* `llm-pi-ai` projection is deleted only while it still carries EXACTLY
|
|
1633
|
+
* this login's settlement-time committed payload, so compensation can
|
|
1634
|
+
* never delete a grant another account owns.
|
|
1635
|
+
*
|
|
1636
|
+
* Provenance unknown (the settlement-time capture found no usable record)
|
|
1637
|
+
* refuses both deletions with a warning: without the committed payload
|
|
1638
|
+
* there is no proof of ownership, and deleting "whatever is live" is the
|
|
1639
|
+
* round-4 defect (I-000083).
|
|
1640
|
+
*/
|
|
1641
|
+
async compensateFailedHarvest(attempt, harvest) {
|
|
1642
|
+
const records = this.credentialRecords();
|
|
1643
|
+
if (records === undefined)
|
|
1644
|
+
return;
|
|
1645
|
+
if (harvest === undefined) {
|
|
1646
|
+
this.warn(`harvest compensation refused for ${attempt.provider}: this attempt's committed grant payload is unknown, so the projection was left untouched`);
|
|
1647
|
+
return;
|
|
1648
|
+
}
|
|
1649
|
+
if (harvest.wroteRef) {
|
|
1650
|
+
try {
|
|
1651
|
+
await records.deleteRecord(harvest.grantRef);
|
|
1652
|
+
}
|
|
1653
|
+
catch (error) {
|
|
1654
|
+
this.warn(`harvest compensation could not delete ${harvest.grantRef}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
await this.revertHarvestProjection(attempt, harvest);
|
|
1658
|
+
}
|
|
1659
|
+
/**
|
|
1660
|
+
* Delete the `llm-pi-ai` projection only while it still carries exactly
|
|
1661
|
+
* this login's settlement-time committed payload. Shared by the
|
|
1662
|
+
* failed-harvest compensation and the flush-failure rollback.
|
|
1663
|
+
*
|
|
1664
|
+
* Resolves `ok: false` only when the compare/delete writes themselves
|
|
1665
|
+
* rejected; an ownership mismatch is the guard doing its job (the live
|
|
1666
|
+
* record belongs to someone else) and counts as a clean skip.
|
|
1667
|
+
*/
|
|
1668
|
+
async revertHarvestProjection(attempt, harvest) {
|
|
1669
|
+
const records = this.credentialRecords();
|
|
1670
|
+
if (records === undefined)
|
|
1671
|
+
return { ok: true };
|
|
1672
|
+
try {
|
|
1673
|
+
const projectionKey = `${OAUTH_GRANT_RECORD_SCOPE}/${attempt.provider}`;
|
|
1674
|
+
// Same atomic decide-and-delete as the demotion path (round-3
|
|
1675
|
+
// finding 1): the compare runs inside `modifyRecord`'s per-key lock
|
|
1676
|
+
// against the FULL settlement-time payload (not merely its refresh
|
|
1677
|
+
// token), so a native writer that replaced the projection — even by a
|
|
1678
|
+
// payload that reused our refresh token — is never matched, and the
|
|
1679
|
+
// delete runs only on a locked-in exact match (no delete-inside-modify
|
|
1680
|
+
// primitive exists on the seam; see decideAndDeleteProjection and
|
|
1681
|
+
// I-000090).
|
|
1682
|
+
const provenance = JSON.stringify(harvest.payload);
|
|
1683
|
+
let owns = false;
|
|
1684
|
+
let decidedAgainst;
|
|
1685
|
+
await records.modifyRecord(projectionKey, async (current) => {
|
|
1686
|
+
owns = current !== undefined && current.kind === 'grant'
|
|
1687
|
+
&& isOAuthGrantPayload(current.payload) && JSON.stringify(current.payload) === provenance;
|
|
1688
|
+
// F2 pre-delete re-read snapshot, identical to the demotion path.
|
|
1689
|
+
if (owns && current !== undefined)
|
|
1690
|
+
decidedAgainst = provenance;
|
|
1691
|
+
return undefined;
|
|
1692
|
+
});
|
|
1693
|
+
if (owns && await this.projectionStillMatches(records, projectionKey, decidedAgainst)) {
|
|
1694
|
+
await records.deleteRecord(projectionKey);
|
|
1695
|
+
}
|
|
1696
|
+
else if (!owns) {
|
|
1697
|
+
this.warn(`harvest compensation kept the projection for ${attempt.provider}: the live grant no longer matches this attempt's committed payload`);
|
|
1698
|
+
}
|
|
1699
|
+
return { ok: true };
|
|
1700
|
+
}
|
|
1701
|
+
catch (error) {
|
|
1702
|
+
const failure = `harvest compensation could not revert the projection for ${attempt.provider}: ${error instanceof Error ? error.message : String(error)}`;
|
|
1703
|
+
this.warn(failure);
|
|
1704
|
+
return { ok: false, failure };
|
|
1705
|
+
}
|
|
1706
|
+
}
|
|
1707
|
+
/** Expire wedged sign-ins and drop long-settled attempt records. */
|
|
1708
|
+
pruneLoginAttempts(now) {
|
|
1709
|
+
for (const [id, attempt] of this.loginAttempts) {
|
|
1710
|
+
if (attempt.settledAt === undefined) {
|
|
1711
|
+
if (now - attempt.updatedAt <= LOGIN_ATTEMPT_TTL_MS)
|
|
1712
|
+
continue;
|
|
1713
|
+
attempt.controller.abort();
|
|
1714
|
+
this.failLoginAttempt(attempt, new Error('The sign-in attempt timed out'));
|
|
1715
|
+
continue;
|
|
1716
|
+
}
|
|
1717
|
+
if (now - attempt.settledAt > ATTEMPT_TTL_MS)
|
|
1718
|
+
this.loginAttempts.delete(id);
|
|
1719
|
+
}
|
|
1720
|
+
}
|
|
1721
|
+
/** Keep the attempt cache bounded: evict the oldest settled entries first. */
|
|
1722
|
+
capLoginAttempts() {
|
|
1723
|
+
while (this.loginAttempts.size > MAX_LOGIN_ATTEMPTS) {
|
|
1724
|
+
let oldestId;
|
|
1725
|
+
let oldestSettledAt = Number.POSITIVE_INFINITY;
|
|
1726
|
+
for (const [id, attempt] of this.loginAttempts) {
|
|
1727
|
+
// In-flight attempts are never eviction candidates.
|
|
1728
|
+
if (attempt.settledAt === undefined)
|
|
1729
|
+
continue;
|
|
1730
|
+
if (attempt.settledAt < oldestSettledAt) {
|
|
1731
|
+
oldestSettledAt = attempt.settledAt;
|
|
1732
|
+
oldestId = id;
|
|
1733
|
+
}
|
|
1734
|
+
}
|
|
1735
|
+
if (oldestId === undefined)
|
|
1736
|
+
break;
|
|
1737
|
+
this.loginAttempts.delete(oldestId);
|
|
1738
|
+
}
|
|
1739
|
+
}
|
|
1740
|
+
/**
|
|
1741
|
+
* Subscribe to the credential seam's post-commit record broadcast
|
|
1742
|
+
* (credentials/src/index.ts:271-277; the event carries the key only, so
|
|
1743
|
+
* the handler re-reads the record for its payload).
|
|
1744
|
+
*/
|
|
1745
|
+
watchGrantRecordUpdated() {
|
|
1746
|
+
const on = this.ctx.on;
|
|
1747
|
+
if (typeof on !== 'function') {
|
|
1748
|
+
this.warnSubscriptionLost('credentials/record-updated');
|
|
1749
|
+
return undefined;
|
|
1750
|
+
}
|
|
1751
|
+
try {
|
|
1752
|
+
return on.call(this.ctx, 'credentials/record-updated', (key) => {
|
|
1753
|
+
if (typeof key === 'string')
|
|
1754
|
+
this.onGrantRecordUpdated(key);
|
|
1755
|
+
});
|
|
1756
|
+
}
|
|
1757
|
+
catch (error) {
|
|
1758
|
+
this.warnSubscriptionLost('credentials/record-updated', error);
|
|
1759
|
+
return undefined;
|
|
1760
|
+
}
|
|
1761
|
+
}
|
|
1762
|
+
/**
|
|
1763
|
+
* Reconcile one `llm-pi-ai/<provider>` record change with the pool.
|
|
1764
|
+
*
|
|
1765
|
+
* The manager writes this record only through {@link activateAccount}
|
|
1766
|
+
* (D1); pi-ai rewrites it on every lazy refresh and owns every login. A
|
|
1767
|
+
* change whose payload belongs to the active account is a rotation and
|
|
1768
|
+
* syncs back into the owning row; anything else the manager did not write
|
|
1769
|
+
* is native management of the route and surfaces as a warning instead of
|
|
1770
|
+
* being adopted silently.
|
|
1771
|
+
*
|
|
1772
|
+
* C6-1: reconciliation writes a plugin-scope record for the rotated
|
|
1773
|
+
* generation and then mutates the owning row. removeAccount /
|
|
1774
|
+
* releaseProvider mutate the same row but cannot be serialized against
|
|
1775
|
+
* that write — the credential seam offers only a per-record lock
|
|
1776
|
+
* (`modifyRecord`), never a transaction across the record write and the
|
|
1777
|
+
* pool row. So {@link reconcileGrantRecord} revalidates the row immediately
|
|
1778
|
+
* before AND after the awaited write and compensates by deleting the record
|
|
1779
|
+
* it just wrote. An orphan record must never outlive a removed account.
|
|
1780
|
+
*
|
|
1781
|
+
* Round-9 finding P1: two echoes for the same provider can
|
|
1782
|
+
* arrive before the first reconciliation commits. Their write-backs then
|
|
1783
|
+
* race and an out-of-order completion could move the row back onto an
|
|
1784
|
+
* older generation, stranding the newer record outside manager state. The
|
|
1785
|
+
* bodies are therefore chained per provider: one observation's
|
|
1786
|
+
* read → write → commit window is never interleaved with another's, and
|
|
1787
|
+
* {@link reconcileGrantRecord} additionally re-checks that its generation
|
|
1788
|
+
* is still current before committing.
|
|
1789
|
+
*/
|
|
1790
|
+
onGrantRecordUpdated(key) {
|
|
1791
|
+
if (!key.startsWith(`${OAUTH_GRANT_RECORD_SCOPE}/`))
|
|
1792
|
+
return;
|
|
1793
|
+
const provider = key.slice(OAUTH_GRANT_RECORD_SCOPE.length + 1);
|
|
1794
|
+
if (this.store.provider(provider) === undefined)
|
|
1795
|
+
return;
|
|
1796
|
+
// Fire-and-forget: the seam event is synchronous and carries no
|
|
1797
|
+
// completion channel, so failures are logged like any async work. The
|
|
1798
|
+
// body is appended to the provider's chain, preserving arrival order
|
|
1799
|
+
// (round-9 P1); the chain itself never rejects, so a failed
|
|
1800
|
+
// reconciliation does not stop the next echo.
|
|
1801
|
+
const next = this.scheduleGrantReconciliation(provider, () => this.reconcileGrantRecord(provider, key));
|
|
1802
|
+
void next.catch(error => {
|
|
1803
|
+
this.warn(`grant reconciliation failed for ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1804
|
+
});
|
|
1805
|
+
}
|
|
1806
|
+
/**
|
|
1807
|
+
* Append one body to the provider's per-route grant chain and return it.
|
|
1808
|
+
*
|
|
1809
|
+
* The chain is the single serialization point for every operation that can
|
|
1810
|
+
* repoint `row.oauth.grantRef` or reproject the route: the echo reconciler
|
|
1811
|
+
* (round-9 P1 ordering) and, since round-8 finding I-000106, activation.
|
|
1812
|
+
* The stored tail never rejects, so one failed body cannot stop the chain;
|
|
1813
|
+
* callers that must observe the failure await the returned promise (the
|
|
1814
|
+
* fire-and-forget echo caller attaches its own catch).
|
|
1815
|
+
*/
|
|
1816
|
+
scheduleGrantReconciliation(provider, body) {
|
|
1817
|
+
const previous = this.grantReconciliations.get(provider) ?? Promise.resolve();
|
|
1818
|
+
const next = previous.then(body, body);
|
|
1819
|
+
const tail = next.then(() => undefined, () => undefined);
|
|
1820
|
+
this.grantReconciliations.set(provider, tail);
|
|
1821
|
+
void tail.then(() => {
|
|
1822
|
+
if (this.grantReconciliations.get(provider) === tail)
|
|
1823
|
+
this.grantReconciliations.delete(provider);
|
|
1824
|
+
});
|
|
1825
|
+
return next;
|
|
1826
|
+
}
|
|
1827
|
+
/**
|
|
1828
|
+
* The body of {@link onGrantRecordUpdated}. Pool state is read live, and
|
|
1829
|
+
* every awaited step around the write is guarded by revalidation: a
|
|
1830
|
+
* removal is allowed to complete while the write is in flight.
|
|
1831
|
+
*/
|
|
1832
|
+
async reconcileGrantRecord(provider, key) {
|
|
1833
|
+
const records = this.credentialRecords();
|
|
1834
|
+
if (records === undefined)
|
|
1835
|
+
return;
|
|
1836
|
+
const providerRecord = this.store.provider(provider);
|
|
1837
|
+
if (providerRecord === undefined)
|
|
1838
|
+
return;
|
|
1839
|
+
const accounts = providerRecord.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
1840
|
+
if (accounts.length === 0)
|
|
1841
|
+
return;
|
|
1842
|
+
const activeId = this.activeAccounts.get(provider);
|
|
1843
|
+
const active = activeId === undefined
|
|
1844
|
+
? undefined
|
|
1845
|
+
: accounts.find(candidate => candidate.id === activeId);
|
|
1846
|
+
// F1 (I-000081): the owner claim below must not overwrite a claim an
|
|
1847
|
+
// activation or login lands while this awaited re-read is in flight —
|
|
1848
|
+
// the same generation guard load-time hydration applies.
|
|
1849
|
+
const generation = this.ownershipGenerations.get(provider) ?? 0;
|
|
1850
|
+
const record = await records.readRecord(key);
|
|
1851
|
+
if (record === undefined) {
|
|
1852
|
+
// This manager's own demotion marked the removal just before
|
|
1853
|
+
// deleting; while the marker is inside its TTL window the echo is
|
|
1854
|
+
// ours — consume the one-shot marker and stay silent. An expired
|
|
1855
|
+
// marker no longer masks anything (and is dropped for hygiene).
|
|
1856
|
+
const markedAt = this.intentionalProjectionRemovals.get(provider);
|
|
1857
|
+
if (markedAt !== undefined && Date.now() - markedAt <= INTENTIONAL_REMOVAL_TTL_MS) {
|
|
1858
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
1859
|
+
return;
|
|
1860
|
+
}
|
|
1861
|
+
if (markedAt !== undefined)
|
|
1862
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
1863
|
+
if (active !== undefined) {
|
|
1864
|
+
// Not our deletion: someone removed the projection, so the route
|
|
1865
|
+
// silently serves the key pool while the account row looks
|
|
1866
|
+
// healthy. Mark, warn, and drop the stale owner entry (round-2
|
|
1867
|
+
// review, step 1): nothing owns the projection anymore.
|
|
1868
|
+
active.valueMissing = true;
|
|
1869
|
+
this.activeAccounts.delete(provider);
|
|
1870
|
+
this.store.touch();
|
|
1871
|
+
this.pushWarning(`The ${provider} OAuth grant was removed outside Keys Manager; the route now resolves through its API-key pool.`);
|
|
1872
|
+
}
|
|
1873
|
+
return;
|
|
1874
|
+
}
|
|
1875
|
+
if (record.kind === 'api-key') {
|
|
1876
|
+
// Native api-key management took the route over: the manager no
|
|
1877
|
+
// longer owns any projection here.
|
|
1878
|
+
this.activeAccounts.delete(provider);
|
|
1879
|
+
this.pushWarning(`The ${provider} route now holds a natively managed API-key credential; the Keys Manager account pool is not serving it.`);
|
|
1880
|
+
return;
|
|
1881
|
+
}
|
|
1882
|
+
if (!isOAuthGrantPayload(record.payload)) {
|
|
1883
|
+
// An unrecognized payload is not ours to interpret: drop the stale
|
|
1884
|
+
// owner entry so failure attribution and demotion never fire on it.
|
|
1885
|
+
this.activeAccounts.delete(provider);
|
|
1886
|
+
this.pushWarning(`The ${provider} OAuth record holds an unrecognized payload; Keys Manager left it untouched.`);
|
|
1887
|
+
return;
|
|
1888
|
+
}
|
|
1889
|
+
const payload = record.payload;
|
|
1890
|
+
const hash = createHash('sha256').update(payload.refresh).digest('hex').slice(0, 12);
|
|
1891
|
+
const rotatedRef = accountRef(provider, hash);
|
|
1892
|
+
// Owner resolution runs through the shared matcher (refresh digest
|
|
1893
|
+
// first, then identity metadata) — the same rule load-time hydration
|
|
1894
|
+
// applies, so a restarted process and the live reconciler agree.
|
|
1895
|
+
const owner = matchOwningAccount(provider, accounts, payload);
|
|
1896
|
+
if (owner === undefined) {
|
|
1897
|
+
// A foreign account's grant serves the route: no pool row owns the
|
|
1898
|
+
// projection, so the previous owner entry must not survive it.
|
|
1899
|
+
this.activeAccounts.delete(provider);
|
|
1900
|
+
this.pushWarning(`An OAuth account for ${provider} is managed natively outside Keys Manager and is not part of the account pool.`);
|
|
1901
|
+
return;
|
|
1902
|
+
}
|
|
1903
|
+
// C6-1 pre-write revalidation: removeAccount / releaseProvider run on
|
|
1904
|
+
// the mutation queue and delete the row (and its grant record) while
|
|
1905
|
+
// this reconciliation is still resolving its owner. Writing the rotated
|
|
1906
|
+
// generation for an account removed in that window would recreate a
|
|
1907
|
+
// secret-bearing record nothing owns, so the write is skipped entirely.
|
|
1908
|
+
// R7-SF3: this proof is adoption-oriented, so a merely demoted row
|
|
1909
|
+
// (`valueMissing`) still qualifies — a projection that reappears for it
|
|
1910
|
+
// is re-adopted below. Only a removal in progress is refused.
|
|
1911
|
+
if (!this.accountRowIsManaged(provider, owner))
|
|
1912
|
+
return;
|
|
1913
|
+
// F1 (I-000081): the matched grant IS the live projection, so this row
|
|
1914
|
+
// owns it from now on — even when no in-process login or activation
|
|
1915
|
+
// ran (the post-restart echo-rotation case: a hydrated row, no owner
|
|
1916
|
+
// entry). Claiming here is what makes request attribution
|
|
1917
|
+
// (oauthOwnerFor), cooling, and demotion act on the serving account.
|
|
1918
|
+
// The guard above keeps unrecognized and foreign payloads on the
|
|
1919
|
+
// warn-and-clear path; ownership is never adopted for them. A newer
|
|
1920
|
+
// activation/login that claimed while the read was in flight wins:
|
|
1921
|
+
// the stale echo claim is discarded exactly like a stale hydration
|
|
1922
|
+
// read (the rotation sync below still runs, but a commit that a newer
|
|
1923
|
+
// claim has superseded is rejected — round-9 P1).
|
|
1924
|
+
let claimedGeneration;
|
|
1925
|
+
if ((this.ownershipGenerations.get(provider) ?? 0) === generation) {
|
|
1926
|
+
this.claimProjectionOwner(provider, owner.id);
|
|
1927
|
+
claimedGeneration = this.ownershipGenerations.get(provider);
|
|
1928
|
+
// R7-SF3: the live projection proves this row's identity again, so a
|
|
1929
|
+
// demoted row is re-adopted — restore its owner claim (above) and end
|
|
1930
|
+
// the demoted state. `valueMissing` means "not serving but still
|
|
1931
|
+
// managed", not "removal in progress": a native sign-in or pi-ai
|
|
1932
|
+
// write that re-projects the grant recovers the account without a
|
|
1933
|
+
// manual re-activation. The flag is persisted with the rest of the
|
|
1934
|
+
// state document.
|
|
1935
|
+
if (owner.valueMissing === true) {
|
|
1936
|
+
owner.valueMissing = false;
|
|
1937
|
+
// R14-SF3: a demotion is earned by a failure, so the row usually
|
|
1938
|
+
// still carries the cooldown `markFailure` wrote. Re-adoption is the
|
|
1939
|
+
// same manual-recovery equivalence `activateAccount` and re-auth
|
|
1940
|
+
// implement: the live projection is serving again, so the old
|
|
1941
|
+
// window ends with it — otherwise the resurrected row renders
|
|
1942
|
+
// "cooling" and stays out of the healthy counts until the timer
|
|
1943
|
+
// expires. Only this proven-adoption branch clears; a row whose
|
|
1944
|
+
// projection did not reappear keeps its window.
|
|
1945
|
+
owner.cooldownUntil = 0;
|
|
1946
|
+
this.store.touch();
|
|
1947
|
+
}
|
|
1948
|
+
}
|
|
1949
|
+
// Rotation sync: the rotated payload is authoritative (the plan's
|
|
1950
|
+
// single-writer contract — "sync them back to the owning plugin-scope
|
|
1951
|
+
// record on record-updated"), so write it back to the row's grant
|
|
1952
|
+
// record instead of trusting that refresh tokens never rotate. The
|
|
1953
|
+
// plugin-scope record is written FIRST; the row's grantRef commits
|
|
1954
|
+
// only after that write resolves, so a failed write leaves the row
|
|
1955
|
+
// pointing at the old, still-present record and Remove keeps working
|
|
1956
|
+
// against the live grant (round-2 review, step 5). This write cannot
|
|
1957
|
+
// re-enter this handler: it is keys-manager/-scope, and the handler
|
|
1958
|
+
// above only reconciles llm-pi-ai/-scope keys.
|
|
1959
|
+
const previousRef = owner.oauth.grantRef;
|
|
1960
|
+
await records.modifyRecord(rotatedRef, () => Promise.resolve({ kind: 'grant', payload: structuredClone(payload) }));
|
|
1961
|
+
// C6-1 post-write revalidation: a removal that landed while the write
|
|
1962
|
+
// was in flight already deleted its own record, so the generation just
|
|
1963
|
+
// written must be compensated or it outlives the account. The removed
|
|
1964
|
+
// row object is detached, so the mutation below would be invisible
|
|
1965
|
+
// anyway; it is skipped, and a stale owner claim is withdrawn.
|
|
1966
|
+
if (!this.accountRowStillCurrent(provider, owner)) {
|
|
1967
|
+
if (this.activeAccounts.get(provider) === owner.id)
|
|
1968
|
+
this.activeAccounts.delete(provider);
|
|
1969
|
+
try {
|
|
1970
|
+
await records.deleteRecord(rotatedRef);
|
|
1971
|
+
}
|
|
1972
|
+
catch (error) {
|
|
1973
|
+
this.warn(`cancelled rotation record cleanup failed for ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
1974
|
+
}
|
|
1975
|
+
return;
|
|
1976
|
+
}
|
|
1977
|
+
// Round-9 P1 commit guard: the awaited write gave a newer echo, an
|
|
1978
|
+
// activation, or a login time to supersede the generation this body
|
|
1979
|
+
// observed. Committing then would point the row back at an older
|
|
1980
|
+
// generation and leave the newer generation's record outside manager
|
|
1981
|
+
// state, so the superseded write-back is removed instead. Only a
|
|
1982
|
+
// distinct record is deleted: in the same-ref case the write landed on
|
|
1983
|
+
// the row's own current record and must stay.
|
|
1984
|
+
const stillCurrent = await this.rotationStillCurrent(records, provider, key, rotatedRef, claimedGeneration);
|
|
1985
|
+
if (!stillCurrent) {
|
|
1986
|
+
if (rotatedRef !== previousRef) {
|
|
1987
|
+
// R5-MF1: the staleness probe is awaited, and a settled harvest can
|
|
1988
|
+
// adopt this very generation into a live row while it is in flight
|
|
1989
|
+
// (same account, same committed refresh token). Removing the record
|
|
1990
|
+
// then would delete the row's own grant — keep anything a live row
|
|
1991
|
+
// still serves.
|
|
1992
|
+
await this.discardRotationRecord(records, provider, rotatedRef, 'cleanup of the superseded rotation record failed for');
|
|
1993
|
+
}
|
|
1994
|
+
return;
|
|
1995
|
+
}
|
|
1996
|
+
// R5-SF1: the staleness probe above is itself awaited, so the post-write
|
|
1997
|
+
// revalidation is already stale by the time the commit runs. A removal
|
|
1998
|
+
// that completes inside that probe is only visible through the
|
|
1999
|
+
// removal-aware check, so it runs again immediately before the commit:
|
|
2000
|
+
// committing would write grantRef onto a detached row and leave the record
|
|
2001
|
+
// just written orphaned. Compensation mirrors the post-write branch.
|
|
2002
|
+
if (!this.accountRowStillCurrent(provider, owner)) {
|
|
2003
|
+
if (this.activeAccounts.get(provider) === owner.id)
|
|
2004
|
+
this.activeAccounts.delete(provider);
|
|
2005
|
+
if (rotatedRef !== previousRef) {
|
|
2006
|
+
await this.discardRotationRecord(records, provider, rotatedRef, 'cancelled rotation record cleanup failed for');
|
|
2007
|
+
}
|
|
2008
|
+
return;
|
|
2009
|
+
}
|
|
2010
|
+
// SF2: once the new payload persists and the row points at it, delete
|
|
2011
|
+
// the superseded generation's plugin-scope record — harvest parity. A
|
|
2012
|
+
// rotated refresh token must not strand its secret-bearing record
|
|
2013
|
+
// outside manager state; cleanup is best-effort with a warning,
|
|
2014
|
+
// exactly like the harvest-time cleanup.
|
|
2015
|
+
if (previousRef !== rotatedRef) {
|
|
2016
|
+
owner.oauth = { ...owner.oauth, grantRef: rotatedRef };
|
|
2017
|
+
try {
|
|
2018
|
+
await records.deleteRecord(previousRef);
|
|
2019
|
+
}
|
|
2020
|
+
catch (error) {
|
|
2021
|
+
this.warn(`cleanup of the superseded grant record failed for ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
// Refresh-derived identity fields are informational and update in place.
|
|
2025
|
+
owner.oauth = {
|
|
2026
|
+
...owner.oauth,
|
|
2027
|
+
...(payload.email === undefined ? {} : { email: payload.email }),
|
|
2028
|
+
...(payload.accountId === undefined ? {} : { accountId: payload.accountId }),
|
|
2029
|
+
...(typeof payload.expires === 'number' && Number.isFinite(payload.expires) ? { expiresAt: payload.expires } : {}),
|
|
2030
|
+
};
|
|
2031
|
+
this.store.touch();
|
|
2032
|
+
}
|
|
2033
|
+
/**
|
|
2034
|
+
* C6-1 revalidation: the row captured before an awaited credential write
|
|
2035
|
+
* must still be the provider's live row — same object identity, still an
|
|
2036
|
+
* OAuth account. A remove/release landing in between detaches the row
|
|
2037
|
+
* object; object identity is what tells "this account is still there"
|
|
2038
|
+
* apart from "some same-id row was recreated".
|
|
2039
|
+
*
|
|
2040
|
+
* C6-1/I1: a removal that is still IN PROGRESS is not visible through
|
|
2041
|
+
* identity alone. A multi-row release deletes each row's grant first and
|
|
2042
|
+
* the provider record last, and it never splices its rows, so after row X's
|
|
2043
|
+
* grant is deleted the provider record and row X both still look live while
|
|
2044
|
+
* the release awaits later rows. Syncing a rotation into that window would
|
|
2045
|
+
* commit `grantRef = rotatedRef` on a row nothing will serve, and the
|
|
2046
|
+
* release's final provider deletion leaves that record orphaned. The
|
|
2047
|
+
* removal-aware check is {@link retiringAccounts}, set before the removal's
|
|
2048
|
+
* first awaited delete.
|
|
2049
|
+
*
|
|
2050
|
+
* R7-SF3: `valueMissing` is NOT a removal signal — it only says the row is
|
|
2051
|
+
* currently not serving. A demoted row is still managed, and a projection
|
|
2052
|
+
* that reappears for it must be re-adopted (the caller proves the
|
|
2053
|
+
* projection belongs to this row before claiming it). This guard therefore
|
|
2054
|
+
* keeps the credential-write semantics strict: writing to a demoted row is
|
|
2055
|
+
* pointless until the adoption path has cleared the flag.
|
|
2056
|
+
*/
|
|
2057
|
+
accountRowStillCurrent(provider, owner) {
|
|
2058
|
+
if (owner.valueMissing === true)
|
|
2059
|
+
return false;
|
|
2060
|
+
return this.accountRowIsManaged(provider, owner);
|
|
2061
|
+
}
|
|
2062
|
+
/**
|
|
2063
|
+
* R7-SF3 adoption proof: the row is live and still managed, whether or not
|
|
2064
|
+
* it is currently serving. "Retiring" (removal in progress) refuses
|
|
2065
|
+
* everything; "demoted" (`valueMissing`) refuses nothing here — the caller
|
|
2066
|
+
* has already proven that the live projection belongs to this exact row
|
|
2067
|
+
* ({@link matchOwningAccount}) and that no newer projection superseded the
|
|
2068
|
+
* observation (generation guard).
|
|
2069
|
+
*/
|
|
2070
|
+
accountRowIsManaged(provider, owner) {
|
|
2071
|
+
if (this.retiringAccounts.has(owner))
|
|
2072
|
+
return false;
|
|
2073
|
+
const providerRecord = this.store.provider(provider);
|
|
2074
|
+
if (providerRecord === undefined)
|
|
2075
|
+
return false;
|
|
2076
|
+
if (owner.kind !== 'oauth' || owner.oauth === null)
|
|
2077
|
+
return false;
|
|
2078
|
+
return providerRecord.keys.some(candidate => candidate === owner);
|
|
2079
|
+
}
|
|
2080
|
+
/**
|
|
2081
|
+
* Round-5 review (R5-MF1): whether a live account row of `provider` still
|
|
2082
|
+
* names `ref` as its grant. The settled harvest can adopt the generation a
|
|
2083
|
+
* reconciliation just wrote while that reconciliation is still awaiting its
|
|
2084
|
+
* staleness probe — the row's `grantRef` then IS the record the cleanup
|
|
2085
|
+
* would delete, so deleting it destroys the account's own live grant.
|
|
2086
|
+
* A row already marked for removal does not count: its removal deletes the
|
|
2087
|
+
* refs it names, so keeping the record would strand it instead.
|
|
2088
|
+
*/
|
|
2089
|
+
grantRefServed(provider, ref) {
|
|
2090
|
+
const providerRecord = this.store.provider(provider);
|
|
2091
|
+
if (providerRecord === undefined)
|
|
2092
|
+
return false;
|
|
2093
|
+
return providerRecord.keys.some(candidate => candidate.kind === 'oauth'
|
|
2094
|
+
&& candidate.oauth !== null
|
|
2095
|
+
&& candidate.oauth.grantRef === ref
|
|
2096
|
+
&& !this.retiringAccounts.has(candidate));
|
|
2097
|
+
}
|
|
2098
|
+
/**
|
|
2099
|
+
* Delete a rotation record a reconciliation wrote but must not keep, unless
|
|
2100
|
+
* a live row already serves it (R5-MF1). Cleanup is best-effort with a
|
|
2101
|
+
* warning, exactly like the harvest-time superseded-record cleanup.
|
|
2102
|
+
*/
|
|
2103
|
+
async discardRotationRecord(records, provider, ref, failure) {
|
|
2104
|
+
if (this.grantRefServed(provider, ref))
|
|
2105
|
+
return;
|
|
2106
|
+
try {
|
|
2107
|
+
await records.deleteRecord(ref);
|
|
2108
|
+
}
|
|
2109
|
+
catch (error) {
|
|
2110
|
+
this.warn(`${failure} ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2111
|
+
}
|
|
2112
|
+
}
|
|
2113
|
+
/**
|
|
2114
|
+
* Round-9 P1 revalidation: the generation a reconciliation observed is
|
|
2115
|
+
* still the one the route holds, and the ownership claim this body made (if
|
|
2116
|
+
* any) is still the newest. Two independent writes can supersede it while
|
|
2117
|
+
* the plugin-scope write-back is in flight: pi-ai refreshing again (the
|
|
2118
|
+
* projection's refresh digest — the ref's identity — moves on) and an
|
|
2119
|
+
* activation or login claiming the projection (the ownership generation
|
|
2120
|
+
* moves). Committing the older ref in either case would move the row
|
|
2121
|
+
* backwards and strand the newer record outside manager state.
|
|
2122
|
+
*
|
|
2123
|
+
* A failed re-read cannot prove staleness, so it must not block the sync:
|
|
2124
|
+
* the commit then proceeds exactly as it did before this guard existed.
|
|
2125
|
+
*/
|
|
2126
|
+
async rotationStillCurrent(records, provider, key, rotatedRef, claimedGeneration) {
|
|
2127
|
+
if (claimedGeneration !== undefined
|
|
2128
|
+
&& (this.ownershipGenerations.get(provider) ?? 0) !== claimedGeneration)
|
|
2129
|
+
return false;
|
|
2130
|
+
try {
|
|
2131
|
+
const current = await records.readRecord(key);
|
|
2132
|
+
if (current === undefined || current.kind !== 'grant' || !isOAuthGrantPayload(current.payload))
|
|
2133
|
+
return false;
|
|
2134
|
+
const hash = createHash('sha256').update(current.payload.refresh).digest('hex').slice(0, 12);
|
|
2135
|
+
return accountRef(provider, hash) === rotatedRef;
|
|
2136
|
+
}
|
|
2137
|
+
catch (error) {
|
|
2138
|
+
this.warn(`rotation staleness check could not re-read ${key}: ${error instanceof Error ? error.message : String(error)}`);
|
|
2139
|
+
return true;
|
|
2140
|
+
}
|
|
2141
|
+
}
|
|
2142
|
+
/**
|
|
2143
|
+
* Make one account's grant payload the route's stored credential
|
|
2144
|
+
* (D1 activation projection).
|
|
2145
|
+
*
|
|
2146
|
+
* Round-8 finding I-000106 (must-fix): the decide snapshot used to be read
|
|
2147
|
+
* before the seam's lock, and `reconcileGrantRecord` runs OUTSIDE the
|
|
2148
|
+
* mutation queue — so a rotation echo could repoint `row.oauth.grantRef` and
|
|
2149
|
+
* delete the record being projected between the read and the write, leaving
|
|
2150
|
+
* the route on an older generation than the row names. The whole
|
|
2151
|
+
* read → project → claim section therefore runs on the provider's grant
|
|
2152
|
+
* chain ({@link scheduleGrantReconciliation}) — the same chain the
|
|
2153
|
+
* reconciler bodies use. A rotation committed before this body is observed
|
|
2154
|
+
* in `row.oauth.grantRef`; one that arrives while the body runs is queued
|
|
2155
|
+
* behind it and reconciles the generation this body projected. Either way
|
|
2156
|
+
* the projection holds exactly the generation the row names.
|
|
2157
|
+
*
|
|
2158
|
+
* The seam's per-key lock still serializes the read-decide-replace steps of
|
|
2159
|
+
* `modifyRecord` (credentials/src/index.ts:236-250) against other writers.
|
|
2160
|
+
* The account whose payload lands becomes the pool's active account and is
|
|
2161
|
+
* cascade-selectable again.
|
|
2162
|
+
*/
|
|
2163
|
+
activateAccount(provider, accountId) {
|
|
2164
|
+
return this.run(async () => {
|
|
2165
|
+
const providerRecord = this.store.provider(provider);
|
|
2166
|
+
if (providerRecord === undefined)
|
|
2167
|
+
throw new Error(`Provider "${provider}" is not managed`);
|
|
2168
|
+
const row = providerRecord.keys.find(candidate => candidate.id === accountId);
|
|
2169
|
+
if (row === undefined)
|
|
2170
|
+
throw new Error(`Account "${accountId}" not found on ${provider}`);
|
|
2171
|
+
if (row.kind !== 'oauth' || row.oauth === null)
|
|
2172
|
+
throw new Error(`Key "${accountId}" is not an OAuth account`);
|
|
2173
|
+
const records = this.credentialRecords();
|
|
2174
|
+
if (records === undefined)
|
|
2175
|
+
throw new Error('The credential store does not support records');
|
|
2176
|
+
await this.scheduleGrantReconciliation(provider, async () => {
|
|
2177
|
+
// Re-read the generation under the chain: a rotation that committed
|
|
2178
|
+
// before this body claimed its slot is already reflected here.
|
|
2179
|
+
const grantRef = row.oauth.grantRef;
|
|
2180
|
+
const stored = await records.readRecord(grantRef);
|
|
2181
|
+
if (stored === undefined || stored.kind !== 'grant' || !isOAuthGrantPayload(stored.payload)) {
|
|
2182
|
+
throw new Error(`The grant payload at ${grantRef} is missing or unusable; sign in to this account again`);
|
|
2183
|
+
}
|
|
2184
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
2185
|
+
await records.modifyRecord(`${OAUTH_GRANT_RECORD_SCOPE}/${provider}`, () => Promise.resolve({ kind: 'grant', payload: structuredClone(stored.payload) }));
|
|
2186
|
+
this.claimProjectionOwner(provider, row.id);
|
|
2187
|
+
});
|
|
2188
|
+
row.valueMissing = false;
|
|
2189
|
+
// Round-5 finding M1: manual recovery ends the cooling window too, the
|
|
2190
|
+
// same reset a successful re-authentication applies. A demoted account
|
|
2191
|
+
// is re-activated while its stale cooldown still sits on the row, which
|
|
2192
|
+
// would render the freshly projected grant as cooling and keep it out
|
|
2193
|
+
// of the healthy counts until the timer expires. Clearing the window
|
|
2194
|
+
// also resets escalation: `applyFailureCooldown` derives the next level
|
|
2195
|
+
// from `cooldownUntil` only (core.ts:168-174).
|
|
2196
|
+
row.cooldownUntil = 0;
|
|
2197
|
+
this.store.touch();
|
|
2198
|
+
// Remote methods hand back the refreshed state view (addKey/deleteKey
|
|
2199
|
+
// precedent), so the client updates its snapshot in one round trip.
|
|
2200
|
+
return this.buildState();
|
|
2201
|
+
});
|
|
2202
|
+
}
|
|
2203
|
+
/**
|
|
2204
|
+
* Demote the cascade to the key pool by deleting the projection record:
|
|
2205
|
+
* pi-ai's per-request resolution prefers the stored credential and only
|
|
2206
|
+
* falls through to the broker `apiKeyEnv` when none is stored
|
|
2207
|
+
* (research, resolve.js:26-55).
|
|
2208
|
+
*
|
|
2209
|
+
* Recovery is deliberately manual: the cooled account stays demoted until
|
|
2210
|
+
* the user re-activates it or signs in again. An automatic re-activation
|
|
2211
|
+
* on cooldown expiry would replay the failing grant every request; the
|
|
2212
|
+
* user (who can see whether the account recovered upstream) is the
|
|
2213
|
+
* arbiter. `fallbackToKeys` marks every account row value-missing so the
|
|
2214
|
+
* live arbitration — projection presence plus probe fail-skip over a
|
|
2215
|
+
* kind-agnostic cursor ring — keeps serving only real candidates.
|
|
2216
|
+
*
|
|
2217
|
+
* `accountId` names the row whose failure triggers this demotion (R7-MF).
|
|
2218
|
+
* It is resolved when the queued body runs, so the demotion deletes only a
|
|
2219
|
+
* projection that still belongs to THAT row: activating another account
|
|
2220
|
+
* while the failing request is in flight must not lose the new account's
|
|
2221
|
+
* healthy projection.
|
|
2222
|
+
*
|
|
2223
|
+
* R14-M3: the targetless form (omitted `accountId`) is TEST-ONLY. It
|
|
2224
|
+
* resolves the provider's active account when the queued body runs — the
|
|
2225
|
+
* same execution-time race R7-MF removed from the failure path — so it is
|
|
2226
|
+
* kept only because the tests drive the demotion without a failure event.
|
|
2227
|
+
* Production callers must name the row; the failure path always passes
|
|
2228
|
+
* `key.id`.
|
|
2229
|
+
*/
|
|
2230
|
+
fallbackToKeys(provider, accountId) {
|
|
2231
|
+
return this.run(() => this.fallbackToKeysUnqueued(provider, accountId));
|
|
2232
|
+
}
|
|
2233
|
+
/**
|
|
2234
|
+
* Atomic decide-and-delete of the `llm-pi-ai/<provider>` projection for one
|
|
2235
|
+
* row (round-3 finding 1): the payload compare runs INSIDE the credential
|
|
2236
|
+
* seam's per-key lock — `modifyRecord` hands `mutate` the record as it
|
|
2237
|
+
* stands while the write is exclusive, so a native sign-in that already
|
|
2238
|
+
* replaced the projection is visible here and the delete is skipped. The
|
|
2239
|
+
* seam exposes no delete-inside-modify primitive: a `mutate` may only
|
|
2240
|
+
* return a replacement or `undefined` (leave untouched), and calling
|
|
2241
|
+
* `deleteRecord` inside the mutate would deadlock the provider's own
|
|
2242
|
+
* serialized operation queue (credentials-local `enqueue`). The closest
|
|
2243
|
+
* equivalent — the exact primitives this method uses — are a decide-only
|
|
2244
|
+
* modify (returns `undefined`, no write) and a second pre-delete re-read
|
|
2245
|
+
* against the decided-against snapshot ({@link projectionStillMatches},
|
|
2246
|
+
* F2 step-1 hardening): a native writer landing between the decide and
|
|
2247
|
+
* the delete is therefore visible before the delete runs, and its grant
|
|
2248
|
+
* survives. The residual window is the sub-microtask gap between the
|
|
2249
|
+
* re-read releasing the lock and the delete's own queue entry; a native
|
|
2250
|
+
* writer landing there is still deleted, because closing that gap needs a
|
|
2251
|
+
* compare-and-delete primitive the seam does not expose (store issue
|
|
2252
|
+
* I-000090). `tests/host.test.ts` ("a native grant landing after the final
|
|
2253
|
+
* comparison is still lost") pins that exact outcome so the seam fix has a
|
|
2254
|
+
* test to flip.
|
|
2255
|
+
*
|
|
2256
|
+
* This is the only delete authority: a natively replaced grant — another
|
|
2257
|
+
* account signed in, a native api-key record, an unrecognized payload —
|
|
2258
|
+
* never matches and must survive the demotion of a stale row, and the
|
|
2259
|
+
* async hydration window must not decide ownership (round-2 review,
|
|
2260
|
+
* steps 1–2).
|
|
2261
|
+
*/
|
|
2262
|
+
async decideAndDeleteProjection(provider, row) {
|
|
2263
|
+
if (row.kind !== 'oauth' || row.oauth === null)
|
|
2264
|
+
return false;
|
|
2265
|
+
const records = this.credentialRecords();
|
|
2266
|
+
if (records === undefined)
|
|
2267
|
+
return false;
|
|
2268
|
+
const accounts = this.store.provider(provider)?.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null) ?? [];
|
|
2269
|
+
const projectionKey = `${OAUTH_GRANT_RECORD_SCOPE}/${provider}`;
|
|
2270
|
+
let owns = false;
|
|
2271
|
+
let decidedAgainst;
|
|
2272
|
+
await records.modifyRecord(projectionKey, async (current) => {
|
|
2273
|
+
owns = current !== undefined && current.kind === 'grant'
|
|
2274
|
+
&& isOAuthGrantPayload(current.payload)
|
|
2275
|
+
&& matchOwningAccount(provider, accounts, current.payload)?.id === row.id;
|
|
2276
|
+
// The decided-against payload snapshot (in-memory JSON, never logged
|
|
2277
|
+
// or persisted): the pre-delete re-read below skips the removal when
|
|
2278
|
+
// the live record no longer holds exactly this content.
|
|
2279
|
+
if (owns && current !== undefined)
|
|
2280
|
+
decidedAgainst = JSON.stringify(current.payload);
|
|
2281
|
+
// Decide only: the removal goes through deleteRecord below, outside
|
|
2282
|
+
// the mutate (see the primitive note above).
|
|
2283
|
+
return undefined;
|
|
2284
|
+
});
|
|
2285
|
+
if (!owns)
|
|
2286
|
+
return false;
|
|
2287
|
+
// F2 (step-1 removal window): re-read the live record under the seam's
|
|
2288
|
+
// per-key lock immediately before the delete and skip when a native
|
|
2289
|
+
// grant landed between the decide and the delete. Residual window,
|
|
2290
|
+
// documented honestly: the seam exposes no delete-inside-modify
|
|
2291
|
+
// primitive (a mutate may only return a replacement or undefined, and
|
|
2292
|
+
// deleteRecord inside the mutate would deadlock the provider's
|
|
2293
|
+
// serialized operation queue), so a native writer landing between THIS
|
|
2294
|
+
// re-read releasing the lock and the delete's own queue entry can still
|
|
2295
|
+
// lose its grant. That writer is the next writer's grant, and the
|
|
2296
|
+
// ownership oracle re-identifies it on the record-updated echo.
|
|
2297
|
+
if (!await this.projectionStillMatches(records, projectionKey, decidedAgainst))
|
|
2298
|
+
return false;
|
|
2299
|
+
// Mark BEFORE the delete: the deletion's echo handler reads this
|
|
2300
|
+
// marker across its own awaited record re-read, whenever that
|
|
2301
|
+
// continuation resumes. The TTL window — not scheduling order —
|
|
2302
|
+
// decides how long the marker stays trustworthy.
|
|
2303
|
+
const markedAt = Date.now();
|
|
2304
|
+
this.intentionalProjectionRemovals.set(provider, markedAt);
|
|
2305
|
+
try {
|
|
2306
|
+
await records.deleteRecord(projectionKey);
|
|
2307
|
+
}
|
|
2308
|
+
catch (error) {
|
|
2309
|
+
// R7-SF2: a delete that threw removed nothing, so no own-removal echo
|
|
2310
|
+
// is coming. Revoke this call's marker before rethrowing — left in
|
|
2311
|
+
// place it would consume a genuine out-of-band removal as the
|
|
2312
|
+
// manager's own for the whole TTL, hiding the warning, the missing
|
|
2313
|
+
// mark, and the stale owner cleanup. The identity check keeps a newer
|
|
2314
|
+
// marker (there is none on the serialized mutation queue) untouched.
|
|
2315
|
+
// R14-SF2: seam assumption — the throwing delete may have removed the
|
|
2316
|
+
// record before throwing; the later echo is then read as external
|
|
2317
|
+
// (warning + stale-owner drop), the conservative direction over
|
|
2318
|
+
// masking a genuine out-of-band removal.
|
|
2319
|
+
if (this.intentionalProjectionRemovals.get(provider) === markedAt) {
|
|
2320
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
2321
|
+
}
|
|
2322
|
+
throw error;
|
|
2323
|
+
}
|
|
2324
|
+
return true;
|
|
2325
|
+
}
|
|
2326
|
+
/**
|
|
2327
|
+
* F2 pre-delete re-read, shared by the demotion and harvest-compensation
|
|
2328
|
+
* decide-then-delete pairs: `true` while the live projection still holds
|
|
2329
|
+
* exactly the decided-against payload. Runs inside `modifyRecord`'s
|
|
2330
|
+
* per-key lock (the same primitive as the decide), so a native writer
|
|
2331
|
+
* that already replaced the record is visible here and the delete is
|
|
2332
|
+
* skipped instead of destroying the foreign grant. The lock is released
|
|
2333
|
+
* before the delete, so a writer landing in that final gap is still lost:
|
|
2334
|
+
* closing it needs the seam's compare-and-delete (store issue I-000090).
|
|
2335
|
+
*/
|
|
2336
|
+
async projectionStillMatches(records, projectionKey, decidedAgainst) {
|
|
2337
|
+
if (decidedAgainst === undefined)
|
|
2338
|
+
return false;
|
|
2339
|
+
let liveMatches = false;
|
|
2340
|
+
await records.modifyRecord(projectionKey, async (current) => {
|
|
2341
|
+
liveMatches = current !== undefined && current.kind === 'grant'
|
|
2342
|
+
&& JSON.stringify(current.payload) === decidedAgainst;
|
|
2343
|
+
return undefined;
|
|
2344
|
+
});
|
|
2345
|
+
return liveMatches;
|
|
2346
|
+
}
|
|
2347
|
+
/**
|
|
2348
|
+
* The fallback body; callers own the mutation queue. `target` narrows the
|
|
2349
|
+
* ownership check to one explicit row: removal call sites pass the removed
|
|
2350
|
+
* row object, the failure path passes the failing row's id (R7-MF) — an
|
|
2351
|
+
* id is re-resolved here because the failure's demotion is queued and the
|
|
2352
|
+
* row object may be detached by the time it runs. Without a target the
|
|
2353
|
+
* provider's active account decides.
|
|
2354
|
+
*/
|
|
2355
|
+
async fallbackToKeysUnqueued(provider, target) {
|
|
2356
|
+
const providerRecord = this.store.provider(provider);
|
|
2357
|
+
if (providerRecord === undefined)
|
|
2358
|
+
return;
|
|
2359
|
+
const activeId = this.activeAccounts.get(provider);
|
|
2360
|
+
const resolveOAuthRow = (candidate) => candidate !== undefined && candidate.kind === 'oauth' && candidate.oauth !== null ? candidate : undefined;
|
|
2361
|
+
const active = resolveOAuthRow(typeof target === 'string'
|
|
2362
|
+
? providerRecord.keys.find(candidate => candidate.id === target)
|
|
2363
|
+
: target ?? (activeId === undefined
|
|
2364
|
+
? undefined
|
|
2365
|
+
: providerRecord.keys.find(candidate => candidate.id === activeId)));
|
|
2366
|
+
// Ownership-verified delete: demote the projection only while it still
|
|
2367
|
+
// belongs to this row. The compare runs inside the seam's per-key lock
|
|
2368
|
+
// and the delete follows it (decideAndDeleteProjection, round-3
|
|
2369
|
+
// finding 1) — both inside this mutation.
|
|
2370
|
+
const ownsProjection = active !== undefined && await this.decideAndDeleteProjection(provider, active);
|
|
2371
|
+
if (ownsProjection) {
|
|
2372
|
+
// Full cascade demotion: no pool row serves the route anymore, so
|
|
2373
|
+
// every account row is demoted until re-activation. The owner entry
|
|
2374
|
+
// must go too (round-3 finding 4): oauthOwnerFor otherwise keeps
|
|
2375
|
+
// naming this row, so the client renders "Serving provider" next to
|
|
2376
|
+
// the missing pill and leaves "Set active" disabled — the documented
|
|
2377
|
+
// re-activation path unreachable until a restart or a new sign-in.
|
|
2378
|
+
this.activeAccounts.delete(provider);
|
|
2379
|
+
for (const row of providerRecord.keys) {
|
|
2380
|
+
if (row.kind === 'oauth')
|
|
2381
|
+
row.valueMissing = true;
|
|
2382
|
+
}
|
|
2383
|
+
}
|
|
2384
|
+
else if (active !== undefined && activeId === active.id) {
|
|
2385
|
+
// The projection is not ours anymore (native replacement): keep the
|
|
2386
|
+
// stale owner entry from routing attribution or demotion at this row.
|
|
2387
|
+
this.activeAccounts.delete(provider);
|
|
2388
|
+
}
|
|
2389
|
+
this.store.touch();
|
|
2390
|
+
}
|
|
2391
|
+
/**
|
|
2392
|
+
* Drop one OAuth account's plugin-scope grant payload and — only while
|
|
2393
|
+
* this row still owns the live projection — demote the cascade to the key
|
|
2394
|
+
* pool first, so the route falls through to the API-key pool instead of
|
|
2395
|
+
* serving a grant whose owning row is gone. The demotion re-reads
|
|
2396
|
+
* `llm-pi-ai/<provider>` inside this mutation and runs it through the
|
|
2397
|
+
* ownership oracle ({@link matchOwningAccount}); a natively replaced
|
|
2398
|
+
* projection (another account signed in, a native api-key record) is left
|
|
2399
|
+
* untouched. Non-oauth rows are a no-op; every OAuth-removal call site
|
|
2400
|
+
* goes through this helper.
|
|
2401
|
+
*/
|
|
2402
|
+
async removeOAuthGrant(provider, row) {
|
|
2403
|
+
if (row.kind !== 'oauth' || row.oauth === null)
|
|
2404
|
+
return;
|
|
2405
|
+
// C6-1/I1: mark the row BEFORE the first awaited delete. A release
|
|
2406
|
+
// deletes every row's credential before it deletes the provider record
|
|
2407
|
+
// and never splices the rows, so from the first delete onward the row
|
|
2408
|
+
// object is no longer a truthful "still serving" signal; the reconciler
|
|
2409
|
+
// consults this mark instead (see accountRowStillCurrent).
|
|
2410
|
+
this.retiringAccounts.add(row);
|
|
2411
|
+
try {
|
|
2412
|
+
await this.fallbackToKeysUnqueued(provider, row);
|
|
2413
|
+
const records = this.credentialRecords();
|
|
2414
|
+
if (records !== undefined)
|
|
2415
|
+
await records.deleteRecord(row.oauth.grantRef);
|
|
2416
|
+
}
|
|
2417
|
+
catch (error) {
|
|
2418
|
+
// Round-7 minor: a throwing awaited step aborts the removal — the
|
|
2419
|
+
// callers rethrow before splicing the row or releasing the provider —
|
|
2420
|
+
// so the row is still live and still managed. Clear the mark or the
|
|
2421
|
+
// row would stay permanently retired and its rotation echoes would be
|
|
2422
|
+
// skipped until a restart. A retried removal re-marks before its own
|
|
2423
|
+
// first delete, and a successful removal never reaches this path.
|
|
2424
|
+
this.retiringAccounts.delete(row);
|
|
2425
|
+
throw error;
|
|
2426
|
+
}
|
|
2427
|
+
}
|
|
2428
|
+
/**
|
|
2429
|
+
* Drop one account: its plugin-scope grant payload, its row, and — when
|
|
2430
|
+
* it was the active projection — the projection itself, so the route
|
|
2431
|
+
* falls back to the key pool.
|
|
2432
|
+
*/
|
|
2433
|
+
removeAccount(provider, accountId) {
|
|
2434
|
+
return this.run(async () => {
|
|
2435
|
+
const providerRecord = this.store.provider(provider);
|
|
2436
|
+
if (providerRecord === undefined)
|
|
2437
|
+
throw new Error(`Provider "${provider}" is not managed`);
|
|
2438
|
+
const index = providerRecord.keys.findIndex(candidate => candidate.id === accountId);
|
|
2439
|
+
if (index < 0)
|
|
2440
|
+
throw new Error(`Account "${accountId}" not found on ${provider}`);
|
|
2441
|
+
const row = providerRecord.keys[index];
|
|
2442
|
+
if (row.kind !== 'oauth' || row.oauth === null)
|
|
2443
|
+
throw new Error(`Key "${accountId}" is not an OAuth account; use deleteKey`);
|
|
2444
|
+
await this.removeOAuthGrant(provider, row);
|
|
2445
|
+
providerRecord.keys.splice(index, 1);
|
|
2446
|
+
this.store.clearBindings((_key, binding) => binding.keyId === row.id);
|
|
2447
|
+
this.store.touch();
|
|
2448
|
+
return this.buildState();
|
|
2449
|
+
});
|
|
2450
|
+
}
|
|
319
2451
|
/** Install the broker layer on one credential provider instance. */
|
|
320
2452
|
attachCredentials(credentials) {
|
|
321
2453
|
const raw = credentials[ORIGINAL] ?? credentials;
|
|
@@ -542,11 +2674,15 @@ export class KeysManagerService {
|
|
|
542
2674
|
const profile = namespace === undefined ? undefined : asProfile(getPath(namespace.value, entry.settingsPath));
|
|
543
2675
|
const managed = this.store.provider(entry.provider);
|
|
544
2676
|
const extra = EXTRA_PROVIDER_MAP.get(entry.provider);
|
|
545
|
-
const
|
|
2677
|
+
const oauthFlow = this.oauthFlowAvailable(entry.provider);
|
|
2678
|
+
const caps = capabilityOf(entry, profile, oauthFlow);
|
|
2679
|
+
const support = { supported: caps.keys, ...(caps.reason === undefined ? {} : { reason: caps.reason }) };
|
|
2680
|
+
const accountSupport = accountSupportOf(entry, oauthFlow);
|
|
546
2681
|
const apiKeyEnv = profile !== undefined && typeof profile.apiKeyEnv === 'string' ? profile.apiKeyEnv : null;
|
|
547
2682
|
const binding = managed === undefined ? undefined : this.bindingState(managed, descriptor);
|
|
548
2683
|
const bindingHealthy = binding === 'healthy' ? true : binding === 'mismatch' ? false : undefined;
|
|
549
|
-
const
|
|
2684
|
+
const projectedAccountId = managed === undefined ? null : this.oauthOwnerFor(managed.provider);
|
|
2685
|
+
const keys = managed !== undefined ? managed.keys.map(key => publicKey(key, now, projectedAccountId)) : [];
|
|
550
2686
|
const warning = binding === 'mismatch'
|
|
551
2687
|
? 'The provider profile no longer points at Keys Manager. Re-adopt it after releasing it, or repair the provider in Models.'
|
|
552
2688
|
: undefined;
|
|
@@ -562,6 +2698,7 @@ export class KeysManagerService {
|
|
|
562
2698
|
active: active.has(entry.provider),
|
|
563
2699
|
supported: support.supported,
|
|
564
2700
|
...(support.reason === undefined ? {} : { reason: support.reason }),
|
|
2701
|
+
accountSupported: accountSupport.supported,
|
|
565
2702
|
settingsNs: entry.settingsNs,
|
|
566
2703
|
settingsPath: [...entry.settingsPath],
|
|
567
2704
|
apiKeyEnv,
|
|
@@ -588,7 +2725,8 @@ export class KeysManagerService {
|
|
|
588
2725
|
continue;
|
|
589
2726
|
const managed = this.store.provider(extra.id);
|
|
590
2727
|
const binding = managed === undefined ? undefined : this.bindingState(managed, descriptor);
|
|
591
|
-
const
|
|
2728
|
+
const projectedAccountId = managed === undefined ? null : this.oauthOwnerFor(managed.provider);
|
|
2729
|
+
const keys = managed !== undefined ? managed.keys.map(key => publicKey(key, now, projectedAccountId)) : [];
|
|
592
2730
|
views.push({
|
|
593
2731
|
provider: extra.id,
|
|
594
2732
|
displayName: extra.name,
|
|
@@ -599,6 +2737,8 @@ export class KeysManagerService {
|
|
|
599
2737
|
configured: managed !== undefined,
|
|
600
2738
|
active: false,
|
|
601
2739
|
supported: true,
|
|
2740
|
+
// Extra providers are key-pool routes; they offer no OAuth flow.
|
|
2741
|
+
accountSupported: false,
|
|
602
2742
|
settingsNs: 'llm-pi-ai',
|
|
603
2743
|
settingsPath: ['providers', extra.id],
|
|
604
2744
|
apiKeyEnv: managed?.brokerRef ?? null,
|
|
@@ -623,7 +2763,8 @@ export class KeysManagerService {
|
|
|
623
2763
|
for (const managed of Object.values(this.store.providers)) {
|
|
624
2764
|
if (seen.has(managed.provider))
|
|
625
2765
|
continue;
|
|
626
|
-
const
|
|
2766
|
+
const projectedAccountId = this.oauthOwnerFor(managed.provider);
|
|
2767
|
+
const keys = managed.keys.map(key => publicKey(key, now, projectedAccountId));
|
|
627
2768
|
const binding = this.bindingState(managed, descriptor);
|
|
628
2769
|
views.push({
|
|
629
2770
|
provider: managed.provider,
|
|
@@ -635,6 +2776,7 @@ export class KeysManagerService {
|
|
|
635
2776
|
configured: false,
|
|
636
2777
|
active: false,
|
|
637
2778
|
supported: true,
|
|
2779
|
+
accountSupported: false,
|
|
638
2780
|
settingsNs: managed.settingsNs,
|
|
639
2781
|
settingsPath: [...managed.settingsPath],
|
|
640
2782
|
apiKeyEnv: managed.brokerRef,
|
|
@@ -779,9 +2921,16 @@ export class KeysManagerService {
|
|
|
779
2921
|
if (namespace === undefined)
|
|
780
2922
|
throw new Error(`Settings namespace "${entry.settingsNs}" is not mounted`);
|
|
781
2923
|
const profile = asProfile(getPath(namespace.value, entry.settingsPath));
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
2924
|
+
// MF1: adoption is reachable through EITHER capability. An OAuth-only
|
|
2925
|
+
// route adopts with zero keys (account-only management); a key-only
|
|
2926
|
+
// route adopts as before; the key-pool table alone no longer blocks
|
|
2927
|
+
// an adoptable account route.
|
|
2928
|
+
const oauthFlow = this.oauthFlowAvailable(entry.provider);
|
|
2929
|
+
const caps = capabilityOf(entry, profile, oauthFlow);
|
|
2930
|
+
const accountSupport = accountSupportOf(entry, oauthFlow);
|
|
2931
|
+
if (!caps.keys && !accountSupport.supported) {
|
|
2932
|
+
throw new Error(caps.reason ?? accountSupport.reason ?? 'This provider cannot be managed');
|
|
2933
|
+
}
|
|
785
2934
|
if (entry.declared === true && profile === undefined) {
|
|
786
2935
|
throw new Error('Add this hand-declared provider in Models first, then adopt it here');
|
|
787
2936
|
}
|
|
@@ -977,6 +3126,16 @@ export class KeysManagerService {
|
|
|
977
3126
|
const record = this.store.provider(provider);
|
|
978
3127
|
if (record === undefined)
|
|
979
3128
|
throw new Error(`Provider "${provider}" is not managed`);
|
|
3129
|
+
// MF1: the key-pool restriction lives on the key entry path. A route
|
|
3130
|
+
// adopted for accounts only (no key-pool credential) refuses new keys.
|
|
3131
|
+
const entry = this.directoryEntry(provider);
|
|
3132
|
+
if (entry !== undefined) {
|
|
3133
|
+
const namespace = this.namespace(this.describeSettings(), entry.settingsNs);
|
|
3134
|
+
const profile = namespace === undefined ? undefined : asProfile(getPath(namespace.value, entry.settingsPath));
|
|
3135
|
+
const caps = capabilityOf(entry, profile, this.oauthFlowAvailable(entry.provider));
|
|
3136
|
+
if (!caps.keys)
|
|
3137
|
+
throw new Error(caps.reason ?? `Provider "${provider}" does not accept API keys`);
|
|
3138
|
+
}
|
|
980
3139
|
const input = validateKeyInput(alias, value);
|
|
981
3140
|
if (record.keys.length >= KEY_LIMIT)
|
|
982
3141
|
throw new Error(`A provider may have at most ${KEY_LIMIT} keys`);
|
|
@@ -1023,9 +3182,17 @@ export class KeysManagerService {
|
|
|
1023
3182
|
return this.buildState();
|
|
1024
3183
|
}
|
|
1025
3184
|
const key = record.keys[index];
|
|
1026
|
-
//
|
|
1027
|
-
//
|
|
1028
|
-
|
|
3185
|
+
// An OAuth account leaves the pool like removeAccount: drop its
|
|
3186
|
+
// plugin-scope payload and, when it was the projection, fall back to
|
|
3187
|
+
// the key pool. API keys remove their broker literal.
|
|
3188
|
+
if (key.kind === 'oauth' && key.oauth !== null) {
|
|
3189
|
+
await this.removeOAuthGrant(provider, key);
|
|
3190
|
+
}
|
|
3191
|
+
else {
|
|
3192
|
+
// Unset first: if the credential backend refuses, keep the metadata so a
|
|
3193
|
+
// later retry can remove both together.
|
|
3194
|
+
await this.unsetOriginal(key.ref);
|
|
3195
|
+
}
|
|
1029
3196
|
record.keys.splice(index, 1);
|
|
1030
3197
|
this.store.clearBindings((_key, binding) => binding.keyId === key.id);
|
|
1031
3198
|
this.store.touch();
|
|
@@ -1054,16 +3221,35 @@ export class KeysManagerService {
|
|
|
1054
3221
|
/** Remove every managed credential, recording progress before a retry. */
|
|
1055
3222
|
async deleteManagedCredentials(record) {
|
|
1056
3223
|
for (const key of record.keys) {
|
|
1057
|
-
|
|
3224
|
+
if (key.kind === 'oauth' && key.oauth !== null) {
|
|
3225
|
+
// Same cleanup as removeAccount/deleteKey: when this row owns the
|
|
3226
|
+
// activation projection, demote it first so the released route
|
|
3227
|
+
// resolves through the restored apiKeyEnv instead of a grant whose
|
|
3228
|
+
// manager is gone (pi-ai resolution prefers a stored credential).
|
|
3229
|
+
await this.removeOAuthGrant(record.provider, key);
|
|
3230
|
+
}
|
|
3231
|
+
else {
|
|
3232
|
+
await this.unsetOriginal(key.ref);
|
|
3233
|
+
}
|
|
1058
3234
|
key.valueMissing = true;
|
|
1059
3235
|
this.store.touch();
|
|
1060
3236
|
}
|
|
3237
|
+
this.activeAccounts.delete(record.provider);
|
|
1061
3238
|
}
|
|
1062
3239
|
/** Restore the provider profile and drop management; caller owns the queue. */
|
|
1063
3240
|
async releaseLocked(provider) {
|
|
1064
3241
|
const record = this.store.provider(provider);
|
|
1065
3242
|
if (record === undefined)
|
|
1066
3243
|
return;
|
|
3244
|
+
// Round-2 review step 4: releasing a provider revokes its in-flight
|
|
3245
|
+
// sign-in attempts. A login whose seam flow is aborted can no longer
|
|
3246
|
+
// harvest rows for a provider that is no longer managed (the harvest
|
|
3247
|
+
// itself also refuses settled attempts).
|
|
3248
|
+
for (const attempt of this.loginAttempts.values()) {
|
|
3249
|
+
if (attempt.provider !== provider || attempt.settledAt !== undefined)
|
|
3250
|
+
continue;
|
|
3251
|
+
this.failLoginAttempt(attempt, new Error(`Provider "${provider}" was released during sign-in`));
|
|
3252
|
+
}
|
|
1067
3253
|
const descriptor = this.describeSettings();
|
|
1068
3254
|
if (!descriptor.readable) {
|
|
1069
3255
|
throw new Error('Settings are currently unreadable; refusing to release a provider whose binding cannot be inspected.');
|
|
@@ -1312,6 +3498,20 @@ export class KeysManagerService {
|
|
|
1312
3498
|
// are cleared when their own request-error path terminates.
|
|
1313
3499
|
}
|
|
1314
3500
|
}
|
|
3501
|
+
/**
|
|
3502
|
+
* Codex P2: record a projection-served success on the owning account row —
|
|
3503
|
+
* `lastUsedAt`/`totalUses` move and the failure streak resets, symmetric to
|
|
3504
|
+
* the MF2 failure attribution.
|
|
3505
|
+
*/
|
|
3506
|
+
recordProjectionSuccess(providerId, accountId) {
|
|
3507
|
+
const provider = this.store.provider(providerId);
|
|
3508
|
+
if (provider === undefined)
|
|
3509
|
+
return;
|
|
3510
|
+
const row = provider.keys.find(candidate => candidate.id === accountId);
|
|
3511
|
+
if (row !== undefined)
|
|
3512
|
+
markKeySuccess(row, Date.now());
|
|
3513
|
+
this.store.touch();
|
|
3514
|
+
}
|
|
1315
3515
|
/** Mark one key cooling, preserving escalation across recovery windows. */
|
|
1316
3516
|
markFailure(providerId, keyId, failure) {
|
|
1317
3517
|
const provider = this.store.provider(providerId);
|
|
@@ -1321,12 +3521,96 @@ export class KeysManagerService {
|
|
|
1321
3521
|
if (key === undefined)
|
|
1322
3522
|
return;
|
|
1323
3523
|
const now = Date.now();
|
|
1324
|
-
|
|
3524
|
+
// R6-SF1/R7-SF1: the projection generation is read at failure time and
|
|
3525
|
+
// stamped on the row's window. It must be stamped on EVERY observed
|
|
3526
|
+
// failure, including one the dedupe below skips: a failure of a freshly
|
|
3527
|
+
// projected grant that lands inside an existing window still belongs to
|
|
3528
|
+
// the new projection, and leaving the old stamp would let a later
|
|
3529
|
+
// harvest clear a window the new projection earned.
|
|
3530
|
+
const generation = this.ownershipGenerations.get(providerId) ?? 0;
|
|
3531
|
+
// D1/D2 pool arbitration: when the OAuth grant itself failed (the pinned
|
|
3532
|
+
// pi-ai `ModelsError("oauth", …)` code), the account cools down AND the
|
|
3533
|
+
// cascade demotes to the key pool by deleting the projection record.
|
|
3534
|
+
const oauthDemotion = key.kind === 'oauth' && classifyOAuthFailure(failure) === 'oauth';
|
|
3535
|
+
if (key.cooldownUntil > now && now - key.lastFailureAt < 10_000) {
|
|
3536
|
+
// Dedupe skips only the cooldown EXTENSION; the attribution stamp
|
|
3537
|
+
// always follows the live projection.
|
|
3538
|
+
this.cooldownGenerations.set(key.id, generation);
|
|
3539
|
+
// R14-M1: the demotion is not part of the extension this dedupe
|
|
3540
|
+
// suppresses. The first attempt may have thrown on its delete (or a
|
|
3541
|
+
// native write re-projected the grant inside the window), leaving the
|
|
3542
|
+
// route on a grant whose row already cooled while C1's await pointed
|
|
3543
|
+
// at that settled attempt. Re-running the idempotent demotion repairs
|
|
3544
|
+
// exactly that; skipping it made C1's awaited demotion pointless for
|
|
3545
|
+
// every deduped failure.
|
|
3546
|
+
if (oauthDemotion)
|
|
3547
|
+
this.demoteProjection(providerId, key.id);
|
|
1325
3548
|
return;
|
|
3549
|
+
}
|
|
1326
3550
|
const result = applyFailureCooldown(key, this.store.settings, now, failure);
|
|
3551
|
+
// R6-SF1: stamp which projection earned this window. The generation is
|
|
3552
|
+
// read at failure time — never derived later from a cooldown timestamp —
|
|
3553
|
+
// so a harvest can clear only a window an older projection earned.
|
|
3554
|
+
this.cooldownGenerations.set(key.id, generation);
|
|
3555
|
+
if (oauthDemotion) {
|
|
3556
|
+
this.warn(`oauth account ${key.id} on ${providerId} failed (${failure.code}); demoting to the key pool for ${Math.round(result.cooldownMs / 1000)}s`);
|
|
3557
|
+
// Persist the applied cooldown on this path too: the deferred
|
|
3558
|
+
// fallbackToKeys touch never runs when its deleteRecord throws, and
|
|
3559
|
+
// this touch is what schedules the debounced write regardless.
|
|
3560
|
+
this.store.touch();
|
|
3561
|
+
// Round-5 finding C1: publish the demotion so the request-error handler
|
|
3562
|
+
// can wait for the projection to actually be gone before its retry
|
|
3563
|
+
// re-enters the stream (otherwise pi-ai serves the stored grant again).
|
|
3564
|
+
// R7-MF: carry THIS row's identity through the queued demotion. The
|
|
3565
|
+
// failure belongs to the account that served the request, not to
|
|
3566
|
+
// whichever account is active when the queue drains — a "Set active"
|
|
3567
|
+
// landing in between must keep its healthy projection.
|
|
3568
|
+
this.demoteProjection(providerId, key.id);
|
|
3569
|
+
return;
|
|
3570
|
+
}
|
|
1327
3571
|
this.warn(`key ${key.id} on ${providerId} cooled for ${Math.round(result.cooldownMs / 1000)}s`);
|
|
1328
3572
|
this.store.touch();
|
|
1329
3573
|
}
|
|
3574
|
+
/**
|
|
3575
|
+
* Queue one projection demotion for a failed account and publish its
|
|
3576
|
+
* promise so the request-error handler can wait for the projection to be
|
|
3577
|
+
* gone (round-5 finding C1). `keyId` is the failing row (R7-MF), so the
|
|
3578
|
+
* queued body deletes only a projection that still belongs to it. R14-M1:
|
|
3579
|
+
* both the cooldown-extension branch and the dedupe branch start the
|
|
3580
|
+
* demotion; it is idempotent when the projection is already gone.
|
|
3581
|
+
*/
|
|
3582
|
+
demoteProjection(providerId, keyId) {
|
|
3583
|
+
const demotion = this.fallbackToKeys(providerId, keyId).catch(error => {
|
|
3584
|
+
this.warn(`key-pool fallback failed for ${providerId}: ${error instanceof Error ? error.message : String(error)}`);
|
|
3585
|
+
});
|
|
3586
|
+
this.projectionDemotions.set(providerId, demotion);
|
|
3587
|
+
void demotion.then(() => {
|
|
3588
|
+
if (this.projectionDemotions.get(providerId) === demotion)
|
|
3589
|
+
this.projectionDemotions.delete(providerId);
|
|
3590
|
+
});
|
|
3591
|
+
}
|
|
3592
|
+
/**
|
|
3593
|
+
* Wait for any in-flight projection demotion on `providerId` (round-5
|
|
3594
|
+
* finding C1). Resolves immediately when nothing is pending, and always
|
|
3595
|
+
* resolves — the recorded promise already absorbed its own error.
|
|
3596
|
+
*/
|
|
3597
|
+
settleProjectionDemotion(providerId) {
|
|
3598
|
+
const pending = this.projectionDemotions.get(providerId);
|
|
3599
|
+
return pending === undefined ? Promise.resolve() : pending;
|
|
3600
|
+
}
|
|
3601
|
+
/**
|
|
3602
|
+
* Whether the key pool can serve a request right now (round-5 finding C1):
|
|
3603
|
+
* a selectable non-account row other than `excludeKeyId`. Account rows are
|
|
3604
|
+
* excluded because their reference is not broker-resolvable — only a
|
|
3605
|
+
* projected grant serves them.
|
|
3606
|
+
*/
|
|
3607
|
+
hasFailoverCandidate(providerId, excludeKeyId) {
|
|
3608
|
+
const provider = this.store.provider(providerId);
|
|
3609
|
+
if (provider === undefined)
|
|
3610
|
+
return false;
|
|
3611
|
+
const now = Date.now();
|
|
3612
|
+
return provider.keys.some(key => key.id !== excludeKeyId && key.kind !== 'oauth' && isSelectable(key, now));
|
|
3613
|
+
}
|
|
1330
3614
|
/** Prune stale attempt records. */
|
|
1331
3615
|
pruneAttempts(now) {
|
|
1332
3616
|
if (this.attempts.size >= 128) {
|
|
@@ -1405,6 +3689,18 @@ export class KeysManagerService {
|
|
|
1405
3689
|
}
|
|
1406
3690
|
return this.wrapStream(options, next, providerId, sessionId, step);
|
|
1407
3691
|
}
|
|
3692
|
+
/**
|
|
3693
|
+
* The account row whose grant payload currently IS the live projection,
|
|
3694
|
+
* verified against the provider's rows (MF2/MF3 read side). A stale map
|
|
3695
|
+
* entry (row removed) resolves to null.
|
|
3696
|
+
*/
|
|
3697
|
+
oauthOwnerFor(provider) {
|
|
3698
|
+
const accountId = this.activeAccounts.get(provider);
|
|
3699
|
+
if (accountId === undefined)
|
|
3700
|
+
return null;
|
|
3701
|
+
const row = this.store.provider(provider)?.keys.find(candidate => candidate.id === accountId);
|
|
3702
|
+
return row !== undefined && row.kind === 'oauth' && row.oauth !== null ? row.id : null;
|
|
3703
|
+
}
|
|
1408
3704
|
/** Wrap one managed stream so credential resolution sees this key. */
|
|
1409
3705
|
wrapStream(_options, next, providerId, sessionId, step) {
|
|
1410
3706
|
const service = this;
|
|
@@ -1418,12 +3714,43 @@ export class KeysManagerService {
|
|
|
1418
3714
|
async next() {
|
|
1419
3715
|
if (!started) {
|
|
1420
3716
|
started = true;
|
|
3717
|
+
// Round-8 finding I-000108: a restarted process serves the
|
|
3718
|
+
// persisted projection before the ownership hydration settles.
|
|
3719
|
+
// Attribution must wait for it — otherwise this request selects
|
|
3720
|
+
// and mutates a key cursor even though pi-ai serves the stored
|
|
3721
|
+
// grant, and an OAuth failure is classified as externally
|
|
3722
|
+
// authenticated. Round-9 should-fix: the wait is bounded and is
|
|
3723
|
+
// skipped entirely for providers that hold no OAuth rows, so a
|
|
3724
|
+
// stalled credential read can neither wedge this request nor
|
|
3725
|
+
// delay an API-key-only provider.
|
|
3726
|
+
const hydrated = await service.awaitProjectionOwnership(providerId);
|
|
3727
|
+
// Round-5 finding C4: while a projection exists, pi-ai serves
|
|
3728
|
+
// the stored grant and never consults the broker, so selecting
|
|
3729
|
+
// a key here would only mutate unused key state (cursor,
|
|
3730
|
+
// lastUsedAt, sticky binding) on every OAuth-backed request.
|
|
3731
|
+
// Skip selection entirely; the request is attributed to the
|
|
3732
|
+
// serving account row below. The retry after a demotion runs
|
|
3733
|
+
// with no projection, so it selects from the key pool.
|
|
3734
|
+
//
|
|
3735
|
+
// When the bounded wait elapsed, attribution is unknown: no
|
|
3736
|
+
// owner, and no key selection either, because a projection may
|
|
3737
|
+
// still be serving (the hydration result lands later under its
|
|
3738
|
+
// generation guard).
|
|
3739
|
+
const oauthAccountId = hydrated ? service.oauthOwnerFor(providerId) : null;
|
|
1421
3740
|
let selection = { key: undefined, value: undefined };
|
|
1422
|
-
|
|
1423
|
-
|
|
3741
|
+
if (hydrated && oauthAccountId === null) {
|
|
3742
|
+
try {
|
|
3743
|
+
selection = await service.selectKey(providerId, sessionId, step);
|
|
3744
|
+
}
|
|
3745
|
+
catch (error) {
|
|
3746
|
+
service.warn(`key selection failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
3747
|
+
}
|
|
1424
3748
|
}
|
|
1425
|
-
|
|
1426
|
-
|
|
3749
|
+
else if (hydrated && sessionId !== undefined && step !== undefined) {
|
|
3750
|
+
// A projection-served attempt cannot consume a step retry pin;
|
|
3751
|
+
// dropping it here keeps a stale pin from leaking into a later
|
|
3752
|
+
// attempt of the same step.
|
|
3753
|
+
service.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
1427
3754
|
}
|
|
1428
3755
|
keyId = selection.key?.id;
|
|
1429
3756
|
store = {
|
|
@@ -1431,8 +3758,12 @@ export class KeysManagerService {
|
|
|
1431
3758
|
keyId: keyId ?? null,
|
|
1432
3759
|
keyValue: selection.value ?? null,
|
|
1433
3760
|
// Only the broker resolver proves that this request actually
|
|
1434
|
-
// used a managed credential.
|
|
3761
|
+
// used a managed key credential.
|
|
1435
3762
|
keyUsed: false,
|
|
3763
|
+
// Captured where the broker resolve path is set up (MF2):
|
|
3764
|
+
// when a projection exists, pi-ai serves the stored grant
|
|
3765
|
+
// and this scope owns the failure attribution.
|
|
3766
|
+
oauthAccountId,
|
|
1436
3767
|
};
|
|
1437
3768
|
if (sessionId !== undefined && keyId !== undefined) {
|
|
1438
3769
|
const key = bindingKey(sessionId, providerId);
|
|
@@ -1461,19 +3792,45 @@ export class KeysManagerService {
|
|
|
1461
3792
|
if (chunk.type === 'finish' && chunk.reason !== undefined) {
|
|
1462
3793
|
const reason = chunk.reason;
|
|
1463
3794
|
const used = store?.keyUsed === true;
|
|
3795
|
+
// Captured where the broker resolve path is set up: when a
|
|
3796
|
+
// projection exists, pi-ai serves the stored grant, so both
|
|
3797
|
+
// terminal kinds attribute to the owning account row.
|
|
3798
|
+
const oauthAccountId = store?.oauthAccountId ?? null;
|
|
1464
3799
|
if (reason.kind === 'error') {
|
|
1465
|
-
|
|
1466
|
-
|
|
1467
|
-
|
|
1468
|
-
|
|
1469
|
-
|
|
3800
|
+
// MF2 attribution: with an active projection, pi-ai served the
|
|
3801
|
+
// stored grant, so an oauth-coded failure belongs to the
|
|
3802
|
+
// account row — cool it and demote the projection exactly as
|
|
3803
|
+
// markFailure's oauth branch does, instead of delegating with
|
|
3804
|
+
// no recovery evidence.
|
|
3805
|
+
const failure = reason.failure;
|
|
3806
|
+
if (oauthAccountId !== null && failure !== undefined && classifyOAuthFailure(failure) === 'oauth') {
|
|
3807
|
+
service.recordFailure(providerId, sessionId, oauthAccountId, failure, true);
|
|
3808
|
+
service.markFailure(providerId, oauthAccountId, failure);
|
|
3809
|
+
// Round-5 finding C1: the demotion alone only helps a LATER
|
|
3810
|
+
// request. Recording request-scoped evidence in the same
|
|
3811
|
+
// linked form the key paths produce lets the manager's retry
|
|
3812
|
+
// policy fail over inside THIS request. It is recorded only
|
|
3813
|
+
// when the pool holds a candidate: with none, neither the
|
|
3814
|
+
// demoted account nor a key can serve the retry, and a retry
|
|
3815
|
+
// would be a phantom loop — delegation stays the answer.
|
|
3816
|
+
if (service.hasFailoverCandidate(providerId, oauthAccountId)) {
|
|
3817
|
+
service.recordStreamAttempt(providerId, sessionId, oauthAccountId, true, failure);
|
|
3818
|
+
}
|
|
1470
3819
|
}
|
|
1471
|
-
else
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1475
|
-
if (
|
|
1476
|
-
service.
|
|
3820
|
+
else {
|
|
3821
|
+
if (reason.failure !== undefined && shouldFailover(reason.failure.code) && service.store.settings.autoSwitch) {
|
|
3822
|
+
service.recordStreamAttempt(providerId, sessionId, keyId, used, reason.failure);
|
|
3823
|
+
}
|
|
3824
|
+
if (used && reason.failure !== undefined) {
|
|
3825
|
+
service.recordFailure(providerId, sessionId, keyId, reason.failure, true);
|
|
3826
|
+
}
|
|
3827
|
+
else if (sessionId !== undefined) {
|
|
3828
|
+
// The request failed on an external credential; do not
|
|
3829
|
+
// attribute it to a managed key.
|
|
3830
|
+
service.attempts.delete(bindingKey(sessionId, providerId));
|
|
3831
|
+
if (step !== undefined)
|
|
3832
|
+
service.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3833
|
+
}
|
|
1477
3834
|
}
|
|
1478
3835
|
service.pruneAttempts(Date.now());
|
|
1479
3836
|
}
|
|
@@ -1484,6 +3841,18 @@ export class KeysManagerService {
|
|
|
1484
3841
|
else if (used) {
|
|
1485
3842
|
service.recordSuccess(providerId, sessionId, keyId, step);
|
|
1486
3843
|
}
|
|
3844
|
+
else if (oauthAccountId !== null) {
|
|
3845
|
+
// Codex P2: symmetric to the failure attribution — a request
|
|
3846
|
+
// served by the projected grant succeeded on the owning
|
|
3847
|
+
// account row, so its counters and failure streak update
|
|
3848
|
+
// there, not on a broker key.
|
|
3849
|
+
service.recordProjectionSuccess(providerId, oauthAccountId);
|
|
3850
|
+
if (sessionId !== undefined) {
|
|
3851
|
+
service.attempts.delete(bindingKey(sessionId, providerId));
|
|
3852
|
+
if (step !== undefined)
|
|
3853
|
+
service.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3854
|
+
}
|
|
3855
|
+
}
|
|
1487
3856
|
else if (sessionId !== undefined) {
|
|
1488
3857
|
service.attempts.delete(bindingKey(sessionId, providerId));
|
|
1489
3858
|
if (step !== undefined)
|
|
@@ -1625,6 +3994,12 @@ export class KeysManagerService {
|
|
|
1625
3994
|
}
|
|
1626
3995
|
if (sessionId === undefined)
|
|
1627
3996
|
return next();
|
|
3997
|
+
// Round-5 finding C1: an OAuth-grant failure cools the account and starts
|
|
3998
|
+
// an asynchronous projection demotion. The retry must not re-enter the
|
|
3999
|
+
// stream while that demotion is pending — pi-ai would serve the stored
|
|
4000
|
+
// grant again and the key-pool failover would never happen. Awaiting here
|
|
4001
|
+
// is what makes the retry attempt land on the pool within this request.
|
|
4002
|
+
await this.settleProjectionDemotion(providerId);
|
|
1628
4003
|
const now = Date.now();
|
|
1629
4004
|
const attemptKey = bindingKey(sessionId, providerId);
|
|
1630
4005
|
const switchKey = `${attemptKey}\u0000${payload.turn}\u0000${payload.step}`;
|