dsh-keys-manager 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +33 -0
- package/README.md +108 -15
- package/README.zh-CN.md +80 -2
- package/bundle/cordis.patch.yml +5 -2
- package/lib/client/types.d.ts +27 -2
- package/lib/client.js +320 -3
- package/lib/domain.d.ts +55 -0
- package/lib/host-types.d.ts +29 -0
- package/lib/index.d.ts +865 -23
- package/lib/index.js +3452 -188
- package/lib/index.js.map +1 -1
- package/lib/providers.js +426 -0
- package/lib/providers.js.map +1 -1
- package/lib/store.js +4 -0
- package/lib/store.js.map +1 -1
- package/locale/en.json +17 -1
- package/locale/zh.json +17 -1
- package/package.json +14 -12
- package/scripts/prepare-release.mjs +161 -74
- package/scripts/verify-install.mjs +156 -0
package/lib/index.js
CHANGED
|
@@ -25,6 +25,13 @@ const SERVICE = 'keysManager';
|
|
|
25
25
|
const NAMESPACE = 'keysManager';
|
|
26
26
|
const BROKER_PREFIX = 'KEYSMGR_';
|
|
27
27
|
const ORIGINAL = Symbol.for('cordis.original');
|
|
28
|
+
/**
|
|
29
|
+
* The capability `close()`'s pre-close restoration writes under (PR review
|
|
30
|
+
* round 36). A module-private token, not a boolean a caller can pass: no
|
|
31
|
+
* cleanup, watchdog or late reconcile can hold it, so the restoration is the
|
|
32
|
+
* only writer allowed while `shutdownRequested` is up.
|
|
33
|
+
*/
|
|
34
|
+
const PRE_CLOSE_RESTORE = Symbol('keys-manager.pre-close restore');
|
|
28
35
|
const MAX_SWITCHES = 6;
|
|
29
36
|
const ATTEMPT_TTL_MS = 60 * 60 * 1000;
|
|
30
37
|
/** A retry pin only lives for this long; an expired pin is treated as absent. */
|
|
@@ -53,6 +60,15 @@ const MAX_ALIAS = 80;
|
|
|
53
60
|
const INTENTIONAL_REMOVAL_TTL_MS = 5_000;
|
|
54
61
|
/** The credential-record scope pi-ai stores its per-provider grants under (llm-pi-ai/src/auth.ts:29). */
|
|
55
62
|
export const OAUTH_GRANT_RECORD_SCOPE = 'llm-pi-ai';
|
|
63
|
+
/**
|
|
64
|
+
* The credential-record scope the llm-omp transports package owns: one
|
|
65
|
+
* `llm-omp-transports/<route>` grant per transport route (its
|
|
66
|
+
* `CURSOR_CREDENTIAL_KEY` is `llm-omp-transports/cursor`). Keys Manager reads
|
|
67
|
+
* those records to list/verify an account and deletes one on removal; it
|
|
68
|
+
* never writes one and never projects one into `llm-pi-ai/<provider>`, so a
|
|
69
|
+
* transport route keeps serving from its own adapter-owned credential.
|
|
70
|
+
*/
|
|
71
|
+
export const TRANSPORT_CREDENTIAL_SCOPE = 'llm-omp-transports';
|
|
56
72
|
/**
|
|
57
73
|
* Upper bound on one stream's wait for projection-ownership hydration
|
|
58
74
|
* (round-9 should-fix). Hydration reads one `readRecord` per OAuth provider
|
|
@@ -65,6 +81,67 @@ export const OAUTH_GRANT_RECORD_SCOPE = 'llm-pi-ai';
|
|
|
65
81
|
* generation guard in {@link KeysManagerService.hydrateProjectionOwnership}.
|
|
66
82
|
*/
|
|
67
83
|
const HYDRATION_WAIT_MS = 500;
|
|
84
|
+
/** Upper bound on one per-row connection test: a stalled provider must not hold the row busy forever. */
|
|
85
|
+
const KEY_TEST_TIMEOUT_MS = 20_000;
|
|
86
|
+
/** The test timeout as user-visible copy; derived so the constant stays the single source of truth. */
|
|
87
|
+
const KEY_TEST_TIMEOUT_LABEL = `${KEY_TEST_TIMEOUT_MS / 1000}s`;
|
|
88
|
+
/**
|
|
89
|
+
* Hard bound on how long an ordinary request waits for an in-flight row test
|
|
90
|
+
* (QC B3). The test's own request timeout covers only `ctx.llm.stream`; its
|
|
91
|
+
* credential-store reads offer no timeout or cancellation, so without a bound
|
|
92
|
+
* of its own one stalled seam read parks every ordinary request for the
|
|
93
|
+
* provider indefinitely. The small margin keeps the wait from cutting short a
|
|
94
|
+
* test whose provider call used its whole budget.
|
|
95
|
+
*/
|
|
96
|
+
const KEY_TEST_ISOLATION_WAIT_MS = KEY_TEST_TIMEOUT_MS + 5_000;
|
|
97
|
+
/**
|
|
98
|
+
* Hard bound on one abandoned test window's cleanup (QC fix wave 2). The
|
|
99
|
+
* borrowed projection must be put back before ordinary traffic may resume on
|
|
100
|
+
* the provider, but the cleanup runs through the same credential seam the
|
|
101
|
+
* wedged test is parked in, so it cannot be trusted to return. Past this bound
|
|
102
|
+
* the provider stays isolated and fenced instead of being released on an
|
|
103
|
+
* unreconciled credential surface.
|
|
104
|
+
*/
|
|
105
|
+
const KEY_TEST_CLEANUP_DEADLINE_MS = 2_000;
|
|
106
|
+
/**
|
|
107
|
+
* Hard bound on the shutdown drain of the per-provider grant chains (PR review
|
|
108
|
+
* round 29). A chain body carries the row test's own provider request, and an
|
|
109
|
+
* adapter that ignores the abort signal keeps that request pending after the
|
|
110
|
+
* watchdog already restored the borrowed credential, so the chain may never
|
|
111
|
+
* settle. Past this bound the pending chain is abandoned: process teardown does
|
|
112
|
+
* not need it, and shutdown must not hang on it.
|
|
113
|
+
*/
|
|
114
|
+
const GRANT_DRAIN_DEADLINE_MS = 2_000;
|
|
115
|
+
/**
|
|
116
|
+
* How many times one binding decision is re-derived after its write settled on
|
|
117
|
+
* a serving source that has moved again (PR review round 32). The write itself
|
|
118
|
+
* is asynchronous, so each settled profile can already describe a source a
|
|
119
|
+
* newer classification replaced; a small bound chases that without letting a
|
|
120
|
+
* genuinely flapping route spawn an unbounded write chain.
|
|
121
|
+
*/
|
|
122
|
+
const BINDING_CORRECTION_LIMIT = 3;
|
|
123
|
+
/** The error-like `name` of an unknown thrown value, if it carries one. */
|
|
124
|
+
function errorNameOf(value) {
|
|
125
|
+
if (typeof value !== 'object' || value === null)
|
|
126
|
+
return undefined;
|
|
127
|
+
const name = value.name;
|
|
128
|
+
return typeof name === 'string' ? name : undefined;
|
|
129
|
+
}
|
|
130
|
+
/**
|
|
131
|
+
* Whether the 20 s test timeout caused the caught failure. The abort signal
|
|
132
|
+
* is a whole-request fact, so an error that merely races the timer keeps its
|
|
133
|
+
* own message (M1): only the timeout's own reason, or the abort the timeout
|
|
134
|
+
* caused, maps to the timeout copy.
|
|
135
|
+
*/
|
|
136
|
+
function isTestTimeoutAbort(error, signal) {
|
|
137
|
+
if (!signal.aborted)
|
|
138
|
+
return false;
|
|
139
|
+
if (error === signal.reason)
|
|
140
|
+
return true;
|
|
141
|
+
if (errorNameOf(signal.reason) !== 'TimeoutError')
|
|
142
|
+
return false;
|
|
143
|
+
return errorNameOf(error) === 'AbortError';
|
|
144
|
+
}
|
|
68
145
|
/**
|
|
69
146
|
* Stable failure code for a harvest whose state flush rejected AND whose
|
|
70
147
|
* credential-side rollback then failed to write.
|
|
@@ -168,6 +245,16 @@ const INVOCATIONS = [
|
|
|
168
245
|
endpoint('listAccountAttempts'),
|
|
169
246
|
endpoint('activateAccount', ['provider', 'accountId']),
|
|
170
247
|
endpoint('removeAccount', ['provider', 'accountId']),
|
|
248
|
+
// One minimal request per key/account row (observation-only: no health,
|
|
249
|
+
// cursor or binding writes) so a user can tell a bad credential from a bad
|
|
250
|
+
// route without waiting for a chat failure.
|
|
251
|
+
endpoint('testKey', ['provider', 'keyId']),
|
|
252
|
+
// Transport accounts (llm-omp-transports/<route>): managed beside the pi-ai
|
|
253
|
+
// pool, never through it. The sign-in reuses the same attempt machinery and
|
|
254
|
+
// is polled through `accountAttempt` above.
|
|
255
|
+
endpoint('beginTransportLogin', ['provider']),
|
|
256
|
+
endpoint('activateTransportAccount', ['provider']),
|
|
257
|
+
endpoint('removeTransportAccount', ['provider']),
|
|
171
258
|
];
|
|
172
259
|
/** Read a nested value by path. */
|
|
173
260
|
function getPath(value, path) {
|
|
@@ -251,6 +338,89 @@ function discoveredModel(model) {
|
|
|
251
338
|
inputModalities: model.inputModalities,
|
|
252
339
|
});
|
|
253
340
|
}
|
|
341
|
+
/** Credential-bearing header names the carrier must never persist as plain settings. */
|
|
342
|
+
const CREDENTIAL_HEADER_NAMES = new Set(['authorization', 'proxy-authorization', 'cookie', 'api-key', 'apikey', 'x-api-key']);
|
|
343
|
+
/** Carrier headers are non-secret metadata; a credential-bearing name is refused. */
|
|
344
|
+
function isCredentialHeader(header) {
|
|
345
|
+
const name = header.toLowerCase();
|
|
346
|
+
return CREDENTIAL_HEADER_NAMES.has(name) || name.endsWith('-api-key');
|
|
347
|
+
}
|
|
348
|
+
/**
|
|
349
|
+
* Resolve one extra provider's declared static request headers from the
|
|
350
|
+
* environment. Every value must be non-blank and legal for Fetch; a missing or
|
|
351
|
+
* unusable value refuses the add before any credential or settings write,
|
|
352
|
+
* because a blank header would only surface later as an opaque
|
|
353
|
+
* profile-resolution error.
|
|
354
|
+
*/
|
|
355
|
+
function resolveHeaderEnv(extra) {
|
|
356
|
+
if (extra.headerEnv === undefined)
|
|
357
|
+
return undefined;
|
|
358
|
+
const headers = {};
|
|
359
|
+
for (const [header, envName] of Object.entries(extra.headerEnv)) {
|
|
360
|
+
if (isCredentialHeader(header)) {
|
|
361
|
+
throw new Error(`${extra.id} declares the credential header ${header}; only non-secret metadata headers can be carried`);
|
|
362
|
+
}
|
|
363
|
+
const value = process.env[envName]?.trim();
|
|
364
|
+
if (value === undefined || value.length === 0) {
|
|
365
|
+
throw new Error(`${extra.id} requires the ${envName} environment variable (${header} header)`);
|
|
366
|
+
}
|
|
367
|
+
// The settings seam materializes the profile through Fetch, which rejects
|
|
368
|
+
// names and values a header cannot carry; refuse them by name here.
|
|
369
|
+
try {
|
|
370
|
+
new Headers([[header, value]]);
|
|
371
|
+
}
|
|
372
|
+
catch {
|
|
373
|
+
throw new Error(`${extra.id} cannot carry the ${envName} value as the ${header} header: it is not a valid Fetch header value`);
|
|
374
|
+
}
|
|
375
|
+
headers[header] = value;
|
|
376
|
+
}
|
|
377
|
+
return headers;
|
|
378
|
+
}
|
|
379
|
+
/** Read a stored profile header map, keeping only the string values a header can carry. */
|
|
380
|
+
function staticHeaders(value) {
|
|
381
|
+
const headers = {};
|
|
382
|
+
if (value === null || typeof value !== 'object')
|
|
383
|
+
return headers;
|
|
384
|
+
for (const [header, entry] of Object.entries(value)) {
|
|
385
|
+
if (typeof entry === 'string')
|
|
386
|
+
headers[header] = entry;
|
|
387
|
+
}
|
|
388
|
+
return headers;
|
|
389
|
+
}
|
|
390
|
+
/** Stored header key matching a declared name; Fetch field names are case-insensitive. */
|
|
391
|
+
function storedHeaderKey(stored, header) {
|
|
392
|
+
const wanted = header.toLowerCase();
|
|
393
|
+
for (const key of Object.keys(stored)) {
|
|
394
|
+
if (key.toLowerCase() === wanted)
|
|
395
|
+
return key;
|
|
396
|
+
}
|
|
397
|
+
return undefined;
|
|
398
|
+
}
|
|
399
|
+
/** Resolve the declared header values the environment currently supplies. */
|
|
400
|
+
function environmentHeaders(extra) {
|
|
401
|
+
const headers = {};
|
|
402
|
+
if (extra?.headerEnv === undefined)
|
|
403
|
+
return headers;
|
|
404
|
+
for (const [header, envName] of Object.entries(extra.headerEnv)) {
|
|
405
|
+
const value = process.env[envName]?.trim();
|
|
406
|
+
if (value !== undefined && value.length > 0)
|
|
407
|
+
headers[header] = value;
|
|
408
|
+
}
|
|
409
|
+
return headers;
|
|
410
|
+
}
|
|
411
|
+
/** Declared headers the stored map does not carry yet; a stored value always wins. */
|
|
412
|
+
function missingHeaders(declared, stored) {
|
|
413
|
+
const missing = {};
|
|
414
|
+
if (declared === undefined)
|
|
415
|
+
return missing;
|
|
416
|
+
for (const [header, value] of Object.entries(declared)) {
|
|
417
|
+
// A stored variant in any case counts as the same Fetch field, so no
|
|
418
|
+
// second case variant is ever added.
|
|
419
|
+
if (storedHeaderKey(stored, header) === undefined)
|
|
420
|
+
missing[header] = value;
|
|
421
|
+
}
|
|
422
|
+
return missing;
|
|
423
|
+
}
|
|
254
424
|
/** Public, secret-free key row. */
|
|
255
425
|
function publicKey(key, now, projectedAccountId) {
|
|
256
426
|
return {
|
|
@@ -285,6 +455,68 @@ function publicKey(key, now, projectedAccountId) {
|
|
|
285
455
|
: {}),
|
|
286
456
|
};
|
|
287
457
|
}
|
|
458
|
+
/**
|
|
459
|
+
* The usable grant of one transport record, or undefined when the record is
|
|
460
|
+
* absent or carries a payload this manager cannot read. The payload belongs
|
|
461
|
+
* to the transports package, so every field is checked structurally.
|
|
462
|
+
*/
|
|
463
|
+
function transportGrantOf(record) {
|
|
464
|
+
if (record === undefined || record.kind !== 'grant')
|
|
465
|
+
return undefined;
|
|
466
|
+
const payload = record.payload;
|
|
467
|
+
if (payload === null || typeof payload !== 'object' || Array.isArray(payload))
|
|
468
|
+
return undefined;
|
|
469
|
+
const source = payload;
|
|
470
|
+
if (typeof source.access !== 'string' || source.access.length === 0)
|
|
471
|
+
return undefined;
|
|
472
|
+
if (typeof source.expires !== 'number' || !Number.isFinite(source.expires))
|
|
473
|
+
return undefined;
|
|
474
|
+
return {
|
|
475
|
+
access: source.access,
|
|
476
|
+
expires: source.expires,
|
|
477
|
+
...(typeof source.refresh === 'string' && source.refresh.length > 0 ? { refresh: source.refresh } : {}),
|
|
478
|
+
...(typeof source.email === 'string' && source.email.length > 0 ? { email: source.email } : {}),
|
|
479
|
+
...(typeof source.accountId === 'string' && source.accountId.length > 0 ? { accountId: source.accountId } : {}),
|
|
480
|
+
};
|
|
481
|
+
}
|
|
482
|
+
/**
|
|
483
|
+
* One transport account row from its route flow and stored record.
|
|
484
|
+
*
|
|
485
|
+
* A transport route keeps exactly ONE grant record, and the route's adapter
|
|
486
|
+
* resolves that record per request — so a usable grant IS the route's live
|
|
487
|
+
* credential. There is no projection to select between: `active` only
|
|
488
|
+
* restates that, and activation can never write a pi-ai record. The row
|
|
489
|
+
* exists while signed out, so the UI can offer the sign-in that writes it.
|
|
490
|
+
*/
|
|
491
|
+
function transportAccountOf(flow, record, now) {
|
|
492
|
+
const grant = transportGrantOf(record);
|
|
493
|
+
const status = grant !== undefined
|
|
494
|
+
? grant.expires > now
|
|
495
|
+
? 'active'
|
|
496
|
+
: grant.refresh === undefined ? 'needs-reauth' : 'expired'
|
|
497
|
+
: record === undefined ? 'signed-out' : 'needs-reauth';
|
|
498
|
+
return {
|
|
499
|
+
id: flow.key.slice(TRANSPORT_CREDENTIAL_SCOPE.length + 1),
|
|
500
|
+
scope: flow.key,
|
|
501
|
+
displayName: flow.label,
|
|
502
|
+
...(grant?.email === undefined ? {} : { email: grant.email }),
|
|
503
|
+
...(grant?.accountId === undefined ? {} : { accountId: grant.accountId }),
|
|
504
|
+
...(grant?.expires === undefined ? {} : { expiresAt: grant.expires }),
|
|
505
|
+
status,
|
|
506
|
+
// Derived, never a second opinion: an expired or unreadable grant is not
|
|
507
|
+
// the route's live credential.
|
|
508
|
+
active: status === 'active',
|
|
509
|
+
};
|
|
510
|
+
}
|
|
511
|
+
/**
|
|
512
|
+
* The method id a transport flow begins with, or undefined when it has none.
|
|
513
|
+
*
|
|
514
|
+
* The transports package owns its flow's method id; this manager only reads
|
|
515
|
+
* the scope prefix, so it must not assume the pi-ai `'oauth'` id.
|
|
516
|
+
*/
|
|
517
|
+
function transportMethodOf(flow) {
|
|
518
|
+
return flow.methods[0]?.id;
|
|
519
|
+
}
|
|
288
520
|
/** Env names that are a single API-key credential, not an OAuth/federation token. */
|
|
289
521
|
function isKeyishEnv(name) {
|
|
290
522
|
if (!/(?:API_KEY|_TOKEN)$/.test(name))
|
|
@@ -427,17 +659,147 @@ export class KeysManagerService {
|
|
|
427
659
|
credentialWrappers = new WeakMap();
|
|
428
660
|
brokerOwners = new Map();
|
|
429
661
|
bindingCache = new Map();
|
|
662
|
+
/**
|
|
663
|
+
* Providers whose managed credentials are being removed right now. While a
|
|
664
|
+
* provider is unmanaged only the unbind direction of
|
|
665
|
+
* {@link syncBrokerBinding} may run: a demotion inside the deletion loop
|
|
666
|
+
* (an `oauth` row iterated before a surviving `api-key` row) must never
|
|
667
|
+
* re-point the released profile at the broker.
|
|
668
|
+
*/
|
|
669
|
+
releasingProviders = new Set();
|
|
430
670
|
lastBindingPruneAt = 0;
|
|
431
671
|
selectionQueues = new Map();
|
|
432
672
|
queue = Promise.resolve();
|
|
433
673
|
closing = false;
|
|
674
|
+
/** The one shutdown this manager runs; a later close() joins it (round 38). */
|
|
675
|
+
closePromise;
|
|
676
|
+
/**
|
|
677
|
+
* The pre-close restoration is running (PR review round 37). Only inside it
|
|
678
|
+
* — set when {@link restoreBorrowedWindows} starts and cleared in its
|
|
679
|
+
* `finally`, the cleanup deadline included — does the module-private
|
|
680
|
+
* {@link PRE_CLOSE_RESTORE} capability let a write past shutdown. A
|
|
681
|
+
* restoration that outlives its deadline resumes after this went down and
|
|
682
|
+
* is refused like any other continuation.
|
|
683
|
+
*/
|
|
684
|
+
preCloseRestorePhase = false;
|
|
685
|
+
/**
|
|
686
|
+
* Shutdown has begun (PR review round 34). Set on `close()`'s first line,
|
|
687
|
+
* before the pre-close restoration runs, so a test arriving while that
|
|
688
|
+
* restoration is putting the outstanding borrows back is refused instead of
|
|
689
|
+
* starting a second window the restoration would not cover. Never cleared:
|
|
690
|
+
* the instance is closing.
|
|
691
|
+
*/
|
|
692
|
+
shutdownRequested = false;
|
|
693
|
+
/**
|
|
694
|
+
* Providers whose in-flight row test borrows the route's credential
|
|
695
|
+
* surface (QC B4: an OAuth row test's projection, and the broker reference
|
|
696
|
+
* it may temporarily release). While a provider is borrowed the binding
|
|
697
|
+
* authority must not derive a real binding from the borrowed world: a
|
|
698
|
+
* mutation landing inside the window (`addKey`, `deleteKey`, an account
|
|
699
|
+
* removal) would otherwise persist an unbind decided about a state that is
|
|
700
|
+
* not the route's. {@link syncBrokerBinding} consults this exactly like
|
|
701
|
+
* {@link releasingProviders}; the test's own window-end sync passes
|
|
702
|
+
* `borrow: true` so the restored state is written back in one step.
|
|
703
|
+
*/
|
|
704
|
+
testBindingBorrows = new Set();
|
|
705
|
+
/**
|
|
706
|
+
* Providers with a borrowing row test in flight, mapped to that test's own
|
|
707
|
+
* promise. The promise is the isolation seam: an ordinary request for the
|
|
708
|
+
* provider waits on it (bounded, see {@link settleProviderTest}) before
|
|
709
|
+
* resolving a credential, and reports its own outcome if the test fails.
|
|
710
|
+
* Only OAuth-row tests borrow (QC N7), so only they register here.
|
|
711
|
+
*/
|
|
712
|
+
testsInFlight = new Map();
|
|
713
|
+
/**
|
|
714
|
+
* Every in-flight row test, by provider — the single-concurrent-test lock
|
|
715
|
+
* for both row kinds. Separate from {@link testsInFlight} because an
|
|
716
|
+
* api-key test borrows nothing and must not isolate ordinary requests.
|
|
717
|
+
*/
|
|
718
|
+
testsActive = new Map();
|
|
719
|
+
/**
|
|
720
|
+
* Every row test window that has not been released yet, by provider (PR
|
|
721
|
+
* review round 33). Shutdown restores each window's borrowed projection
|
|
722
|
+
* before `closing` refuses the in-window cleanup, so the map outlives
|
|
723
|
+
* {@link testsActive} for a window whose request never settles.
|
|
724
|
+
*/
|
|
725
|
+
testWindows = new Map();
|
|
726
|
+
/**
|
|
727
|
+
* Upper bound on one ordinary request's wait for an in-flight row test
|
|
728
|
+
* (QC B3). Injectable so tests can exercise the bound quickly.
|
|
729
|
+
*/
|
|
730
|
+
isolationWaitMs = KEY_TEST_ISOLATION_WAIT_MS;
|
|
731
|
+
/**
|
|
732
|
+
* Providers whose abandoned test window is still being cleaned up (QC fix
|
|
733
|
+
* wave 2). The fence stays until a cleanup completed: a new test would
|
|
734
|
+
* borrow a projection that is not back yet, and an ordinary request must not
|
|
735
|
+
* be served by a borrowed grant or an unbound route.
|
|
736
|
+
*/
|
|
737
|
+
cleaningTests = new Set();
|
|
738
|
+
/**
|
|
739
|
+
* Nesting depth of an in-flight removal per provider (PR review round 12).
|
|
740
|
+
* {@link settleTestBeforeRemoval} can only wait for a test that already
|
|
741
|
+
* exists, so a test starting after that check would read the grant and the
|
|
742
|
+
* projection and then re-project an account the removal deleted. {@link
|
|
743
|
+
* testKey} refuses while this marker is up. A depth, because `deleteKey`
|
|
744
|
+
* releases through `releaseLocked` inside its own removal.
|
|
745
|
+
*/
|
|
746
|
+
removalsInProgress = new Map();
|
|
747
|
+
/**
|
|
748
|
+
* Generation of the newest cleanup attempt per provider (PR review round 2).
|
|
749
|
+
* An attempt that outlives its deadline keeps writing while a later attempt
|
|
750
|
+
* for the same window starts. Only the newest generation may release
|
|
751
|
+
* provider-wide state — the fence, the borrow marker, the in-flight maps —
|
|
752
|
+
* so a superseded attempt can never clear isolation a newer window owns.
|
|
753
|
+
*/
|
|
754
|
+
cleanupAttempts = new Map();
|
|
755
|
+
/**
|
|
756
|
+
* The abandoned window currently fencing each provider (QC fix wave 3,
|
|
757
|
+
* W-002), so a later test attempt can retry its cleanup. Without this the
|
|
758
|
+
* fence could only ever clear from the abandoned promise's settlement, and a
|
|
759
|
+
* promise that settled while the seam was still wedged left the provider
|
|
760
|
+
* isolated forever.
|
|
761
|
+
*/
|
|
762
|
+
abandonedWindows = new Map();
|
|
763
|
+
/** Hard bound on one abandoned window's cleanup. Injectable for tests. */
|
|
764
|
+
testCleanupDeadlineMs = KEY_TEST_CLEANUP_DEADLINE_MS;
|
|
765
|
+
/** Hard bound on the shutdown drain of the grant chains. Injectable for tests. */
|
|
766
|
+
grantDrainDeadlineMs = GRANT_DRAIN_DEADLINE_MS;
|
|
767
|
+
/** Providers whose isolation-deadline warning already fired (warn once). */
|
|
768
|
+
isolationWaitWarned = new Set();
|
|
434
769
|
/** Interactive sign-in attempts, by attempt id. */
|
|
435
770
|
loginAttempts = new Map();
|
|
436
771
|
/** The provider → attempt id index for the single-attempt rule. */
|
|
437
772
|
loginsInFlight = new Map();
|
|
438
773
|
loginSequence = 0;
|
|
774
|
+
/** The route → attempt id index for the single transport sign-in rule. */
|
|
775
|
+
transportInFlight = new Map();
|
|
776
|
+
/**
|
|
777
|
+
* Per-route removal epoch. A removal bumps it before withdrawing the live
|
|
778
|
+
* attempt, so that attempt's settle path can tell the account is gone and
|
|
779
|
+
* drop a grant its commit had already started writing.
|
|
780
|
+
*/
|
|
781
|
+
transportRemovals = new Map();
|
|
782
|
+
/**
|
|
783
|
+
* Transport-account rows, hydrated from the `llm-omp-transports/<route>`
|
|
784
|
+
* records. Kept out of the pi-ai pool document on purpose: the rows are a
|
|
785
|
+
* read-only projection of another package's records, rebuilt by
|
|
786
|
+
* {@link refreshTransportAccounts} and rendered by `buildState`.
|
|
787
|
+
*/
|
|
788
|
+
transportRows = [];
|
|
789
|
+
/** Tail of the serialized transport read chain. */
|
|
790
|
+
transportRefresh;
|
|
439
791
|
/** The account row whose grant payload currently IS the projection, per provider. */
|
|
440
792
|
activeAccounts = new Map();
|
|
793
|
+
/**
|
|
794
|
+
* Providers whose live projection EXISTS but no managed row owns it (PR
|
|
795
|
+
* review round 3). `activeAccounts` alone cannot tell "no projection" from
|
|
796
|
+
* "a projection this process does not manage": an orphan or foreign grant
|
|
797
|
+
* (hydration found no matching row) still serves ahead of any broker
|
|
798
|
+
* reference, so the route must not be treated as the key pool's. Set by
|
|
799
|
+
* hydration and by the reconciler when a record is present but unattributed,
|
|
800
|
+
* cleared when the record is deleted or becomes owned.
|
|
801
|
+
*/
|
|
802
|
+
foreignProjections = new Set();
|
|
441
803
|
/**
|
|
442
804
|
* Account rows committed to removal (C6-1/I1 removal-aware oracle), marked
|
|
443
805
|
* by {@link removeOAuthGrant} before its first awaited delete. A multi-row
|
|
@@ -538,6 +900,16 @@ export class KeysManagerService {
|
|
|
538
900
|
* to the requested provider's own unresolved read.
|
|
539
901
|
*/
|
|
540
902
|
hydratedProviders = new Set();
|
|
903
|
+
/**
|
|
904
|
+
* Providers whose hydration ownership read FAILED (PR review round 8). A
|
|
905
|
+
* failed read knows nothing about the route, so it is not the "no projection"
|
|
906
|
+
* answer a successful empty read is: treating it as one let the startup
|
|
907
|
+
* repair read an empty owner/foreign map as proof that the pool serves and
|
|
908
|
+
* bind the broker over a persisted OAuth projection it could not read.
|
|
909
|
+
* Readiness therefore stays false for these providers and the repair skips
|
|
910
|
+
* them; only a restart can retry the read.
|
|
911
|
+
*/
|
|
912
|
+
ownershipReadFailures = new Set();
|
|
541
913
|
/**
|
|
542
914
|
* One resolver set per provider still being waited on (round-10 should-fix
|
|
543
915
|
* SF2). A wait registers exactly one entry here and removes it when the wait
|
|
@@ -549,11 +921,27 @@ export class KeysManagerService {
|
|
|
549
921
|
hydrationWaiters = new Map();
|
|
550
922
|
/**
|
|
551
923
|
* True once the load-time hydration read is over (round-10 SF1). From then
|
|
552
|
-
* on no
|
|
553
|
-
*
|
|
554
|
-
*
|
|
924
|
+
* on no hydration read can land. It is NOT evidence that a given provider's
|
|
925
|
+
* projection was read (PR review round 17): a provider adopted after the
|
|
926
|
+
* snapshot was never in it, so readiness is decided per provider
|
|
927
|
+
* ({@link projectionOwnershipKnown}) and adoption reads its own projection.
|
|
555
928
|
*/
|
|
556
929
|
hydrationSettled = false;
|
|
930
|
+
/**
|
|
931
|
+
* Credential-store generation of the ownership hydration (PR review round
|
|
932
|
+
* 27). A remounted credentials service starts a new pass while the previous
|
|
933
|
+
* store's reads can still be in flight; their answers describe records the
|
|
934
|
+
* new backend does not hold, so the epoch gates every write and settle of
|
|
935
|
+
* the pass that started it.
|
|
936
|
+
*/
|
|
937
|
+
hydrationEpoch = 0;
|
|
938
|
+
/**
|
|
939
|
+
* Whether a credential store was ever attached (PR review round 27). The
|
|
940
|
+
* first attach is the startup mount and keeps its behaviour; every later one
|
|
941
|
+
* replaces the credential backend, so the ownership classification loaded
|
|
942
|
+
* from the previous one is stale and the read restarts from unknown.
|
|
943
|
+
*/
|
|
944
|
+
credentialsMounted = false;
|
|
557
945
|
/** Providers whose bounded-wait warning already fired (warn once). */
|
|
558
946
|
hydrationWaitWarned = new Set();
|
|
559
947
|
/** Channels whose lost-subscription warning already fired (warn once). */
|
|
@@ -569,8 +957,16 @@ export class KeysManagerService {
|
|
|
569
957
|
this.typertRemote = Object.freeze({ service: this, serviceKey: SERVICE, namespace: NAMESPACE });
|
|
570
958
|
this.rebuildBrokerMap();
|
|
571
959
|
this.projectionHydration = this.hydrateProjectionOwnership();
|
|
960
|
+
// Startup self-heal: repair every managed provider whose persisted binding
|
|
961
|
+
// the serving credential no longer justifies. The pass waits per provider
|
|
962
|
+
// for its own ownership read (see {@link repairBrokerBindings}), so a
|
|
963
|
+
// wedged OAuth read cannot starve an api-key-only route.
|
|
964
|
+
void this.repairBrokerBindings().catch(error => {
|
|
965
|
+
this.warn(`startup broker binding repair failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
966
|
+
});
|
|
572
967
|
this.settledDisposer = this.watchAuthorizationSettled();
|
|
573
968
|
this.recordUpdatedDisposer = this.watchGrantRecordUpdated();
|
|
969
|
+
this.refreshTransportAccountsIfIdle();
|
|
574
970
|
}
|
|
575
971
|
/**
|
|
576
972
|
* MF3 load-time hydration: a projection outlives this process — pi-ai
|
|
@@ -582,7 +978,17 @@ export class KeysManagerService {
|
|
|
582
978
|
* after a restart. An orphan projection — no matching row, e.g. a foreign
|
|
583
979
|
* or stale grant — is left untouched: the manager never adopts a grant it
|
|
584
980
|
* does not own, and out-of-band updates keep surfacing the native-
|
|
585
|
-
* management warning.
|
|
981
|
+
* management warning. A live record that is not a managed grant at all (a
|
|
982
|
+
* native api-key, an unrecognized payload) is foreign for the same reason
|
|
983
|
+
* (PR review round 4): it serves the route, so the startup repair must not
|
|
984
|
+
* bind the broker over it.
|
|
985
|
+
*
|
|
986
|
+
* EVERY managed provider's projection is read (PR review round 13), not
|
|
987
|
+
* only those with OAuth rows: a key-only pool can be overridden by a
|
|
988
|
+
* native grant persisted for the same route, and a skipped read left the
|
|
989
|
+
* startup repair free to bind the broker over it. The reads run
|
|
990
|
+
* concurrently so one provider's stalled read cannot delay another
|
|
991
|
+
* provider's own readiness (round 9).
|
|
586
992
|
*
|
|
587
993
|
* Returns its own completion (round-8 finding I-000108), and the returned
|
|
588
994
|
* promise is already caught, so a failed hydration degrades to "no owner"
|
|
@@ -591,18 +997,23 @@ export class KeysManagerService {
|
|
|
591
997
|
* each request waits on its own provider's landing instead.
|
|
592
998
|
*/
|
|
593
999
|
hydrateProjectionOwnership() {
|
|
1000
|
+
const epoch = this.hydrationEpoch;
|
|
1001
|
+
const records = this.credentialRecords();
|
|
1002
|
+
if (records === undefined) {
|
|
1003
|
+
// The credentials service can mount after this plugin (PR review round
|
|
1004
|
+
// 23): nothing was read, so ownership stays UNKNOWN. Settling here marked
|
|
1005
|
+
// every provider hydrated, and the startup repair read the empty
|
|
1006
|
+
// owner/foreign maps as "the key pool serves", binding the broker over a
|
|
1007
|
+
// persisted projection it never saw. attachCredentials re-runs this read
|
|
1008
|
+
// (and the repair) once the record half is reachable.
|
|
1009
|
+
this.warnSubscriptionLost('credentials record access');
|
|
1010
|
+
return Promise.resolve();
|
|
1011
|
+
}
|
|
594
1012
|
return (async () => {
|
|
595
1013
|
try {
|
|
596
|
-
|
|
597
|
-
if (records === undefined) {
|
|
598
|
-
this.warnSubscriptionLost('credentials record access');
|
|
599
|
-
return;
|
|
600
|
-
}
|
|
601
|
-
for (const providerRecord of Object.values(this.store.providers)) {
|
|
1014
|
+
await Promise.all(Object.values(this.store.providers).map(async (providerRecord) => {
|
|
602
1015
|
const provider = providerRecord.provider;
|
|
603
1016
|
const accounts = providerRecord.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
604
|
-
if (accounts.length === 0)
|
|
605
|
-
continue;
|
|
606
1017
|
try {
|
|
607
1018
|
const grantKey = `${OAUTH_GRANT_RECORD_SCOPE}/${provider}`;
|
|
608
1019
|
// Generation tag (round-3 finding 3): the read below is awaited, and
|
|
@@ -614,41 +1025,87 @@ export class KeysManagerService {
|
|
|
614
1025
|
record = await records.readRecord(grantKey);
|
|
615
1026
|
}
|
|
616
1027
|
catch (error) {
|
|
1028
|
+
// PR review round 8: a failed read is not an absent projection.
|
|
1029
|
+
// Record it so readiness stays false and no repair binds the
|
|
1030
|
+
// broker over a grant this process could not see.
|
|
1031
|
+
// PR review round 23: unless a newer claim landed while this read
|
|
1032
|
+
// was in flight — a stale failure must not mark an established
|
|
1033
|
+
// owner unknown (same generation guard as the hydrated write).
|
|
1034
|
+
if (this.hydrationEpoch !== epoch || (this.ownershipGenerations.get(provider) ?? 0) !== generation)
|
|
1035
|
+
return;
|
|
1036
|
+
this.ownershipReadFailures.add(provider);
|
|
617
1037
|
this.warn(`projection ownership hydration could not read ${grantKey}: ${error instanceof Error ? error.message : String(error)}`);
|
|
618
|
-
|
|
1038
|
+
return;
|
|
619
1039
|
}
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
1040
|
+
// No record: nothing projects this route, so the pool may serve it.
|
|
1041
|
+
if (record === undefined)
|
|
1042
|
+
return;
|
|
1043
|
+
// A live record this pool cannot interpret — a native api-key, an
|
|
1044
|
+
// unrecognized payload, or (for a key-only pool) any grant at all —
|
|
1045
|
+
// serves the route ahead of any broker reference exactly like an
|
|
1046
|
+
// unowned grant, so it is classified as foreign below (PR review
|
|
1047
|
+
// rounds 4 and 13).
|
|
1048
|
+
const owner = record.kind === 'grant' && isOAuthGrantPayload(record.payload)
|
|
1049
|
+
? matchOwningAccount(provider, accounts, record.payload)
|
|
1050
|
+
: undefined;
|
|
623
1051
|
// A newer activation/login claimed the owner while this read was in
|
|
624
|
-
// flight: the stale read is discarded, never written over it.
|
|
625
|
-
|
|
626
|
-
|
|
1052
|
+
// flight: the stale read is discarded, never written over it. A
|
|
1053
|
+
// remount replaced the whole store: its classification is gone too.
|
|
1054
|
+
if (this.hydrationEpoch !== epoch || (this.ownershipGenerations.get(provider) ?? 0) !== generation)
|
|
1055
|
+
return;
|
|
627
1056
|
// A concurrent removal may have dropped the row while this read was
|
|
628
1057
|
// in flight: never resurrect an owner entry for a row that is gone.
|
|
629
1058
|
if (owner !== undefined && providerRecord.keys.some(candidate => candidate.id === owner.id)) {
|
|
630
1059
|
this.activeAccounts.set(provider, owner.id);
|
|
1060
|
+
this.foreignProjections.delete(provider);
|
|
1061
|
+
}
|
|
1062
|
+
else {
|
|
1063
|
+
// Present but not a managed OAuth grant (PR review rounds 3–4):
|
|
1064
|
+
// the projection still serves, so the pool is not the route's
|
|
1065
|
+
// credential source and the startup repair must not bind it.
|
|
1066
|
+
this.foreignProjections.add(provider);
|
|
631
1067
|
}
|
|
632
1068
|
}
|
|
633
1069
|
finally {
|
|
634
1070
|
// This provider's own read is over (owner or not): its requests are
|
|
635
1071
|
// attributable now and must not wait behind another provider's
|
|
636
|
-
// stalled read (round-10 should-fix SF1).
|
|
637
|
-
|
|
1072
|
+
// stalled read (round-10 should-fix SF1). A superseded pass does
|
|
1073
|
+
// not report readiness for a store it no longer describes.
|
|
1074
|
+
if (this.hydrationEpoch === epoch)
|
|
1075
|
+
this.noteProviderHydrated(provider);
|
|
638
1076
|
}
|
|
639
|
-
}
|
|
1077
|
+
}));
|
|
640
1078
|
}
|
|
641
1079
|
finally {
|
|
642
1080
|
// The read is over (completed or abandoned): from now on no result can
|
|
643
1081
|
// land, so every provider is ready — including one whose OAuth rows
|
|
644
1082
|
// appear later, which this read could not have projected.
|
|
645
|
-
this.
|
|
646
|
-
|
|
1083
|
+
if (this.hydrationEpoch === epoch) {
|
|
1084
|
+
this.hydrationSettled = true;
|
|
1085
|
+
this.settleHydrationWaits();
|
|
1086
|
+
}
|
|
647
1087
|
}
|
|
648
1088
|
})().catch(error => {
|
|
649
1089
|
this.warn(`projection ownership hydration failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
650
1090
|
});
|
|
651
1091
|
}
|
|
1092
|
+
/**
|
|
1093
|
+
* Drop every ownership fact loaded from a previous credential store (PR
|
|
1094
|
+
* review round 27). The classification is per backend: a record that existed
|
|
1095
|
+
* in the old store says nothing about the remounted one, and a route whose
|
|
1096
|
+
* old store held a managed projection would otherwise stay attributed to
|
|
1097
|
+
* that account and off its surviving key pool. The epoch bump discards a read
|
|
1098
|
+
* still in flight from the replaced store instead of letting its answer land
|
|
1099
|
+
* over the new one; ownership stays unknown until the new pass reads.
|
|
1100
|
+
*/
|
|
1101
|
+
invalidateProjectionOwnership() {
|
|
1102
|
+
this.hydrationEpoch += 1;
|
|
1103
|
+
this.activeAccounts.clear();
|
|
1104
|
+
this.foreignProjections.clear();
|
|
1105
|
+
this.hydratedProviders.clear();
|
|
1106
|
+
this.ownershipReadFailures.clear();
|
|
1107
|
+
this.hydrationSettled = false;
|
|
1108
|
+
}
|
|
652
1109
|
/**
|
|
653
1110
|
* Record that one provider's hydration read is over — owner or not (SF1) —
|
|
654
1111
|
* and wake the requests waiting on that provider alone (SF2).
|
|
@@ -662,6 +1119,21 @@ export class KeysManagerService {
|
|
|
662
1119
|
for (const wake of [...waiters])
|
|
663
1120
|
wake();
|
|
664
1121
|
}
|
|
1122
|
+
/**
|
|
1123
|
+
* An authoritative projection read answered for this provider, so a failed
|
|
1124
|
+
* startup hydration no longer leaves ownership unknown (PR review round 13:
|
|
1125
|
+
* the marker used to survive until a restart — no attribution, no key
|
|
1126
|
+
* selection, and row tests refused with "ownership is still loading"). The
|
|
1127
|
+
* binding re-derive is queued behind the reconciliation that proved the
|
|
1128
|
+
* read: its attribution decides the route's serving shape, and the startup
|
|
1129
|
+
* repair was skipped while that shape was unknown.
|
|
1130
|
+
*/
|
|
1131
|
+
noteProjectionReadSucceeded(provider) {
|
|
1132
|
+
if (!this.ownershipReadFailures.delete(provider))
|
|
1133
|
+
return;
|
|
1134
|
+
this.noteProviderHydrated(provider);
|
|
1135
|
+
void this.scheduleGrantReconciliation(provider, () => this.syncBindingAfterProjectionChange(provider));
|
|
1136
|
+
}
|
|
665
1137
|
/** Release every provider's readiness wait when hydration cannot proceed. */
|
|
666
1138
|
settleHydrationWaits() {
|
|
667
1139
|
for (const provider of Object.keys(this.store.providers))
|
|
@@ -680,6 +1152,10 @@ export class KeysManagerService {
|
|
|
680
1152
|
* (I-000108).
|
|
681
1153
|
*/
|
|
682
1154
|
waitForProviderHydration(provider, ms) {
|
|
1155
|
+
// A failed read never becomes ready (PR review round 8): waiting cannot
|
|
1156
|
+
// produce an answer this process does not have.
|
|
1157
|
+
if (this.ownershipReadFailures.has(provider))
|
|
1158
|
+
return Promise.resolve(false);
|
|
683
1159
|
if (this.hydrationSettled || this.hydratedProviders.has(provider))
|
|
684
1160
|
return Promise.resolve(true);
|
|
685
1161
|
return new Promise(resolve => {
|
|
@@ -701,7 +1177,9 @@ export class KeysManagerService {
|
|
|
701
1177
|
resolve(landed);
|
|
702
1178
|
};
|
|
703
1179
|
const onLanded = () => {
|
|
704
|
-
|
|
1180
|
+
// A failed read wakes the wait with "not known", never with a guess
|
|
1181
|
+
// about the projection (PR review round 8).
|
|
1182
|
+
wake(!this.ownershipReadFailures.has(provider));
|
|
705
1183
|
};
|
|
706
1184
|
timer = setTimeout(() => {
|
|
707
1185
|
wake(false);
|
|
@@ -724,10 +1202,10 @@ export class KeysManagerService {
|
|
|
724
1202
|
* Wait (bounded) until projection-ownership hydration can attribute this
|
|
725
1203
|
* request, and report whether it did.
|
|
726
1204
|
*
|
|
727
|
-
*
|
|
728
|
-
*
|
|
729
|
-
*
|
|
730
|
-
*
|
|
1205
|
+
* Readiness is decided PER PROVIDER (round-10 should-fix SF1): every managed
|
|
1206
|
+
* provider's own projection read lands independently (PR review round 13), so
|
|
1207
|
+
* a provider whose read has landed needs no wait and a stalled read for
|
|
1208
|
+
* another provider cannot delay this one.
|
|
731
1209
|
*
|
|
732
1210
|
* Returns false when the bound elapsed first. The caller then treats
|
|
733
1211
|
* attribution as unknown (no owner) and keeps key state untouched: a
|
|
@@ -735,34 +1213,45 @@ export class KeysManagerService {
|
|
|
735
1213
|
* repeat the I-000108 corruption. The hydration promise is not cancelled,
|
|
736
1214
|
* so a late result still lands under the generation guard and later
|
|
737
1215
|
* requests are attributed normally.
|
|
738
|
-
*
|
|
739
|
-
* Readiness is decided PER PROVIDER (round-10 should-fix SF1). Hydration
|
|
740
|
-
* writes owners progressively, so racing the one global promise would
|
|
741
|
-
* discard an owner this provider already confirmed just because another
|
|
742
|
-
* provider's read stalled. A provider whose own read has landed — with an
|
|
743
|
-
* owner or with none — is ready without waiting, and the "unknown" fallback
|
|
744
|
-
* is confined to the requested provider's own unresolved read.
|
|
745
1216
|
*/
|
|
746
1217
|
async awaitProjectionOwnership(providerId) {
|
|
747
|
-
|
|
748
|
-
if (provider === undefined)
|
|
749
|
-
return true;
|
|
750
|
-
const hasOauthRows = provider.keys.some(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
751
|
-
if (!hasOauthRows)
|
|
752
|
-
return true;
|
|
753
|
-
// This provider's own read already landed (or a newer claim already owns
|
|
754
|
-
// it, or the whole read is over): nothing can arrive later, so no wait.
|
|
755
|
-
if (this.hydrationSettled || this.hydratedProviders.has(providerId) || this.activeAccounts.has(providerId))
|
|
1218
|
+
if (this.projectionOwnershipKnown(providerId))
|
|
756
1219
|
return true;
|
|
757
1220
|
const landed = await this.waitForProviderHydration(providerId, this.hydrationWaitMs);
|
|
758
1221
|
if (landed)
|
|
759
1222
|
return true;
|
|
760
1223
|
if (!this.hydrationWaitWarned.has(providerId)) {
|
|
761
1224
|
this.hydrationWaitWarned.add(providerId);
|
|
762
|
-
this.warn(
|
|
1225
|
+
this.warn(this.ownershipReadFailures.has(providerId)
|
|
1226
|
+
? `projection ownership for ${providerId} could not be read; requests proceed without an attributed account until a restart`
|
|
1227
|
+
: `projection ownership hydration did not settle within ${this.hydrationWaitMs}ms for ${providerId}; requests proceed without an attributed account until it does`);
|
|
763
1228
|
}
|
|
764
1229
|
return false;
|
|
765
1230
|
}
|
|
1231
|
+
/**
|
|
1232
|
+
* Whether this provider's projection ownership can already be decided
|
|
1233
|
+
* without waiting (round-10 SF1, made the single readiness rule by PR review
|
|
1234
|
+
* round 3). Every managed provider's projection is read at hydration (a
|
|
1235
|
+
* key-only pool can be overridden by a persisted native grant, PR review
|
|
1236
|
+
* round 13), so the rule is the same for all of them: this provider's own
|
|
1237
|
+
* read landed, or a claim already owns it.
|
|
1238
|
+
*
|
|
1239
|
+
* The process-wide settled flag is deliberately NOT part of this rule (PR
|
|
1240
|
+
* review round 17): a provider adopted after the snapshot was never read by
|
|
1241
|
+
* it, so "hydration is over" says nothing about THAT route's projection.
|
|
1242
|
+
* Adoption reads it itself ({@link readAdoptionProjection}) and an extra
|
|
1243
|
+
* provider's profile is created by its own call, so every managed provider
|
|
1244
|
+
* still answers through its own marker.
|
|
1245
|
+
*/
|
|
1246
|
+
projectionOwnershipKnown(providerId) {
|
|
1247
|
+
if (this.store.provider(providerId) === undefined)
|
|
1248
|
+
return true;
|
|
1249
|
+
// A failed read is unknown even after the whole hydration settled: the
|
|
1250
|
+
// settled flag says no read can land, not that this one answered.
|
|
1251
|
+
if (this.ownershipReadFailures.has(providerId))
|
|
1252
|
+
return false;
|
|
1253
|
+
return this.hydratedProviders.has(providerId) || this.activeAccounts.has(providerId);
|
|
1254
|
+
}
|
|
766
1255
|
/**
|
|
767
1256
|
* Record that a newly committed grant became the provider's projection and
|
|
768
1257
|
* return that projection's generation (round-6 finding SF1).
|
|
@@ -788,6 +1277,7 @@ export class KeysManagerService {
|
|
|
788
1277
|
*/
|
|
789
1278
|
claimProjectionOwner(provider, accountId) {
|
|
790
1279
|
this.activeAccounts.set(provider, accountId);
|
|
1280
|
+
this.foreignProjections.delete(provider);
|
|
791
1281
|
this.noteProjectionCommitted(provider);
|
|
792
1282
|
}
|
|
793
1283
|
/**
|
|
@@ -841,8 +1331,47 @@ export class KeysManagerService {
|
|
|
841
1331
|
this.pruneAttempts(now);
|
|
842
1332
|
this.pruneLoginAttempts(now);
|
|
843
1333
|
}
|
|
1334
|
+
/**
|
|
1335
|
+
* Whether a continuation that resumes from an await must stop writing
|
|
1336
|
+
* (PR review round 35). `shutdownRequested` is up for `close()`'s own
|
|
1337
|
+
* pre-close restoration, which writes by design: it alone carries the
|
|
1338
|
+
* module-private {@link PRE_CLOSE_RESTORE} capability (round 36), which is
|
|
1339
|
+
* honoured only while that phase is running (round 37). Both facts are read
|
|
1340
|
+
* here, at the caller's write decision, so a capability the phase left
|
|
1341
|
+
* behind cannot pass a check made earlier.
|
|
1342
|
+
*/
|
|
1343
|
+
persistenceRefused(capability) {
|
|
1344
|
+
if (!this.closing && !this.shutdownRequested)
|
|
1345
|
+
return false;
|
|
1346
|
+
// PR review round 38: the capability is scoped to the one restoration pass
|
|
1347
|
+
// that runs before shutdown closes the store. Once `closing` is set the
|
|
1348
|
+
// pass is over for good, so a capability left over from a timed-out pass
|
|
1349
|
+
// (or handed to a repeated `close()`) cannot write again.
|
|
1350
|
+
return !(capability === PRE_CLOSE_RESTORE && this.preCloseRestorePhase && !this.closing);
|
|
1351
|
+
}
|
|
844
1352
|
/** Stop accepting mutations, drain the queue, then persist the final snapshot. */
|
|
845
|
-
|
|
1353
|
+
close() {
|
|
1354
|
+
// PR review round 38: shutdown is single-entry. A second call (or a
|
|
1355
|
+
// concurrent one) joins the first instead of replaying the restoration,
|
|
1356
|
+
// which would otherwise write to a store the first call already closed.
|
|
1357
|
+
// Not `async`: callers share the identical promise, so a repeated close
|
|
1358
|
+
// cannot observe a second shutdown.
|
|
1359
|
+
if (this.closePromise !== undefined)
|
|
1360
|
+
return this.closePromise;
|
|
1361
|
+
const closing = this.closeOnce();
|
|
1362
|
+
this.closePromise = closing;
|
|
1363
|
+
return closing;
|
|
1364
|
+
}
|
|
1365
|
+
/** The body of the one shutdown this manager runs. */
|
|
1366
|
+
async closeOnce() {
|
|
1367
|
+
// PR review round 34: the entry marker goes up before the restoration, so
|
|
1368
|
+
// the window this function spends putting borrowed projections back cannot
|
|
1369
|
+
// accept a new test and a new borrow.
|
|
1370
|
+
this.shutdownRequested = true;
|
|
1371
|
+
// PR review round 33: `closing` refuses the in-window cleanup, so every
|
|
1372
|
+
// outstanding borrow is put back first. A test whose request ignores the
|
|
1373
|
+
// abort otherwise leaves the tested account projected past unload.
|
|
1374
|
+
await this.restoreBorrowedWindows();
|
|
846
1375
|
this.closing = true;
|
|
847
1376
|
this.settledDisposer?.();
|
|
848
1377
|
this.settledDisposer = undefined;
|
|
@@ -861,9 +1390,96 @@ export class KeysManagerService {
|
|
|
861
1390
|
// Round-9 P1: the per-provider reconciliation chains are independent of
|
|
862
1391
|
// the mutation queue, so drain them before the store closes — a queued
|
|
863
1392
|
// echo must not write a grant record into a manager that is shutting down.
|
|
864
|
-
await
|
|
1393
|
+
await this.drainGrantReconciliations();
|
|
865
1394
|
await this.store.close();
|
|
866
1395
|
}
|
|
1396
|
+
/**
|
|
1397
|
+
* Put every outstanding window's borrowed projection back before shutdown
|
|
1398
|
+
* closes the store (PR review round 33).
|
|
1399
|
+
*
|
|
1400
|
+
* `close()` sets `closing`, which refuses {@link cleanupTestWindow} — the one
|
|
1401
|
+
* identity-checked restoration — so a test whose request ignores the abort
|
|
1402
|
+
* used to leave the tested account projected and the binding it released
|
|
1403
|
+
* released, past unload and into the next startup. Each window's captured
|
|
1404
|
+
* facts are restored through the same compare-by-payload path as the late
|
|
1405
|
+
* reconcile, under one cleanup bound and the attempt fence (a superseded
|
|
1406
|
+
* attempt writes nothing), and a window that released the binding re-syncs it
|
|
1407
|
+
* against the restored serving source.
|
|
1408
|
+
*
|
|
1409
|
+
* The request itself is never waited for — an adapter can still rewrite the
|
|
1410
|
+
* projection after its restore — so a restoration that cannot finish in the
|
|
1411
|
+
* bound warns once and leaves the state to the next startup's hydration.
|
|
1412
|
+
*/
|
|
1413
|
+
async restoreBorrowedWindows() {
|
|
1414
|
+
// PR review round 37: the capability is scoped to this phase. The flag goes
|
|
1415
|
+
// up before the first await and down however the phase ends — a restoration
|
|
1416
|
+
// the cleanup deadline abandoned is still parked, and it resumes with the
|
|
1417
|
+
// phase already over.
|
|
1418
|
+
this.preCloseRestorePhase = true;
|
|
1419
|
+
try {
|
|
1420
|
+
for (const window of [...this.testWindows.values()]) {
|
|
1421
|
+
const facts = window.facts;
|
|
1422
|
+
if (facts === undefined)
|
|
1423
|
+
continue;
|
|
1424
|
+
const attempt = (this.cleanupAttempts.get(window.provider) ?? 0) + 1;
|
|
1425
|
+
this.cleanupAttempts.set(window.provider, attempt);
|
|
1426
|
+
const stillCurrent = () => this.cleanupAttempts.get(window.provider) === attempt;
|
|
1427
|
+
const failure = await this.withinCleanupDeadline(async () => {
|
|
1428
|
+
// QC fix wave 3 (W-001): a borrow or binding release still in flight
|
|
1429
|
+
// reads as the pre-borrow payload, so the restore would call the route
|
|
1430
|
+
// clean and let the write land afterwards.
|
|
1431
|
+
await this.awaitWindowMutations(window);
|
|
1432
|
+
// The window is still outstanding, so this is an in-window restore (not
|
|
1433
|
+
// the late one): the attempt fence is the only identity it needs. This
|
|
1434
|
+
// is also the one writer that holds the pre-close capability.
|
|
1435
|
+
const restored = await this.restoreProjection(facts.records, window.provider, window.row, facts.key, facts.previous, facts.borrowed, stillCurrent, false, PRE_CLOSE_RESTORE);
|
|
1436
|
+
if (restored !== undefined)
|
|
1437
|
+
return restored;
|
|
1438
|
+
if (!stillCurrent())
|
|
1439
|
+
return 'a newer cleanup attempt took over this provider';
|
|
1440
|
+
if (!window.releasedBinding)
|
|
1441
|
+
return undefined;
|
|
1442
|
+
// QC B4: the borrow released the profile's binding, so the restored
|
|
1443
|
+
// serving source must be written back to it.
|
|
1444
|
+
return await this.syncBrokerBinding(window.provider, { borrow: true })
|
|
1445
|
+
? undefined
|
|
1446
|
+
: 'the route credential binding could not be restored';
|
|
1447
|
+
});
|
|
1448
|
+
if (failure !== undefined) {
|
|
1449
|
+
this.warn(`shutdown could not put the borrowed credential for ${window.provider} back (${failure}); the projection stays as the store holds it and the next startup's hydration restores it`);
|
|
1450
|
+
}
|
|
1451
|
+
}
|
|
1452
|
+
}
|
|
1453
|
+
finally {
|
|
1454
|
+
this.preCloseRestorePhase = false;
|
|
1455
|
+
}
|
|
1456
|
+
}
|
|
1457
|
+
/**
|
|
1458
|
+
* Drain the grant chains under a hard bound (PR review round 29).
|
|
1459
|
+
*
|
|
1460
|
+
* The work is not cancelled — the credential seam offers no cancellation and
|
|
1461
|
+
* a wedged provider request may never settle — it is only stopped from
|
|
1462
|
+
* holding shutdown: the chain past the deadline is abandoned, and its late
|
|
1463
|
+
* write can only land in a store that is closing anyway.
|
|
1464
|
+
*/
|
|
1465
|
+
async drainGrantReconciliations() {
|
|
1466
|
+
// The drain never rejects: a chain failure is that chain's own result.
|
|
1467
|
+
const drained = Promise.allSettled([...this.grantReconciliations.values()]).then(() => false);
|
|
1468
|
+
let timer;
|
|
1469
|
+
const deadline = new Promise(resolve => {
|
|
1470
|
+
timer = setTimeout(() => resolve(true), this.grantDrainDeadlineMs);
|
|
1471
|
+
timer.unref?.();
|
|
1472
|
+
});
|
|
1473
|
+
try {
|
|
1474
|
+
if (await Promise.race([drained, deadline])) {
|
|
1475
|
+
this.warn('shutdown stopped waiting for a grant reconciliation still held by a connection test request that never settled');
|
|
1476
|
+
}
|
|
1477
|
+
}
|
|
1478
|
+
finally {
|
|
1479
|
+
if (timer !== undefined)
|
|
1480
|
+
clearTimeout(timer);
|
|
1481
|
+
}
|
|
1482
|
+
}
|
|
867
1483
|
/** Push a user-facing warning that rides the next state response. */
|
|
868
1484
|
pushWarning(message) {
|
|
869
1485
|
this.warnings.push(message);
|
|
@@ -875,12 +1491,53 @@ export class KeysManagerService {
|
|
|
875
1491
|
this.warnings = [];
|
|
876
1492
|
return warnings;
|
|
877
1493
|
}
|
|
1494
|
+
/**
|
|
1495
|
+
* Report declared headers whose stored value no longer matches what the
|
|
1496
|
+
* environment supplies. The stored value is never replaced — the user may
|
|
1497
|
+
* have set it deliberately — but the divergence must be visible, because
|
|
1498
|
+
* otherwise the route keeps sending the stale value forever.
|
|
1499
|
+
*/
|
|
1500
|
+
warnHeaderDrift(extra, declared, stored) {
|
|
1501
|
+
if (declared === undefined)
|
|
1502
|
+
return;
|
|
1503
|
+
for (const [header, value] of Object.entries(declared)) {
|
|
1504
|
+
const storedKey = storedHeaderKey(stored, header);
|
|
1505
|
+
if (storedKey === undefined || stored[storedKey] === value)
|
|
1506
|
+
continue;
|
|
1507
|
+
const envName = extra.headerEnv?.[header] ?? header;
|
|
1508
|
+
this.pushWarning(`The stored ${storedKey} header on the ${extra.id} route differs from the value ${envName} now supplies; the stored value was kept.`);
|
|
1509
|
+
}
|
|
1510
|
+
}
|
|
878
1511
|
/** Current settings service, if the composition mounted one. */
|
|
879
1512
|
settingsService() {
|
|
880
1513
|
if (this.settings !== undefined)
|
|
881
1514
|
return this.settings;
|
|
882
1515
|
return this.ctx.get('settings');
|
|
883
1516
|
}
|
|
1517
|
+
/**
|
|
1518
|
+
* Mount the settings service — the host's `ctx.inject(['settings'])` seam.
|
|
1519
|
+
*
|
|
1520
|
+
* A mount that lands after construction retries the binding repair: the
|
|
1521
|
+
* startup pass could not read a profile yet and would otherwise leave the
|
|
1522
|
+
* route broken for the whole session.
|
|
1523
|
+
*/
|
|
1524
|
+
attachSettings(settings) {
|
|
1525
|
+
this.settings = settings;
|
|
1526
|
+
// The repair decides from projection ownership, which the constructor
|
|
1527
|
+
// hydrates asynchronously: a persisted projection read from an empty owner
|
|
1528
|
+
// map would make the route look key-pool-served and bind it to the broker
|
|
1529
|
+
// (PR review round 2). The pass therefore waits per provider for its own
|
|
1530
|
+
// read — never for the process-wide promise, which one wedged OAuth read
|
|
1531
|
+
// can hold forever (PR review round 3).
|
|
1532
|
+
void this.repairBrokerBindings().catch(error => {
|
|
1533
|
+
this.warn(`broker binding repair after the settings mount failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1534
|
+
});
|
|
1535
|
+
}
|
|
1536
|
+
/** Unmount the settings service that a re-mount replaced. */
|
|
1537
|
+
detachSettings(settings) {
|
|
1538
|
+
if (this.settings === settings)
|
|
1539
|
+
this.settings = undefined;
|
|
1540
|
+
}
|
|
884
1541
|
/** Current raw credential provider. */
|
|
885
1542
|
credentialsRaw() {
|
|
886
1543
|
const current = this.ctx.get('credentials');
|
|
@@ -935,6 +1592,29 @@ export class KeysManagerService {
|
|
|
935
1592
|
oauthFlowAvailable(provider) {
|
|
936
1593
|
return this.oauthFlowFor(provider) !== undefined;
|
|
937
1594
|
}
|
|
1595
|
+
/**
|
|
1596
|
+
* Resolve one transport route's sign-in flow.
|
|
1597
|
+
*
|
|
1598
|
+
* The transports package registers `llm-omp-transports/<route>` itself, so
|
|
1599
|
+
* the live flow list is also this manager's inventory of transport routes:
|
|
1600
|
+
* a route with no registered flow is not a transport account surface, and a
|
|
1601
|
+
* route whose record exists but whose flow is gone is not listed at all
|
|
1602
|
+
* (there would be nothing to sign in with).
|
|
1603
|
+
*
|
|
1604
|
+
* The method id is the flow's own (`methods[0].id`), never a literal
|
|
1605
|
+
* `'oauth'`: the scope prefix is the whole cross-package contract, and the
|
|
1606
|
+
* transports package may name a method that is not OAuth. A flow with no
|
|
1607
|
+
* method cannot be begun and is not a sign-in surface.
|
|
1608
|
+
*/
|
|
1609
|
+
transportFlowFor(route) {
|
|
1610
|
+
return (this.ctx.authorization?.list() ?? [])
|
|
1611
|
+
.find(entry => entry.key === `${TRANSPORT_CREDENTIAL_SCOPE}/${route}` && transportMethodOf(entry) !== undefined);
|
|
1612
|
+
}
|
|
1613
|
+
/** Every registered transport flow, in seam order. */
|
|
1614
|
+
transportFlows() {
|
|
1615
|
+
return (this.ctx.authorization?.list() ?? [])
|
|
1616
|
+
.filter(entry => entry.key.startsWith(`${TRANSPORT_CREDENTIAL_SCOPE}/`) && transportMethodOf(entry) !== undefined);
|
|
1617
|
+
}
|
|
938
1618
|
/**
|
|
939
1619
|
* Warn once per channel: a lost subscription silently disables a whole
|
|
940
1620
|
* feature (harvest never runs; rotations never sync), so the degradation
|
|
@@ -1004,6 +1684,42 @@ export class KeysManagerService {
|
|
|
1004
1684
|
else
|
|
1005
1685
|
this.failLoginAttempt(attempt, new Error(`the sign-in attempt failed (${settlement || 'unknown outcome'})`));
|
|
1006
1686
|
}
|
|
1687
|
+
/**
|
|
1688
|
+
* The seam interaction one attempt hands to its flow.
|
|
1689
|
+
*
|
|
1690
|
+
* Shared by the pi-ai and transport sign-ins: both surfaces park notices and
|
|
1691
|
+
* prompts on the same {@link LoginAttempt} and are polled through
|
|
1692
|
+
* `accountAttempt`; only what happens after `authorized` differs.
|
|
1693
|
+
*/
|
|
1694
|
+
attemptInteraction(attempt) {
|
|
1695
|
+
return {
|
|
1696
|
+
notify: (notice) => {
|
|
1697
|
+
attempt.notices.push(noticeView(notice));
|
|
1698
|
+
if (attempt.notices.length > 32)
|
|
1699
|
+
attempt.notices.shift();
|
|
1700
|
+
attempt.updatedAt = Date.now();
|
|
1701
|
+
},
|
|
1702
|
+
prompt: (prompt) => new Promise((resolve, reject) => {
|
|
1703
|
+
if (attempt.settledAt !== undefined) {
|
|
1704
|
+
reject(new Error('This sign-in attempt has already ended'));
|
|
1705
|
+
return;
|
|
1706
|
+
}
|
|
1707
|
+
if (prompt.signal?.aborted === true) {
|
|
1708
|
+
reject(new Error('The sign-in prompt was withdrawn'));
|
|
1709
|
+
return;
|
|
1710
|
+
}
|
|
1711
|
+
attempt.status = 'prompt';
|
|
1712
|
+
attempt.updatedAt = Date.now();
|
|
1713
|
+
attempt.prompt = promptView(prompt);
|
|
1714
|
+
attempt.resolvePrompt = resolve;
|
|
1715
|
+
attempt.rejectPrompt = reject;
|
|
1716
|
+
// Cancellation propagates to the awaited prompt: a seam-side
|
|
1717
|
+
// withdrawal of this signal releases a flow parked here instead
|
|
1718
|
+
// of leaving it pending forever.
|
|
1719
|
+
prompt.signal?.addEventListener('abort', () => reject(new Error('The sign-in prompt was withdrawn')), { once: true });
|
|
1720
|
+
}),
|
|
1721
|
+
};
|
|
1722
|
+
}
|
|
1007
1723
|
/**
|
|
1008
1724
|
* Start one interactive OAuth sign-in. Returns immediately with the
|
|
1009
1725
|
* attempt view; the attempt walks `pending → (prompt ⇄ pending) →
|
|
@@ -1079,33 +1795,7 @@ export class KeysManagerService {
|
|
|
1079
1795
|
this.loginsInFlight.delete(provider);
|
|
1080
1796
|
throw new Error('The credential store does not support records; OAuth sign-in is unavailable');
|
|
1081
1797
|
}
|
|
1082
|
-
const interaction =
|
|
1083
|
-
notify: (notice) => {
|
|
1084
|
-
attempt.notices.push(noticeView(notice));
|
|
1085
|
-
if (attempt.notices.length > 32)
|
|
1086
|
-
attempt.notices.shift();
|
|
1087
|
-
attempt.updatedAt = Date.now();
|
|
1088
|
-
},
|
|
1089
|
-
prompt: (prompt) => new Promise((resolve, reject) => {
|
|
1090
|
-
if (attempt.settledAt !== undefined) {
|
|
1091
|
-
reject(new Error('This sign-in attempt has already ended'));
|
|
1092
|
-
return;
|
|
1093
|
-
}
|
|
1094
|
-
if (prompt.signal?.aborted === true) {
|
|
1095
|
-
reject(new Error('The sign-in prompt was withdrawn'));
|
|
1096
|
-
return;
|
|
1097
|
-
}
|
|
1098
|
-
attempt.status = 'prompt';
|
|
1099
|
-
attempt.updatedAt = Date.now();
|
|
1100
|
-
attempt.prompt = promptView(prompt);
|
|
1101
|
-
attempt.resolvePrompt = resolve;
|
|
1102
|
-
attempt.rejectPrompt = reject;
|
|
1103
|
-
// Cancellation propagates to the awaited prompt: a seam-side
|
|
1104
|
-
// withdrawal of this signal releases a flow parked here instead
|
|
1105
|
-
// of leaving it pending forever.
|
|
1106
|
-
prompt.signal?.addEventListener('abort', () => reject(new Error('The sign-in prompt was withdrawn')), { once: true });
|
|
1107
|
-
}),
|
|
1108
|
-
};
|
|
1798
|
+
const interaction = this.attemptInteraction(attempt);
|
|
1109
1799
|
const beginPromise = this.ctx.authorization.begin({
|
|
1110
1800
|
key: flow.key,
|
|
1111
1801
|
method: flow.method,
|
|
@@ -1156,29 +1846,235 @@ export class KeysManagerService {
|
|
|
1156
1846
|
return [...this.loginAttempts.values()].sort((a, b) => b.createdAt - a.createdAt).map(attemptView);
|
|
1157
1847
|
}
|
|
1158
1848
|
/**
|
|
1159
|
-
*
|
|
1160
|
-
*
|
|
1161
|
-
*
|
|
1162
|
-
*
|
|
1849
|
+
* Re-read every `llm-omp-transports/<route>` grant into
|
|
1850
|
+
* {@link transportRows}.
|
|
1851
|
+
*
|
|
1852
|
+
* Every caller gets a pass appended after the one already in flight, so an
|
|
1853
|
+
* awaiting caller observes a read that started after its own write (a
|
|
1854
|
+
* removal, a sign-in commit). Poll-only callers use
|
|
1855
|
+
* {@link refreshTransportAccountsIfIdle} instead, so a slow store cannot
|
|
1856
|
+
* accumulate one pass per page tick. A failed read leaves that route's
|
|
1857
|
+
* previous row alone instead of claiming the account disappeared; the next
|
|
1858
|
+
* pass retries.
|
|
1163
1859
|
*/
|
|
1164
|
-
|
|
1165
|
-
const
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1860
|
+
refreshTransportAccounts() {
|
|
1861
|
+
const previous = this.transportRefresh ?? Promise.resolve();
|
|
1862
|
+
const pass = previous.then(() => this.readTransportAccounts(), () => this.readTransportAccounts());
|
|
1863
|
+
const tail = pass.then(() => undefined, () => undefined);
|
|
1864
|
+
this.transportRefresh = tail;
|
|
1865
|
+
void tail.then(() => {
|
|
1866
|
+
if (this.transportRefresh === tail)
|
|
1867
|
+
this.transportRefresh = undefined;
|
|
1868
|
+
});
|
|
1869
|
+
return pass;
|
|
1174
1870
|
}
|
|
1175
|
-
/**
|
|
1176
|
-
|
|
1177
|
-
if (
|
|
1178
|
-
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1871
|
+
/** Poll hint: start a pass only when none is in flight. */
|
|
1872
|
+
refreshTransportAccountsIfIdle() {
|
|
1873
|
+
if (this.transportRefresh === undefined)
|
|
1874
|
+
void this.refreshTransportAccounts();
|
|
1875
|
+
}
|
|
1876
|
+
/** One read pass over every registered transport route. */
|
|
1877
|
+
async readTransportAccounts() {
|
|
1878
|
+
try {
|
|
1879
|
+
const flows = this.transportFlows();
|
|
1880
|
+
if (flows.length === 0) {
|
|
1881
|
+
this.transportRows = [];
|
|
1882
|
+
return;
|
|
1883
|
+
}
|
|
1884
|
+
const records = this.credentialRecords();
|
|
1885
|
+
const now = Date.now();
|
|
1886
|
+
const rows = [];
|
|
1887
|
+
for (const flow of flows) {
|
|
1888
|
+
let record;
|
|
1889
|
+
if (records !== undefined) {
|
|
1890
|
+
try {
|
|
1891
|
+
record = await records.readRecord(flow.key);
|
|
1892
|
+
}
|
|
1893
|
+
catch (error) {
|
|
1894
|
+
const previous = this.transportRows.find(row => row.scope === flow.key);
|
|
1895
|
+
if (previous !== undefined)
|
|
1896
|
+
rows.push(previous);
|
|
1897
|
+
this.warn(`transport account ${flow.key} could not be read: ${error instanceof Error ? error.message : String(error)}`);
|
|
1898
|
+
continue;
|
|
1899
|
+
}
|
|
1900
|
+
}
|
|
1901
|
+
rows.push(transportAccountOf(flow, record, now));
|
|
1902
|
+
}
|
|
1903
|
+
this.transportRows = rows;
|
|
1904
|
+
}
|
|
1905
|
+
catch (error) {
|
|
1906
|
+
// Fire-and-forget callers (getState, the record watcher) must never see
|
|
1907
|
+
// a rejection; the previous rows stay until the next pass.
|
|
1908
|
+
this.warn(`transport account refresh failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
1909
|
+
}
|
|
1910
|
+
}
|
|
1911
|
+
/**
|
|
1912
|
+
* Remote: start one transport sign-in.
|
|
1913
|
+
*
|
|
1914
|
+
* The flow lives on the transports package's own key
|
|
1915
|
+
* (`llm-omp-transports/<route>`), so the grant it commits stays that route's
|
|
1916
|
+
* credential: nothing here harvests it into a pool row or a `llm-pi-ai`
|
|
1917
|
+
* projection. The attempt is otherwise the same record the pi-ai dialog
|
|
1918
|
+
* polls through `accountAttempt`, so the client reuses that dialog; an
|
|
1919
|
+
* authorized outcome is terminal here (the flow has already committed).
|
|
1920
|
+
*/
|
|
1921
|
+
async beginTransportLogin(route) {
|
|
1922
|
+
if (typeof route !== 'string' || route.length === 0)
|
|
1923
|
+
throw new TypeError('provider is required');
|
|
1924
|
+
const flow = this.transportFlowFor(route);
|
|
1925
|
+
if (flow === undefined)
|
|
1926
|
+
throw new Error(`Provider "${route}" offers no transport sign-in flow`);
|
|
1927
|
+
const pendingId = this.transportInFlight.get(route);
|
|
1928
|
+
if (pendingId !== undefined) {
|
|
1929
|
+
const pendingAttempt = this.loginAttempts.get(pendingId);
|
|
1930
|
+
if (pendingAttempt !== undefined && pendingAttempt.settledAt === undefined)
|
|
1931
|
+
return attemptView(pendingAttempt);
|
|
1932
|
+
this.transportInFlight.delete(route);
|
|
1933
|
+
}
|
|
1934
|
+
const now = Date.now();
|
|
1935
|
+
const attempt = {
|
|
1936
|
+
id: `la_${(++this.loginSequence).toString(36)}_${randomBytes(4).toString('hex')}`,
|
|
1937
|
+
provider: route,
|
|
1938
|
+
status: 'pending',
|
|
1939
|
+
notices: [],
|
|
1940
|
+
createdAt: now,
|
|
1941
|
+
updatedAt: now,
|
|
1942
|
+
controller: new AbortController(),
|
|
1943
|
+
};
|
|
1944
|
+
this.loginAttempts.set(attempt.id, attempt);
|
|
1945
|
+
this.capLoginAttempts();
|
|
1946
|
+
this.transportInFlight.set(route, attempt.id);
|
|
1947
|
+
const generation = this.transportGenerationOf(route);
|
|
1948
|
+
// `transportFlowFor` already proved the seam is mounted and the flow has a
|
|
1949
|
+
// method; the method id is the flow's own, not the pi-ai `'oauth'` literal.
|
|
1950
|
+
const settled = this.ctx.authorization.begin({
|
|
1951
|
+
key: flow.key,
|
|
1952
|
+
method: transportMethodOf(flow),
|
|
1953
|
+
interaction: this.attemptInteraction(attempt),
|
|
1954
|
+
signal: attempt.controller.signal,
|
|
1955
|
+
}).then(async (outcome) => {
|
|
1956
|
+
// A removal during the attempt bumped the route's epoch: the account
|
|
1957
|
+
// is gone, so a grant this flow wrote while being withdrawn must not
|
|
1958
|
+
// recreate it. The seam refuses a commit after the abort, but a commit
|
|
1959
|
+
// that had already started can still land.
|
|
1960
|
+
if (this.transportGenerationOf(route) !== generation) {
|
|
1961
|
+
await this.dropRemovedTransportGrant(flow.key);
|
|
1962
|
+
this.finishLoginAttempt(attempt, 'cancelled');
|
|
1963
|
+
return;
|
|
1964
|
+
}
|
|
1965
|
+
if (outcome.status === 'authorized') {
|
|
1966
|
+
this.finishLoginAttempt(attempt, 'ready');
|
|
1967
|
+
void this.refreshTransportAccounts();
|
|
1968
|
+
return;
|
|
1969
|
+
}
|
|
1970
|
+
this.finishLoginAttempt(attempt, 'cancelled');
|
|
1971
|
+
}, error => this.failLoginAttempt(attempt, error));
|
|
1972
|
+
// A removal awaits this, so it deletes the grant only after the withdrawn
|
|
1973
|
+
// attempt (and any late write it drops) is done. The route keeps its
|
|
1974
|
+
// pending entry until the seam's `begin` resolves: an earlier settle
|
|
1975
|
+
// (prune, release, cancel) must not hide a commit that is still in flight.
|
|
1976
|
+
attempt.settled = settled.then(() => undefined, () => undefined).finally(() => {
|
|
1977
|
+
if (this.transportInFlight.get(route) === attempt.id)
|
|
1978
|
+
this.transportInFlight.delete(route);
|
|
1979
|
+
});
|
|
1980
|
+
return attemptView(attempt);
|
|
1981
|
+
}
|
|
1982
|
+
/**
|
|
1983
|
+
* Remote: confirm one transport route's stored grant.
|
|
1984
|
+
*
|
|
1985
|
+
* A transport route has exactly one grant record and its adapter resolves
|
|
1986
|
+
* that record per request, so the stored grant already IS the route's live
|
|
1987
|
+
* credential. Activation is therefore a state-level re-read: it writes
|
|
1988
|
+
* nothing (in particular no `llm-pi-ai/<route>` projection, which the
|
|
1989
|
+
* transport route never consults) and reports the current status so the UI
|
|
1990
|
+
* can tell "active" from "expired" or "sign in again".
|
|
1991
|
+
*/
|
|
1992
|
+
async activateTransportAccount(route) {
|
|
1993
|
+
if (this.transportFlowFor(route) === undefined)
|
|
1994
|
+
throw new Error(`Provider "${route}" offers no transport account`);
|
|
1995
|
+
await this.refreshTransportAccounts();
|
|
1996
|
+
return this.buildState();
|
|
1997
|
+
}
|
|
1998
|
+
/**
|
|
1999
|
+
* Remote: drop one transport route's grant record.
|
|
2000
|
+
*
|
|
2001
|
+
* The record is the whole account — the transports package re-creates none —
|
|
2002
|
+
* so deleting it leaves the route signed out until the next sign-in. A live
|
|
2003
|
+
* sign-in is withdrawn first: left running, its commit would recreate the
|
|
2004
|
+
* grant this removal just deleted. No manager state is written, hence no
|
|
2005
|
+
* `run` queue and no state flush.
|
|
2006
|
+
*/
|
|
2007
|
+
async removeTransportAccount(route) {
|
|
2008
|
+
const flow = this.transportFlowFor(route);
|
|
2009
|
+
if (flow === undefined)
|
|
2010
|
+
throw new Error(`Provider "${route}" offers no transport account`);
|
|
2011
|
+
const records = this.credentialRecords();
|
|
2012
|
+
if (records === undefined)
|
|
2013
|
+
throw new Error('The credential store does not support records');
|
|
2014
|
+
const pendingId = this.transportInFlight.get(route);
|
|
2015
|
+
const pending = pendingId === undefined ? undefined : this.loginAttempts.get(pendingId);
|
|
2016
|
+
// Key on the seam promise, not `settledAt`: the prune timeout, a provider
|
|
2017
|
+
// release and a cancel settle the attempt record while `begin` is still
|
|
2018
|
+
// pending (the seam refuses a withdraw during a commit). The epoch bump
|
|
2019
|
+
// and the await must cover that window, or a late `authorized` recreates
|
|
2020
|
+
// the grant this removal deleted.
|
|
2021
|
+
if (pending !== undefined && pending.settled !== undefined) {
|
|
2022
|
+
// Bump the epoch before the abort: the attempt's settle path reads it to
|
|
2023
|
+
// know this removal owns the record, and this await covers that path.
|
|
2024
|
+
this.transportRemovals.set(route, this.transportGenerationOf(route) + 1);
|
|
2025
|
+
pending.controller.abort();
|
|
2026
|
+
await pending.settled;
|
|
2027
|
+
}
|
|
2028
|
+
await records.deleteRecord(flow.key);
|
|
2029
|
+
await this.refreshTransportAccounts();
|
|
2030
|
+
return this.buildState();
|
|
2031
|
+
}
|
|
2032
|
+
/** The route's removal epoch; absent means never removed. */
|
|
2033
|
+
transportGenerationOf(route) {
|
|
2034
|
+
return this.transportRemovals.get(route) ?? 0;
|
|
2035
|
+
}
|
|
2036
|
+
/**
|
|
2037
|
+
* Drop a grant a withdrawn sign-in wrote after its account was removed.
|
|
2038
|
+
*
|
|
2039
|
+
* The seam refuses a commit once the attempt's signal aborts, but a commit
|
|
2040
|
+
* that had already started can still land. The account was deleted, so that
|
|
2041
|
+
* write is deleted too instead of being reported as a signed-in route.
|
|
2042
|
+
*/
|
|
2043
|
+
async dropRemovedTransportGrant(key) {
|
|
2044
|
+
const records = this.credentialRecords();
|
|
2045
|
+
if (records === undefined)
|
|
2046
|
+
return;
|
|
2047
|
+
try {
|
|
2048
|
+
await records.deleteRecord(key);
|
|
2049
|
+
}
|
|
2050
|
+
catch (error) {
|
|
2051
|
+
this.warn(`transport account ${key} could not be dropped after its removal: ${error instanceof Error ? error.message : String(error)}`);
|
|
2052
|
+
}
|
|
2053
|
+
}
|
|
2054
|
+
/**
|
|
2055
|
+
* Detach the parked prompt. With a reason the flow's awaited promise is
|
|
2056
|
+
* rejected too — a parked prompt dropped un-rejected would leave the
|
|
2057
|
+
* seam's flow (and the provider's single-flight slot) wedged until its
|
|
2058
|
+
* own timeout, so every attempt-ending path passes one.
|
|
2059
|
+
*/
|
|
2060
|
+
dropPrompt(attempt, reason) {
|
|
2061
|
+
const reject = attempt.rejectPrompt;
|
|
2062
|
+
attempt.resolvePrompt = undefined;
|
|
2063
|
+
attempt.rejectPrompt = undefined;
|
|
2064
|
+
attempt.prompt = undefined;
|
|
2065
|
+
if (attempt.status === 'prompt')
|
|
2066
|
+
attempt.status = 'pending';
|
|
2067
|
+
attempt.updatedAt = Date.now();
|
|
2068
|
+
if (reject !== undefined && reason !== undefined)
|
|
2069
|
+
reject(reason);
|
|
2070
|
+
}
|
|
2071
|
+
/** Finalize an attempt with a terminal status exactly once. */
|
|
2072
|
+
finishLoginAttempt(attempt, status, error) {
|
|
2073
|
+
if (attempt.settledAt !== undefined)
|
|
2074
|
+
return;
|
|
2075
|
+
attempt.settledAt = Date.now();
|
|
2076
|
+
attempt.status = status;
|
|
2077
|
+
if (error !== undefined) {
|
|
1182
2078
|
attempt.error = error instanceof Error ? error.message : String(error);
|
|
1183
2079
|
}
|
|
1184
2080
|
this.dropPrompt(attempt, new Error(status === 'failed'
|
|
@@ -1211,6 +2107,11 @@ export class KeysManagerService {
|
|
|
1211
2107
|
* recorded at that same broadcast (R6-SF1): it names the projection whose
|
|
1212
2108
|
* failures own a cooldown, so the stale-window reset below clears only a
|
|
1213
2109
|
* window attributed to an OLDER projection.
|
|
2110
|
+
*
|
|
2111
|
+
* A harvest that finds the attempt already settled writes neither the
|
|
2112
|
+
* plugin-scope copy nor a row, but the projection the seam committed for
|
|
2113
|
+
* this attempt is still demoted (ownership-checked, I-000122): a
|
|
2114
|
+
* failed/cancelled login must not keep serving through the route.
|
|
1214
2115
|
*/
|
|
1215
2116
|
async harvestLogin(attempt, committed, projectionGeneration) {
|
|
1216
2117
|
// A revoke (provider release) may have finalized this attempt before
|
|
@@ -1304,11 +2205,20 @@ export class KeysManagerService {
|
|
|
1304
2205
|
// user sees is failed while the harvest would write the grant and
|
|
1305
2206
|
// the account row. Revalidate settlement immediately before the
|
|
1306
2207
|
// first credential write: a settled attempt must never grow a grant
|
|
1307
|
-
// copy or a row.
|
|
1308
|
-
//
|
|
1309
|
-
//
|
|
1310
|
-
|
|
2208
|
+
// copy or a row.
|
|
2209
|
+
//
|
|
2210
|
+
// I-000122: aborting the write is not enough. The seam committed
|
|
2211
|
+
// THIS attempt's grant as the provider's `llm-pi-ai` projection at
|
|
2212
|
+
// the settlement broadcast, so returning here would leave the route
|
|
2213
|
+
// serving a grant whose login failed/cancelled. Compensate that
|
|
2214
|
+
// projection through the same ownership-checked path the failure
|
|
2215
|
+
// handler uses: only this attempt's captured payload is deleted and
|
|
2216
|
+
// a foreign or newer live grant is never touched. The release
|
|
2217
|
+
// already published the terminal it owns.
|
|
2218
|
+
if (attempt.settledAt !== undefined) {
|
|
2219
|
+
await this.compensateFailedHarvest(attempt, harvest);
|
|
1311
2220
|
return;
|
|
2221
|
+
}
|
|
1312
2222
|
await records.modifyRecord(grantRef, () => Promise.resolve({ kind: 'grant', payload: structuredClone(payload) }));
|
|
1313
2223
|
facts.wroteRef = true;
|
|
1314
2224
|
const now = Date.now();
|
|
@@ -1431,6 +2341,9 @@ export class KeysManagerService {
|
|
|
1431
2341
|
// owns it until activation or fallback says otherwise.
|
|
1432
2342
|
if (attempt.accountId !== undefined)
|
|
1433
2343
|
this.claimProjectionOwner(attempt.provider, attempt.accountId);
|
|
2344
|
+
// The committed grant now serves this route, so any broker reference
|
|
2345
|
+
// carried over from the key pool must go.
|
|
2346
|
+
await this.syncBrokerBinding(attempt.provider);
|
|
1434
2347
|
// No terminal here: `run` flushes the state document after this
|
|
1435
2348
|
// callback resolves, and the ready terminal publishes below — only
|
|
1436
2349
|
// once persistence has actually resolved (round-2 review, step 3).
|
|
@@ -1477,6 +2390,10 @@ export class KeysManagerService {
|
|
|
1477
2390
|
else {
|
|
1478
2391
|
await this.compensateFailedHarvest(attempt, harvest);
|
|
1479
2392
|
}
|
|
2393
|
+
// The rollback restored the rows and the owner map, so the binding must
|
|
2394
|
+
// follow whatever serves the route now: this attempt's projection is
|
|
2395
|
+
// gone, and the surviving pool must serve again (Codex P2).
|
|
2396
|
+
await this.syncBindingAfterProjectionChange(attempt.provider);
|
|
1480
2397
|
}
|
|
1481
2398
|
}
|
|
1482
2399
|
/**
|
|
@@ -1664,6 +2581,30 @@ export class KeysManagerService {
|
|
|
1664
2581
|
* Resolves `ok: false` only when the compare/delete writes themselves
|
|
1665
2582
|
* rejected; an ownership mismatch is the guard doing its job (the live
|
|
1666
2583
|
* record belongs to someone else) and counts as a clean skip.
|
|
2584
|
+
*
|
|
2585
|
+
* Every manager-initiated projection deletion registers the one-shot
|
|
2586
|
+
* {@link intentionalProjectionRemovals} marker before the delete, exactly
|
|
2587
|
+
* as {@link decideAndDeleteProjection} does (I-000125). Unmarked, the
|
|
2588
|
+
* removal's own `credentials/record-updated` echo reconciles as an
|
|
2589
|
+
* external deletion: the route's previously active managed account is
|
|
2590
|
+
* flagged `valueMissing`, its owner claim is dropped and a misleading
|
|
2591
|
+
* "removed outside Keys Manager" warning is pushed — even though that
|
|
2592
|
+
* row and its plugin-scope grant are untouched.
|
|
2593
|
+
*
|
|
2594
|
+
* The marker only silences that echo; the route's state is corrected
|
|
2595
|
+
* explicitly. The removal leaves the route with no stored projection, so the
|
|
2596
|
+
* provider's owner claim — whatever it named before this attempt's grant
|
|
2597
|
+
* replaced the record — is untruthful: {@link oauthOwnerFor} keeps naming a
|
|
2598
|
+
* row pi-ai no longer resolves, `wrapStream` skips key-pool selection, oauth
|
|
2599
|
+
* failures are attributed to that row, and the client keeps "Set active"
|
|
2600
|
+
* disabled — the account cannot be re-activated. The claim is therefore
|
|
2601
|
+
* invalidated (I-000125), and ONLY the claim: the surviving row keeps its
|
|
2602
|
+
* `grantRef` and its `valueMissing: false`, so it stays managed and
|
|
2603
|
+
* re-activatable while the route falls back to its API-key pool.
|
|
2604
|
+
*
|
|
2605
|
+
* The marker is armed only while the provider is still managed: a released
|
|
2606
|
+
* provider has no echo consumer, and a marker nothing consumes would stay
|
|
2607
|
+
* armed for its whole TTL and mask a genuine out-of-band removal.
|
|
1667
2608
|
*/
|
|
1668
2609
|
async revertHarvestProjection(attempt, harvest) {
|
|
1669
2610
|
const records = this.credentialRecords();
|
|
@@ -1691,7 +2632,55 @@ export class KeysManagerService {
|
|
|
1691
2632
|
return undefined;
|
|
1692
2633
|
});
|
|
1693
2634
|
if (owns && await this.projectionStillMatches(records, projectionKey, decidedAgainst)) {
|
|
1694
|
-
|
|
2635
|
+
// The claim this removal makes untruthful, captured before the awaited
|
|
2636
|
+
// delete. A newer activation or login claiming the route while the
|
|
2637
|
+
// delete is in flight wins: its generation moves, and its claim names
|
|
2638
|
+
// whatever that writer actually put in place.
|
|
2639
|
+
const ownerBeforeRemoval = this.activeAccounts.get(attempt.provider);
|
|
2640
|
+
const generation = this.ownershipGenerations.get(attempt.provider) ?? 0;
|
|
2641
|
+
// Mark BEFORE the delete, same one-shot bookkeeping the demotion
|
|
2642
|
+
// path uses: the deletion's echo handler reads this marker across
|
|
2643
|
+
// its own awaited record re-read, so the removal stays silent for
|
|
2644
|
+
// this manager's own demotion instead of demoting the active
|
|
2645
|
+
// account.
|
|
2646
|
+
const markedAt = Date.now();
|
|
2647
|
+
// Only arm the marker where an echo can reach its consumer: the
|
|
2648
|
+
// dispatcher drops record-updates for an unmanaged provider, and a
|
|
2649
|
+
// marker nothing consumes would mask a genuine out-of-band removal for
|
|
2650
|
+
// its whole TTL.
|
|
2651
|
+
const providerManaged = this.store.provider(attempt.provider) !== undefined;
|
|
2652
|
+
if (providerManaged)
|
|
2653
|
+
this.intentionalProjectionRemovals.set(attempt.provider, markedAt);
|
|
2654
|
+
try {
|
|
2655
|
+
await records.deleteRecord(projectionKey);
|
|
2656
|
+
}
|
|
2657
|
+
catch (error) {
|
|
2658
|
+
// A delete that threw removed nothing, so no own-removal echo is
|
|
2659
|
+
// coming. Revoke this call's marker before the failure leaves this
|
|
2660
|
+
// method — left in place it would consume a genuine out-of-band
|
|
2661
|
+
// removal as the manager's own for the whole TTL, hiding the
|
|
2662
|
+
// warning, the missing mark, and the stale owner cleanup. The
|
|
2663
|
+
// identity check keeps a newer marker untouched.
|
|
2664
|
+
if (providerManaged && this.intentionalProjectionRemovals.get(attempt.provider) === markedAt) {
|
|
2665
|
+
this.intentionalProjectionRemovals.delete(attempt.provider);
|
|
2666
|
+
}
|
|
2667
|
+
throw error;
|
|
2668
|
+
}
|
|
2669
|
+
// The removal succeeded: the projection the rows were claimed against
|
|
2670
|
+
// is gone, so the provider's owner claim must go with it. The row is
|
|
2671
|
+
// deliberately left untouched — same `grantRef`, still `valueMissing:
|
|
2672
|
+
// false` — because it is still managed and its grant is still live;
|
|
2673
|
+
// re-activating it re-projects the route.
|
|
2674
|
+
if (ownerBeforeRemoval !== undefined
|
|
2675
|
+
&& (this.ownershipGenerations.get(attempt.provider) ?? 0) === generation) {
|
|
2676
|
+
this.activeAccounts.delete(attempt.provider);
|
|
2677
|
+
this.store.touch();
|
|
2678
|
+
}
|
|
2679
|
+
// The projection the pool was hidden behind is gone, so the route
|
|
2680
|
+
// falls back to whatever still serves it: rebind its pool here rather
|
|
2681
|
+
// than leaving the profile unbound for pi-ai to bypass the broker
|
|
2682
|
+
// (PR review round 2 must-fix).
|
|
2683
|
+
await this.syncBindingAfterProjectionChange(attempt.provider);
|
|
1695
2684
|
}
|
|
1696
2685
|
else if (!owns) {
|
|
1697
2686
|
this.warn(`harvest compensation kept the projection for ${attempt.provider}: the live grant no longer matches this attempt's committed payload`);
|
|
@@ -1788,6 +2777,16 @@ export class KeysManagerService {
|
|
|
1788
2777
|
* is still current before committing.
|
|
1789
2778
|
*/
|
|
1790
2779
|
onGrantRecordUpdated(key) {
|
|
2780
|
+
// A transport route's record is another package's credential: a write or
|
|
2781
|
+
// removal there (its adapter re-authenticating, or a sign-in on another
|
|
2782
|
+
// surface) only re-reads the account list. It must never reach the pi-ai
|
|
2783
|
+
// reconciliation path below, which would look for a pool row that does not
|
|
2784
|
+
// exist and warn about native management of a route this manager does not
|
|
2785
|
+
// own.
|
|
2786
|
+
if (key.startsWith(`${TRANSPORT_CREDENTIAL_SCOPE}/`)) {
|
|
2787
|
+
void this.refreshTransportAccounts();
|
|
2788
|
+
return;
|
|
2789
|
+
}
|
|
1791
2790
|
if (!key.startsWith(`${OAUTH_GRANT_RECORD_SCOPE}/`))
|
|
1792
2791
|
return;
|
|
1793
2792
|
const provider = key.slice(OAUTH_GRANT_RECORD_SCOPE.length + 1);
|
|
@@ -1830,6 +2829,17 @@ export class KeysManagerService {
|
|
|
1830
2829
|
* removal is allowed to complete while the write is in flight.
|
|
1831
2830
|
*/
|
|
1832
2831
|
async reconcileGrantRecord(provider, key) {
|
|
2832
|
+
// PR review round 30: the shutdown drain abandons a chain that has not
|
|
2833
|
+
// settled, and this body resumes after `close()`. Its whole result is a
|
|
2834
|
+
// credential write-back plus a binding sync, so persistence is refused
|
|
2835
|
+
// once shutdown started.
|
|
2836
|
+
if (this.closing)
|
|
2837
|
+
return;
|
|
2838
|
+
// PR review round 32: a remount replaces the credential backend while this
|
|
2839
|
+
// body awaits its read, and the answer then describes records the new store
|
|
2840
|
+
// does not hold. The epoch gates every decision below — owner, foreign
|
|
2841
|
+
// marker, readiness and the binding sync — exactly like the hydration pass.
|
|
2842
|
+
const epoch = this.hydrationEpoch;
|
|
1833
2843
|
const records = this.credentialRecords();
|
|
1834
2844
|
if (records === undefined)
|
|
1835
2845
|
return;
|
|
@@ -1837,8 +2847,6 @@ export class KeysManagerService {
|
|
|
1837
2847
|
if (providerRecord === undefined)
|
|
1838
2848
|
return;
|
|
1839
2849
|
const accounts = providerRecord.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null);
|
|
1840
|
-
if (accounts.length === 0)
|
|
1841
|
-
return;
|
|
1842
2850
|
const activeId = this.activeAccounts.get(provider);
|
|
1843
2851
|
const active = activeId === undefined
|
|
1844
2852
|
? undefined
|
|
@@ -1848,7 +2856,29 @@ export class KeysManagerService {
|
|
|
1848
2856
|
// the same generation guard load-time hydration applies.
|
|
1849
2857
|
const generation = this.ownershipGenerations.get(provider) ?? 0;
|
|
1850
2858
|
const record = await records.readRecord(key);
|
|
2859
|
+
if (this.hydrationEpoch !== epoch)
|
|
2860
|
+
return;
|
|
2861
|
+
// PR review round 35: shutdown abandons this chain, so the body resumes
|
|
2862
|
+
// here after `close()`. The read may finish; the writes below may not.
|
|
2863
|
+
if (this.persistenceRefused())
|
|
2864
|
+
return;
|
|
2865
|
+
// PR review round 13: this read is authoritative for the provider's
|
|
2866
|
+
// projection, so it answers what a failed startup hydration could not —
|
|
2867
|
+
// ownership stops being unknown and the binding repair it skipped runs.
|
|
2868
|
+
this.noteProjectionReadSucceeded(provider);
|
|
1851
2869
|
if (record === undefined) {
|
|
2870
|
+
// PR review round 31: a harvest outside this chain can commit and claim
|
|
2871
|
+
// a newer projection while this read is in flight, so the absence above
|
|
2872
|
+
// can already be superseded. The newer claim owns the route and settled
|
|
2873
|
+
// the binding itself: this stale removal decides nothing — no owner
|
|
2874
|
+
// delete, no rebind — it only consumes its own one-shot removal marker.
|
|
2875
|
+
if ((this.ownershipGenerations.get(provider) ?? 0) !== generation) {
|
|
2876
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
2877
|
+
return;
|
|
2878
|
+
}
|
|
2879
|
+
// The projection is gone, so nothing foreign serves this route any more
|
|
2880
|
+
// (PR review round 3).
|
|
2881
|
+
this.foreignProjections.delete(provider);
|
|
1852
2882
|
// This manager's own demotion marked the removal just before
|
|
1853
2883
|
// deleting; while the marker is inside its TTL window the echo is
|
|
1854
2884
|
// ours — consume the one-shot marker and stay silent. An expired
|
|
@@ -1856,6 +2886,11 @@ export class KeysManagerService {
|
|
|
1856
2886
|
const markedAt = this.intentionalProjectionRemovals.get(provider);
|
|
1857
2887
|
if (markedAt !== undefined && Date.now() - markedAt <= INTENTIONAL_REMOVAL_TTL_MS) {
|
|
1858
2888
|
this.intentionalProjectionRemovals.delete(provider);
|
|
2889
|
+
// The manager's own removal left no projection: converge the binding
|
|
2890
|
+
// with the state the removal created, so the surviving pool serves
|
|
2891
|
+
// through the broker instead of pi-ai's bare credential (PR review
|
|
2892
|
+
// round 2).
|
|
2893
|
+
await this.syncBindingAfterProjectionChange(provider);
|
|
1859
2894
|
return;
|
|
1860
2895
|
}
|
|
1861
2896
|
if (markedAt !== undefined)
|
|
@@ -1870,20 +2905,52 @@ export class KeysManagerService {
|
|
|
1870
2905
|
this.store.touch();
|
|
1871
2906
|
this.pushWarning(`The ${provider} OAuth grant was removed outside Keys Manager; the route now resolves through its API-key pool.`);
|
|
1872
2907
|
}
|
|
2908
|
+
// The row is demoted (or was never the owner) and nothing projects the
|
|
2909
|
+
// route, so the binding must name the key pool now — an unbound profile
|
|
2910
|
+
// here read as a mismatch. This runs for EVERY confirmed deletion, not
|
|
2911
|
+
// only an owned one (PR review round 3): a native writer can replace the
|
|
2912
|
+
// managed projection first and delete it later, leaving the pool to
|
|
2913
|
+
// serve an unbound profile.
|
|
2914
|
+
await this.syncBindingAfterProjectionChange(provider);
|
|
1873
2915
|
return;
|
|
1874
2916
|
}
|
|
2917
|
+
// I-000125 marker hygiene: the removal branch above is the marker's ONLY
|
|
2918
|
+
// consumer, so nothing may return before it. A provider that holds no
|
|
2919
|
+
// OAuth row cannot be demoted (nothing to mark and nobody to warn), but
|
|
2920
|
+
// its own removal echo still lands here — returning first would leave the
|
|
2921
|
+
// marker armed for its whole TTL and mask a genuine out-of-band removal of
|
|
2922
|
+
// the projection that serves a row added inside that window.
|
|
2923
|
+
// PR review round 19 (I-000182): the classification below runs for a
|
|
2924
|
+
// key-only provider too. An early return on `accounts.length === 0` left
|
|
2925
|
+
// the foreign marker unset while the native api-key / unrecognized record
|
|
2926
|
+
// still served the route ahead of the broker, so the queued binding repair
|
|
2927
|
+
// treated the pool as the serving source and bound/kept the broker
|
|
2928
|
+
// reference over it. With no OAuth row the unowned-grant branch below
|
|
2929
|
+
// classifies a valid grant the same way.
|
|
1875
2930
|
if (record.kind === 'api-key') {
|
|
1876
2931
|
// Native api-key management took the route over: the manager no
|
|
1877
2932
|
// longer owns any projection here.
|
|
1878
2933
|
this.activeAccounts.delete(provider);
|
|
2934
|
+
// A live record Keys Manager does not own still serves ahead of the
|
|
2935
|
+
// broker reference, so the pool is not the route's source (round 3).
|
|
2936
|
+
this.foreignProjections.add(provider);
|
|
2937
|
+
// A pending hydration read of the previous owner is now stale: the
|
|
2938
|
+
// route serves a record this pool does not own (round 40).
|
|
2939
|
+
this.noteProjectionCommitted(provider);
|
|
1879
2940
|
this.pushWarning(`The ${provider} route now holds a natively managed API-key credential; the Keys Manager account pool is not serving it.`);
|
|
2941
|
+
// pi-ai resolves the broker reference before this record, so a route
|
|
2942
|
+
// left bound would keep serving the pool (PR review round 4).
|
|
2943
|
+
await this.syncBindingAfterProjectionChange(provider);
|
|
1880
2944
|
return;
|
|
1881
2945
|
}
|
|
1882
2946
|
if (!isOAuthGrantPayload(record.payload)) {
|
|
1883
2947
|
// An unrecognized payload is not ours to interpret: drop the stale
|
|
1884
2948
|
// owner entry so failure attribution and demotion never fire on it.
|
|
1885
2949
|
this.activeAccounts.delete(provider);
|
|
2950
|
+
this.foreignProjections.add(provider);
|
|
2951
|
+
this.noteProjectionCommitted(provider);
|
|
1886
2952
|
this.pushWarning(`The ${provider} OAuth record holds an unrecognized payload; Keys Manager left it untouched.`);
|
|
2953
|
+
await this.syncBindingAfterProjectionChange(provider);
|
|
1887
2954
|
return;
|
|
1888
2955
|
}
|
|
1889
2956
|
const payload = record.payload;
|
|
@@ -1895,9 +2962,13 @@ export class KeysManagerService {
|
|
|
1895
2962
|
const owner = matchOwningAccount(provider, accounts, payload);
|
|
1896
2963
|
if (owner === undefined) {
|
|
1897
2964
|
// A foreign account's grant serves the route: no pool row owns the
|
|
1898
|
-
// projection, so the previous owner entry must not survive it
|
|
2965
|
+
// projection, so the previous owner entry must not survive it — and the
|
|
2966
|
+
// live record keeps the pool from claiming the route (round 3).
|
|
1899
2967
|
this.activeAccounts.delete(provider);
|
|
2968
|
+
this.foreignProjections.add(provider);
|
|
2969
|
+
this.noteProjectionCommitted(provider);
|
|
1900
2970
|
this.pushWarning(`An OAuth account for ${provider} is managed natively outside Keys Manager and is not part of the account pool.`);
|
|
2971
|
+
await this.syncBindingAfterProjectionChange(provider);
|
|
1901
2972
|
return;
|
|
1902
2973
|
}
|
|
1903
2974
|
// C6-1 pre-write revalidation: removeAccount / releaseProvider run on
|
|
@@ -1958,6 +3029,15 @@ export class KeysManagerService {
|
|
|
1958
3029
|
// above only reconciles llm-pi-ai/-scope keys.
|
|
1959
3030
|
const previousRef = owner.oauth.grantRef;
|
|
1960
3031
|
await records.modifyRecord(rotatedRef, () => Promise.resolve({ kind: 'grant', payload: structuredClone(payload) }));
|
|
3032
|
+
// PR review round 32: the write above landed in the store this body started
|
|
3033
|
+
// on; a remount since then makes every following decision (the owner
|
|
3034
|
+
// cleanup, the generation commit) describe the replaced backend.
|
|
3035
|
+
if (this.hydrationEpoch !== epoch)
|
|
3036
|
+
return;
|
|
3037
|
+
// PR review round 35: the write above is the last one this body may start
|
|
3038
|
+
// once shutdown began; the compensation and commit below are refused here.
|
|
3039
|
+
if (this.persistenceRefused())
|
|
3040
|
+
return;
|
|
1961
3041
|
// C6-1 post-write revalidation: a removal that landed while the write
|
|
1962
3042
|
// was in flight already deleted its own record, so the generation just
|
|
1963
3043
|
// written must be compensated or it outlives the account. The removed
|
|
@@ -1982,6 +3062,14 @@ export class KeysManagerService {
|
|
|
1982
3062
|
// distinct record is deleted: in the same-ref case the write landed on
|
|
1983
3063
|
// the row's own current record and must stay.
|
|
1984
3064
|
const stillCurrent = await this.rotationStillCurrent(records, provider, key, rotatedRef, claimedGeneration);
|
|
3065
|
+
// PR review round 32: the staleness probe is an await; the store may have
|
|
3066
|
+
// been replaced under it.
|
|
3067
|
+
if (this.hydrationEpoch !== epoch)
|
|
3068
|
+
return;
|
|
3069
|
+
// PR review round 35: the probe is another await; a shutdown that started
|
|
3070
|
+
// under it refuses the discard/compensation below.
|
|
3071
|
+
if (this.persistenceRefused())
|
|
3072
|
+
return;
|
|
1985
3073
|
if (!stillCurrent) {
|
|
1986
3074
|
if (rotatedRef !== previousRef) {
|
|
1987
3075
|
// R5-MF1: the staleness probe is awaited, and a settled harvest can
|
|
@@ -2007,28 +3095,24 @@ export class KeysManagerService {
|
|
|
2007
3095
|
}
|
|
2008
3096
|
return;
|
|
2009
3097
|
}
|
|
2010
|
-
// SF2: once the new payload persists
|
|
2011
|
-
// the superseded generation's plugin-scope record
|
|
2012
|
-
// rotated refresh token must not strand its secret-bearing
|
|
2013
|
-
// outside manager state; cleanup is best-effort with a warning,
|
|
2014
|
-
// exactly like the harvest-time cleanup.
|
|
2015
|
-
|
|
2016
|
-
|
|
2017
|
-
|
|
2018
|
-
|
|
2019
|
-
|
|
2020
|
-
|
|
2021
|
-
|
|
2022
|
-
|
|
2023
|
-
|
|
2024
|
-
//
|
|
2025
|
-
|
|
2026
|
-
|
|
2027
|
-
|
|
2028
|
-
...(payload.accountId === undefined ? {} : { accountId: payload.accountId }),
|
|
2029
|
-
...(typeof payload.expires === 'number' && Number.isFinite(payload.expires) ? { expiresAt: payload.expires } : {}),
|
|
2030
|
-
};
|
|
2031
|
-
this.store.touch();
|
|
3098
|
+
// SF2: once the new payload persists, the commit repoints the row at it
|
|
3099
|
+
// and deletes the superseded generation's plugin-scope record (harvest
|
|
3100
|
+
// parity). A rotated refresh token must not strand its secret-bearing
|
|
3101
|
+
// record outside manager state; cleanup is best-effort with a warning,
|
|
3102
|
+
// exactly like the harvest-time cleanup. QC N1: the ref scheme, identity
|
|
3103
|
+
// merge and cleanup are the one write-back owned by
|
|
3104
|
+
// {@link commitGrantGeneration}, shared with the connection test.
|
|
3105
|
+
// PR review round 35: the commit is this body's persisted state write; a
|
|
3106
|
+
// shutdown that began under any of the awaits above refuses it.
|
|
3107
|
+
if (this.persistenceRefused())
|
|
3108
|
+
return;
|
|
3109
|
+
await this.commitGrantGeneration(provider, owner, payload, rotatedRef);
|
|
3110
|
+
// PR review round 44: the projection now serves this managed account again,
|
|
3111
|
+
// so a profile that still names the broker would make pi-ai resolve the
|
|
3112
|
+
// pool first and fail with MISSING_CREDENTIAL. Converge the binding with the
|
|
3113
|
+
// serving source, exactly like activation and harvest do — off the grant
|
|
3114
|
+
// chain, which this body runs on (a settings write inside it would re-enter).
|
|
3115
|
+
void this.syncBindingAfterProjectionChange(provider);
|
|
2032
3116
|
}
|
|
2033
3117
|
/**
|
|
2034
3118
|
* C6-1 revalidation: the row captured before an awaited credential write
|
|
@@ -2184,6 +3268,10 @@ export class KeysManagerService {
|
|
|
2184
3268
|
this.intentionalProjectionRemovals.delete(provider);
|
|
2185
3269
|
await records.modifyRecord(`${OAUTH_GRANT_RECORD_SCOPE}/${provider}`, () => Promise.resolve({ kind: 'grant', payload: structuredClone(stored.payload) }));
|
|
2186
3270
|
this.claimProjectionOwner(provider, row.id);
|
|
3271
|
+
// The re-activated grant is the serving credential again, so a broker
|
|
3272
|
+
// reference left by the key pool must go (see syncBrokerBinding): the
|
|
3273
|
+
// committed projection is not rolled back by a settings failure.
|
|
3274
|
+
await this.syncBrokerBinding(provider);
|
|
2187
3275
|
});
|
|
2188
3276
|
row.valueMissing = false;
|
|
2189
3277
|
// Round-5 finding M1: manual recovery ends the cooling window too, the
|
|
@@ -2200,6 +3288,1389 @@ export class KeysManagerService {
|
|
|
2200
3288
|
return this.buildState();
|
|
2201
3289
|
});
|
|
2202
3290
|
}
|
|
3291
|
+
/**
|
|
3292
|
+
* Remote: send one minimal request with the selected key or account row.
|
|
3293
|
+
*
|
|
3294
|
+
* Observation-only by construction: the request runs under a test-only
|
|
3295
|
+
* request scope, so the broker answers with the pinned credential and the
|
|
3296
|
+
* stream listener skips selection and every health/binding write. An OAuth
|
|
3297
|
+
* row's grant is projected only for the duration of the request; the
|
|
3298
|
+
* projection (and the in-memory owner) that served the route before is
|
|
3299
|
+
* restored even when the request fails, so a test never changes which
|
|
3300
|
+
* account the route is on.
|
|
3301
|
+
*
|
|
3302
|
+
* Every expected refusal is a structured `{ ok: false, error }` result
|
|
3303
|
+
* (M2): the inline row line is the only failure surface, so a refusal must
|
|
3304
|
+
* not also raise the page-level banner a thrown RPC error produces.
|
|
3305
|
+
*/
|
|
3306
|
+
async testKey(provider, keyId) {
|
|
3307
|
+
// PR review round 34: shutdown marks its entry before it restores the
|
|
3308
|
+
// outstanding borrows, and that restoration is not in this test's window —
|
|
3309
|
+
// so a test arriving now would borrow during it and outlive it.
|
|
3310
|
+
if (this.shutdownRequested)
|
|
3311
|
+
return this.shutdownEntryResult();
|
|
3312
|
+
// PR review round 12: a removal holds the provider from its entry, before
|
|
3313
|
+
// its first await, so a test cannot start in the window this check would
|
|
3314
|
+
// otherwise miss — read the grant and the projection while the removal is
|
|
3315
|
+
// deleting them, then re-project the removed account.
|
|
3316
|
+
if (this.removalsInProgress.has(provider)) {
|
|
3317
|
+
return {
|
|
3318
|
+
ok: false,
|
|
3319
|
+
latencyMs: 0,
|
|
3320
|
+
error: `Provider "${provider}" is being removed; run the connection test again once the removal finishes.`,
|
|
3321
|
+
};
|
|
3322
|
+
}
|
|
3323
|
+
const record = this.store.provider(provider);
|
|
3324
|
+
if (record === undefined)
|
|
3325
|
+
return { ok: false, latencyMs: 0, error: `Provider "${provider}" is not managed` };
|
|
3326
|
+
const row = record.keys.find(candidate => candidate.id === keyId);
|
|
3327
|
+
if (row === undefined)
|
|
3328
|
+
return { ok: false, latencyMs: 0, error: `Key "${keyId}" not found on ${provider}` };
|
|
3329
|
+
// QC fix wave 2: an abandoned window owns this provider until its cleanup
|
|
3330
|
+
// completed. Refused before the running-test check below, because the
|
|
3331
|
+
// abandoned test keeps that entry on purpose: the route's projection is
|
|
3332
|
+
// still unreconciled, so a second test would borrow it twice.
|
|
3333
|
+
if (this.cleaningTests.has(provider)) {
|
|
3334
|
+
// QC fix wave 3 (W-002): the fence must be able to clear itself, so this
|
|
3335
|
+
// attempt first runs one bounded cleanup of the abandoned window instead
|
|
3336
|
+
// of refusing on it forever. A failed attempt still refuses; a successful
|
|
3337
|
+
// one releases the isolation and the requested test proceeds.
|
|
3338
|
+
if (!await this.recoverAbandonedWindow(provider)) {
|
|
3339
|
+
return { ok: false, latencyMs: 0, error: 'A previous test is still being cleaned up' };
|
|
3340
|
+
}
|
|
3341
|
+
}
|
|
3342
|
+
// PR review round 15: the recovery above awaits the abandoned window's
|
|
3343
|
+
// cleanup, and that cleanup is what lifts the fence a removal's own
|
|
3344
|
+
// precheck reads. A removal beginning inside that await therefore passes
|
|
3345
|
+
// its precheck (no fence, no registered test), parks in its credential
|
|
3346
|
+
// deletion, and this attempt would register, read the still-present grant
|
|
3347
|
+
// and re-project the account the removal is deleting. The entry check
|
|
3348
|
+
// cannot cover the await, so the markers are read again here.
|
|
3349
|
+
if (this.removalsInProgress.has(provider) || this.retiringAccounts.has(row)) {
|
|
3350
|
+
return {
|
|
3351
|
+
ok: false,
|
|
3352
|
+
latencyMs: 0,
|
|
3353
|
+
error: `Provider "${provider}" is being removed; run the connection test again once the removal finishes.`,
|
|
3354
|
+
};
|
|
3355
|
+
}
|
|
3356
|
+
// Concurrent tests would race the projection save/restore; one at a time
|
|
3357
|
+
// per provider is enough for a per-row action and keeps the restore exact
|
|
3358
|
+
// (QC N7: the lock covers both row kinds, the isolation guard only the
|
|
3359
|
+
// kind that actually borrows).
|
|
3360
|
+
if (this.testsActive.has(provider)) {
|
|
3361
|
+
return { ok: false, latencyMs: 0, error: 'A test for this provider is already running' };
|
|
3362
|
+
}
|
|
3363
|
+
// Everything that awaits — the ownership readiness wait (QC B6) and the
|
|
3364
|
+
// request — belongs to the guarded promise, so a second call cannot slip
|
|
3365
|
+
// past the lock during the wait.
|
|
3366
|
+
const window = {
|
|
3367
|
+
provider,
|
|
3368
|
+
row,
|
|
3369
|
+
borrowBinding: false,
|
|
3370
|
+
releasedBinding: false,
|
|
3371
|
+
mutations: [],
|
|
3372
|
+
cleaned: false,
|
|
3373
|
+
settled: false,
|
|
3374
|
+
held: false,
|
|
3375
|
+
lateReconcileRequested: false,
|
|
3376
|
+
};
|
|
3377
|
+
// PR review round 21: the request is over when the guarded promise settles,
|
|
3378
|
+
// and that is what a held cleanup waits for. The marker is registered before
|
|
3379
|
+
// any settlement listener (the abandon retry), so that retry observes a
|
|
3380
|
+
// settled window instead of re-holding it forever.
|
|
3381
|
+
const test = this.runRowTest(record, row, window).finally(() => {
|
|
3382
|
+
window.settled = true;
|
|
3383
|
+
});
|
|
3384
|
+
window.test = test;
|
|
3385
|
+
this.testsActive.set(provider, test);
|
|
3386
|
+
// PR review round 33: registered with the lock, because shutdown restores
|
|
3387
|
+
// this window's borrow even when the test promise never settles.
|
|
3388
|
+
this.testWindows.set(provider, window);
|
|
3389
|
+
// The guard holds the whole test as a promise, not just a membership
|
|
3390
|
+
// flag: `streamListener` awaits it so no ordinary request is served by —
|
|
3391
|
+
// or attributed to — the grant an OAuth row test has borrowed. An api-key
|
|
3392
|
+
// test borrows nothing, so it never isolates ordinary traffic (QC N7).
|
|
3393
|
+
if (row.kind === 'oauth')
|
|
3394
|
+
this.testsInFlight.set(provider, test);
|
|
3395
|
+
// QC B3 + QC fix wave 2: this watchdog, not only the finally below, ends a
|
|
3396
|
+
// test wedged in an untimed seam call — and it ends the window the way the
|
|
3397
|
+
// test itself would, because releasing the provider without putting the
|
|
3398
|
+
// borrowed projection and binding back hands ordinary traffic the tested
|
|
3399
|
+
// account's credential.
|
|
3400
|
+
const watchdog = setTimeout(() => {
|
|
3401
|
+
this.abandonTestWindow(window);
|
|
3402
|
+
}, this.isolationWaitMs);
|
|
3403
|
+
watchdog.unref?.();
|
|
3404
|
+
try {
|
|
3405
|
+
return await test;
|
|
3406
|
+
}
|
|
3407
|
+
finally {
|
|
3408
|
+
clearTimeout(watchdog);
|
|
3409
|
+
if (this.testsActive.get(provider) === test)
|
|
3410
|
+
this.testsActive.delete(provider);
|
|
3411
|
+
if (this.testWindows.get(provider)?.test === test)
|
|
3412
|
+
this.testWindows.delete(provider);
|
|
3413
|
+
// PR review MF1: a window whose cleanup could not put the route back
|
|
3414
|
+
// (normal path or watchdog) keeps its isolation entry until a recovery
|
|
3415
|
+
// attempt succeeds, so ordinary traffic is never served by the borrowed
|
|
3416
|
+
// grant while the fence is up.
|
|
3417
|
+
if (this.testsInFlight.get(provider) === test && !this.cleaningTests.has(provider))
|
|
3418
|
+
this.testsInFlight.delete(provider);
|
|
3419
|
+
}
|
|
3420
|
+
}
|
|
3421
|
+
/**
|
|
3422
|
+
* Take over a test window that did not settle within the isolation deadline
|
|
3423
|
+
* (QC fix wave 2) or whose own cleanup could not restore the route (PR
|
|
3424
|
+
* review MF1).
|
|
3425
|
+
*
|
|
3426
|
+
* The fence is set synchronously, before the first cleanup await: the wedged
|
|
3427
|
+
* test may resume at any await from here, and it must observe the
|
|
3428
|
+
* abandonment before it starts a borrow or restores from its own stale
|
|
3429
|
+
* snapshot. The provider keeps its isolation entry until the cleanup
|
|
3430
|
+
* completed — see {@link settleAbandonedWindow}.
|
|
3431
|
+
*/
|
|
3432
|
+
abandonTestWindow(window, origin = 'deadline') {
|
|
3433
|
+
// One cleanup owner per window: the watchdog and the normal path can both
|
|
3434
|
+
// observe the same failure, and a second owner would put two cleanups on
|
|
3435
|
+
// one route.
|
|
3436
|
+
if (window.cleaned)
|
|
3437
|
+
return;
|
|
3438
|
+
window.cleaned = true;
|
|
3439
|
+
this.cleaningTests.add(window.provider);
|
|
3440
|
+
this.abandonedWindows.set(window.provider, window);
|
|
3441
|
+
void this.settleAbandonedWindow(window, origin);
|
|
3442
|
+
}
|
|
3443
|
+
/**
|
|
3444
|
+
* Clean up one abandoned window and release its provider only on success.
|
|
3445
|
+
*
|
|
3446
|
+
* A cleanup that cannot finish needs the same untimed seam the wedged test is
|
|
3447
|
+
* parked in, so it fails exactly like the test did. Releasing the provider
|
|
3448
|
+
* then would serve ordinary traffic from the borrowed grant (or, on a
|
|
3449
|
+
* broker-bound profile, from a route whose binding was released and never
|
|
3450
|
+
* restored), so the isolation entry and the fence both stay. Two events can
|
|
3451
|
+
* retry the cleanup: the abandoned promise's settlement, and the next test
|
|
3452
|
+
* attempt ({@link recoverAbandonedWindow}, QC fix wave 3 W-002) — a promise
|
|
3453
|
+
* that settles while the seam is still wedged must not be the only path.
|
|
3454
|
+
*/
|
|
3455
|
+
async settleAbandonedWindow(window, origin) {
|
|
3456
|
+
const failure = await this.attemptAbandonedCleanup(window);
|
|
3457
|
+
const cause = origin === 'deadline'
|
|
3458
|
+
? `did not settle within ${this.isolationWaitMs}ms`
|
|
3459
|
+
: 'could not be cleaned up when it finished';
|
|
3460
|
+
if (failure === undefined && !window.held) {
|
|
3461
|
+
this.warn(`connection test for ${window.provider} ${cause}; its borrowed credential was put back and the provider slot is released`);
|
|
3462
|
+
return;
|
|
3463
|
+
}
|
|
3464
|
+
if (failure === undefined) {
|
|
3465
|
+
// PR review round 21: the route is back, but the request that borrowed it
|
|
3466
|
+
// is still running and can refresh and rewrite that projection. The
|
|
3467
|
+
// provider stays isolated until the settlement reconciles the late write
|
|
3468
|
+
// and releases it.
|
|
3469
|
+
this.warn(`connection test for ${window.provider} ${cause}; its borrowed credential was put back, but its request has not settled, so the provider stays isolated until it does`);
|
|
3470
|
+
}
|
|
3471
|
+
else {
|
|
3472
|
+
this.warn(`connection test for ${window.provider} ${cause} and its cleanup did not complete (${failure}); the provider stays isolated`);
|
|
3473
|
+
}
|
|
3474
|
+
const pending = window.test;
|
|
3475
|
+
if (pending === undefined)
|
|
3476
|
+
return;
|
|
3477
|
+
const retry = () => { void this.retryAbandonedWindow(window); };
|
|
3478
|
+
void pending.then(retry, retry);
|
|
3479
|
+
}
|
|
3480
|
+
/** One later cleanup attempt for an abandoned window, once the seam answers again. */
|
|
3481
|
+
async retryAbandonedWindow(window) {
|
|
3482
|
+
if (!this.cleaningTests.has(window.provider))
|
|
3483
|
+
return;
|
|
3484
|
+
if (this.abandonedWindows.get(window.provider) !== window)
|
|
3485
|
+
return;
|
|
3486
|
+
await this.recoverAbandonedWindow(window.provider, 'settled');
|
|
3487
|
+
}
|
|
3488
|
+
/**
|
|
3489
|
+
* One bounded cleanup attempt for a fenced provider (QC fix wave 3, W-002).
|
|
3490
|
+
*
|
|
3491
|
+
* Returns whether the provider is servable again. The caller's attempt is
|
|
3492
|
+
* what makes the fence self-clearing: `testKey` calls this before refusing,
|
|
3493
|
+
* so isolation no longer depends on the abandoned promise settling after the
|
|
3494
|
+
* seam recovered. A failed attempt keeps the fence and warns exactly once.
|
|
3495
|
+
*/
|
|
3496
|
+
async recoverAbandonedWindow(provider, origin = 'next-attempt') {
|
|
3497
|
+
const window = this.abandonedWindows.get(provider);
|
|
3498
|
+
if (window === undefined)
|
|
3499
|
+
return !this.cleaningTests.has(provider);
|
|
3500
|
+
const failure = await this.attemptAbandonedCleanup(window);
|
|
3501
|
+
if (failure === undefined && !window.held) {
|
|
3502
|
+
this.warn(origin === 'settled'
|
|
3503
|
+
? `the abandoned connection test for ${provider} settled; its borrowed credential was put back and the provider slot is released`
|
|
3504
|
+
: `the abandoned connection test for ${provider} was cleaned up on the next attempt; its borrowed credential was put back and the provider slot is released`);
|
|
3505
|
+
return true;
|
|
3506
|
+
}
|
|
3507
|
+
if (failure === undefined) {
|
|
3508
|
+
// PR review round 21: the cleanup put the route back, but the request it
|
|
3509
|
+
// borrowed it for is still running and can rewrite the projection, so the
|
|
3510
|
+
// provider is not servable yet.
|
|
3511
|
+
this.warn(`the abandoned connection test for ${provider} was cleaned up, but its request has not settled; the provider stays isolated until it does`);
|
|
3512
|
+
return false;
|
|
3513
|
+
}
|
|
3514
|
+
this.warn(`the abandoned connection test for ${provider} is still fenced (${failure}); the provider stays isolated`);
|
|
3515
|
+
return false;
|
|
3516
|
+
}
|
|
3517
|
+
/**
|
|
3518
|
+
* Run the window's cleanup under its hard deadline and drop the provider's
|
|
3519
|
+
* isolation only when it completed. Returns the reason it could not finish.
|
|
3520
|
+
*
|
|
3521
|
+
* PR review round 21: the request that borrowed the route is not part of the
|
|
3522
|
+
* cleanup, so a cleanup can finish while that request is still able to
|
|
3523
|
+
* refresh and rewrite the borrowed projection. Such a window is held
|
|
3524
|
+
* ({@link RowTestWindow.held}) — the fence and the isolation entries stay —
|
|
3525
|
+
* and only its settlement reconciles the late write and releases it.
|
|
3526
|
+
*/
|
|
3527
|
+
async attemptAbandonedCleanup(window) {
|
|
3528
|
+
const provider = window.provider;
|
|
3529
|
+
const attempt = (this.cleanupAttempts.get(provider) ?? 0) + 1;
|
|
3530
|
+
this.cleanupAttempts.set(provider, attempt);
|
|
3531
|
+
const stillCurrent = () => this.cleanupAttempts.get(provider) === attempt;
|
|
3532
|
+
this.cleaningTests.add(provider);
|
|
3533
|
+
const failure = await this.withinCleanupDeadline(() => this.cleanupTestWindow(window, stillCurrent));
|
|
3534
|
+
if (failure !== undefined)
|
|
3535
|
+
return failure;
|
|
3536
|
+
// The work outlived its deadline and a newer attempt took the provider
|
|
3537
|
+
// over: its writes may finish (they were decided against the live store),
|
|
3538
|
+
// but it may not release state that attempt now owns.
|
|
3539
|
+
if (!stillCurrent())
|
|
3540
|
+
return 'a newer cleanup attempt took over this provider';
|
|
3541
|
+
if (!window.settled && window.facts !== undefined) {
|
|
3542
|
+
// PR review round 21: the route is back, but the request that borrowed
|
|
3543
|
+
// this projection is still running and can refresh and rewrite it. Hold
|
|
3544
|
+
// the isolation (the fence and the entries above stay) until that
|
|
3545
|
+
// settlement reconciles the late write and releases the provider.
|
|
3546
|
+
window.held = true;
|
|
3547
|
+
return undefined;
|
|
3548
|
+
}
|
|
3549
|
+
// PR review round 28: the request settled while this cleanup ran, so its
|
|
3550
|
+
// settlement could not reconcile the late write itself — a running cleanup
|
|
3551
|
+
// reads as the route's owner. Reconcile that write here, before the release
|
|
3552
|
+
// hands the route back: a failure keeps the fence for the next attempt.
|
|
3553
|
+
if (window.lateReconcileRequested) {
|
|
3554
|
+
window.lateReconcileRequested = false;
|
|
3555
|
+
const late = await this.withinCleanupDeadline(() => this.reconcileLateWindowProjection(window, true, stillCurrent));
|
|
3556
|
+
if (!stillCurrent())
|
|
3557
|
+
return 'a newer cleanup attempt took over this provider';
|
|
3558
|
+
if (late !== undefined)
|
|
3559
|
+
return late;
|
|
3560
|
+
}
|
|
3561
|
+
this.releaseAbandonedWindow(window);
|
|
3562
|
+
return undefined;
|
|
3563
|
+
}
|
|
3564
|
+
/** Drop every isolation entry this window still owns, identity-checked. */
|
|
3565
|
+
releaseAbandonedWindow(window) {
|
|
3566
|
+
if (this.testsActive.get(window.provider) === window.test)
|
|
3567
|
+
this.testsActive.delete(window.provider);
|
|
3568
|
+
if (this.testsInFlight.get(window.provider) === window.test)
|
|
3569
|
+
this.testsInFlight.delete(window.provider);
|
|
3570
|
+
if (this.abandonedWindows.get(window.provider) === window)
|
|
3571
|
+
this.abandonedWindows.delete(window.provider);
|
|
3572
|
+
// PR review round 33: a released window is no longer outstanding, so
|
|
3573
|
+
// shutdown does not restore over the newer window's borrow.
|
|
3574
|
+
if (this.testWindows.get(window.provider) === window)
|
|
3575
|
+
this.testWindows.delete(window.provider);
|
|
3576
|
+
window.held = false;
|
|
3577
|
+
this.cleaningTests.delete(window.provider);
|
|
3578
|
+
}
|
|
3579
|
+
/**
|
|
3580
|
+
* Put the route back the way the window found it: the mutations this window
|
|
3581
|
+
* started, then the projection (compare-by-payload and rotation-aware, so
|
|
3582
|
+
* nothing a writer stored meanwhile is overwritten), then the binding — the
|
|
3583
|
+
* original finally's order, because the binding decision reads the restored
|
|
3584
|
+
* owner map — then the borrow marker. Returns the reason the cleanup could
|
|
3585
|
+
* not complete.
|
|
3586
|
+
*
|
|
3587
|
+
* `stillCurrent` is passed by abandoned-window attempts only: a superseded
|
|
3588
|
+
* attempt that outlived its deadline may finish its writes, but the
|
|
3589
|
+
* provider-wide borrow marker belongs to the newest attempt (PR review
|
|
3590
|
+
* round 2) — deleting it would let ordinary traffic reach a newer window's
|
|
3591
|
+
* borrowed projection.
|
|
3592
|
+
*/
|
|
3593
|
+
async cleanupTestWindow(window, stillCurrent) {
|
|
3594
|
+
// PR review round 30: the watchdog can abandon this window during or after
|
|
3595
|
+
// shutdown. Every step below writes (credential record, binding, state), so
|
|
3596
|
+
// the cleanup is refused instead of landing after `close()`; the process is
|
|
3597
|
+
// going away, which leaves the borrowed projection as the store holds it.
|
|
3598
|
+
if (this.closing)
|
|
3599
|
+
return 'Keys Manager is shutting down; the test cleanup was skipped';
|
|
3600
|
+
// QC fix wave 3 (W-001): a re-read cannot tell the pre-borrow payload from
|
|
3601
|
+
// a write that has not landed yet, so the cleanup first waits for the
|
|
3602
|
+
// borrow/binding mutations this window started. Past the caller's deadline
|
|
3603
|
+
// an unsettled mutation is a failed cleanup, not a clean route.
|
|
3604
|
+
await this.awaitWindowMutations(window);
|
|
3605
|
+
// PR review round 35: the wait above is an await, and shutdown's own
|
|
3606
|
+
// restoration owns every outstanding window from its entry — this cleanup
|
|
3607
|
+
// must not write after it.
|
|
3608
|
+
if (this.persistenceRefused())
|
|
3609
|
+
return 'Keys Manager is shutting down; the test cleanup was skipped';
|
|
3610
|
+
const facts = window.facts;
|
|
3611
|
+
if (facts !== undefined) {
|
|
3612
|
+
// PR review round 28: a restore that starts now reads the state the
|
|
3613
|
+
// route carries at this moment, so it also covers a settlement that
|
|
3614
|
+
// asked for a late reconcile before it began; only a settlement landing
|
|
3615
|
+
// while this restore runs still needs the cleanup to re-read afterwards.
|
|
3616
|
+
window.lateReconcileRequested = false;
|
|
3617
|
+
const restored = await this.restoreProjection(facts.records, window.provider, window.row, facts.key, facts.previous, facts.borrowed, stillCurrent);
|
|
3618
|
+
if (restored !== undefined)
|
|
3619
|
+
return restored;
|
|
3620
|
+
}
|
|
3621
|
+
// PR review round 3: a superseded attempt may not persist a binding
|
|
3622
|
+
// decision derived from a world a newer window now owns.
|
|
3623
|
+
if (stillCurrent !== undefined && !stillCurrent())
|
|
3624
|
+
return 'a newer cleanup attempt took over this provider';
|
|
3625
|
+
// PR review round 35: the restore above awaited; a shutdown that started
|
|
3626
|
+
// under it refuses the binding write below.
|
|
3627
|
+
if (this.persistenceRefused())
|
|
3628
|
+
return 'Keys Manager is shutting down; the test cleanup was skipped';
|
|
3629
|
+
// QC B4: one sync at window end, after the owner map and the projection
|
|
3630
|
+
// are back, so the persisted binding matches the state the route actually
|
|
3631
|
+
// restored to.
|
|
3632
|
+
// QC fix wave 3 (S-003): unconditionally, because a binding decision the
|
|
3633
|
+
// borrow made {@link syncBrokerBinding} skip (an `addKey` inside the
|
|
3634
|
+
// window) is only observed here. It stays a no-op when nothing changed and
|
|
3635
|
+
// never overwrites a value Keys Manager did not write; only a window that
|
|
3636
|
+
// actually released the binding reports a refused write as a failure.
|
|
3637
|
+
const rebound = await this.syncBrokerBinding(window.provider, { borrow: true });
|
|
3638
|
+
if (!rebound && window.releasedBinding) {
|
|
3639
|
+
return 'The route credential binding could not be restored after the test; repair it in Models before relying on this route.';
|
|
3640
|
+
}
|
|
3641
|
+
if (stillCurrent === undefined || stillCurrent())
|
|
3642
|
+
this.testBindingBorrows.delete(window.provider);
|
|
3643
|
+
return undefined;
|
|
3644
|
+
}
|
|
3645
|
+
/**
|
|
3646
|
+
* Reconcile the projection once an abandoned window's request finally
|
|
3647
|
+
* settles (PR review round 13).
|
|
3648
|
+
*
|
|
3649
|
+
* The watchdog's cleanup can complete before the wedged request resumes: an
|
|
3650
|
+
* adapter that ignored the abort then refreshes and rewrites the borrowed
|
|
3651
|
+
* grant, and the abandoned window used to skip cleanup on settlement — so
|
|
3652
|
+
* the tested account stayed projected over the restored one and served
|
|
3653
|
+
* ordinary traffic. The shared compare-by-payload restore is idempotent and
|
|
3654
|
+
* only touches a projection this test left (the borrowed grant or a
|
|
3655
|
+
* rotation of the tested row): a foreign payload survives, and a newer
|
|
3656
|
+
* generation is written back to the tested row before the pre-test
|
|
3657
|
+
* projection is restored.
|
|
3658
|
+
*
|
|
3659
|
+
* Returns the failure, so a caller that releases the provider afterwards (PR
|
|
3660
|
+
* review round 21) can keep the fence instead. `ownsFence` is passed by the
|
|
3661
|
+
* settlement of a held window and by the cleanup that releases a window whose
|
|
3662
|
+
* settlement it deferred (PR review round 28): both already own the
|
|
3663
|
+
* provider's fence, which the default guards would read as a competing
|
|
3664
|
+
* cleanup and skip.
|
|
3665
|
+
*/
|
|
3666
|
+
async reconcileLateWindowProjection(window, ownsFence = false, stillCurrent) {
|
|
3667
|
+
const facts = window.facts;
|
|
3668
|
+
if (facts === undefined)
|
|
3669
|
+
return undefined;
|
|
3670
|
+
const provider = window.provider;
|
|
3671
|
+
if (!ownsFence) {
|
|
3672
|
+
// A cleanup that is still running — or one that gave up and left the
|
|
3673
|
+
// provider fenced — owns the route: its own attempt reads the late state
|
|
3674
|
+
// and reconciles it, and this late promise must not race it. PR review
|
|
3675
|
+
// round 28: that attempt only knows about the late write through this
|
|
3676
|
+
// marker, set synchronously with `window.settled` in the caller, so a
|
|
3677
|
+
// cleanup that observes a settled window also observes this request.
|
|
3678
|
+
if (this.cleaningTests.has(provider)) {
|
|
3679
|
+
if (this.abandonedWindows.get(provider) === window)
|
|
3680
|
+
window.lateReconcileRequested = true;
|
|
3681
|
+
return undefined;
|
|
3682
|
+
}
|
|
3683
|
+
const active = this.testsActive.get(provider);
|
|
3684
|
+
if (active !== undefined && active !== window.test)
|
|
3685
|
+
return undefined;
|
|
3686
|
+
}
|
|
3687
|
+
// PR review round 32: the phase runs under its own deadline, so the attempt
|
|
3688
|
+
// that started it can already be superseded when it resumes — the restore
|
|
3689
|
+
// and the binding sync are both gated on that attempt identity, not merely
|
|
3690
|
+
// checked by the caller afterwards.
|
|
3691
|
+
const failure = await this.restoreProjection(facts.records, provider, window.row, facts.key, facts.previous, facts.borrowed, stillCurrent, true);
|
|
3692
|
+
if (failure !== undefined) {
|
|
3693
|
+
this.warn(`the late projection write of the abandoned connection test for ${provider} could not be reconciled: ${failure}`);
|
|
3694
|
+
return failure;
|
|
3695
|
+
}
|
|
3696
|
+
if (stillCurrent !== undefined && !stillCurrent())
|
|
3697
|
+
return undefined;
|
|
3698
|
+
await this.syncBindingAfterProjectionChange(provider);
|
|
3699
|
+
return undefined;
|
|
3700
|
+
}
|
|
3701
|
+
/**
|
|
3702
|
+
* Record one in-flight window mutation before it is awaited (QC fix wave 3,
|
|
3703
|
+
* W-001). The normalized promise never rejects: the mutation's own outcome
|
|
3704
|
+
* belongs to the code awaiting it, while the cleanup only needs to know when
|
|
3705
|
+
* the write is over.
|
|
3706
|
+
*/
|
|
3707
|
+
recordWindowMutation(window, mutation) {
|
|
3708
|
+
window.mutations.push(mutation.then(() => undefined, () => undefined));
|
|
3709
|
+
}
|
|
3710
|
+
/**
|
|
3711
|
+
* Await every mutation this window recorded, including one the continuation
|
|
3712
|
+
* of an earlier mutation records (a borrow that settles right before its
|
|
3713
|
+
* binding release starts). The microtask hop after each settling round lets
|
|
3714
|
+
* that continuation run before the cleanup judges the route clean. A
|
|
3715
|
+
* mutation the seam never answers is bounded by the caller's deadline, not
|
|
3716
|
+
* here.
|
|
3717
|
+
*/
|
|
3718
|
+
async awaitWindowMutations(window) {
|
|
3719
|
+
let awaited = 0;
|
|
3720
|
+
for (;;) {
|
|
3721
|
+
while (awaited < window.mutations.length) {
|
|
3722
|
+
await window.mutations[awaited];
|
|
3723
|
+
awaited += 1;
|
|
3724
|
+
}
|
|
3725
|
+
await Promise.resolve();
|
|
3726
|
+
if (awaited >= window.mutations.length)
|
|
3727
|
+
return;
|
|
3728
|
+
}
|
|
3729
|
+
}
|
|
3730
|
+
/**
|
|
3731
|
+
* Run one cleanup under its hard deadline. The work is not cancelled — the
|
|
3732
|
+
* credential seam offers no cancellation — it is only stopped from holding
|
|
3733
|
+
* the provider past the bound; a late result is ignored.
|
|
3734
|
+
*/
|
|
3735
|
+
async withinCleanupDeadline(work) {
|
|
3736
|
+
// The bound never rejects: a seam failure is a cleanup result, not a
|
|
3737
|
+
// control-flow error.
|
|
3738
|
+
const running = work().then(failure => failure, (error) => `the credential store refused the cleanup: ${error instanceof Error ? error.message : String(error)}`);
|
|
3739
|
+
let timer;
|
|
3740
|
+
const deadline = new Promise(resolve => {
|
|
3741
|
+
timer = setTimeout(() => resolve('the credential store did not answer before the cleanup deadline'), this.testCleanupDeadlineMs);
|
|
3742
|
+
timer.unref?.();
|
|
3743
|
+
});
|
|
3744
|
+
try {
|
|
3745
|
+
return await Promise.race([running, deadline]);
|
|
3746
|
+
}
|
|
3747
|
+
finally {
|
|
3748
|
+
if (timer !== undefined)
|
|
3749
|
+
clearTimeout(timer);
|
|
3750
|
+
}
|
|
3751
|
+
}
|
|
3752
|
+
/**
|
|
3753
|
+
* Decide the route's serving shape and run one row test. Separate from
|
|
3754
|
+
* {@link testKey} so the concurrency lock is registered before the first
|
|
3755
|
+
* await.
|
|
3756
|
+
*/
|
|
3757
|
+
async runRowTest(record, row, window) {
|
|
3758
|
+
const provider = record.provider;
|
|
3759
|
+
// QC B6: the serving source is decided from projection ownership, which a
|
|
3760
|
+
// restarted process hydrates asynchronously. Until that read lands the
|
|
3761
|
+
// shape is unknown, so the honest answer is a refusal with no request
|
|
3762
|
+
// rather than a test attributed to the wrong credential.
|
|
3763
|
+
if (!await this.awaitProjectionOwnership(provider)) {
|
|
3764
|
+
return { ok: false, latencyMs: 0, error: 'Account ownership is still loading; try again in a moment' };
|
|
3765
|
+
}
|
|
3766
|
+
const route = this.testRouteProfile(record);
|
|
3767
|
+
if ('error' in route)
|
|
3768
|
+
return { ok: false, latencyMs: 0, error: route.error };
|
|
3769
|
+
const profile = route.profile;
|
|
3770
|
+
const apiKeyEnv = typeof profile.apiKeyEnv === 'string' ? profile.apiKeyEnv : undefined;
|
|
3771
|
+
// QC B1: derive the route's serving shape once, from the profile value
|
|
3772
|
+
// read at entry, and only test a row whose credential this shape carries.
|
|
3773
|
+
// pi-ai resolves a named `apiKeyEnv` override before its stored grant, so
|
|
3774
|
+
// `bound` decides that the broker (key pool) authenticates the request and
|
|
3775
|
+
// `projected` that the account's grant does.
|
|
3776
|
+
const bound = apiKeyEnv === record.brokerRef;
|
|
3777
|
+
const projected = this.oauthOwnerFor(provider) !== null;
|
|
3778
|
+
const foreignReference = apiKeyEnv !== undefined && !bound;
|
|
3779
|
+
if (foreignReference) {
|
|
3780
|
+
return {
|
|
3781
|
+
ok: false,
|
|
3782
|
+
latencyMs: 0,
|
|
3783
|
+
error: `This route is served by the credential reference "${apiKeyEnv}" set in the provider settings, which Keys Manager does not manage. Repair the binding in Models, then test again.`,
|
|
3784
|
+
};
|
|
3785
|
+
}
|
|
3786
|
+
// C1: a route served by a projected OAuth account resolves that account's
|
|
3787
|
+
// stored grant, never the API-key pool. `syncBrokerBinding` unsets the
|
|
3788
|
+
// profile's broker reference while a projection exists, so the adapter has
|
|
3789
|
+
// no `apiKey` override to hand pi-ai and the test scope's pinned literal is
|
|
3790
|
+
// never consulted: an api-key row test here would report a pass for a key
|
|
3791
|
+
// the request did not carry. Refuse with the serving source instead of
|
|
3792
|
+
// borrowing (or deleting) the projection — a diagnostic action must not
|
|
3793
|
+
// rewrite the route's credential binding. `oauthOwnerFor` is the read-only
|
|
3794
|
+
// ownership oracle; a stale map entry resolves to null and does not refuse.
|
|
3795
|
+
if (row.kind === 'api-key' && projected) {
|
|
3796
|
+
return {
|
|
3797
|
+
ok: false,
|
|
3798
|
+
latencyMs: 0,
|
|
3799
|
+
error: 'This route is currently served by its OAuth account; the API key pool is not in use. Deactivate the account or let the projection fall back, then test again.',
|
|
3800
|
+
};
|
|
3801
|
+
}
|
|
3802
|
+
if (row.kind === 'api-key' && !bound) {
|
|
3803
|
+
// No `apiKeyEnv` override means pi-ai reads its own stored credential:
|
|
3804
|
+
// the pinned literal never reaches the request, so a pass would be
|
|
3805
|
+
// reported for a key this route did not carry.
|
|
3806
|
+
return {
|
|
3807
|
+
ok: false,
|
|
3808
|
+
latencyMs: 0,
|
|
3809
|
+
error: 'This route is not served by its API-key pool (the provider profile holds no Keys Manager binding). Re-add a key or repair the binding in Models, then test again.',
|
|
3810
|
+
};
|
|
3811
|
+
}
|
|
3812
|
+
const model = normalizeModels(profile.models)[0]?.id;
|
|
3813
|
+
if (model === undefined)
|
|
3814
|
+
return { ok: false, latencyMs: 0, error: 'No model is configured for this provider' };
|
|
3815
|
+
// QC B1: a broker-bound profile hides the borrowed projection from pi-ai,
|
|
3816
|
+
// so the binding is released for the test window through the one authority
|
|
3817
|
+
// and restored at window end (see runOAuthRowTest).
|
|
3818
|
+
const borrowBinding = row.kind === 'oauth' && bound;
|
|
3819
|
+
if (row.kind !== 'oauth')
|
|
3820
|
+
return await this.runApiKeyRowTest(provider, row, model, window);
|
|
3821
|
+
// The window records what this test borrows, so the watchdog can put the
|
|
3822
|
+
// route back exactly as the test's own finally would (QC fix wave 2).
|
|
3823
|
+
window.borrowBinding = borrowBinding;
|
|
3824
|
+
return await this.runOAuthRowTest(provider, row, model, window);
|
|
3825
|
+
}
|
|
3826
|
+
/**
|
|
3827
|
+
* The live provider profile one test must reason about, read once at entry
|
|
3828
|
+
* (QC B1/N6), or the reason it cannot be used. An unreadable settings
|
|
3829
|
+
* snapshot and a missing profile are distinct causes, so the row never
|
|
3830
|
+
* claims a model is missing when the settings service is merely absent.
|
|
3831
|
+
*/
|
|
3832
|
+
testRouteProfile(record) {
|
|
3833
|
+
const descriptor = this.describeSettings();
|
|
3834
|
+
if (!descriptor.readable) {
|
|
3835
|
+
return { error: 'The provider settings could not be read, so the test cannot tell which credential serves this route. Try again in a moment.' };
|
|
3836
|
+
}
|
|
3837
|
+
const namespace = this.namespace(descriptor, record.settingsNs);
|
|
3838
|
+
if (namespace === undefined) {
|
|
3839
|
+
return { error: `The settings namespace "${record.settingsNs}" is not mounted; configure this provider in Models and test again.` };
|
|
3840
|
+
}
|
|
3841
|
+
const profile = asProfile(getPath(namespace.value, record.settingsPath));
|
|
3842
|
+
if (profile === undefined) {
|
|
3843
|
+
return { error: 'This provider has no settings profile yet; configure a model for it and test again.' };
|
|
3844
|
+
}
|
|
3845
|
+
return { profile };
|
|
3846
|
+
}
|
|
3847
|
+
/** One request pinned to the row's API-key literal. */
|
|
3848
|
+
async runApiKeyRowTest(provider, row, model, window) {
|
|
3849
|
+
const probe = await this.probeKey(row.ref);
|
|
3850
|
+
// PR review round 10 (I-000179): the probe is an untimed seam call, so the
|
|
3851
|
+
// window can be abandoned — and a release, refused or not, can have
|
|
3852
|
+
// removed the row — while the test is parked here. Re-assert both right
|
|
3853
|
+
// before the send: a late probe result must not send a deleted literal.
|
|
3854
|
+
if (window.cleaned)
|
|
3855
|
+
return this.abandonedTestResult();
|
|
3856
|
+
const live = this.store.provider(provider);
|
|
3857
|
+
if (live === undefined || !live.keys.some(candidate => candidate.id === row.id)) {
|
|
3858
|
+
return { ok: false, latencyMs: 0, error: `Key "${row.id}" was removed from ${provider} before its test could send` };
|
|
3859
|
+
}
|
|
3860
|
+
if (probe.hit === undefined) {
|
|
3861
|
+
return {
|
|
3862
|
+
ok: false,
|
|
3863
|
+
latencyMs: 0,
|
|
3864
|
+
error: probe.failed ? `Could not read credential ${row.ref}` : 'The stored credential is missing',
|
|
3865
|
+
};
|
|
3866
|
+
}
|
|
3867
|
+
// PR review round 40: a native OAuth sign-in (or a demotion) can change the
|
|
3868
|
+
// route's serving source while the probe is parked. Sending now would test a
|
|
3869
|
+
// key the request does not carry, so refuse instead of reporting a false pass.
|
|
3870
|
+
if (this.oauthOwnerFor(provider) !== null || this.foreignProjections.has(provider)) {
|
|
3871
|
+
return {
|
|
3872
|
+
ok: false,
|
|
3873
|
+
latencyMs: 0,
|
|
3874
|
+
error: `The ${provider} route is now served by another credential source; run the test again once it settles.`,
|
|
3875
|
+
};
|
|
3876
|
+
}
|
|
3877
|
+
return await this.runTestRequest(provider, model, {
|
|
3878
|
+
provider,
|
|
3879
|
+
keyId: row.id,
|
|
3880
|
+
keyValue: probe.hit.value,
|
|
3881
|
+
keyUsed: false,
|
|
3882
|
+
oauthAccountId: null,
|
|
3883
|
+
test: true,
|
|
3884
|
+
});
|
|
3885
|
+
}
|
|
3886
|
+
/** One request with the row's grant temporarily projected as the route's credential. */
|
|
3887
|
+
async runOAuthRowTest(provider, row, model, window) {
|
|
3888
|
+
const grant = row.oauth;
|
|
3889
|
+
if (grant === null)
|
|
3890
|
+
return { ok: false, latencyMs: 0, error: `Key "${row.id}" is not an OAuth account` };
|
|
3891
|
+
const records = this.credentialRecords();
|
|
3892
|
+
if (records === undefined)
|
|
3893
|
+
return { ok: false, latencyMs: 0, error: 'The credential store does not support records' };
|
|
3894
|
+
const projectionKey = `${OAUTH_GRANT_RECORD_SCOPE}/${provider}`;
|
|
3895
|
+
let result = { ok: false, latencyMs: 0, error: 'The test did not run' };
|
|
3896
|
+
// The projection write, the request and the restore share the provider's
|
|
3897
|
+
// grant chain, so a rotation echo cannot observe or overwrite the
|
|
3898
|
+
// temporary projection between them.
|
|
3899
|
+
await this.scheduleGrantReconciliation(provider, async () => {
|
|
3900
|
+
const stored = await records.readRecord(grant.grantRef);
|
|
3901
|
+
if (window.cleaned) {
|
|
3902
|
+
// QC fix wave 2: the watchdog abandoned this window while the read was
|
|
3903
|
+
// in flight. Starting (or resuming) the borrow now would land it after
|
|
3904
|
+
// the cleanup that already ran, so the test stops here.
|
|
3905
|
+
result = this.abandonedTestResult();
|
|
3906
|
+
return;
|
|
3907
|
+
}
|
|
3908
|
+
if (stored === undefined || stored.kind !== 'grant' || !isOAuthGrantPayload(stored.payload)) {
|
|
3909
|
+
result = {
|
|
3910
|
+
ok: false,
|
|
3911
|
+
latencyMs: 0,
|
|
3912
|
+
error: `The grant payload at ${grant.grantRef} is missing or unusable; sign in to this account again`,
|
|
3913
|
+
};
|
|
3914
|
+
return;
|
|
3915
|
+
}
|
|
3916
|
+
const borrowed = structuredClone(stored.payload);
|
|
3917
|
+
const captured = await records.readRecord(projectionKey);
|
|
3918
|
+
// PR review round 14: the snapshot and its comparison key are frozen
|
|
3919
|
+
// here. The seam hands back a live record, so a writer mutating that
|
|
3920
|
+
// stored object in place would otherwise change `previous` along with
|
|
3921
|
+
// the updater's `current` and make the borrow look unchanged.
|
|
3922
|
+
const previous = captured === undefined ? undefined : structuredClone(captured);
|
|
3923
|
+
const previousKey = previous === undefined ? undefined : JSON.stringify(previous);
|
|
3924
|
+
if (window.cleaned) {
|
|
3925
|
+
result = this.abandonedTestResult();
|
|
3926
|
+
return;
|
|
3927
|
+
}
|
|
3928
|
+
if (this.shutdownRequested) {
|
|
3929
|
+
// PR review round 30: shutdown abandons the chain parked here, so the
|
|
3930
|
+
// borrow write below must not land after `close()`. Round 34: the
|
|
3931
|
+
// marker is up from shutdown's entry, so the pre-close restoration is
|
|
3932
|
+
// covered too — it does not borrow, and this test must not either.
|
|
3933
|
+
result = this.shutdownEntryResult();
|
|
3934
|
+
return;
|
|
3935
|
+
}
|
|
3936
|
+
// PR review round 12: both reads above are awaits, so a removal can have
|
|
3937
|
+
// deleted the provider or this row while they were in flight (a test that
|
|
3938
|
+
// registered and one that did not are covered: the marker refuses the
|
|
3939
|
+
// latter, this re-check the former). Never write the borrow — and never
|
|
3940
|
+
// re-project the removed account's grant — on that state.
|
|
3941
|
+
// PR review round 15: the existence check alone still holds while a
|
|
3942
|
+
// removal is deleting credentials (the row is spliced only after its
|
|
3943
|
+
// grant delete), so the row's retirement mark is part of the assertion.
|
|
3944
|
+
// The provider-wide marker is deliberately not read here: a removal that
|
|
3945
|
+
// began while this test is registered waits for it in
|
|
3946
|
+
// {@link settleTestBeforeRemoval} and is the designed handshake, while
|
|
3947
|
+
// the row mark is only set by a removal that already passed that check.
|
|
3948
|
+
const live = this.store.provider(provider);
|
|
3949
|
+
const removing = this.retiringAccounts.has(row);
|
|
3950
|
+
if (removing || live === undefined || !live.keys.some(candidate => candidate.id === row.id)) {
|
|
3951
|
+
result = removing
|
|
3952
|
+
? { ok: false, latencyMs: 0, error: `Provider "${provider}" is being removed; the test did not borrow its credential` }
|
|
3953
|
+
: { ok: false, latencyMs: 0, error: `Key "${row.id}" was removed from ${provider} before its test could borrow it` };
|
|
3954
|
+
return;
|
|
3955
|
+
}
|
|
3956
|
+
// QC fix wave 2: record the borrow before the write, so an abandoned
|
|
3957
|
+
// window can restore a borrow whose write lands after the watchdog
|
|
3958
|
+
// looked. The restore decides by payload, so the facts are safe to hold
|
|
3959
|
+
// whether or not the write landed.
|
|
3960
|
+
window.facts = { records, key: projectionKey, previous, borrowed };
|
|
3961
|
+
// QC B4 + PR review round 8: the borrow marker is registered BEFORE the
|
|
3962
|
+
// write is awaited. A mutation landing while the write is still in
|
|
3963
|
+
// flight would otherwise derive a binding from the pre-borrow world — on
|
|
3964
|
+
// an unbound route it binds the broker over the projection this test is
|
|
3965
|
+
// about to borrow, and the window (whose `borrowBinding` was decided at
|
|
3966
|
+
// entry) never releases it.
|
|
3967
|
+
this.testBindingBorrows.add(provider);
|
|
3968
|
+
// QC fix wave 3 (W-001): the write is recorded before it is awaited, so
|
|
3969
|
+
// a cleanup that runs while it is still in flight waits for it instead
|
|
3970
|
+
// of re-reading the pre-borrow payload and calling the route clean.
|
|
3971
|
+
// PR review round 13: the pre-borrow read is awaited, so a native or
|
|
3972
|
+
// manager sign-in can commit a newer projection before this write runs.
|
|
3973
|
+
// The write is therefore a compare-and-swap against the record it
|
|
3974
|
+
// captured: borrowing over that sign-in — and restoring the stale
|
|
3975
|
+
// `previous` at window end — would drop it for good.
|
|
3976
|
+
let borrowRefused = false;
|
|
3977
|
+
const borrowWrite = records.modifyRecord(projectionKey, current => {
|
|
3978
|
+
const unchanged = previous === undefined
|
|
3979
|
+
? current === undefined
|
|
3980
|
+
: current !== undefined && JSON.stringify(current) === previousKey;
|
|
3981
|
+
if (!unchanged) {
|
|
3982
|
+
borrowRefused = true;
|
|
3983
|
+
return Promise.resolve(undefined);
|
|
3984
|
+
}
|
|
3985
|
+
return Promise.resolve({ kind: 'grant', payload: structuredClone(borrowed) });
|
|
3986
|
+
});
|
|
3987
|
+
this.recordWindowMutation(window, borrowWrite);
|
|
3988
|
+
try {
|
|
3989
|
+
await borrowWrite;
|
|
3990
|
+
}
|
|
3991
|
+
catch (error) {
|
|
3992
|
+
// Nothing was borrowed, so the fence goes back — unless an abandoned
|
|
3993
|
+
// window's cleanup owns it (it removes the marker when it completes).
|
|
3994
|
+
if (!window.cleaned)
|
|
3995
|
+
this.testBindingBorrows.delete(provider);
|
|
3996
|
+
result = {
|
|
3997
|
+
ok: false,
|
|
3998
|
+
latencyMs: 0,
|
|
3999
|
+
error: `The grant could not be projected for the test: ${error instanceof Error ? error.message : String(error)}`,
|
|
4000
|
+
};
|
|
4001
|
+
return;
|
|
4002
|
+
}
|
|
4003
|
+
if (this.shutdownRequested) {
|
|
4004
|
+
// PR review round 30: the borrow write settled after shutdown started,
|
|
4005
|
+
// so no binding release, request or restore may follow it. Round 34:
|
|
4006
|
+
// shutdown's entry marker makes that true during the pre-close
|
|
4007
|
+
// restoration as well, which waits this write out and puts it back.
|
|
4008
|
+
result = this.shutdownTestResult();
|
|
4009
|
+
return;
|
|
4010
|
+
}
|
|
4011
|
+
if (borrowRefused) {
|
|
4012
|
+
// The projection belongs to a newer writer, so this window borrowed
|
|
4013
|
+
// nothing and has nothing to restore. An abandoned window's cleanup
|
|
4014
|
+
// owns the fence and the marker.
|
|
4015
|
+
if (!window.cleaned) {
|
|
4016
|
+
window.facts = undefined;
|
|
4017
|
+
this.testBindingBorrows.delete(provider);
|
|
4018
|
+
result = {
|
|
4019
|
+
ok: false,
|
|
4020
|
+
latencyMs: 0,
|
|
4021
|
+
error: `The credential projection for ${provider} changed while the test was starting; the test did not borrow it. Run the test again.`,
|
|
4022
|
+
};
|
|
4023
|
+
return;
|
|
4024
|
+
}
|
|
4025
|
+
result = this.abandonedTestResult();
|
|
4026
|
+
return;
|
|
4027
|
+
}
|
|
4028
|
+
if (window.cleaned) {
|
|
4029
|
+
// QC fix wave 2: the window was conceded while this write was in
|
|
4030
|
+
// flight, so the watchdog's cleanup may have read the projection
|
|
4031
|
+
// before it landed. Undo the borrow through the same compare-by-payload
|
|
4032
|
+
// restore (not the stale snapshot) and keep the fence if that fails.
|
|
4033
|
+
// QC fix wave 3 (W-002): only while this window still owns the fence.
|
|
4034
|
+
// A cleanup that completed on a later test attempt already restored
|
|
4035
|
+
// this write; running again now would put a projection back out from
|
|
4036
|
+
// under that newer window.
|
|
4037
|
+
if (this.abandonedWindows.get(provider) === window)
|
|
4038
|
+
await this.attemptAbandonedCleanup(window);
|
|
4039
|
+
result = this.abandonedTestResult();
|
|
4040
|
+
return;
|
|
4041
|
+
}
|
|
4042
|
+
// QC fix wave 3 (W-002): the owner claim sits after the fence check — an
|
|
4043
|
+
// abandoned window's repair already reconciled the owner from the
|
|
4044
|
+
// restored payload, and claiming it again would name a row the cleanup
|
|
4045
|
+
// no longer serves. QC B4: the borrow marker that hides this window's
|
|
4046
|
+
// credential surface from the binding authority is already set, before
|
|
4047
|
+
// the borrow write (PR review round 8); the window-end sync below writes
|
|
4048
|
+
// the restored state.
|
|
4049
|
+
this.claimProjectionOwner(provider, row.id);
|
|
4050
|
+
let restoreFailure;
|
|
4051
|
+
try {
|
|
4052
|
+
if (window.borrowBinding) {
|
|
4053
|
+
// QC B1: a broker-bound profile hands pi-ai the broker reference, so
|
|
4054
|
+
// the projection would be inert for the request. Release it for the
|
|
4055
|
+
// window through the single binding authority (a failed release
|
|
4056
|
+
// refuses the test instead of borrowing a second writer).
|
|
4057
|
+
// QC fix wave 2: the attempt is recorded before it is awaited — it
|
|
4058
|
+
// may land partially, and a cleanup must re-sync what it touched.
|
|
4059
|
+
// QC fix wave 3 (W-001): it is also recorded as an in-flight window
|
|
4060
|
+
// mutation, so a cleanup does not sync a route whose release is
|
|
4061
|
+
// still on its way to the profile.
|
|
4062
|
+
window.releasedBinding = true;
|
|
4063
|
+
const release = this.syncBrokerBinding(provider, { borrow: true });
|
|
4064
|
+
this.recordWindowMutation(window, release);
|
|
4065
|
+
const released = await release;
|
|
4066
|
+
if (window.cleaned) {
|
|
4067
|
+
// The abandonment landed while this release was in flight, so the
|
|
4068
|
+
// watchdog's re-sync may have run before the release did and left
|
|
4069
|
+
// the profile unbound. Re-sync now that the release is complete —
|
|
4070
|
+
// unless a later attempt already cleaned this window up (W-002).
|
|
4071
|
+
if (this.abandonedWindows.get(provider) === window)
|
|
4072
|
+
await this.attemptAbandonedCleanup(window);
|
|
4073
|
+
result = this.abandonedTestResult();
|
|
4074
|
+
return;
|
|
4075
|
+
}
|
|
4076
|
+
if (!released) {
|
|
4077
|
+
restoreFailure = 'The route is served by its API-key pool and its credential binding could not be released for the test; the account was not tested.';
|
|
4078
|
+
}
|
|
4079
|
+
}
|
|
4080
|
+
if (restoreFailure === undefined) {
|
|
4081
|
+
result = await this.runTestRequest(provider, model, {
|
|
4082
|
+
provider,
|
|
4083
|
+
keyId: null,
|
|
4084
|
+
keyValue: null,
|
|
4085
|
+
keyUsed: false,
|
|
4086
|
+
oauthAccountId: row.id,
|
|
4087
|
+
test: true,
|
|
4088
|
+
});
|
|
4089
|
+
}
|
|
4090
|
+
}
|
|
4091
|
+
finally {
|
|
4092
|
+
// PR review round 21: the request is over. A cleanup that completed
|
|
4093
|
+
// while it was still running held the provider's isolation for exactly
|
|
4094
|
+
// this point, so the late write is reconciled before that isolation is
|
|
4095
|
+
// released; a failed reconcile keeps the fence (the existing
|
|
4096
|
+
// borrow-could-not-be-undone path).
|
|
4097
|
+
window.settled = true;
|
|
4098
|
+
if (this.closing) {
|
|
4099
|
+
// PR review round 30: the shutdown drain abandoned this chain; the
|
|
4100
|
+
// restore and the binding sync below are persistence, and the store
|
|
4101
|
+
// is closed already. Nothing is written, so the borrowed projection
|
|
4102
|
+
// stays as the credential store holds it.
|
|
4103
|
+
restoreFailure ??= this.shutdownTestResult().error;
|
|
4104
|
+
}
|
|
4105
|
+
else if (window.cleaned) {
|
|
4106
|
+
// QC fix wave 2: this window was abandoned and its cleanup already
|
|
4107
|
+
// ran (or is running) under the watchdog. Restoring from this
|
|
4108
|
+
// promise's own pre-test snapshot now would race whatever state the
|
|
4109
|
+
// route carries by then, so the late settlement writes nothing and
|
|
4110
|
+
// reports the failure its own result cannot claim.
|
|
4111
|
+
restoreFailure = this.abandonedTestResult().error;
|
|
4112
|
+
// PR review round 13: the abandoned request may itself have
|
|
4113
|
+
// refreshed and rewritten the borrowed projection after that
|
|
4114
|
+
// cleanup ran. Reconcile that late write against what the window
|
|
4115
|
+
// found before the route counts as handed back.
|
|
4116
|
+
const late = await this.reconcileLateWindowProjection(window, window.held);
|
|
4117
|
+
if (window.held && late === undefined && this.abandonedWindows.get(provider) === window) {
|
|
4118
|
+
this.releaseAbandonedWindow(window);
|
|
4119
|
+
}
|
|
4120
|
+
}
|
|
4121
|
+
else {
|
|
4122
|
+
const restored = await this.cleanupTestWindow(window);
|
|
4123
|
+
if (restoreFailure === undefined)
|
|
4124
|
+
restoreFailure = restored;
|
|
4125
|
+
// PR review MF1: a cleanup that failed on the normal path must enter
|
|
4126
|
+
// the same fenced recovery the watchdog uses. Reporting the failure
|
|
4127
|
+
// alone would strand the borrow marker (every later
|
|
4128
|
+
// `syncBrokerBinding` returns early) and let the borrowed grant keep
|
|
4129
|
+
// serving ordinary requests. `window.cleaned` guards the case where
|
|
4130
|
+
// the watchdog took the window over while this cleanup ran — that
|
|
4131
|
+
// owner is already retrying.
|
|
4132
|
+
if (restored !== undefined && !window.cleaned)
|
|
4133
|
+
this.abandonTestWindow(window, 'cleanup');
|
|
4134
|
+
}
|
|
4135
|
+
}
|
|
4136
|
+
if (restoreFailure !== undefined) {
|
|
4137
|
+
result = { ok: false, latencyMs: result.latencyMs, error: restoreFailure };
|
|
4138
|
+
}
|
|
4139
|
+
});
|
|
4140
|
+
return result;
|
|
4141
|
+
}
|
|
4142
|
+
/** The refusal an abandoned test reports instead of a stale success. */
|
|
4143
|
+
abandonedTestResult() {
|
|
4144
|
+
return {
|
|
4145
|
+
ok: false,
|
|
4146
|
+
latencyMs: 0,
|
|
4147
|
+
error: `The connection test was abandoned after ${this.isolationWaitMs}ms and its borrowed credential was cleaned up; run the test again.`,
|
|
4148
|
+
};
|
|
4149
|
+
}
|
|
4150
|
+
/** The refusal a chain reports when shutdown refuses its persistence. */
|
|
4151
|
+
shutdownTestResult() {
|
|
4152
|
+
return {
|
|
4153
|
+
ok: false,
|
|
4154
|
+
latencyMs: 0,
|
|
4155
|
+
error: 'Keys Manager is shutting down; the connection test did not finish and its cleanup was skipped.',
|
|
4156
|
+
};
|
|
4157
|
+
}
|
|
4158
|
+
/** The refusal a test reports when it arrives after shutdown began (round 34). */
|
|
4159
|
+
shutdownEntryResult() {
|
|
4160
|
+
return {
|
|
4161
|
+
ok: false,
|
|
4162
|
+
latencyMs: 0,
|
|
4163
|
+
error: 'Keys Manager is shutting down; the connection test was refused before it borrowed a credential.',
|
|
4164
|
+
};
|
|
4165
|
+
}
|
|
4166
|
+
/**
|
|
4167
|
+
* Mark one provider as being removed for the whole removal body (PR review
|
|
4168
|
+
* round 12). The marker is up before the body's first await — the removal's
|
|
4169
|
+
* own test wait included — so {@link testKey} can refuse a test that would
|
|
4170
|
+
* otherwise start inside the removal. Nesting is a depth because a
|
|
4171
|
+
* last-key `deleteKey` releases through `releaseLocked`.
|
|
4172
|
+
*/
|
|
4173
|
+
async withRemovalInProgress(provider, work) {
|
|
4174
|
+
const depth = this.removalsInProgress.get(provider) ?? 0;
|
|
4175
|
+
this.removalsInProgress.set(provider, depth + 1);
|
|
4176
|
+
try {
|
|
4177
|
+
return await work();
|
|
4178
|
+
}
|
|
4179
|
+
finally {
|
|
4180
|
+
if (depth === 0)
|
|
4181
|
+
this.removalsInProgress.delete(provider);
|
|
4182
|
+
else
|
|
4183
|
+
this.removalsInProgress.set(provider, depth);
|
|
4184
|
+
}
|
|
4185
|
+
}
|
|
4186
|
+
/**
|
|
4187
|
+
* Hold a row/account removal until the provider's running row test settles
|
|
4188
|
+
* (PR review round 9, R8-2).
|
|
4189
|
+
*
|
|
4190
|
+
* `testKey` captures its row, its credential and the route's serving shape
|
|
4191
|
+
* before the first await, so a removal landing inside that window can delete
|
|
4192
|
+
* the credential the test is about to send. For a borrowed OAuth account the
|
|
4193
|
+
* removal is worse than a stale read: it compares its decision against the
|
|
4194
|
+
* test's temporary projection, deletes it and demotes every row, and the
|
|
4195
|
+
* test's own restore then writes the removed account's grant back. The
|
|
4196
|
+
* removal therefore waits for the window instead of racing it. The wait is
|
|
4197
|
+
* bounded by {@link isolationWaitMs} — a test wedged in an untimed seam call
|
|
4198
|
+
* must not park the removal forever — and a provider whose abandoned window
|
|
4199
|
+
* still owns the route's records refuses the removal outright. A wait that
|
|
4200
|
+
* hits its own bound refuses too (PR review round 10, I-000179): proceeding
|
|
4201
|
+
* would delete the credential the parked test is about to send. The user can
|
|
4202
|
+
* retry once the abandoned window has been cleaned up.
|
|
4203
|
+
*/
|
|
4204
|
+
async settleTestBeforeRemoval(provider) {
|
|
4205
|
+
const refusal = `A previous connection test for ${provider} is still being cleaned up; try again in a moment.`;
|
|
4206
|
+
if (this.cleaningTests.has(provider))
|
|
4207
|
+
throw new Error(refusal);
|
|
4208
|
+
const pending = this.testsActive.get(provider);
|
|
4209
|
+
if (pending === undefined)
|
|
4210
|
+
return;
|
|
4211
|
+
let settled = false;
|
|
4212
|
+
let timer;
|
|
4213
|
+
const bound = new Promise(resolve => {
|
|
4214
|
+
timer = setTimeout(resolve, this.isolationWaitMs);
|
|
4215
|
+
timer.unref?.();
|
|
4216
|
+
});
|
|
4217
|
+
try {
|
|
4218
|
+
await Promise.race([pending.then(() => { settled = true; }, () => { settled = true; }), bound]);
|
|
4219
|
+
}
|
|
4220
|
+
finally {
|
|
4221
|
+
if (timer !== undefined)
|
|
4222
|
+
clearTimeout(timer);
|
|
4223
|
+
}
|
|
4224
|
+
if (this.cleaningTests.has(provider))
|
|
4225
|
+
throw new Error(refusal);
|
|
4226
|
+
if (!settled) {
|
|
4227
|
+
throw new Error(`A connection test for ${provider} did not settle within ${this.isolationWaitMs}ms, so the removal was refused rather than delete a credential its request may still carry; try again in a moment.`);
|
|
4228
|
+
}
|
|
4229
|
+
}
|
|
4230
|
+
/**
|
|
4231
|
+
* Put the projection back as it was before the test borrowed it, without
|
|
4232
|
+
* losing or overwriting anything that changed meanwhile.
|
|
4233
|
+
*
|
|
4234
|
+
* `late` marks the abandoned window's settlement, after the watchdog's
|
|
4235
|
+
* cleanup already released the provider: a removal no longer waits for this
|
|
4236
|
+
* test, so the pre-test projection's owning row may be gone by the time this
|
|
4237
|
+
* runs and must not be projected again (PR review round 19).
|
|
4238
|
+
*
|
|
4239
|
+
* The decision is compare-by-payload against the borrowed grant: only a
|
|
4240
|
+
* live record that still carries it — or a refresh-rotated version of it
|
|
4241
|
+
* that {@link matchOwningAccount} attributes to the tested row — is
|
|
4242
|
+
* replaced. A rotation is reconciled into the tested row first (the same
|
|
4243
|
+
* plugin-scope write-back {@link reconcileGrantRecord} performs), because
|
|
4244
|
+
* the provider may already have consumed the superseded refresh token. A
|
|
4245
|
+
* foreign or newer payload (another account, a native writer) is left
|
|
4246
|
+
* untouched, and a record removed during the window stays removed. In every
|
|
4247
|
+
* branch the owner map ends consistent with what the store actually holds.
|
|
4248
|
+
*
|
|
4249
|
+
* Returns a failure message when the restore or the rotation write-back
|
|
4250
|
+
* did not land, so the caller reports `{ ok: false }` instead of a false
|
|
4251
|
+
* pass.
|
|
4252
|
+
*/
|
|
4253
|
+
async restoreProjection(records, provider, row, key, previous, borrowed, stillCurrent, late, capability) {
|
|
4254
|
+
let live;
|
|
4255
|
+
try {
|
|
4256
|
+
live = await records.readRecord(key);
|
|
4257
|
+
}
|
|
4258
|
+
catch (error) {
|
|
4259
|
+
const message = `The credential projection at ${key} could not be re-read after the test: ${error instanceof Error ? error.message : String(error)}`;
|
|
4260
|
+
this.pushWarning(message);
|
|
4261
|
+
return message;
|
|
4262
|
+
}
|
|
4263
|
+
// PR review round 3: a cleanup that outlived its deadline can be
|
|
4264
|
+
// superseded by a newer window that borrowed the SAME account meanwhile,
|
|
4265
|
+
// so the compare by payload below still matches a state this attempt no
|
|
4266
|
+
// longer owns. `stillCurrent` is that attempt identity; only an abandoned
|
|
4267
|
+
// attempt has one. Every write decision — the projection, the owner map
|
|
4268
|
+
// and the binding sync — is gated on it.
|
|
4269
|
+
const superseded = () => stillCurrent !== undefined && !stillCurrent();
|
|
4270
|
+
const takeover = 'a newer cleanup attempt took over this provider';
|
|
4271
|
+
// PR review round 36: the re-read above is an await a cleanup can park in
|
|
4272
|
+
// across `close()`, so every write decision refuses persistence from the
|
|
4273
|
+
// moment shutdown starts — unless the caller is the pre-close restoration,
|
|
4274
|
+
// which is the only holder of that capability.
|
|
4275
|
+
const shutdown = 'Keys Manager is shutting down; the test cleanup was skipped';
|
|
4276
|
+
if (this.persistenceRefused(capability))
|
|
4277
|
+
return shutdown;
|
|
4278
|
+
// PR review round 8: the fence is checked in each branch that writes the
|
|
4279
|
+
// owner map, instead of once before every branch. The pre-branch check ran
|
|
4280
|
+
// after the only await, so it pre-empted the branch fences below and made
|
|
4281
|
+
// them untestable; every write is still guarded, at the write itself.
|
|
4282
|
+
const liveGrant = live !== undefined && live.kind === 'grant' && isOAuthGrantPayload(live.payload)
|
|
4283
|
+
? live.payload
|
|
4284
|
+
: undefined;
|
|
4285
|
+
const previousGrant = previous !== undefined && previous.kind === 'grant' && isOAuthGrantPayload(previous.payload)
|
|
4286
|
+
? previous.payload
|
|
4287
|
+
: undefined;
|
|
4288
|
+
const carriesBorrowed = liveGrant !== undefined && JSON.stringify(liveGrant) === JSON.stringify(borrowed);
|
|
4289
|
+
// R2-1: the pre-test projection was deleted while this test ran. That
|
|
4290
|
+
// removal came from a writer outside the provider's grant chain — the
|
|
4291
|
+
// manager's own projection release, or a native sign-out — so the
|
|
4292
|
+
// pre-test payload is not this test's to put back: resurrecting it would
|
|
4293
|
+
// restore a credential its owner just stopped serving. The record stays
|
|
4294
|
+
// absent and the owner map follows the (empty) live store. The
|
|
4295
|
+
// `previous === undefined` case is different — that projection was this
|
|
4296
|
+
// test's own write — and is deleted below.
|
|
4297
|
+
if (live === undefined && previous !== undefined) {
|
|
4298
|
+
// PR review round 7: the re-read above is an await, so the attempt fence
|
|
4299
|
+
// is re-checked at this write instead of inherited from an earlier site.
|
|
4300
|
+
if (superseded())
|
|
4301
|
+
return takeover;
|
|
4302
|
+
this.reconcileProjectionOwner(provider, undefined);
|
|
4303
|
+
return undefined;
|
|
4304
|
+
}
|
|
4305
|
+
const rows = this.store.provider(provider)?.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null) ?? [];
|
|
4306
|
+
const previousOwner = previousGrant === undefined ? undefined : matchOwningAccount(provider, rows, previousGrant);
|
|
4307
|
+
if (live !== undefined && !carriesBorrowed) {
|
|
4308
|
+
const rotationOwner = liveGrant === undefined ? undefined : matchOwningAccount(provider, rows, liveGrant);
|
|
4309
|
+
if (rotationOwner?.id === row.id && this.accountRowIsManaged(provider, row)) {
|
|
4310
|
+
// PR review round 8: this branch's write-back starts an awaited write,
|
|
4311
|
+
// so it checks the attempt fence where it is about to write.
|
|
4312
|
+
if (superseded())
|
|
4313
|
+
return takeover;
|
|
4314
|
+
if (this.persistenceRefused(capability))
|
|
4315
|
+
return shutdown;
|
|
4316
|
+
try {
|
|
4317
|
+
await this.writeBackRotation(provider, row, liveGrant, capability, stillCurrent);
|
|
4318
|
+
}
|
|
4319
|
+
catch (error) {
|
|
4320
|
+
// PR review round 38: the helper throws when this attempt was
|
|
4321
|
+
// superseded mid-write or shutdown started; neither may claim the
|
|
4322
|
+
// route, and nothing may be written on their behalf.
|
|
4323
|
+
if (superseded())
|
|
4324
|
+
return takeover;
|
|
4325
|
+
if (this.persistenceRefused(capability))
|
|
4326
|
+
return shutdown;
|
|
4327
|
+
// The live projection is the only copy of the rotated payload:
|
|
4328
|
+
// overwriting it with the pre-test payload would strand the account
|
|
4329
|
+
// on a consumed refresh token, so it stays and owns the route —
|
|
4330
|
+
// unless this attempt is superseded, and the route is no longer its
|
|
4331
|
+
// to claim (PR review round 6).
|
|
4332
|
+
if (superseded())
|
|
4333
|
+
return takeover;
|
|
4334
|
+
this.claimProjectionOwner(provider, row.id);
|
|
4335
|
+
const message = `The rotated grant for ${provider} could not be written back after the test: ${error instanceof Error ? error.message : String(error)}`;
|
|
4336
|
+
this.pushWarning(message);
|
|
4337
|
+
return message;
|
|
4338
|
+
}
|
|
4339
|
+
}
|
|
4340
|
+
else {
|
|
4341
|
+
// A foreign or newer payload — another account, a native writer, an
|
|
4342
|
+
// unrecognized record — is not this test's to restore. Only the owner
|
|
4343
|
+
// map follows what is actually stored, and only while this attempt
|
|
4344
|
+
// still owns the provider (PR review round 7: fenced at the write).
|
|
4345
|
+
if (superseded())
|
|
4346
|
+
return takeover;
|
|
4347
|
+
this.reconcileProjectionOwner(provider, live);
|
|
4348
|
+
return undefined;
|
|
4349
|
+
}
|
|
4350
|
+
}
|
|
4351
|
+
// PR review round 6: the rotation write-back is an await. A cleanup that
|
|
4352
|
+
// outlived its deadline can resume after a newer window borrowed another
|
|
4353
|
+
// account, so the owner map it would reconcile belongs to that window's
|
|
4354
|
+
// world now: finish the write, skip the owner write, report the takeover.
|
|
4355
|
+
if (superseded())
|
|
4356
|
+
return takeover;
|
|
4357
|
+
// QC B2: the pre-test projection already belonged to the tested row, so
|
|
4358
|
+
// the live payload (the borrowed grant, or a rotation just written back
|
|
4359
|
+
// above) IS that row's current grant. Writing `previous` back would revert
|
|
4360
|
+
// it: its echo re-matches the same row, moves `grantRef` to the superseded
|
|
4361
|
+
// generation and deletes the rotated record — leaving the account on a
|
|
4362
|
+
// consumed refresh token.
|
|
4363
|
+
//
|
|
4364
|
+
// PR review round 28: ownership alone cannot decide that, because the
|
|
4365
|
+
// captured `previous` is a *different generation* of the same account
|
|
4366
|
+
// whenever a re-authentication projected a newer payload before the borrow
|
|
4367
|
+
// (the borrow's compare-and-swap then replaced it with the older stored
|
|
4368
|
+
// grant). Keeping the borrowed payload there strands the route on the
|
|
4369
|
+
// stale generation, and the next echo adopts it and deletes the newer one.
|
|
4370
|
+
// Only the payload identity decides: identical payloads are the pre-test
|
|
4371
|
+
// projection itself, and a different one that the store still carries as
|
|
4372
|
+
// the borrow is the newer generation this restore puts back.
|
|
4373
|
+
const staleBorrow = previousGrant !== undefined && JSON.stringify(previousGrant) !== JSON.stringify(borrowed);
|
|
4374
|
+
if (previousOwner?.id === row.id && !(carriesBorrowed && staleBorrow)) {
|
|
4375
|
+
this.reconcileProjectionOwner(provider, live);
|
|
4376
|
+
return undefined;
|
|
4377
|
+
}
|
|
4378
|
+
// PR review round 19 (I-000179 late restore): only a late restore can find
|
|
4379
|
+
// its pre-test projection's owner gone — the watchdog's cleanup released
|
|
4380
|
+
// the provider, so a removal no longer waited for this test and `rows` no
|
|
4381
|
+
// longer matches that payload, leaving nothing else to reject the write.
|
|
4382
|
+
// Restoring it would serve a deleted account's credential. An in-window
|
|
4383
|
+
// restore is protected by settleTestBeforeRemoval and keeps restoring a
|
|
4384
|
+
// pre-test projection no pool row owns (a foreign grant it borrowed over).
|
|
4385
|
+
if (late === true && (previousOwner === undefined || !this.accountRowIsManaged(provider, previousOwner))) {
|
|
4386
|
+
if (superseded())
|
|
4387
|
+
return takeover;
|
|
4388
|
+
this.reconcileProjectionOwner(provider, live);
|
|
4389
|
+
const message = `The credential projection at ${key} was not restored after the test: the account whose grant it carried is no longer managed. The route keeps whichever credential those records hold now.`;
|
|
4390
|
+
this.pushWarning(message);
|
|
4391
|
+
return message;
|
|
4392
|
+
}
|
|
4393
|
+
// The record this restore left in place: the tail reconciles the owner map
|
|
4394
|
+
// against the live record, never against the payload it hoped to restore
|
|
4395
|
+
// (PR review round 8 — the store may hold a present non-grant record).
|
|
4396
|
+
let left;
|
|
4397
|
+
try {
|
|
4398
|
+
if (previous === undefined) {
|
|
4399
|
+
// PR review MF2: the projection was this test's own write, so the
|
|
4400
|
+
// route goes back to having none — but only while the record still
|
|
4401
|
+
// carries what this test left (the borrowed grant, or a rotation of
|
|
4402
|
+
// the tested row). The decision runs inside the seam's per-key lock
|
|
4403
|
+
// and the removal is preceded by {@link projectionStillMatches}, so a
|
|
4404
|
+
// native writer's grant landing after the re-read above survives as
|
|
4405
|
+
// that writer's projection instead of being deleted as a leftover.
|
|
4406
|
+
let decidedAgainst = undefined;
|
|
4407
|
+
await records.modifyRecord(key, async (current) => {
|
|
4408
|
+
// A superseded attempt, or a cleanup the shutdown overtook, leaves
|
|
4409
|
+
// the record exactly as it found it.
|
|
4410
|
+
if (superseded() || this.persistenceRefused(capability))
|
|
4411
|
+
return undefined;
|
|
4412
|
+
if (current === undefined)
|
|
4413
|
+
return undefined;
|
|
4414
|
+
const currentGrant = current.kind === 'grant' && isOAuthGrantPayload(current.payload)
|
|
4415
|
+
? current.payload
|
|
4416
|
+
: undefined;
|
|
4417
|
+
const testLeft = currentGrant !== undefined && (JSON.stringify(currentGrant) === JSON.stringify(borrowed)
|
|
4418
|
+
|| matchOwningAccount(provider, rows, currentGrant)?.id === row.id);
|
|
4419
|
+
if (testLeft)
|
|
4420
|
+
decidedAgainst = JSON.stringify(current.payload);
|
|
4421
|
+
return undefined;
|
|
4422
|
+
});
|
|
4423
|
+
if (superseded())
|
|
4424
|
+
return takeover;
|
|
4425
|
+
if (this.persistenceRefused(capability))
|
|
4426
|
+
return shutdown;
|
|
4427
|
+
if (decidedAgainst === undefined || !await this.projectionStillMatches(records, key, decidedAgainst)) {
|
|
4428
|
+
const liveAfter = await records.readRecord(key);
|
|
4429
|
+
// PR review round 6: this reconcile follows an awaited re-read; a
|
|
4430
|
+
// superseded attempt may not repoint the owner from a world a newer
|
|
4431
|
+
// window now owns.
|
|
4432
|
+
if (superseded())
|
|
4433
|
+
return takeover;
|
|
4434
|
+
// PR review round 35: the re-read is the cleanup's last await before
|
|
4435
|
+
// it would write; a shutdown that began under it stops the restore.
|
|
4436
|
+
if (this.persistenceRefused(capability))
|
|
4437
|
+
return shutdown;
|
|
4438
|
+
this.reconcileProjectionOwner(provider, liveAfter);
|
|
4439
|
+
// Already gone: the route has no projection, which is the state the
|
|
4440
|
+
// test found. Anything else is a grant this test may not remove —
|
|
4441
|
+
// it stays and owns the route, and the test reports that its own
|
|
4442
|
+
// borrow could not be undone.
|
|
4443
|
+
if (liveAfter === undefined)
|
|
4444
|
+
return undefined;
|
|
4445
|
+
const message = `The credential projection at ${key} changed during the test and was left in place; the route serves whichever credential those records hold now.`;
|
|
4446
|
+
this.pushWarning(message);
|
|
4447
|
+
return message;
|
|
4448
|
+
}
|
|
4449
|
+
// The projection this test borrowed may already be a newer window's.
|
|
4450
|
+
if (superseded())
|
|
4451
|
+
return takeover;
|
|
4452
|
+
if (this.persistenceRefused(capability))
|
|
4453
|
+
return shutdown;
|
|
4454
|
+
await records.deleteRecord(key);
|
|
4455
|
+
}
|
|
4456
|
+
else {
|
|
4457
|
+
// QC N2: the compare travels with the write, so a foreign payload
|
|
4458
|
+
// stored between the re-read above and this write is never
|
|
4459
|
+
// overwritten. Only a record that still carries what this test left
|
|
4460
|
+
// (the borrowed grant, or a rotation of the tested row) is replaced.
|
|
4461
|
+
const snapshot = structuredClone(previous);
|
|
4462
|
+
left = await records.modifyRecord(key, current => {
|
|
4463
|
+
// A superseded attempt, or a cleanup the shutdown overtook, leaves
|
|
4464
|
+
// the record exactly as it found it.
|
|
4465
|
+
if (superseded() || this.persistenceRefused(capability))
|
|
4466
|
+
return Promise.resolve(current);
|
|
4467
|
+
const currentGrant = current !== undefined && current.kind === 'grant' && isOAuthGrantPayload(current.payload)
|
|
4468
|
+
? current.payload
|
|
4469
|
+
: undefined;
|
|
4470
|
+
const testLeft = currentGrant !== undefined && (JSON.stringify(currentGrant) === JSON.stringify(borrowed)
|
|
4471
|
+
|| matchOwningAccount(provider, rows, currentGrant)?.id === row.id);
|
|
4472
|
+
return Promise.resolve(testLeft ? snapshot : current);
|
|
4473
|
+
});
|
|
4474
|
+
if (superseded())
|
|
4475
|
+
return takeover;
|
|
4476
|
+
if (this.persistenceRefused(capability))
|
|
4477
|
+
return shutdown;
|
|
4478
|
+
}
|
|
4479
|
+
}
|
|
4480
|
+
catch (error) {
|
|
4481
|
+
// The store still holds what the re-read saw: keep the owner map on
|
|
4482
|
+
// that record rather than claiming the pre-test account (I2).
|
|
4483
|
+
if (!superseded())
|
|
4484
|
+
this.reconcileProjectionOwner(provider, live);
|
|
4485
|
+
const message = `The credential projection at ${key} could not be restored after the test: ${error instanceof Error ? error.message : String(error)}`;
|
|
4486
|
+
this.pushWarning(message);
|
|
4487
|
+
return message;
|
|
4488
|
+
}
|
|
4489
|
+
if (superseded())
|
|
4490
|
+
return takeover;
|
|
4491
|
+
this.reconcileProjectionOwner(provider, left);
|
|
4492
|
+
return undefined;
|
|
4493
|
+
}
|
|
4494
|
+
/**
|
|
4495
|
+
* Point the owner map at the OAuth row the stored record identifies.
|
|
4496
|
+
*
|
|
4497
|
+
* The argument is the live record, not its grant payload (PR review round 8):
|
|
4498
|
+
* a payload-only signature cannot tell a removed projection from a PRESENT
|
|
4499
|
+
* record this pool cannot interpret (a native api-key, an unrecognized
|
|
4500
|
+
* payload), so a surviving native record reached this method as `undefined`,
|
|
4501
|
+
* cleared the foreign marker, and `keyPoolServes` then let the window-end
|
|
4502
|
+
* sync bind the broker over it. Only an absent record is a confirmed removal
|
|
4503
|
+
* (I-000167); a present record no row owns keeps the route foreign (round 3).
|
|
4504
|
+
*/
|
|
4505
|
+
reconcileProjectionOwner(provider, live) {
|
|
4506
|
+
if (live === undefined) {
|
|
4507
|
+
this.activeAccounts.delete(provider);
|
|
4508
|
+
// A confirmed absence: the foreign marker must clear with the owner, or
|
|
4509
|
+
// `keyPoolServes` keeps the surviving pool unbound forever (I-000167).
|
|
4510
|
+
this.foreignProjections.delete(provider);
|
|
4511
|
+
return;
|
|
4512
|
+
}
|
|
4513
|
+
const rows = this.store.provider(provider)?.keys.filter(candidate => candidate.kind === 'oauth' && candidate.oauth !== null) ?? [];
|
|
4514
|
+
const owner = live.kind === 'grant' && isOAuthGrantPayload(live.payload)
|
|
4515
|
+
? matchOwningAccount(provider, rows, live.payload)
|
|
4516
|
+
: undefined;
|
|
4517
|
+
if (owner !== undefined) {
|
|
4518
|
+
this.claimProjectionOwner(provider, owner.id);
|
|
4519
|
+
return;
|
|
4520
|
+
}
|
|
4521
|
+
this.activeAccounts.delete(provider);
|
|
4522
|
+
// Present but not a managed grant: a native api-key or an unowned grant
|
|
4523
|
+
// still serves the route ahead of any broker reference.
|
|
4524
|
+
this.foreignProjections.add(provider);
|
|
4525
|
+
}
|
|
4526
|
+
/**
|
|
4527
|
+
* Persist one authoritative rotated grant payload for a row: write the
|
|
4528
|
+
* plugin-scope record for the rotated generation, then hand the commit to
|
|
4529
|
+
* {@link commitGrantGeneration} — the same tail {@link reconcileGrantRecord}
|
|
4530
|
+
* performs. The caller runs inside the provider's grant chain and owns the
|
|
4531
|
+
* decision, including the echo-specific generation guards, so only the
|
|
4532
|
+
* write-back semantics live here.
|
|
4533
|
+
*/
|
|
4534
|
+
async writeBackRotation(provider, row, payload, capability, stillCurrent) {
|
|
4535
|
+
const records = this.credentialRecords();
|
|
4536
|
+
if (records === undefined)
|
|
4537
|
+
throw new Error('The credential store does not support records');
|
|
4538
|
+
if (row.oauth === null)
|
|
4539
|
+
throw new Error(`Key "${row.id}" is not an OAuth account`);
|
|
4540
|
+
// PR review round 38: this helper's three steps are separated by awaits, so
|
|
4541
|
+
// both gates are read again at each of them. A superseded attempt may not
|
|
4542
|
+
// write the rotated record, and shutdown refuses the persistence outright
|
|
4543
|
+
// rather than only at the commit below.
|
|
4544
|
+
if (this.persistenceRefused(capability) || (stillCurrent !== undefined && !stillCurrent())) {
|
|
4545
|
+
throw new Error('Keys Manager is shutting down or the test cleanup was superseded');
|
|
4546
|
+
}
|
|
4547
|
+
const hash = createHash('sha256').update(payload.refresh).digest('hex').slice(0, 12);
|
|
4548
|
+
const rotatedRef = accountRef(provider, hash);
|
|
4549
|
+
// PR review round 39: the seam may park this call before it invokes the
|
|
4550
|
+
// updater, so the gates are read inside the updater itself — a write
|
|
4551
|
+
// decided before shutdown/supersession must not land after them.
|
|
4552
|
+
let written = false;
|
|
4553
|
+
await records.modifyRecord(rotatedRef, async () => {
|
|
4554
|
+
if (this.persistenceRefused(capability) || (stillCurrent !== undefined && !stillCurrent()))
|
|
4555
|
+
return undefined;
|
|
4556
|
+
written = true;
|
|
4557
|
+
return { kind: 'grant', payload: structuredClone(payload) };
|
|
4558
|
+
});
|
|
4559
|
+
if (!written)
|
|
4560
|
+
throw new Error('Keys Manager is shutting down or the test cleanup was superseded');
|
|
4561
|
+
// PR review round 35: the write above is an await, and the commit below is
|
|
4562
|
+
// the persisted state write that follows it. Round 36: only the restore
|
|
4563
|
+
// holding the pre-close capability may continue past shutdown entry.
|
|
4564
|
+
if (this.persistenceRefused(capability) || (stillCurrent !== undefined && !stillCurrent())) {
|
|
4565
|
+
throw new Error('Keys Manager is shutting down or the test cleanup was superseded');
|
|
4566
|
+
}
|
|
4567
|
+
await this.commitGrantGeneration(provider, row, payload, rotatedRef, stillCurrent, capability);
|
|
4568
|
+
}
|
|
4569
|
+
/**
|
|
4570
|
+
* The single owner of `row.oauth.grantRef` semantics (QC N1): point the row
|
|
4571
|
+
* at the record holding its rotated generation, merge the refresh-derived
|
|
4572
|
+
* identity fields, and remove the superseded record. Both the echo
|
|
4573
|
+
* reconciler and the connection test's rotation write-back commit through
|
|
4574
|
+
* here, so the ref scheme, the identity merge and the cleanup policy have
|
|
4575
|
+
* exactly one implementation.
|
|
4576
|
+
*
|
|
4577
|
+
* The record write stays with the caller: the reconciler must revalidate the
|
|
4578
|
+
* row between that write and this commit (C6-1), so only the tail is shared.
|
|
4579
|
+
*/
|
|
4580
|
+
async commitGrantGeneration(provider, owner, payload, rotatedRef, guard, capability) {
|
|
4581
|
+
// PR review round 38: the rotation write that precedes this call is an
|
|
4582
|
+
// await. An attempt superseded meanwhile must not repoint the row or delete
|
|
4583
|
+
// the record it would find here — that record may be a newer window's
|
|
4584
|
+
// already-committed generation — and shutdown must refuse both writes even
|
|
4585
|
+
// when it began after the caller's own check.
|
|
4586
|
+
if (this.persistenceRefused(capability) || (guard !== undefined && !guard()))
|
|
4587
|
+
return;
|
|
4588
|
+
const current = owner.oauth;
|
|
4589
|
+
if (current === null)
|
|
4590
|
+
return;
|
|
4591
|
+
const previousRef = current.grantRef;
|
|
4592
|
+
owner.oauth = {
|
|
4593
|
+
...current,
|
|
4594
|
+
...(previousRef === rotatedRef ? {} : { grantRef: rotatedRef }),
|
|
4595
|
+
...(payload.email === undefined ? {} : { email: payload.email }),
|
|
4596
|
+
...(payload.accountId === undefined ? {} : { accountId: payload.accountId }),
|
|
4597
|
+
...(typeof payload.expires === 'number' && Number.isFinite(payload.expires) ? { expiresAt: payload.expires } : {}),
|
|
4598
|
+
};
|
|
4599
|
+
if (previousRef !== rotatedRef) {
|
|
4600
|
+
// The row repoint above is synchronous, but the delete is a second write:
|
|
4601
|
+
// re-read both gates immediately before it so a shutdown or a takeover
|
|
4602
|
+
// that landed in between cannot remove a record this attempt never owned.
|
|
4603
|
+
if (this.persistenceRefused(capability) || (guard !== undefined && !guard())) {
|
|
4604
|
+
this.store.touch();
|
|
4605
|
+
return;
|
|
4606
|
+
}
|
|
4607
|
+
const records = this.credentialRecords();
|
|
4608
|
+
try {
|
|
4609
|
+
if (records === undefined)
|
|
4610
|
+
throw new Error('The credential store does not support records');
|
|
4611
|
+
await records.deleteRecord(previousRef);
|
|
4612
|
+
}
|
|
4613
|
+
catch (error) {
|
|
4614
|
+
this.warn(`cleanup of the superseded grant record failed for ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
4615
|
+
}
|
|
4616
|
+
}
|
|
4617
|
+
this.store.touch();
|
|
4618
|
+
}
|
|
4619
|
+
/**
|
|
4620
|
+
* Send the minimal request under the given scope and map its terminal
|
|
4621
|
+
* chunk. A thrown error or the 20-second abort is reported, never swallowed.
|
|
4622
|
+
*/
|
|
4623
|
+
async runTestRequest(provider, model, scope) {
|
|
4624
|
+
const started = Date.now();
|
|
4625
|
+
const signal = AbortSignal.timeout(KEY_TEST_TIMEOUT_MS);
|
|
4626
|
+
try {
|
|
4627
|
+
const outcome = await this.als.run(scope, async () => {
|
|
4628
|
+
const stream = this.ctx.llm.stream({
|
|
4629
|
+
provider,
|
|
4630
|
+
model,
|
|
4631
|
+
messages: [{ role: 'user', content: [{ type: 'text', text: 'ping' }] }],
|
|
4632
|
+
maxTokens: 1,
|
|
4633
|
+
signal,
|
|
4634
|
+
});
|
|
4635
|
+
for await (const chunk of stream) {
|
|
4636
|
+
if (chunk.type !== 'finish')
|
|
4637
|
+
continue;
|
|
4638
|
+
const kind = chunk.reason?.kind;
|
|
4639
|
+
if (kind === 'error') {
|
|
4640
|
+
const failure = chunk.reason?.failure;
|
|
4641
|
+
return { ok: false, error: failure?.message ?? failure?.code ?? 'The provider reported a failure' };
|
|
4642
|
+
}
|
|
4643
|
+
// PR review SF3 + round 8: only a real completion is a pass. The ping
|
|
4644
|
+
// asks for one token, so `length` is the expected completion; a
|
|
4645
|
+
// finish chunk with NO reason, `aborted`, or any kind this broker
|
|
4646
|
+
// does not know means the provider did not answer, so reporting a
|
|
4647
|
+
// pass would claim a credential it never served.
|
|
4648
|
+
if (kind === 'stop' || kind === 'length' || kind === 'toolUse') {
|
|
4649
|
+
// PR review round 42: a successful ping only proves the route
|
|
4650
|
+
// answered. An api-key row test must also prove the pinned literal
|
|
4651
|
+
// was the credential that served it — a takeover during the stream
|
|
4652
|
+
// would otherwise be reported as this key's success.
|
|
4653
|
+
if (scope.keyId !== null && scope.oauthAccountId === null && scope.keyUsed !== true) {
|
|
4654
|
+
return { ok: false, error: `The ${provider} route was served by another credential source; the selected key was not used.` };
|
|
4655
|
+
}
|
|
4656
|
+
return { ok: true };
|
|
4657
|
+
}
|
|
4658
|
+
if (kind === undefined)
|
|
4659
|
+
return { ok: false, error: 'The provider returned no completion reason' };
|
|
4660
|
+
return { ok: false, error: kind === 'aborted' ? 'The request was aborted' : `The request ended with "${kind}"` };
|
|
4661
|
+
}
|
|
4662
|
+
return { ok: false, error: 'The provider returned no result' };
|
|
4663
|
+
});
|
|
4664
|
+
return { ...outcome, latencyMs: Date.now() - started };
|
|
4665
|
+
}
|
|
4666
|
+
catch (error) {
|
|
4667
|
+
return {
|
|
4668
|
+
ok: false,
|
|
4669
|
+
latencyMs: Date.now() - started,
|
|
4670
|
+
error: isTestTimeoutAbort(error, signal) ? `Timed out after ${KEY_TEST_TIMEOUT_LABEL}` : error instanceof Error ? error.message : String(error),
|
|
4671
|
+
};
|
|
4672
|
+
}
|
|
4673
|
+
}
|
|
2203
4674
|
/**
|
|
2204
4675
|
* Demote the cascade to the key pool by deleting the projection record:
|
|
2205
4676
|
* pi-ai's per-request resolution prefers the stored credential and only
|
|
@@ -2376,6 +4847,7 @@ export class KeysManagerService {
|
|
|
2376
4847
|
// the missing pill and leaves "Set active" disabled — the documented
|
|
2377
4848
|
// re-activation path unreachable until a restart or a new sign-in.
|
|
2378
4849
|
this.activeAccounts.delete(provider);
|
|
4850
|
+
this.foreignProjections.delete(provider);
|
|
2379
4851
|
for (const row of providerRecord.keys) {
|
|
2380
4852
|
if (row.kind === 'oauth')
|
|
2381
4853
|
row.valueMissing = true;
|
|
@@ -2385,7 +4857,12 @@ export class KeysManagerService {
|
|
|
2385
4857
|
// The projection is not ours anymore (native replacement): keep the
|
|
2386
4858
|
// stale owner entry from routing attribution or demotion at this row.
|
|
2387
4859
|
this.activeAccounts.delete(provider);
|
|
4860
|
+
// …and the replacement still serves ahead of the pool (round 3).
|
|
4861
|
+
this.foreignProjections.add(provider);
|
|
2388
4862
|
}
|
|
4863
|
+
// The projection row is gone: a surviving key pool is the serving source
|
|
4864
|
+
// again, and a route with no API-key row stays unbound.
|
|
4865
|
+
await this.syncBrokerBinding(provider);
|
|
2389
4866
|
this.store.touch();
|
|
2390
4867
|
}
|
|
2391
4868
|
/**
|
|
@@ -2431,7 +4908,10 @@ export class KeysManagerService {
|
|
|
2431
4908
|
* falls back to the key pool.
|
|
2432
4909
|
*/
|
|
2433
4910
|
removeAccount(provider, accountId) {
|
|
2434
|
-
return this.run(async () => {
|
|
4911
|
+
return this.run(() => this.withRemovalInProgress(provider, async () => {
|
|
4912
|
+
// R8-2: the removal decides against the route's live records, so it must
|
|
4913
|
+
// not run inside a row test's borrow window.
|
|
4914
|
+
await this.settleTestBeforeRemoval(provider);
|
|
2435
4915
|
const providerRecord = this.store.provider(provider);
|
|
2436
4916
|
if (providerRecord === undefined)
|
|
2437
4917
|
throw new Error(`Provider "${provider}" is not managed`);
|
|
@@ -2446,7 +4926,7 @@ export class KeysManagerService {
|
|
|
2446
4926
|
this.store.clearBindings((_key, binding) => binding.keyId === row.id);
|
|
2447
4927
|
this.store.touch();
|
|
2448
4928
|
return this.buildState();
|
|
2449
|
-
});
|
|
4929
|
+
}));
|
|
2450
4930
|
}
|
|
2451
4931
|
/** Install the broker layer on one credential provider instance. */
|
|
2452
4932
|
attachCredentials(credentials) {
|
|
@@ -2522,6 +5002,23 @@ export class KeysManagerService {
|
|
|
2522
5002
|
this.credentialWrappers.set(raw, wrappers);
|
|
2523
5003
|
if (this.rawCredentials === undefined)
|
|
2524
5004
|
this.rawCredentials = raw;
|
|
5005
|
+
// A remount replaces the credential backend (PR review round 27): every
|
|
5006
|
+
// projection fact loaded so far describes the previous store, so drop the
|
|
5007
|
+
// classification and restart the ownership read before repairing. The
|
|
5008
|
+
// first mount is startup, whose fresh maps need no invalidation.
|
|
5009
|
+
if (this.credentialsMounted)
|
|
5010
|
+
this.invalidateProjectionOwnership();
|
|
5011
|
+
this.credentialsMounted = true;
|
|
5012
|
+
// The credentials service can mount after this plugin (PR review round 23):
|
|
5013
|
+
// the load-time hydration then read nothing and left ownership unknown, so
|
|
5014
|
+
// the startup repair bound nothing. Re-read the projections now that the
|
|
5015
|
+
// record half is reachable, then repair — the same convergence as startup,
|
|
5016
|
+
// idempotent when the seam was mounted all along. The read lands through
|
|
5017
|
+
// {@link noteProviderHydrated}, so a repair already waiting wakes with it.
|
|
5018
|
+
this.projectionHydration = this.hydrateProjectionOwnership();
|
|
5019
|
+
void this.repairBrokerBindings().catch(error => {
|
|
5020
|
+
this.warn(`broker binding repair after the credentials mount failed: ${error instanceof Error ? error.message : String(error)}`);
|
|
5021
|
+
});
|
|
2525
5022
|
}
|
|
2526
5023
|
/** Restore the credential provider methods when our fiber unloads. */
|
|
2527
5024
|
detachCredentials(credentials) {
|
|
@@ -2632,8 +5129,15 @@ export class KeysManagerService {
|
|
|
2632
5129
|
namespace(descriptor, ns) {
|
|
2633
5130
|
return descriptor.namespaces.find(entry => entry.ns === ns);
|
|
2634
5131
|
}
|
|
2635
|
-
/**
|
|
2636
|
-
|
|
5132
|
+
/**
|
|
5133
|
+
* Apply one path-addressed settings write through the live revision.
|
|
5134
|
+
*
|
|
5135
|
+
* `stillWrites` runs after the descriptor re-read and immediately before the
|
|
5136
|
+
* mutation: a decision derived from an earlier snapshot can be stale by then
|
|
5137
|
+
* ({@link syncBrokerBinding}'s bind gate, PR review round 25). A declining
|
|
5138
|
+
* guard skips the write and reports it.
|
|
5139
|
+
*/
|
|
5140
|
+
async mutateSettings(ns, ops, stillWrites) {
|
|
2637
5141
|
const settings = this.settingsService();
|
|
2638
5142
|
if (settings === undefined)
|
|
2639
5143
|
throw new Error('Settings are unavailable in this composition');
|
|
@@ -2648,7 +5152,10 @@ export class KeysManagerService {
|
|
|
2648
5152
|
}
|
|
2649
5153
|
if (descriptor === undefined)
|
|
2650
5154
|
throw new Error(`Settings namespace "${ns}" is not mounted`);
|
|
5155
|
+
if (stillWrites !== undefined && !stillWrites())
|
|
5156
|
+
return false;
|
|
2651
5157
|
await settings.mutate(ns, ops, descriptor.revision);
|
|
5158
|
+
return true;
|
|
2652
5159
|
}
|
|
2653
5160
|
/** Prune expired bindings at most once a minute, without a restart. */
|
|
2654
5161
|
pruneBindings(now) {
|
|
@@ -2825,20 +5332,51 @@ export class KeysManagerService {
|
|
|
2825
5332
|
pausedKeyCount: allKeys.filter(key => key.status === 'paused').length,
|
|
2826
5333
|
bindingsCount: Object.keys(this.store.state.sessionBindings).length,
|
|
2827
5334
|
warnings,
|
|
5335
|
+
transportAccounts: this.transportRows.map(row => ({ ...row })),
|
|
2828
5336
|
};
|
|
2829
5337
|
}
|
|
2830
|
-
/**
|
|
5338
|
+
/**
|
|
5339
|
+
* Remote: complete manager state.
|
|
5340
|
+
*
|
|
5341
|
+
* The transport rows come from another package's records, and reading them
|
|
5342
|
+
* is async while this endpoint is synchronous, so every poll starts a
|
|
5343
|
+
* refresh for the next one (the client ticks every 30 s), unless one is
|
|
5344
|
+
* already in flight. The constructor warms the first read and record
|
|
5345
|
+
* updates refresh immediately, so the one-tick lag only shows on a cold
|
|
5346
|
+
* start.
|
|
5347
|
+
*/
|
|
2831
5348
|
getState() {
|
|
5349
|
+
this.refreshTransportAccountsIfIdle();
|
|
2832
5350
|
return this.buildState();
|
|
2833
5351
|
}
|
|
2834
5352
|
/** Find one directory entry by route id. */
|
|
2835
5353
|
directoryEntry(provider) {
|
|
2836
5354
|
return this.ctx.llm.listConfigurableProviders().find(entry => entry.provider === provider);
|
|
2837
5355
|
}
|
|
5356
|
+
/**
|
|
5357
|
+
* Whether the key pool is the credential source that serves this route: no
|
|
5358
|
+
* OAuth projection owns it and at least one API-key row still holds a value.
|
|
5359
|
+
* This is the one arbiter for both the verdict below and the binding write
|
|
5360
|
+
* ({@link syncBrokerBinding}).
|
|
5361
|
+
*/
|
|
5362
|
+
keyPoolServes(record) {
|
|
5363
|
+
return this.oauthOwnerFor(record.provider) === null
|
|
5364
|
+
// A projection no managed row owns still serves ahead of the broker
|
|
5365
|
+
// reference: binding the pool here would take the foreign account out of
|
|
5366
|
+
// service (PR review round 3).
|
|
5367
|
+
&& !this.foreignProjections.has(record.provider)
|
|
5368
|
+
&& record.keys.some(key => key.kind === 'api-key' && key.valueMissing !== true);
|
|
5369
|
+
}
|
|
2838
5370
|
/**
|
|
2839
5371
|
* Classify the live binding. An unreadable settings snapshot is `unknown`,
|
|
2840
5372
|
* not a confirmed mismatch: transient settings failures must not disable
|
|
2841
5373
|
* key-pool recovery for a provider that is still managed.
|
|
5374
|
+
*
|
|
5375
|
+
* Only a value Keys Manager never wrote is a `mismatch`: the warning now
|
|
5376
|
+
* means "the user took the profile back", not "the OAuth route is broken".
|
|
5377
|
+
* An unbound profile is healthy while the key pool is not the serving
|
|
5378
|
+
* source (an OAuth projection serves, or no API-key row holds a value) —
|
|
5379
|
+
* that is the OAuth-only route's native shape.
|
|
2842
5380
|
*/
|
|
2843
5381
|
bindingState(record, descriptor = this.describeSettings()) {
|
|
2844
5382
|
if (!descriptor.readable) {
|
|
@@ -2848,10 +5386,246 @@ export class KeysManagerService {
|
|
|
2848
5386
|
}
|
|
2849
5387
|
const namespace = this.namespace(descriptor, record.settingsNs);
|
|
2850
5388
|
const profile = namespace === undefined ? undefined : asProfile(getPath(namespace.value, record.settingsPath));
|
|
2851
|
-
const
|
|
5389
|
+
const current = profile !== undefined && typeof profile.apiKeyEnv === 'string' ? profile.apiKeyEnv : undefined;
|
|
5390
|
+
const state = current === record.brokerRef
|
|
5391
|
+
// The broker reference is Keys Manager's own value; a projection that
|
|
5392
|
+
// claimed the route is repaired by the binding sync, not by the user.
|
|
5393
|
+
? 'healthy'
|
|
5394
|
+
: current !== undefined
|
|
5395
|
+
? 'mismatch'
|
|
5396
|
+
: this.keyPoolServes(record) ? 'mismatch' : 'healthy';
|
|
2852
5397
|
this.bindingCache.set(record.provider, state);
|
|
2853
5398
|
return state;
|
|
2854
5399
|
}
|
|
5400
|
+
/**
|
|
5401
|
+
* The single authority for a managed provider's pi-ai `apiKeyEnv`.
|
|
5402
|
+
*
|
|
5403
|
+
* pi-ai resolves a named credential synchronously and throws
|
|
5404
|
+
* `MISSING_CREDENTIAL` when the credential seam cannot answer it, so the
|
|
5405
|
+
* broker reference may only sit on a profile whose requests the key pool
|
|
5406
|
+
* actually serves ({@link keyPoolServes}). Every other shape must be
|
|
5407
|
+
* unbound, or pi-ai never reaches its own stored OAuth grant.
|
|
5408
|
+
*
|
|
5409
|
+
* Each direction has exactly one provably owned write:
|
|
5410
|
+
* - bind: a profile that names no credential, or — with `takeover`, passed
|
|
5411
|
+
* solely by {@link adoptProvider} — the pre-adoption reference adoption
|
|
5412
|
+
* recorded and deliberately replaces;
|
|
5413
|
+
* - unbind: only the broker reference Keys Manager itself wrote.
|
|
5414
|
+
*
|
|
5415
|
+
* Any other value is a hand-edit (or the pre-adoption reference release
|
|
5416
|
+
* restored): it is never touched, and the `mismatch` verdict stays the
|
|
5417
|
+
* signal that the user took the profile back.
|
|
5418
|
+
*
|
|
5419
|
+
* `write: false` is the startup repair: it only touches a profile that
|
|
5420
|
+
* already exists, so a first run never materializes one from nothing. The
|
|
5421
|
+
* release direction is single-sourced in {@link releasingProviders}: a
|
|
5422
|
+
* provider being unmanaged is never bound again, whatever the row order of
|
|
5423
|
+
* the pool it is deleting.
|
|
5424
|
+
*
|
|
5425
|
+
* A read-side problem or a refused write degrades to a warning: the
|
|
5426
|
+
* callers' committed change must not fail on a settings problem, the user
|
|
5427
|
+
* still sees that the route may resolve the wrong credential source, and
|
|
5428
|
+
* the next repair pass retries.
|
|
5429
|
+
*
|
|
5430
|
+
* Returns whether the profile ends in the shape the serving source justifies
|
|
5431
|
+
* (`true` also when no write was needed). The connection test's borrow is the
|
|
5432
|
+
* one caller that acts on it: a failed release must refuse the test instead
|
|
5433
|
+
* of sending a request the wrong credential source would answer.
|
|
5434
|
+
*
|
|
5435
|
+
* While a row test borrows the route (QC B4) no caller may persist a decision
|
|
5436
|
+
* derived from that borrowed world — an `addKey` landing inside the window
|
|
5437
|
+
* would otherwise unbind the profile for good. The test's own writes pass
|
|
5438
|
+
* `borrow: true`; the window-end sync then writes the restored state.
|
|
5439
|
+
*/
|
|
5440
|
+
async syncBrokerBinding(provider, options = {}) {
|
|
5441
|
+
const record = this.store.provider(provider);
|
|
5442
|
+
if (record === undefined)
|
|
5443
|
+
return false;
|
|
5444
|
+
// A borrow hides its world from this authority, except while the provider
|
|
5445
|
+
// is being unmanaged: release is a real state change, and the unbind
|
|
5446
|
+
// direction is already one-way there (a release that skipped it would
|
|
5447
|
+
// leave a dangling broker reference on a profile nothing repairs).
|
|
5448
|
+
if (options.borrow !== true && this.testBindingBorrows.has(provider) && !this.releasingProviders.has(provider)) {
|
|
5449
|
+
return false;
|
|
5450
|
+
}
|
|
5451
|
+
const descriptor = this.describeSettings();
|
|
5452
|
+
if (!descriptor.readable) {
|
|
5453
|
+
// QC N3: a composition that has not mounted the settings service yet is
|
|
5454
|
+
// a not-yet, not a repair failure — the mount seam re-runs this pass and
|
|
5455
|
+
// owns the warning if the write then still fails.
|
|
5456
|
+
if (this.settingsService() === undefined)
|
|
5457
|
+
return false;
|
|
5458
|
+
this.bindingWriteRefused(provider, 'the settings snapshot is unreadable');
|
|
5459
|
+
return false;
|
|
5460
|
+
}
|
|
5461
|
+
const namespace = this.namespace(descriptor, record.settingsNs);
|
|
5462
|
+
if (namespace === undefined) {
|
|
5463
|
+
this.bindingWriteRefused(provider, `settings namespace "${record.settingsNs}" is not mounted`);
|
|
5464
|
+
return false;
|
|
5465
|
+
}
|
|
5466
|
+
const profile = asProfile(getPath(namespace.value, record.settingsPath));
|
|
5467
|
+
if (options.write === false && profile === undefined)
|
|
5468
|
+
return true;
|
|
5469
|
+
const current = profile !== undefined && typeof profile.apiKeyEnv === 'string' ? profile.apiKeyEnv : undefined;
|
|
5470
|
+
const desired = this.keyPoolServes(record) ? record.brokerRef : undefined;
|
|
5471
|
+
// Binding is a guess while this process cannot attribute the projection:
|
|
5472
|
+
// the empty owner/foreign maps otherwise read as "the pool serves" and
|
|
5473
|
+
// overwrite a persisted grant nobody could see (PR review round 24). The
|
|
5474
|
+
// unbind direction is the safe one and still runs.
|
|
5475
|
+
if (desired !== undefined && !this.projectionOwnershipKnown(provider))
|
|
5476
|
+
return false;
|
|
5477
|
+
if (current === desired)
|
|
5478
|
+
return true;
|
|
5479
|
+
if (desired === undefined) {
|
|
5480
|
+
// Never strip a value Keys Manager did not write: a user hand-edit and
|
|
5481
|
+
// a restored pre-adoption reference look exactly like this.
|
|
5482
|
+
if (current !== record.brokerRef)
|
|
5483
|
+
return false;
|
|
5484
|
+
}
|
|
5485
|
+
else {
|
|
5486
|
+
// Never bind over a value Keys Manager did not write — except adoption,
|
|
5487
|
+
// which recorded this exact pre-adoption reference to replace it.
|
|
5488
|
+
if (this.releasingProviders.has(provider))
|
|
5489
|
+
return false;
|
|
5490
|
+
if (current !== undefined && !(options.takeover === true && current === record.originalApiKeyEnv))
|
|
5491
|
+
return false;
|
|
5492
|
+
}
|
|
5493
|
+
const path = [...record.settingsPath, 'apiKeyEnv'];
|
|
5494
|
+
// PR review round 35: the last point before the settings write. Only
|
|
5495
|
+
// `closing` is read: shutdown's own pre-close restoration writes here by
|
|
5496
|
+
// design and runs before that flag goes up.
|
|
5497
|
+
if (this.closing)
|
|
5498
|
+
return false;
|
|
5499
|
+
try {
|
|
5500
|
+
// The entry gate above read a snapshot; ownership can turn unknown while
|
|
5501
|
+
// this write re-reads its descriptor (PR review round 25). Binding is a
|
|
5502
|
+
// guess then, so the gate closes again at the last point before the
|
|
5503
|
+
// mutation. Unbinding stays allowed.
|
|
5504
|
+
const written = await this.mutateSettings(record.settingsNs, [desired === undefined ? { op: 'unset', path } : { op: 'set', path, value: desired }], () => desired === undefined || this.projectionOwnershipKnown(provider));
|
|
5505
|
+
if (!written)
|
|
5506
|
+
return false;
|
|
5507
|
+
}
|
|
5508
|
+
catch (error) {
|
|
5509
|
+
this.bindingWriteRefused(provider, error instanceof Error ? error.message : String(error));
|
|
5510
|
+
return false;
|
|
5511
|
+
}
|
|
5512
|
+
// The mutation is asynchronous, so the serving source can flip while it
|
|
5513
|
+
// settles: a native OAuth write classified foreign after the gates above
|
|
5514
|
+
// unbinds an already-unbound profile and returns, then this older bind
|
|
5515
|
+
// commits over that projection (PR review round 31). Each settled write
|
|
5516
|
+
// therefore re-derives the shape from the live source, and the correction
|
|
5517
|
+
// itself retries — bounded — because the source can move again while the
|
|
5518
|
+
// corrective write settles (PR review round 32). The counter, not a flag,
|
|
5519
|
+
// keeps a genuinely flapping route from cycling: past the bound the next
|
|
5520
|
+
// repair pass owns it.
|
|
5521
|
+
const correction = options.correction ?? 0;
|
|
5522
|
+
if (correction < BINDING_CORRECTION_LIMIT) {
|
|
5523
|
+
// PR review round 35: the mutation above settled across an await, so the
|
|
5524
|
+
// corrective write it may recurse into is refused once shutdown began.
|
|
5525
|
+
if (this.closing)
|
|
5526
|
+
return false;
|
|
5527
|
+
const settled = this.store.provider(provider);
|
|
5528
|
+
if (settled !== undefined && this.keyPoolServes(settled) !== (desired !== undefined)) {
|
|
5529
|
+
return this.syncBrokerBinding(provider, { ...options, correction: correction + 1 });
|
|
5530
|
+
}
|
|
5531
|
+
}
|
|
5532
|
+
// The profile matches the serving source now, by construction.
|
|
5533
|
+
this.bindingCache.set(provider, 'healthy');
|
|
5534
|
+
return true;
|
|
5535
|
+
}
|
|
5536
|
+
/**
|
|
5537
|
+
* Repair every managed provider's binding through the one authority. The
|
|
5538
|
+
* startup pass and the settings-mount seam both run it, so a profile that
|
|
5539
|
+
* was unreadable when hydration settled is still healed in this process
|
|
5540
|
+
* instead of waiting for the next start.
|
|
5541
|
+
*/
|
|
5542
|
+
async repairBrokerBindings() {
|
|
5543
|
+
const providers = Object.keys(this.store.providers);
|
|
5544
|
+
const deferred = [];
|
|
5545
|
+
// A provider whose ownership is already known repairs first (PR review
|
|
5546
|
+
// round 3): one wedged OAuth read must not delay a pure key-pool route,
|
|
5547
|
+
// whose shape can never depend on a projection read. The skipped set is
|
|
5548
|
+
// carried to the second pass (PR review round 5): re-deriving readiness
|
|
5549
|
+
// there would drop a provider whose read landed while pass 1 awaited
|
|
5550
|
+
// another provider.
|
|
5551
|
+
for (const provider of providers) {
|
|
5552
|
+
if (!this.projectionOwnershipKnown(provider)) {
|
|
5553
|
+
deferred.push(provider);
|
|
5554
|
+
continue;
|
|
5555
|
+
}
|
|
5556
|
+
await this.repairBinding(provider);
|
|
5557
|
+
}
|
|
5558
|
+
for (const provider of deferred) {
|
|
5559
|
+
// An OAuth provider whose own read has not landed is never repaired
|
|
5560
|
+
// from a guessed shape; the read is bounded, so a provider that never
|
|
5561
|
+
// answers only skips itself.
|
|
5562
|
+
if (!await this.awaitProjectionOwnership(provider))
|
|
5563
|
+
continue;
|
|
5564
|
+
await this.repairBinding(provider);
|
|
5565
|
+
}
|
|
5566
|
+
}
|
|
5567
|
+
/**
|
|
5568
|
+
* One provider's repair pass (PR review round 42). A recorded adoption
|
|
5569
|
+
* takeover debt is settled here only when the authoritative readiness says
|
|
5570
|
+
* no projection serves the route — the case a restart leaves behind when the
|
|
5571
|
+
* foreign record disappeared while the process was down.
|
|
5572
|
+
*/
|
|
5573
|
+
async repairBinding(provider) {
|
|
5574
|
+
const record = this.store.provider(provider);
|
|
5575
|
+
const takeover = record?.takeoverPending === true
|
|
5576
|
+
&& !this.foreignProjections.has(provider)
|
|
5577
|
+
&& this.oauthOwnerFor(provider) === null;
|
|
5578
|
+
const bound = await this.syncBrokerBinding(provider, { write: false, takeover });
|
|
5579
|
+
if (takeover && bound && record !== undefined) {
|
|
5580
|
+
record.takeoverPending = false;
|
|
5581
|
+
this.store.touch();
|
|
5582
|
+
}
|
|
5583
|
+
}
|
|
5584
|
+
/**
|
|
5585
|
+
* Re-derive a managed route's binding after its projection changed outside
|
|
5586
|
+
* this manager's account pool: a removal, a committed harvest that handed
|
|
5587
|
+
* the route back to its API-key pool (PR review round 2), or a foreign
|
|
5588
|
+
* projection taking a bound route over (round 4).
|
|
5589
|
+
*
|
|
5590
|
+
* The route's serving source is no longer the managed pool, or the pool is
|
|
5591
|
+
* the only source left: either way `syncBrokerBinding` derives the one
|
|
5592
|
+
* profile shape pi-ai's resolution order requires. A store refusal only
|
|
5593
|
+
* warns — these are compensation paths and must never fail the change that
|
|
5594
|
+
* triggered them; the next repair pass retries.
|
|
5595
|
+
*/
|
|
5596
|
+
async syncBindingAfterProjectionChange(provider) {
|
|
5597
|
+
// PR review round 30: this body is chained behind a hydration read that can
|
|
5598
|
+
// land after `close()` has drained the chains; the binding write it ends in
|
|
5599
|
+
// is persistence, so shutdown refuses it.
|
|
5600
|
+
if (this.closing)
|
|
5601
|
+
return;
|
|
5602
|
+
try {
|
|
5603
|
+
// PR review round 40: adoption keeps the pre-adoption reference while a
|
|
5604
|
+
// foreign projection serves the route and records that it still owes the
|
|
5605
|
+
// takeover on the provider record (so the debt survives a restart). Only
|
|
5606
|
+
// that recorded debt lets this sync pass `takeover`, and it is cleared
|
|
5607
|
+
// once the bind lands; `originalApiKeyEnv` alone proves nothing.
|
|
5608
|
+
const record = this.store.provider(provider);
|
|
5609
|
+
const takeover = record?.takeoverPending === true;
|
|
5610
|
+
const bound = await this.syncBrokerBinding(provider, { takeover });
|
|
5611
|
+
if (takeover && bound && record !== undefined) {
|
|
5612
|
+
record.takeoverPending = false;
|
|
5613
|
+
this.store.touch();
|
|
5614
|
+
}
|
|
5615
|
+
}
|
|
5616
|
+
catch (error) {
|
|
5617
|
+
this.warn(`could not re-derive the pi-ai binding for ${provider} after its projection changed: ${error instanceof Error ? error.message : String(error)}`);
|
|
5618
|
+
}
|
|
5619
|
+
}
|
|
5620
|
+
/**
|
|
5621
|
+
* A refused binding write must not fail the change that triggered it, but
|
|
5622
|
+
* it cannot stay in the log alone either: the route may resolve the wrong
|
|
5623
|
+
* credential source until the next repair.
|
|
5624
|
+
*/
|
|
5625
|
+
bindingWriteRefused(provider, reason) {
|
|
5626
|
+
this.warn(`could not update the pi-ai broker binding for ${provider}: ${reason}`);
|
|
5627
|
+
this.pushWarning(`The pi-ai credential reference for ${provider} was not updated (${reason}). Requests may use a different credential source until this is repaired.`);
|
|
5628
|
+
}
|
|
2855
5629
|
/** Candidate credential refs from which an existing key can be imported. */
|
|
2856
5630
|
importRefs(entry, profile) {
|
|
2857
5631
|
const refs = [];
|
|
@@ -2939,25 +5713,14 @@ export class KeysManagerService {
|
|
|
2939
5713
|
if (collision !== undefined)
|
|
2940
5714
|
throw new Error(`Broker reference collision for "${provider}"`);
|
|
2941
5715
|
const originalApiKeyEnv = profile !== undefined && typeof profile.apiKeyEnv === 'string' ? profile.apiKeyEnv : null;
|
|
2942
|
-
|
|
5716
|
+
// A read-only profile cannot carry the binding; refuse before any
|
|
5717
|
+
// managed state or imported credential exists.
|
|
5718
|
+
if (!descriptor.writable)
|
|
5719
|
+
throw new Error('The active profile is read-only');
|
|
2943
5720
|
const hit = await this.findImport(entry, profile);
|
|
2944
5721
|
let imported;
|
|
2945
5722
|
if (hit !== undefined)
|
|
2946
5723
|
imported = await this.createKeyRecord(provider, 'Imported', hit.value, hit.ref);
|
|
2947
|
-
try {
|
|
2948
|
-
await this.mutateSettings(entry.settingsNs, [{ op: 'set', path, value: brokerRef }]);
|
|
2949
|
-
}
|
|
2950
|
-
catch (error) {
|
|
2951
|
-
if (imported !== undefined) {
|
|
2952
|
-
try {
|
|
2953
|
-
await this.unsetOriginal(imported.ref);
|
|
2954
|
-
}
|
|
2955
|
-
catch (cleanupError) {
|
|
2956
|
-
this.warn(`adoption cleanup left credential ${imported.ref}: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`);
|
|
2957
|
-
}
|
|
2958
|
-
}
|
|
2959
|
-
throw error;
|
|
2960
|
-
}
|
|
2961
5724
|
const record = {
|
|
2962
5725
|
provider,
|
|
2963
5726
|
displayName: extra?.name ?? entry.displayName ?? provider,
|
|
@@ -2979,14 +5742,165 @@ export class KeysManagerService {
|
|
|
2979
5742
|
};
|
|
2980
5743
|
this.store.providers[provider] = record;
|
|
2981
5744
|
this.rebuildBrokerMap();
|
|
5745
|
+
// PR review round 17: this provider was not in the startup hydration
|
|
5746
|
+
// snapshot, so its projection is read HERE, before the first binding
|
|
5747
|
+
// sync. Without the read, "not in the snapshot" looked exactly like "no
|
|
5748
|
+
// projection" and the sync bound the pool over the route's native grant.
|
|
5749
|
+
let ownershipRead = await this.readAdoptionProjection(provider);
|
|
5750
|
+
// PR review round 28 (I-000186): a read a projection event overtook lands
|
|
5751
|
+
// nothing, and the reconciliation queued behind it repairs the binding
|
|
5752
|
+
// WITHOUT the takeover only this adoption may perform — so a superseded
|
|
5753
|
+
// answer left a committed adoption on its pre-adoption reference forever
|
|
5754
|
+
// (the pool never serves, and re-adopting returns immediately because the
|
|
5755
|
+
// provider is managed). The retry only reads, and the chain is drained by
|
|
5756
|
+
// definition once this read is its tail: the adoption waits for that
|
|
5757
|
+
// authoritative answer instead of committing an undecided bind.
|
|
5758
|
+
// No deadlock: no grant-chain body awaits the mutation queue this body holds.
|
|
5759
|
+
while (ownershipRead === 'superseded')
|
|
5760
|
+
ownershipRead = await this.readAdoptionProjection(provider);
|
|
5761
|
+
// The binding follows the serving credential, never the route's ability
|
|
5762
|
+
// to hold keys: an OAuth-only route (`caps.keys === false`, no key row)
|
|
5763
|
+
// keeps its profile unbound so pi-ai reaches the stored grant, while a
|
|
5764
|
+
// route with an imported API key binds through the one authority.
|
|
5765
|
+
// Adoption is the only caller allowed to take over the pre-adoption
|
|
5766
|
+
// reference it recorded as `originalApiKeyEnv`.
|
|
5767
|
+
const bound = ownershipRead === 'known'
|
|
5768
|
+
? await this.syncBrokerBinding(provider, { takeover: true })
|
|
5769
|
+
: false;
|
|
5770
|
+
// A refused write the pool's serving shape required leaves a state no
|
|
5771
|
+
// later repair may touch (it cannot tell that value from a hand edit), so
|
|
5772
|
+
// undo the adoption entirely (PR review round 2). A foreign projection or
|
|
5773
|
+
// an unreadable one needs no write: the profile is not the pool's to
|
|
5774
|
+
// rewrite, and the adoption itself is complete.
|
|
5775
|
+
if (ownershipRead === 'known' && !bound && this.keyPoolServes(record)) {
|
|
5776
|
+
await this.rollbackAdoption(record, imported);
|
|
5777
|
+
throw new Error(`The ${provider} profile refused the credential binding write; the adoption was rolled back. Repair the profile in Models and adopt again.`);
|
|
5778
|
+
}
|
|
5779
|
+
if (ownershipRead === 'known' && this.foreignProjections.has(provider) && record.originalApiKeyEnv !== null) {
|
|
5780
|
+
// A foreign projection serves the route, so this adoption binds
|
|
5781
|
+
// nothing; the pre-adoption reference it kept may only be taken over
|
|
5782
|
+
// when that projection disappears (round 40).
|
|
5783
|
+
record.takeoverPending = true;
|
|
5784
|
+
}
|
|
5785
|
+
if (ownershipRead === 'unknown') {
|
|
5786
|
+
// Ownership stays unknown and nothing was bound; a later authoritative
|
|
5787
|
+
// read clears the marker and re-derives the binding.
|
|
5788
|
+
this.pushWarning(`The ${provider} route was adopted, but its stored credential could not be read; its models profile was left unbound until the credential store answers.`);
|
|
5789
|
+
}
|
|
2982
5790
|
if (hit !== undefined)
|
|
2983
5791
|
this.pushWarning(`Imported the existing credential ${hit.ref}.`);
|
|
2984
5792
|
this.store.touch();
|
|
2985
5793
|
return this.buildState();
|
|
2986
5794
|
});
|
|
2987
5795
|
}
|
|
2988
|
-
/**
|
|
5796
|
+
/**
|
|
5797
|
+
* Read and classify one provider's live projection at adoption (PR review
|
|
5798
|
+
* round 17) — the same read hydration does, for the one route hydration
|
|
5799
|
+
* cannot cover: a provider that did not exist when the snapshot was taken.
|
|
5800
|
+
*
|
|
5801
|
+
* An adoption owns no account row yet, so any live record is one no row of
|
|
5802
|
+
* this pool owns: a native api-key, an unrecognized payload, and a grant left
|
|
5803
|
+
* behind by a released account all serve the route ahead of the broker
|
|
5804
|
+
* reference, so the route is marked foreign and the pool is not bound over it
|
|
5805
|
+
* (PR review rounds 3–4). An absent record clears any stale marker: the pool
|
|
5806
|
+
* may serve. A failed read is not an absent projection (round 8): ownership
|
|
5807
|
+
* stays unknown, so nothing may be bound and no repair touches the route
|
|
5808
|
+
* until an authoritative read answers ({@link noteProjectionReadSucceeded}).
|
|
5809
|
+
*
|
|
5810
|
+
* Returns `unknown` for that failure; the adoption itself is kept — there is
|
|
5811
|
+
* no write to roll back, and only the binding is undecided.
|
|
5812
|
+
*
|
|
5813
|
+
* The classification runs inside the route's grant chain (PR review round
|
|
5814
|
+
* 22, Codex P1). A native write or deletion reconciled while the read is
|
|
5815
|
+
* outstanding is therefore either already visible to it or queued behind it:
|
|
5816
|
+
* a read whose snapshot a newer event overtook reports `superseded` and lands
|
|
5817
|
+
* nothing, so a stale absence can never bind the broker over a newer
|
|
5818
|
+
* projection state and a stale presence can never leave the pool unbound.
|
|
5819
|
+
*/
|
|
5820
|
+
readAdoptionProjection(provider) {
|
|
5821
|
+
return this.scheduleGrantReconciliation(provider, () => this.classifyAdoptionProjection(provider));
|
|
5822
|
+
}
|
|
5823
|
+
/** Body of {@link readAdoptionProjection}: read, then land the classification. */
|
|
5824
|
+
async classifyAdoptionProjection(provider) {
|
|
5825
|
+
// This body's own chain entry. A projection event reconciled during the
|
|
5826
|
+
// read appends a newer tail, which re-reads the live state.
|
|
5827
|
+
const chainTail = this.grantReconciliations.get(provider);
|
|
5828
|
+
const grantKey = `${OAUTH_GRANT_RECORD_SCOPE}/${provider}`;
|
|
5829
|
+
const records = this.credentialRecords();
|
|
5830
|
+
if (records === undefined) {
|
|
5831
|
+
this.ownershipReadFailures.add(provider);
|
|
5832
|
+
this.warn(`projection ownership for ${provider} could not be read during adoption: the credential store is unavailable`);
|
|
5833
|
+
return 'unknown';
|
|
5834
|
+
}
|
|
5835
|
+
let record;
|
|
5836
|
+
try {
|
|
5837
|
+
record = await records.readRecord(grantKey);
|
|
5838
|
+
}
|
|
5839
|
+
catch (error) {
|
|
5840
|
+
this.ownershipReadFailures.add(provider);
|
|
5841
|
+
this.warn(`projection ownership for ${provider} could not be read during adoption: ${error instanceof Error ? error.message : String(error)}`);
|
|
5842
|
+
return 'unknown';
|
|
5843
|
+
}
|
|
5844
|
+
if (this.grantReconciliations.get(provider) !== chainTail) {
|
|
5845
|
+
// Ownership stays unknown until the queued read answers; nothing here may
|
|
5846
|
+
// land, and the adoption binds nothing.
|
|
5847
|
+
this.ownershipReadFailures.add(provider);
|
|
5848
|
+
return 'superseded';
|
|
5849
|
+
}
|
|
5850
|
+
this.ownershipReadFailures.delete(provider);
|
|
5851
|
+
this.noteProviderHydrated(provider);
|
|
5852
|
+
if (record === undefined)
|
|
5853
|
+
this.foreignProjections.delete(provider);
|
|
5854
|
+
else
|
|
5855
|
+
this.foreignProjections.add(provider);
|
|
5856
|
+
return 'known';
|
|
5857
|
+
}
|
|
5858
|
+
/**
|
|
5859
|
+
* Undo an adoption whose initial binding write was refused (PR review
|
|
5860
|
+
* round 2): drop the managed record and its broker entry, delete the
|
|
5861
|
+
* plugin-scope copy this call created, and put back a pre-adoption
|
|
5862
|
+
* reference that landed despite the refusal. The source credential and the
|
|
5863
|
+
* profile's other fields are never touched — adoption only read them.
|
|
5864
|
+
*/
|
|
5865
|
+
async rollbackAdoption(record, imported) {
|
|
5866
|
+
const provider = record.provider;
|
|
5867
|
+
delete this.store.providers[provider];
|
|
5868
|
+
this.rebuildBrokerMap();
|
|
5869
|
+
if (imported !== undefined) {
|
|
5870
|
+
try {
|
|
5871
|
+
await this.unsetOriginal(imported.ref);
|
|
5872
|
+
}
|
|
5873
|
+
catch (error) {
|
|
5874
|
+
this.warn(`adoption rollback could not delete the imported credential ${imported.ref} for ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
5875
|
+
}
|
|
5876
|
+
}
|
|
5877
|
+
const descriptor = this.describeSettings();
|
|
5878
|
+
const namespace = this.namespace(descriptor, record.settingsNs);
|
|
5879
|
+
const profile = asProfile(namespace === undefined ? undefined : getPath(namespace.value, record.settingsPath));
|
|
5880
|
+
const current = profile !== undefined && typeof profile.apiKeyEnv === 'string' ? profile.apiKeyEnv : undefined;
|
|
5881
|
+
if (current === record.brokerRef) {
|
|
5882
|
+
try {
|
|
5883
|
+
const path = [...record.settingsPath, 'apiKeyEnv'];
|
|
5884
|
+
await this.mutateSettings(record.settingsNs, [
|
|
5885
|
+
record.originalApiKeyEnv === null ? { op: 'unset', path } : { op: 'set', path, value: record.originalApiKeyEnv },
|
|
5886
|
+
]);
|
|
5887
|
+
}
|
|
5888
|
+
catch (error) {
|
|
5889
|
+
this.warn(`adoption rollback could not restore the pre-adoption reference for ${provider}: ${error instanceof Error ? error.message : String(error)}`);
|
|
5890
|
+
}
|
|
5891
|
+
}
|
|
5892
|
+
this.store.touch();
|
|
5893
|
+
}
|
|
5894
|
+
/**
|
|
5895
|
+
* Discover models for a new extra provider, falling back to the seed catalog.
|
|
5896
|
+
* A route that declares a static header installs the seed catalog directly:
|
|
5897
|
+
* the discovery request has no header slot, so probing such an endpoint can
|
|
5898
|
+
* only fail (and would hold the serialized mutation queue for the probe's
|
|
5899
|
+
* whole timeout).
|
|
5900
|
+
*/
|
|
2989
5901
|
async discoverModels(extra, value) {
|
|
5902
|
+
if (extra.headerEnv !== undefined)
|
|
5903
|
+
return { models: normalizeModels(extra.models) };
|
|
2990
5904
|
let discovered = [];
|
|
2991
5905
|
let failure;
|
|
2992
5906
|
try {
|
|
@@ -3010,7 +5924,14 @@ export class KeysManagerService {
|
|
|
3010
5924
|
return { models, failure };
|
|
3011
5925
|
return { models: normalizeModels(extra.models), failure };
|
|
3012
5926
|
}
|
|
3013
|
-
/**
|
|
5927
|
+
/**
|
|
5928
|
+
* Remote: install and adopt one extra built-in provider.
|
|
5929
|
+
*
|
|
5930
|
+
* Deliberate exemption from the one-authority rule
|
|
5931
|
+
* ({@link syncBrokerBinding}): this call creates the profile it binds and
|
|
5932
|
+
* the key that serves it in the same mutation, so the write below is the
|
|
5933
|
+
* route's first binding rather than a takeover of a value the user owns.
|
|
5934
|
+
*/
|
|
3014
5935
|
addExtraProvider(extraId, alias, value) {
|
|
3015
5936
|
return this.run(async () => {
|
|
3016
5937
|
if (typeof extraId !== 'string' || extraId.length === 0)
|
|
@@ -3019,9 +5940,18 @@ export class KeysManagerService {
|
|
|
3019
5940
|
const extra = EXTRA_PROVIDER_MAP.get(extraId);
|
|
3020
5941
|
if (extra === undefined)
|
|
3021
5942
|
throw new Error(`Unknown extra provider "${extraId}"`);
|
|
3022
|
-
|
|
5943
|
+
// Resolved before the first credential/settings write on every add path,
|
|
5944
|
+
// so a missing variable refuses the add without leaving a key or a
|
|
5945
|
+
// partial profile.
|
|
5946
|
+
const headers = resolveHeaderEnv(extra);
|
|
5947
|
+
const managed = this.store.provider(extraId);
|
|
5948
|
+
if (managed !== undefined) {
|
|
3023
5949
|
this.pushWarning('Provider is already managed; the key was added instead.');
|
|
3024
|
-
|
|
5950
|
+
// The shared managed-key path repairs the declared headers, so every
|
|
5951
|
+
// add-key flow restores them and the repair keeps one implementation.
|
|
5952
|
+
// Hand it the value resolved above: the entry refusal stays the single
|
|
5953
|
+
// decision for this surface, and the environment is read once per add.
|
|
5954
|
+
await this.addKeyLocked(extraId, input.alias, input.value, headers);
|
|
3025
5955
|
this.store.touch();
|
|
3026
5956
|
return this.buildState();
|
|
3027
5957
|
}
|
|
@@ -3052,8 +5982,20 @@ export class KeysManagerService {
|
|
|
3052
5982
|
keys: [],
|
|
3053
5983
|
};
|
|
3054
5984
|
const key = await this.createKeyRecord(extraId, input.alias, input.value);
|
|
5985
|
+
const ops = [{ op: 'set', path: ['providers', extraId, 'apiKeyEnv'], value: record.brokerRef }];
|
|
5986
|
+
// Fill the declared headers the stored profile does not carry; a value
|
|
5987
|
+
// the profile already sets for the same header is never overwritten.
|
|
5988
|
+
const storedHeaders = staticHeaders(existing.headers);
|
|
5989
|
+
this.warnHeaderDrift(extra, headers, storedHeaders);
|
|
5990
|
+
const missing = missingHeaders(headers, storedHeaders);
|
|
5991
|
+
if (Object.keys(missing).length > 0) {
|
|
5992
|
+
ops.push({ op: 'set', path: ['providers', extraId, 'headers'], value: { ...storedHeaders, ...missing } });
|
|
5993
|
+
}
|
|
3055
5994
|
try {
|
|
3056
|
-
|
|
5995
|
+
// First binding of a profile this manager does not own yet, written
|
|
5996
|
+
// together with the key that serves it (see the method comment); the
|
|
5997
|
+
// declared-header repair above travels in the same write.
|
|
5998
|
+
await this.mutateSettings('llm-pi-ai', ops);
|
|
3057
5999
|
}
|
|
3058
6000
|
catch (error) {
|
|
3059
6001
|
try {
|
|
@@ -3067,6 +6009,10 @@ export class KeysManagerService {
|
|
|
3067
6009
|
record.keys.push(key);
|
|
3068
6010
|
this.store.providers[extraId] = record;
|
|
3069
6011
|
this.rebuildBrokerMap();
|
|
6012
|
+
// Readiness is per provider now (PR review round 17): this route was
|
|
6013
|
+
// created here and its binding is written by this very call, so no
|
|
6014
|
+
// snapshot will ever read it.
|
|
6015
|
+
this.noteProviderHydrated(extraId);
|
|
3070
6016
|
this.store.touch();
|
|
3071
6017
|
return this.buildState();
|
|
3072
6018
|
}
|
|
@@ -3081,10 +6027,13 @@ export class KeysManagerService {
|
|
|
3081
6027
|
api: extra.api,
|
|
3082
6028
|
baseURL: extra.baseURL,
|
|
3083
6029
|
apiKeyEnv: brokerRef,
|
|
6030
|
+
...(headers === undefined ? {} : { headers }),
|
|
3084
6031
|
models,
|
|
3085
6032
|
};
|
|
3086
6033
|
const key = await this.createKeyRecord(extraId, input.alias, input.value);
|
|
3087
6034
|
try {
|
|
6035
|
+
// The profile and its binding are created together (see the method
|
|
6036
|
+
// comment); no user value exists to take over here.
|
|
3088
6037
|
await this.mutateSettings('llm-pi-ai', [{ op: 'set', path: ['providers', extraId], value: profile }]);
|
|
3089
6038
|
}
|
|
3090
6039
|
catch (error) {
|
|
@@ -3117,12 +6066,71 @@ export class KeysManagerService {
|
|
|
3117
6066
|
};
|
|
3118
6067
|
this.store.providers[extraId] = record;
|
|
3119
6068
|
this.rebuildBrokerMap();
|
|
6069
|
+
// Same as the take-over branch above: the profile and its binding are
|
|
6070
|
+
// this call's own write, so the route is attributable immediately.
|
|
6071
|
+
this.noteProviderHydrated(extraId);
|
|
3120
6072
|
this.store.touch();
|
|
3121
6073
|
return this.buildState();
|
|
3122
6074
|
});
|
|
3123
6075
|
}
|
|
6076
|
+
/**
|
|
6077
|
+
* Restore a managed extra route's declared static headers.
|
|
6078
|
+
*
|
|
6079
|
+
* A managed profile can lose its declared headers through a Models edit;
|
|
6080
|
+
* every managed add-key flow repairs them instead of leaving a route whose
|
|
6081
|
+
* required header is silently absent. Only a live profile with a healthy
|
|
6082
|
+
* binding is ours to repair: a deleted profile would gain a header-only
|
|
6083
|
+
* half-profile, and a replaced one is the state release refuses to touch. A
|
|
6084
|
+
* stored value for the same header always wins. The caller may pass the
|
|
6085
|
+
* headers it already resolved at its own entry (where a missing variable can
|
|
6086
|
+
* still refuse the add before any write); otherwise this method resolves
|
|
6087
|
+
* them. Either way a failure only warns, because the key has already been
|
|
6088
|
+
* added: a route whose headers cannot be resolved, and a settings write that
|
|
6089
|
+
* cannot be performed, both leave the repair pending for the next add so a
|
|
6090
|
+
* half-applied add is never turned into a rejected one.
|
|
6091
|
+
*/
|
|
6092
|
+
async repairDeclaredHeaders(record, declared) {
|
|
6093
|
+
const extra = record.extraId === null ? undefined : EXTRA_PROVIDER_MAP.get(record.extraId);
|
|
6094
|
+
if (extra?.headerEnv === undefined)
|
|
6095
|
+
return;
|
|
6096
|
+
let headers = declared;
|
|
6097
|
+
if (headers === undefined) {
|
|
6098
|
+
try {
|
|
6099
|
+
headers = resolveHeaderEnv(extra);
|
|
6100
|
+
}
|
|
6101
|
+
catch (error) {
|
|
6102
|
+
this.pushWarning(`Did not restore the declared headers on the ${extra.id} route: ${error instanceof Error ? error.message : String(error)}`);
|
|
6103
|
+
return;
|
|
6104
|
+
}
|
|
6105
|
+
}
|
|
6106
|
+
if (headers === undefined)
|
|
6107
|
+
return;
|
|
6108
|
+
const liveProfile = this.profileAt(record.settingsNs, record.settingsPath);
|
|
6109
|
+
if (liveProfile === undefined || this.bindingState(record) === 'mismatch') {
|
|
6110
|
+
this.pushWarning(`Did not restore the ${Object.keys(headers).join(', ')} header on the ${extra.id} route: ${liveProfile === undefined ? 'its Models profile no longer exists' : 'its Models profile was replaced'}.`);
|
|
6111
|
+
return;
|
|
6112
|
+
}
|
|
6113
|
+
const stored = staticHeaders(liveProfile.headers);
|
|
6114
|
+
this.warnHeaderDrift(extra, headers, stored);
|
|
6115
|
+
const missing = missingHeaders(headers, stored);
|
|
6116
|
+
if (Object.keys(missing).length === 0)
|
|
6117
|
+
return;
|
|
6118
|
+
try {
|
|
6119
|
+
await this.mutateSettings(record.settingsNs, [
|
|
6120
|
+
{ op: 'set', path: [...record.settingsPath, 'headers'], value: { ...stored, ...missing } },
|
|
6121
|
+
]);
|
|
6122
|
+
}
|
|
6123
|
+
catch (error) {
|
|
6124
|
+
// The key and its credential are already written; a failing settings
|
|
6125
|
+
// write must not reject the add. The repair stays pending, so the next
|
|
6126
|
+
// add-key flow retries it.
|
|
6127
|
+
this.pushWarning(`Did not restore the ${Object.keys(missing).join(', ')} header on the ${extra.id} route: ${error instanceof Error ? error.message : String(error)}`);
|
|
6128
|
+
return;
|
|
6129
|
+
}
|
|
6130
|
+
this.pushWarning(`Restored the missing ${Object.keys(missing).join(', ')} header on the managed profile.`);
|
|
6131
|
+
}
|
|
3124
6132
|
/** Add one key; caller already owns the mutation queue. */
|
|
3125
|
-
async addKeyLocked(provider, alias, value) {
|
|
6133
|
+
async addKeyLocked(provider, alias, value, declaredHeaders) {
|
|
3126
6134
|
const record = this.store.provider(provider);
|
|
3127
6135
|
if (record === undefined)
|
|
3128
6136
|
throw new Error(`Provider "${provider}" is not managed`);
|
|
@@ -3141,6 +6149,15 @@ export class KeysManagerService {
|
|
|
3141
6149
|
throw new Error(`A provider may have at most ${KEY_LIMIT} keys`);
|
|
3142
6150
|
const key = await this.createKeyRecord(provider, input.alias, input.value);
|
|
3143
6151
|
record.keys.push(key);
|
|
6152
|
+
// After the key add: a refused key (capability or capacity) must leave the
|
|
6153
|
+
// settings document untouched, so the repair never runs ahead of it.
|
|
6154
|
+
await this.repairDeclaredHeaders(record, declaredHeaders);
|
|
6155
|
+
// A route adopted for accounts only, or one whose pool lost its last key,
|
|
6156
|
+
// binds as soon as a usable API-key row exists again. An extra route's
|
|
6157
|
+
// profile is its whole route, so a deleted one is never re-materialized
|
|
6158
|
+
// from a binding write; a native route keeps its directory defaults and
|
|
6159
|
+
// may adopt the binding as its first profile content.
|
|
6160
|
+
await this.syncBrokerBinding(provider, record.extraId === null ? {} : { write: false });
|
|
3144
6161
|
}
|
|
3145
6162
|
/** Remote: add one key to a managed provider. */
|
|
3146
6163
|
addKey(provider, alias, value) {
|
|
@@ -3170,7 +6187,9 @@ export class KeysManagerService {
|
|
|
3170
6187
|
}
|
|
3171
6188
|
/** Remote: delete one key, releasing the provider when it was the last one. */
|
|
3172
6189
|
deleteKey(provider, keyId) {
|
|
3173
|
-
return this.run(async () => {
|
|
6190
|
+
return this.run(() => this.withRemovalInProgress(provider, async () => {
|
|
6191
|
+
// R8-2: a test may still be about to send this row's credential.
|
|
6192
|
+
await this.settleTestBeforeRemoval(provider);
|
|
3174
6193
|
const record = this.store.provider(provider);
|
|
3175
6194
|
if (record === undefined)
|
|
3176
6195
|
throw new Error(`Provider "${provider}" is not managed`);
|
|
@@ -3195,17 +6214,27 @@ export class KeysManagerService {
|
|
|
3195
6214
|
}
|
|
3196
6215
|
record.keys.splice(index, 1);
|
|
3197
6216
|
this.store.clearBindings((_key, binding) => binding.keyId === key.id);
|
|
6217
|
+
// Removing the last usable API-key row leaves nothing for a broker
|
|
6218
|
+
// reference to resolve; a surviving pool keeps its binding.
|
|
6219
|
+
await this.syncBrokerBinding(provider);
|
|
3198
6220
|
this.store.touch();
|
|
3199
6221
|
return this.buildState();
|
|
3200
|
-
});
|
|
6222
|
+
}));
|
|
3201
6223
|
}
|
|
3202
|
-
/** Read one live profile for release-time cleanup decisions. */
|
|
6224
|
+
/** Read one live profile for repair and release-time cleanup decisions. */
|
|
3203
6225
|
profileAt(ns, path) {
|
|
3204
6226
|
const descriptor = this.describeSettings();
|
|
3205
6227
|
const namespace = this.namespace(descriptor, ns);
|
|
3206
6228
|
return namespace === undefined ? undefined : asProfile(getPath(namespace.value, path));
|
|
3207
6229
|
}
|
|
3208
|
-
/**
|
|
6230
|
+
/**
|
|
6231
|
+
* Restore the exact live profile captured before release mutated it.
|
|
6232
|
+
*
|
|
6233
|
+
* Deliberate exemption from the one-authority rule: release is unmanaging,
|
|
6234
|
+
* so this write puts the pre-adoption document back instead of deriving a
|
|
6235
|
+
* binding from a key pool that is being deleted. Every other caller goes
|
|
6236
|
+
* through {@link syncBrokerBinding}.
|
|
6237
|
+
*/
|
|
3209
6238
|
async restoreProfileSnapshot(record, snapshot) {
|
|
3210
6239
|
if (record.settingsPath.length === 0) {
|
|
3211
6240
|
const apiKeyEnv = snapshot !== undefined && typeof snapshot.apiKeyEnv === 'string' ? snapshot.apiKeyEnv : record.brokerRef;
|
|
@@ -3218,29 +6247,73 @@ export class KeysManagerService {
|
|
|
3218
6247
|
: { op: 'set', path: record.settingsPath, value: snapshot },
|
|
3219
6248
|
]);
|
|
3220
6249
|
}
|
|
3221
|
-
/**
|
|
6250
|
+
/**
|
|
6251
|
+
* Remove every managed credential, recording progress before a retry.
|
|
6252
|
+
*
|
|
6253
|
+
* The provider is flagged as releasing for the whole loop: a demotion
|
|
6254
|
+
* inside it (an `oauth` row iterated before a surviving `api-key` row) must
|
|
6255
|
+
* not re-bind the profile, or the reference release just restored would be
|
|
6256
|
+
* overwritten and then stripped by the closing unbind. While unmanaging the
|
|
6257
|
+
* binding only ever goes one way — off.
|
|
6258
|
+
*/
|
|
3222
6259
|
async deleteManagedCredentials(record) {
|
|
3223
|
-
|
|
3224
|
-
|
|
3225
|
-
|
|
3226
|
-
|
|
3227
|
-
|
|
3228
|
-
|
|
3229
|
-
|
|
3230
|
-
|
|
3231
|
-
|
|
3232
|
-
|
|
6260
|
+
this.releasingProviders.add(record.provider);
|
|
6261
|
+
try {
|
|
6262
|
+
for (const key of record.keys) {
|
|
6263
|
+
if (key.kind === 'oauth' && key.oauth !== null) {
|
|
6264
|
+
// Same cleanup as removeAccount/deleteKey: when this row owns the
|
|
6265
|
+
// activation projection, demote it first so the released route
|
|
6266
|
+
// resolves through the restored apiKeyEnv instead of a grant whose
|
|
6267
|
+
// manager is gone (pi-ai resolution prefers a stored credential).
|
|
6268
|
+
await this.removeOAuthGrant(record.provider, key);
|
|
6269
|
+
}
|
|
6270
|
+
else {
|
|
6271
|
+
await this.unsetOriginal(key.ref);
|
|
6272
|
+
}
|
|
6273
|
+
key.valueMissing = true;
|
|
6274
|
+
this.store.touch();
|
|
3233
6275
|
}
|
|
3234
|
-
|
|
3235
|
-
this.
|
|
6276
|
+
this.activeAccounts.delete(record.provider);
|
|
6277
|
+
this.foreignProjections.delete(record.provider);
|
|
6278
|
+
// Every projection row is gone and every key is marked missing. This is
|
|
6279
|
+
// also releaseLocked's binding step: its profile already carries the
|
|
6280
|
+
// captured original, so only a lingering broker reference of our own is
|
|
6281
|
+
// removed and a released route keeps the value the user had.
|
|
6282
|
+
await this.syncBrokerBinding(record.provider);
|
|
6283
|
+
}
|
|
6284
|
+
finally {
|
|
6285
|
+
this.releasingProviders.delete(record.provider);
|
|
3236
6286
|
}
|
|
3237
|
-
|
|
6287
|
+
}
|
|
6288
|
+
/**
|
|
6289
|
+
* Drop any armed intentional-removal marker for a provider being released.
|
|
6290
|
+
*
|
|
6291
|
+
* The marker only exists to silence the record-updated echo of the manager's
|
|
6292
|
+
* own projection removal, and that echo is dropped outright once the provider
|
|
6293
|
+
* is unmanaged: {@link onGrantRecordUpdated} returns at event time and
|
|
6294
|
+
* {@link reconcileGrantRecord} at body time. A release can land between the
|
|
6295
|
+
* arm and that echo — the abort compensation arms inside the mutation queue
|
|
6296
|
+
* while its echo is still chained behind an in-flight body — leaving the
|
|
6297
|
+
* marker armed for its whole TTL with nothing left to consume it. A provider
|
|
6298
|
+
* re-adopted inside that window would then read a genuine out-of-band
|
|
6299
|
+
* removal as the manager's own: no warning, no `valueMissing`, no
|
|
6300
|
+
* stale-owner cleanup.
|
|
6301
|
+
*
|
|
6302
|
+
* Called only on the successful release path: a release that throws leaves
|
|
6303
|
+
* the provider managed, and its echo may still legitimately arrive.
|
|
6304
|
+
*/
|
|
6305
|
+
dropIntentionalRemoval(provider) {
|
|
6306
|
+
this.intentionalProjectionRemovals.delete(provider);
|
|
3238
6307
|
}
|
|
3239
6308
|
/** Restore the provider profile and drop management; caller owns the queue. */
|
|
3240
6309
|
async releaseLocked(provider) {
|
|
3241
6310
|
const record = this.store.provider(provider);
|
|
3242
6311
|
if (record === undefined)
|
|
3243
6312
|
return;
|
|
6313
|
+
// PR review round 10 (I-000179): release is a removal path too — it deletes
|
|
6314
|
+
// every managed credential below, so it holds the same window coordination
|
|
6315
|
+
// as removeAccount/deleteKey instead of racing a running row test.
|
|
6316
|
+
await this.settleTestBeforeRemoval(provider);
|
|
3244
6317
|
// Round-2 review step 4: releasing a provider revokes its in-flight
|
|
3245
6318
|
// sign-in attempts. A login whose seam flow is aborted can no longer
|
|
3246
6319
|
// harvest rows for a provider that is no longer managed (the harvest
|
|
@@ -3266,6 +6339,7 @@ export class KeysManagerService {
|
|
|
3266
6339
|
// overwrite the live profile.
|
|
3267
6340
|
if (binding === 'mismatch') {
|
|
3268
6341
|
await this.deleteManagedCredentials(record);
|
|
6342
|
+
this.dropIntentionalRemoval(provider);
|
|
3269
6343
|
delete this.store.providers[provider];
|
|
3270
6344
|
this.store.clearBindings(key => key.endsWith(`\u0000${provider}`));
|
|
3271
6345
|
this.rebuildBrokerMap();
|
|
@@ -3281,14 +6355,54 @@ export class KeysManagerService {
|
|
|
3281
6355
|
profileMutated = true;
|
|
3282
6356
|
}
|
|
3283
6357
|
else {
|
|
3284
|
-
// Native and pre-existing extra routes own only
|
|
3285
|
-
//
|
|
6358
|
+
// Native and pre-existing extra routes own only the values this plugin
|
|
6359
|
+
// wrote: apiKeyEnv, plus the declared static headers it injected into
|
|
6360
|
+
// an adopted profile. Leave every other profile field (baseURL,
|
|
6361
|
+
// models, hand edits) exactly as it is.
|
|
6362
|
+
// Exempt from the one-authority rule by design: release restores the
|
|
6363
|
+
// pre-adoption document, and the deletion loop that follows is
|
|
6364
|
+
// unbind-only ({@link deleteManagedCredentials}).
|
|
3286
6365
|
const path = [...record.settingsPath, 'apiKeyEnv'];
|
|
3287
|
-
|
|
6366
|
+
const ops = [
|
|
3288
6367
|
record.originalApiKeyEnv !== null
|
|
3289
6368
|
? { op: 'set', path, value: record.originalApiKeyEnv }
|
|
3290
6369
|
: { op: 'unset', path },
|
|
3291
|
-
]
|
|
6370
|
+
];
|
|
6371
|
+
// An injected header is identified by the environment value the add
|
|
6372
|
+
// wrote into it, so no extra record field has to remember the write.
|
|
6373
|
+
// Anything else stays, and release says so instead of guessing.
|
|
6374
|
+
const storedHeaders = staticHeaders(liveProfile?.headers);
|
|
6375
|
+
const extra = record.extraId === null ? undefined : EXTRA_PROVIDER_MAP.get(record.extraId);
|
|
6376
|
+
const envHeaders = environmentHeaders(extra);
|
|
6377
|
+
const injected = [];
|
|
6378
|
+
for (const [header, envName] of Object.entries(extra?.headerEnv ?? {})) {
|
|
6379
|
+
// Fetch field names are case-insensitive: attribute the stored
|
|
6380
|
+
// variant whatever its case, and unset that same stored key.
|
|
6381
|
+
const storedKey = storedHeaderKey(storedHeaders, header);
|
|
6382
|
+
if (storedKey === undefined)
|
|
6383
|
+
continue;
|
|
6384
|
+
const envValue = envHeaders[header];
|
|
6385
|
+
if (envValue === undefined) {
|
|
6386
|
+
this.pushWarning(`Release left the ${storedKey} header in the ${provider} profile: ${envName} is no longer set, so an injected value cannot be identified.`);
|
|
6387
|
+
}
|
|
6388
|
+
else if (storedHeaders[storedKey] === envValue) {
|
|
6389
|
+
injected.push(storedKey);
|
|
6390
|
+
}
|
|
6391
|
+
else {
|
|
6392
|
+
this.pushWarning(`Release left the ${storedKey} header in the ${provider} profile: it is not the value ${envName} currently supplies.`);
|
|
6393
|
+
}
|
|
6394
|
+
}
|
|
6395
|
+
if (injected.length > 0) {
|
|
6396
|
+
const remaining = Object.keys(storedHeaders).filter(header => !injected.includes(header));
|
|
6397
|
+
if (remaining.length === 0) {
|
|
6398
|
+
ops.push({ op: 'unset', path: [...record.settingsPath, 'headers'] });
|
|
6399
|
+
}
|
|
6400
|
+
else {
|
|
6401
|
+
for (const header of injected)
|
|
6402
|
+
ops.push({ op: 'unset', path: [...record.settingsPath, 'headers', header] });
|
|
6403
|
+
}
|
|
6404
|
+
}
|
|
6405
|
+
await this.mutateSettings(record.settingsNs, ops);
|
|
3292
6406
|
profileMutated = true;
|
|
3293
6407
|
// Remove an auto-created profile only when it is still empty after
|
|
3294
6408
|
// clearing apiKeyEnv; a profile with later user edits stays.
|
|
@@ -3314,6 +6428,7 @@ export class KeysManagerService {
|
|
|
3314
6428
|
}
|
|
3315
6429
|
throw error;
|
|
3316
6430
|
}
|
|
6431
|
+
this.dropIntentionalRemoval(provider);
|
|
3317
6432
|
delete this.store.providers[provider];
|
|
3318
6433
|
this.store.clearBindings(key => key.endsWith(`\u0000${provider}`));
|
|
3319
6434
|
this.rebuildBrokerMap();
|
|
@@ -3321,12 +6436,12 @@ export class KeysManagerService {
|
|
|
3321
6436
|
}
|
|
3322
6437
|
/** Remote: stop managing a provider and restore its original key reference. */
|
|
3323
6438
|
releaseProvider(provider) {
|
|
3324
|
-
return this.run(async () => {
|
|
6439
|
+
return this.run(() => this.withRemovalInProgress(provider, async () => {
|
|
3325
6440
|
if (this.store.provider(provider) === undefined)
|
|
3326
6441
|
throw new Error(`Provider "${provider}" is not managed`);
|
|
3327
6442
|
await this.releaseLocked(provider);
|
|
3328
6443
|
return this.buildState();
|
|
3329
|
-
});
|
|
6444
|
+
}));
|
|
3330
6445
|
}
|
|
3331
6446
|
/** Remote: update manager policy. */
|
|
3332
6447
|
setSettings(patch) {
|
|
@@ -3664,6 +6779,12 @@ export class KeysManagerService {
|
|
|
3664
6779
|
/** Waterfall listener: bind a key to the request and observe its terminal chunk. */
|
|
3665
6780
|
streamListener(options, next) {
|
|
3666
6781
|
const providerId = options.provider;
|
|
6782
|
+
// A per-row connection test owns its credential pin; it must not select a
|
|
6783
|
+
// key, move a cursor or write health state, so it never enters the
|
|
6784
|
+
// bookkeeping path below.
|
|
6785
|
+
const testScope = this.als.getStore();
|
|
6786
|
+
if (testScope?.test === true)
|
|
6787
|
+
return this.wrapTestStream(next, testScope);
|
|
3667
6788
|
const sessionId = options.sessionId === undefined ? undefined : String(options.sessionId);
|
|
3668
6789
|
// `options.provider` is the route the agent/request waterfall finally
|
|
3669
6790
|
// resolved; noteRequestStep keys its record by the same
|
|
@@ -3685,9 +6806,18 @@ export class KeysManagerService {
|
|
|
3685
6806
|
if (binding === 'mismatch') {
|
|
3686
6807
|
if (sessionId !== undefined && step !== undefined)
|
|
3687
6808
|
this.clearRetry(providerId, sessionId, step.turn, step.step);
|
|
3688
|
-
|
|
6809
|
+
// QC B5: the provider stream is handed over as a thunk on both
|
|
6810
|
+
// branches, so it is constructed after the isolation wait — a mismatch
|
|
6811
|
+
// profile still resolves pi-ai's stored grant, so the borrowed
|
|
6812
|
+
// projection reaches it too.
|
|
6813
|
+
return this.isolateFromTest(providerId, () => next());
|
|
3689
6814
|
}
|
|
3690
|
-
|
|
6815
|
+
// An ordinary request must never be served by the grant a row test has
|
|
6816
|
+
// borrowed for this provider, nor be attributed to the tested account.
|
|
6817
|
+
// The isolation wrapper waits for that test to settle before the normal
|
|
6818
|
+
// path resolves its credential; a test-scope request returned above, so
|
|
6819
|
+
// it never waits on its own guard.
|
|
6820
|
+
return this.isolateFromTest(providerId, () => this.wrapStream(options, next, providerId, sessionId, step));
|
|
3691
6821
|
}
|
|
3692
6822
|
/**
|
|
3693
6823
|
* The account row whose grant payload currently IS the live projection,
|
|
@@ -3701,6 +6831,140 @@ export class KeysManagerService {
|
|
|
3701
6831
|
const row = this.store.provider(provider)?.keys.find(candidate => candidate.id === accountId);
|
|
3702
6832
|
return row !== undefined && row.kind === 'oauth' && row.oauth !== null ? row.id : null;
|
|
3703
6833
|
}
|
|
6834
|
+
/**
|
|
6835
|
+
* Keep an ordinary request out of the window where a row test borrows (or
|
|
6836
|
+
* is restoring) this provider's projection. The provider stream is built
|
|
6837
|
+
* only after the bounded wait, so neither branch can start a request inside
|
|
6838
|
+
* the borrowed window; a consumer that cancels during the wait constructs
|
|
6839
|
+
* nothing at all. A test-scope request returns from `streamListener` before
|
|
6840
|
+
* this wrapper, so a test never waits on its own guard.
|
|
6841
|
+
*/
|
|
6842
|
+
isolateFromTest(providerId, open) {
|
|
6843
|
+
const service = this;
|
|
6844
|
+
let iterator;
|
|
6845
|
+
let waited = false;
|
|
6846
|
+
let closed = false;
|
|
6847
|
+
return {
|
|
6848
|
+
[Symbol.asyncIterator]() {
|
|
6849
|
+
return {
|
|
6850
|
+
async next() {
|
|
6851
|
+
if (closed)
|
|
6852
|
+
return { done: true, value: undefined };
|
|
6853
|
+
if (!waited) {
|
|
6854
|
+
waited = true;
|
|
6855
|
+
await service.settleProviderTest(providerId);
|
|
6856
|
+
}
|
|
6857
|
+
// QC N5: the consumer cancelled while this pull waited; there is
|
|
6858
|
+
// no one left to close a stream built here, so never build it.
|
|
6859
|
+
if (closed)
|
|
6860
|
+
return { done: true, value: undefined };
|
|
6861
|
+
if (iterator === undefined)
|
|
6862
|
+
iterator = open()[Symbol.asyncIterator]();
|
|
6863
|
+
return iterator.next();
|
|
6864
|
+
},
|
|
6865
|
+
async return(value) {
|
|
6866
|
+
closed = true;
|
|
6867
|
+
if (iterator?.return !== undefined)
|
|
6868
|
+
return iterator.return(value);
|
|
6869
|
+
return { done: true, value: undefined };
|
|
6870
|
+
},
|
|
6871
|
+
async throw(error) {
|
|
6872
|
+
closed = true;
|
|
6873
|
+
if (iterator?.throw !== undefined)
|
|
6874
|
+
return iterator.throw(error);
|
|
6875
|
+
throw error;
|
|
6876
|
+
},
|
|
6877
|
+
};
|
|
6878
|
+
},
|
|
6879
|
+
};
|
|
6880
|
+
}
|
|
6881
|
+
/**
|
|
6882
|
+
* Wait (bounded) for the in-flight borrowing test on this provider to
|
|
6883
|
+
* settle. The bound is {@link KEY_TEST_ISOLATION_WAIT_MS}, not the test's
|
|
6884
|
+
* own request timeout: the awaited promise also covers untimed
|
|
6885
|
+
* credential-store reads, so without this an ordinary request could park
|
|
6886
|
+
* here forever. A failed or timed-out test never fails the waiting request.
|
|
6887
|
+
*
|
|
6888
|
+
* QC fix wave 2: the one exception is a provider whose abandoned window is
|
|
6889
|
+
* still fenced. The wait is over, but the route's credential surface has not
|
|
6890
|
+
* been reconciled, so serving the request would answer it with the borrowed
|
|
6891
|
+
* grant (or, after a released binding, with no bound route at all). Such a
|
|
6892
|
+
* request is refused instead — the fence is what keeps cross-use impossible
|
|
6893
|
+
* until the cleanup completes.
|
|
6894
|
+
*/
|
|
6895
|
+
async settleProviderTest(providerId) {
|
|
6896
|
+
// QC fix wave 3 (S-004): a fence that is already known at entry is not a
|
|
6897
|
+
// reason to wait — the wait exists for a live test, and this route cannot
|
|
6898
|
+
// be served until its abandoned window is reconciled. Acting on the known
|
|
6899
|
+
// fence keeps the isolation deadline (25 s) off the refusal path.
|
|
6900
|
+
if (this.cleaningTests.has(providerId))
|
|
6901
|
+
throw this.testCleanupRefusal(providerId);
|
|
6902
|
+
const pending = this.testsInFlight.get(providerId);
|
|
6903
|
+
if (pending === undefined)
|
|
6904
|
+
return;
|
|
6905
|
+
const settled = await new Promise(resolve => {
|
|
6906
|
+
let done = false;
|
|
6907
|
+
let timer;
|
|
6908
|
+
const finish = (value) => {
|
|
6909
|
+
if (done)
|
|
6910
|
+
return;
|
|
6911
|
+
done = true;
|
|
6912
|
+
if (timer !== undefined)
|
|
6913
|
+
clearTimeout(timer);
|
|
6914
|
+
resolve(value);
|
|
6915
|
+
};
|
|
6916
|
+
timer = setTimeout(() => {
|
|
6917
|
+
if (!this.isolationWaitWarned.has(providerId)) {
|
|
6918
|
+
this.isolationWaitWarned.add(providerId);
|
|
6919
|
+
this.warn(`connection test for ${providerId} did not settle within ${this.isolationWaitMs}ms; the waiting request proceeds without it`);
|
|
6920
|
+
}
|
|
6921
|
+
finish(false);
|
|
6922
|
+
}, this.isolationWaitMs);
|
|
6923
|
+
timer.unref?.();
|
|
6924
|
+
void pending.then(() => finish(true), () => finish(true));
|
|
6925
|
+
});
|
|
6926
|
+
// A fence set while the request waited (the watchdog abandoned the test, or
|
|
6927
|
+
// its cleanup failed after settling) still owns the route, whatever the
|
|
6928
|
+
// test's own settlement says.
|
|
6929
|
+
if (this.cleaningTests.has(providerId))
|
|
6930
|
+
throw this.testCleanupRefusal(providerId);
|
|
6931
|
+
if (settled)
|
|
6932
|
+
return;
|
|
6933
|
+
}
|
|
6934
|
+
/** The refusal an ordinary request gets while its route is still unreconciled. */
|
|
6935
|
+
testCleanupRefusal(providerId) {
|
|
6936
|
+
return new Error(`The connection test for ${providerId} is still being cleaned up; this request was not sent.`);
|
|
6937
|
+
}
|
|
6938
|
+
/**
|
|
6939
|
+
* Wrap one connection-test stream: keep the pinned credential scope alive
|
|
6940
|
+
* for every chunk and do nothing else. No selection, no cursor, no health
|
|
6941
|
+
* writes — the request is pure observation.
|
|
6942
|
+
*/
|
|
6943
|
+
wrapTestStream(next, scope) {
|
|
6944
|
+
const service = this;
|
|
6945
|
+
let iterator;
|
|
6946
|
+
return {
|
|
6947
|
+
[Symbol.asyncIterator]() {
|
|
6948
|
+
return {
|
|
6949
|
+
async next() {
|
|
6950
|
+
if (iterator === undefined)
|
|
6951
|
+
iterator = await service.als.run(scope, () => next()[Symbol.asyncIterator]());
|
|
6952
|
+
return service.als.run(scope, () => iterator.next());
|
|
6953
|
+
},
|
|
6954
|
+
async return(value) {
|
|
6955
|
+
if (iterator?.return !== undefined)
|
|
6956
|
+
return service.als.run(scope, () => iterator.return(value));
|
|
6957
|
+
return { done: true, value: undefined };
|
|
6958
|
+
},
|
|
6959
|
+
async throw(error) {
|
|
6960
|
+
if (iterator?.throw !== undefined)
|
|
6961
|
+
return service.als.run(scope, () => iterator.throw(error));
|
|
6962
|
+
throw error;
|
|
6963
|
+
},
|
|
6964
|
+
};
|
|
6965
|
+
},
|
|
6966
|
+
};
|
|
6967
|
+
}
|
|
3704
6968
|
/** Wrap one managed stream so credential resolution sees this key. */
|
|
3705
6969
|
wrapStream(_options, next, providerId, sessionId, step) {
|
|
3706
6970
|
const service = this;
|
|
@@ -3719,10 +6983,9 @@ export class KeysManagerService {
|
|
|
3719
6983
|
// Attribution must wait for it — otherwise this request selects
|
|
3720
6984
|
// and mutates a key cursor even though pi-ai serves the stored
|
|
3721
6985
|
// grant, and an OAuth failure is classified as externally
|
|
3722
|
-
// authenticated. Round-9 should-fix: the wait is bounded
|
|
3723
|
-
//
|
|
3724
|
-
//
|
|
3725
|
-
// delay an API-key-only provider.
|
|
6986
|
+
// authenticated. Round-9 should-fix: the wait is bounded per
|
|
6987
|
+
// provider, so a stalled credential read can neither wedge this
|
|
6988
|
+
// request nor delay a provider whose own read already landed.
|
|
3726
6989
|
const hydrated = await service.awaitProjectionOwnership(providerId);
|
|
3727
6990
|
// Round-5 finding C4: while a projection exists, pi-ai serves
|
|
3728
6991
|
// the stored grant and never consults the broker, so selecting
|
|
@@ -3764,6 +7027,7 @@ export class KeysManagerService {
|
|
|
3764
7027
|
// when a projection exists, pi-ai serves the stored grant
|
|
3765
7028
|
// and this scope owns the failure attribution.
|
|
3766
7029
|
oauthAccountId,
|
|
7030
|
+
test: false,
|
|
3767
7031
|
};
|
|
3768
7032
|
if (sessionId !== undefined && keyId !== undefined) {
|
|
3769
7033
|
const key = bindingKey(sessionId, providerId);
|
|
@@ -4122,11 +7386,11 @@ export function apply(ctx) {
|
|
|
4122
7386
|
return () => service.detachCredentials(credentials);
|
|
4123
7387
|
});
|
|
4124
7388
|
ctx.inject(['settings'], child => {
|
|
4125
|
-
|
|
4126
|
-
|
|
4127
|
-
|
|
4128
|
-
|
|
4129
|
-
|
|
7389
|
+
const settings = child.settings;
|
|
7390
|
+
if (settings === undefined)
|
|
7391
|
+
return;
|
|
7392
|
+
service.attachSettings(settings);
|
|
7393
|
+
return () => service.detachSettings(settings);
|
|
4130
7394
|
});
|
|
4131
7395
|
const on = ctx.on;
|
|
4132
7396
|
on('llm/stream', (options, next) => service.streamListener(options, next));
|