@indigoai-us/hq-cloud 6.14.12 → 6.14.14

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. package/.claude/policies/hq-cloud-strip-types-no-parameter-properties.md +22 -0
  2. package/dist/bin/sync-runner-company.d.ts.map +1 -1
  3. package/dist/bin/sync-runner-company.js +6 -0
  4. package/dist/bin/sync-runner-company.js.map +1 -1
  5. package/dist/bin/sync-runner-planning.d.ts.map +1 -1
  6. package/dist/bin/sync-runner-planning.js +3 -1
  7. package/dist/bin/sync-runner-planning.js.map +1 -1
  8. package/dist/bin/sync-runner-watch-loop.d.ts +10 -0
  9. package/dist/bin/sync-runner-watch-loop.d.ts.map +1 -1
  10. package/dist/bin/sync-runner-watch-loop.js +42 -3
  11. package/dist/bin/sync-runner-watch-loop.js.map +1 -1
  12. package/dist/bin/sync-runner-watch-routes.d.ts +1 -0
  13. package/dist/bin/sync-runner-watch-routes.d.ts.map +1 -1
  14. package/dist/bin/sync-runner-watch-routes.js +3 -0
  15. package/dist/bin/sync-runner-watch-routes.js.map +1 -1
  16. package/dist/bin/sync-runner.d.ts +17 -0
  17. package/dist/bin/sync-runner.d.ts.map +1 -1
  18. package/dist/bin/sync-runner.js +83 -33
  19. package/dist/bin/sync-runner.js.map +1 -1
  20. package/dist/bin/sync-runner.test.js +193 -5
  21. package/dist/bin/sync-runner.test.js.map +1 -1
  22. package/dist/cli/reindex.d.ts.map +1 -1
  23. package/dist/cli/reindex.js +23 -60
  24. package/dist/cli/reindex.js.map +1 -1
  25. package/dist/cli/reindex.test.d.ts +2 -2
  26. package/dist/cli/reindex.test.js +60 -109
  27. package/dist/cli/reindex.test.js.map +1 -1
  28. package/dist/cli/rescue.reindex.test.js +3 -2
  29. package/dist/cli/rescue.reindex.test.js.map +1 -1
  30. package/dist/cli/share.d.ts.map +1 -1
  31. package/dist/cli/share.js +12 -1
  32. package/dist/cli/share.js.map +1 -1
  33. package/dist/cli/share.test.js +14 -0
  34. package/dist/cli/share.test.js.map +1 -1
  35. package/dist/cli/sync.d.ts.map +1 -1
  36. package/dist/cli/sync.js +9 -1
  37. package/dist/cli/sync.js.map +1 -1
  38. package/dist/cli/sync.test.js +8 -0
  39. package/dist/cli/sync.test.js.map +1 -1
  40. package/dist/cognito-auth.d.ts +44 -12
  41. package/dist/cognito-auth.d.ts.map +1 -1
  42. package/dist/cognito-auth.js +440 -76
  43. package/dist/cognito-auth.js.map +1 -1
  44. package/dist/cognito-auth.test.js +174 -0
  45. package/dist/cognito-auth.test.js.map +1 -1
  46. package/dist/index.d.ts +1 -1
  47. package/dist/index.d.ts.map +1 -1
  48. package/dist/index.js +3 -2
  49. package/dist/index.js.map +1 -1
  50. package/dist/machine-auth.test.js +683 -5
  51. package/dist/machine-auth.test.js.map +1 -1
  52. package/dist/vault-client.d.ts +2 -1
  53. package/dist/vault-client.d.ts.map +1 -1
  54. package/dist/vault-client.js +45 -9
  55. package/dist/vault-client.js.map +1 -1
  56. package/dist/vault-client.test.js +77 -7
  57. package/dist/vault-client.test.js.map +1 -1
  58. package/package.json +2 -1
  59. package/src/bin/sync-runner-company.ts +5 -0
  60. package/src/bin/sync-runner-planning.ts +2 -1
  61. package/src/bin/sync-runner-watch-loop.ts +51 -4
  62. package/src/bin/sync-runner-watch-routes.ts +2 -0
  63. package/src/bin/sync-runner.test.ts +244 -4
  64. package/src/bin/sync-runner.ts +99 -32
  65. package/src/cli/reindex.test.ts +63 -120
  66. package/src/cli/reindex.ts +23 -62
  67. package/src/cli/rescue.reindex.test.ts +3 -2
  68. package/src/cli/share.test.ts +20 -0
  69. package/src/cli/share.ts +7 -1
  70. package/src/cli/sync.test.ts +14 -0
  71. package/src/cli/sync.ts +5 -1
  72. package/src/cognito-auth.test.ts +227 -0
  73. package/src/cognito-auth.ts +533 -71
  74. package/src/index.ts +5 -1
  75. package/src/machine-auth.test.ts +808 -5
  76. package/src/vault-client.test.ts +100 -11
  77. package/src/vault-client.ts +55 -10
  78. package/test/share-sync.integration.test.ts +12 -9
@@ -28,17 +28,77 @@ export class CognitoAuthError extends Error {
28
28
  this.name = "CognitoAuthError";
29
29
  }
30
30
  }
31
+ /**
32
+ * A failed refresh attempt with enough classification for callers to decide
33
+ * whether the cached session is still worth preserving. Network/5xx/429
34
+ * failures remain recoverable; an invalid/expired refresh token means the
35
+ * stored session can no longer make progress and should be cleared.
36
+ */
37
+ export class CognitoRefreshError extends CognitoAuthError {
38
+ requiresReauth;
39
+ statusCode;
40
+ sessionFingerprint;
41
+ constructor(message, requiresReauth, statusCode, sessionFingerprint) {
42
+ super(message);
43
+ this.name = "CognitoRefreshError";
44
+ this.requiresReauth = requiresReauth;
45
+ this.statusCode = statusCode;
46
+ this.sessionFingerprint = sessionFingerprint;
47
+ }
48
+ }
31
49
  // ---------------------------------------------------------------------------
32
50
  // Token cache (~/.hq/cognito-tokens.json)
33
51
  // ---------------------------------------------------------------------------
34
52
  const HQ_DIR = path.join(os.homedir(), ".hq");
35
53
  const TOKEN_FILE = path.join(HQ_DIR, "cognito-tokens.json");
54
+ const TOKEN_INVALIDATION_PREFIX = `${TOKEN_FILE}.invalid.`;
55
+ /** Stable, non-secret identity for one access-token generation. */
56
+ export function accessTokenFingerprint(accessToken) {
57
+ return crypto.createHash("sha256").update(accessToken).digest("hex");
58
+ }
59
+ function invalidationPathForFingerprint(fingerprint) {
60
+ return /^[a-f0-9]{64}$/i.test(fingerprint)
61
+ ? `${TOKEN_INVALIDATION_PREFIX}${fingerprint.toLowerCase()}`
62
+ : null;
63
+ }
64
+ /**
65
+ * Mark only the rejected token generation unusable. Automatic auth failures
66
+ * must not unlink the shared cache: another process may have completed a
67
+ * refresh or login after the failing request started.
68
+ */
69
+ export function invalidateCachedTokensByFingerprint(fingerprint) {
70
+ if (!fingerprint)
71
+ return;
72
+ const invalidationPath = invalidationPathForFingerprint(fingerprint);
73
+ if (!invalidationPath)
74
+ return;
75
+ if (!fs.existsSync(HQ_DIR)) {
76
+ fs.mkdirSync(HQ_DIR, { recursive: true, mode: 0o700 });
77
+ }
78
+ try {
79
+ fs.writeFileSync(invalidationPath, "", { mode: 0o600, flag: "wx" });
80
+ }
81
+ catch (err) {
82
+ const code = err && typeof err === "object" && "code" in err
83
+ ? err.code
84
+ : undefined;
85
+ if (code !== "EEXIST")
86
+ throw err;
87
+ }
88
+ }
89
+ function tokenIsInvalidated(accessToken) {
90
+ const marker = invalidationPathForFingerprint(accessTokenFingerprint(accessToken));
91
+ return marker !== null && fs.existsSync(marker);
92
+ }
36
93
  export function loadCachedTokens() {
37
94
  if (!fs.existsSync(TOKEN_FILE))
38
95
  return null;
39
96
  try {
40
97
  const raw = fs.readFileSync(TOKEN_FILE, "utf-8");
41
- return JSON.parse(raw);
98
+ const tokens = JSON.parse(raw);
99
+ if (!tokens.accessToken || tokenIsInvalidated(tokens.accessToken))
100
+ return null;
101
+ return tokens;
42
102
  }
43
103
  catch {
44
104
  return null;
@@ -48,6 +108,10 @@ export function saveCachedTokens(tokens) {
48
108
  if (!fs.existsSync(HQ_DIR)) {
49
109
  fs.mkdirSync(HQ_DIR, { recursive: true, mode: 0o700 });
50
110
  }
111
+ const invalidationPath = invalidationPathForFingerprint(accessTokenFingerprint(tokens.accessToken));
112
+ if (invalidationPath && fs.existsSync(invalidationPath)) {
113
+ fs.unlinkSync(invalidationPath);
114
+ }
51
115
  const tmpPath = path.join(HQ_DIR, `.cognito-tokens.json.tmp.${process.pid}`);
52
116
  fs.writeFileSync(tmpPath, JSON.stringify(tokens, null, 2), { mode: 0o600 });
53
117
  fs.renameSync(tmpPath, TOKEN_FILE);
@@ -55,6 +119,133 @@ export function saveCachedTokens(tokens) {
55
119
  export function clearCachedTokens() {
56
120
  if (fs.existsSync(TOKEN_FILE))
57
121
  fs.unlinkSync(TOKEN_FILE);
122
+ if (!fs.existsSync(HQ_DIR))
123
+ return;
124
+ const markerPrefix = `${path.basename(TOKEN_FILE)}.invalid.`;
125
+ for (const entry of fs.readdirSync(HQ_DIR)) {
126
+ if (entry.startsWith(markerPrefix)) {
127
+ fs.unlinkSync(path.join(HQ_DIR, entry));
128
+ }
129
+ }
130
+ }
131
+ /** Advisory lockfile guarding the read→refresh→save cycle (see withRefreshLock). */
132
+ const REFRESH_LOCK_FILE = `${TOKEN_FILE}.lock`;
133
+ /** Thrown when the refresh lock can't be acquired within the deadline. */
134
+ export class RefreshLockTimeoutError extends Error {
135
+ constructor() {
136
+ super("timed out acquiring the token refresh lock");
137
+ this.name = "RefreshLockTimeoutError";
138
+ }
139
+ }
140
+ /** The pid recorded in the lockfile, or null if absent/empty/malformed. */
141
+ function readLockOwnerPid() {
142
+ try {
143
+ const pid = Number.parseInt(fs.readFileSync(REFRESH_LOCK_FILE, "utf-8").trim(), 10);
144
+ return Number.isInteger(pid) && pid > 0 ? pid : null;
145
+ }
146
+ catch {
147
+ return null;
148
+ }
149
+ }
150
+ /**
151
+ * Whether `pid` is a live process. All contenders for this lock share one host
152
+ * (the cache is `~/.hq`), so pid liveness is authoritative. `EPERM` means the
153
+ * process exists but is owned by another user — still alive; any other error
154
+ * (`ESRCH`, bad pid) means it's gone.
155
+ */
156
+ function processAlive(pid) {
157
+ try {
158
+ process.kill(pid, 0);
159
+ return true;
160
+ }
161
+ catch (err) {
162
+ return err.code === "EPERM";
163
+ }
164
+ }
165
+ /**
166
+ * Serialize the token refresh/mint cycle across every process sharing this
167
+ * cache (~/.hq/cognito-tokens.json). Boxes run several hq processes (the
168
+ * auth-refresh timer, the sync runner, replica-sync) that refresh or mint the
169
+ * same session concurrently; without coordination they race on the shared
170
+ * cache. And if the app client ever issues single-use tokens (refresh-token
171
+ * rotation), two unsynchronized refreshers each send the same token — the first
172
+ * consumes it, the second gets `invalid_grant`. An advisory lockfile (atomic
173
+ * hard-link publication) makes them take turns; the caller MUST re-read the cache
174
+ * inside `fn` so a waiter reuses the winner's fresh token, never a consumed one.
175
+ * This is also the lock the per-box machine-identity mint path reuses (ADR-0009).
176
+ *
177
+ * Correctness over availability at the edges:
178
+ * - A held lock is reclaimed ONLY when its recorded holder pid is dead (a real
179
+ * crash) or the lockfile is malformed — NEVER because a live holder is merely
180
+ * slow (a slow Cognito response must not let a second refresher race in).
181
+ * - Release unlinks ONLY our own lock (pid re-checked), so we can't delete a
182
+ * successor's lock.
183
+ * - If we can't acquire within `timeoutMs` (a live holder never released — a
184
+ * hung network), we THROW rather than refresh unlocked; the caller prefers a
185
+ * still-valid cached token and otherwise retries next cycle. Proceeding
186
+ * unlocked would reintroduce the exact rotation race this guards.
187
+ */
188
+ async function withRefreshLock(fn) {
189
+ const timeoutMs = 20_000;
190
+ const pollMs = 50;
191
+ if (!fs.existsSync(HQ_DIR)) {
192
+ fs.mkdirSync(HQ_DIR, { recursive: true, mode: 0o700 });
193
+ }
194
+ // Publish a fully-written inode with an atomic hard link. Creating the lock
195
+ // itself and then writing the pid leaves an observable empty-file window in
196
+ // which a contender can misclassify and unlink a live holder's lock.
197
+ const candidate = `${REFRESH_LOCK_FILE}.candidate.${process.pid}.${crypto.randomBytes(8).toString("hex")}`;
198
+ fs.writeFileSync(candidate, String(process.pid), {
199
+ flag: "wx",
200
+ mode: 0o600,
201
+ });
202
+ const deadline = Date.now() + timeoutMs;
203
+ let held = false;
204
+ try {
205
+ for (;;) {
206
+ try {
207
+ fs.linkSync(candidate, REFRESH_LOCK_FILE);
208
+ held = true;
209
+ break;
210
+ }
211
+ catch (err) {
212
+ if (err.code !== "EEXIST")
213
+ throw err;
214
+ const owner = readLockOwnerPid();
215
+ if (owner === null || (owner !== process.pid && !processAlive(owner))) {
216
+ try {
217
+ fs.unlinkSync(REFRESH_LOCK_FILE); // crashed/malformed holder — reclaim
218
+ }
219
+ catch {
220
+ /* raced with another reclaimer */
221
+ }
222
+ continue;
223
+ }
224
+ if (Date.now() >= deadline)
225
+ throw new RefreshLockTimeoutError();
226
+ await sleep(pollMs);
227
+ }
228
+ }
229
+ return await fn();
230
+ }
231
+ finally {
232
+ // Release ONLY our own lock — never clobber a successor's.
233
+ if (held && readLockOwnerPid() === process.pid) {
234
+ try {
235
+ fs.unlinkSync(REFRESH_LOCK_FILE);
236
+ }
237
+ catch {
238
+ /* already released */
239
+ }
240
+ }
241
+ // Remove the private proposal inode name on success or acquisition failure.
242
+ try {
243
+ fs.unlinkSync(candidate);
244
+ }
245
+ catch {
246
+ /* already removed */
247
+ }
248
+ }
58
249
  }
59
250
  /**
60
251
  * Parse `expiresAt` to epoch-ms. Canonical on-disk shape is epoch milliseconds
@@ -113,7 +304,7 @@ function decodeJwtClaims(token) {
113
304
  return null;
114
305
  }
115
306
  }
116
- function cachedTokensMatchMachineIdentity(tokens, creds, expectedClientId) {
307
+ function machineTokenPairMatchesIdentity(tokens, creds, expectedClientId) {
117
308
  const accessClaims = decodeJwtClaims(tokens.accessToken);
118
309
  const idClaims = decodeJwtClaims(tokens.idToken);
119
310
  if (!accessClaims || !idClaims)
@@ -129,8 +320,23 @@ function cachedTokensMatchMachineIdentity(tokens, creds, expectedClientId) {
129
320
  const accessSub = stringClaim(accessClaims, "sub");
130
321
  const idUsername = stringClaim(idClaims, "cognito:username") ?? stringClaim(idClaims, "username");
131
322
  const idSub = stringClaim(idClaims, "sub");
132
- const entityType = idClaims["custom:entityType"];
133
- const entityUid = idClaims["custom:entityUid"];
323
+ const entityType = stringClaim(idClaims, "custom:entityType");
324
+ const entityUid = stringClaim(idClaims, "custom:entityUid");
325
+ const delegatedSub = stringClaim(idClaims, "custom:delegatedSub");
326
+ const delegatedEmail = stringClaim(idClaims, "custom:delegatedEmail");
327
+ const hasValidMachineClaimShape = (entityType === "agent" &&
328
+ entityUid !== null &&
329
+ entityUid.startsWith("agt_")) ||
330
+ (entityType === "outpost" &&
331
+ entityUid !== null &&
332
+ entityUid.startsWith("out_") &&
333
+ delegatedSub !== null &&
334
+ delegatedEmail !== null);
335
+ // Legacy agent credentials did not persist entityType. If a newer creds
336
+ // file adds only its agt_* uid, infer the only valid type for that uid.
337
+ const expectedEntityType = creds.entityType ?? "agent";
338
+ const entityMatchesCreds = entityType === expectedEntityType &&
339
+ (creds.entityUid === undefined || entityUid === creds.entityUid);
134
340
  const idTokenMatchesCreds = idUsername === creds.username || idSub === creds.username;
135
341
  const subjectBindings = [];
136
342
  if (accessUsername !== null && idUsername !== null) {
@@ -144,9 +350,8 @@ function cachedTokensMatchMachineIdentity(tokens, creds, expectedClientId) {
144
350
  accessClientId === expectedClientId &&
145
351
  idClaims.token_use === "id" &&
146
352
  idAudience === expectedClientId &&
147
- entityType === "agent" &&
148
- typeof entityUid === "string" &&
149
- entityUid.startsWith("agt_") &&
353
+ hasValidMachineClaimShape &&
354
+ entityMatchesCreds &&
150
355
  idTokenMatchesCreds &&
151
356
  tokensShareSubject);
152
357
  }
@@ -169,15 +374,35 @@ export function loadMachineCreds() {
169
374
  // gate. (6.7.0 required a "machine-" prefix, but live boxes are
170
375
  // provisioned with "agt-<ulid>@agents.getindigo.ai", so detection
171
376
  // never engaged on the first real agent box, 2026-06-12.)
377
+ const validEntityIdentity = (raw.entityType === undefined && raw.entityUid === undefined) ||
378
+ (raw.entityType === undefined &&
379
+ typeof raw.entityUid === "string" &&
380
+ raw.entityUid.startsWith("agt_") &&
381
+ raw.entityUid.length > "agt_".length) ||
382
+ (raw.entityType === "agent" &&
383
+ typeof raw.entityUid === "string" &&
384
+ raw.entityUid.startsWith("agt_") &&
385
+ raw.entityUid.length > "agt_".length) ||
386
+ (raw.entityType === "outpost" &&
387
+ typeof raw.entityUid === "string" &&
388
+ raw.entityUid.startsWith("out_") &&
389
+ raw.entityUid.length > "out_".length);
172
390
  if (typeof raw.username === "string" &&
173
391
  raw.username.length > 0 &&
174
392
  typeof raw.secret === "string" &&
175
- raw.secret.length > 0) {
393
+ raw.secret.length > 0 &&
394
+ validEntityIdentity) {
176
395
  return {
177
396
  username: raw.username,
178
397
  secret: raw.secret,
179
398
  clientId: typeof raw.clientId === "string" ? raw.clientId : undefined,
180
399
  region: typeof raw.region === "string" ? raw.region : undefined,
400
+ ...(raw.entityType === "agent" || raw.entityType === "outpost"
401
+ ? { entityType: raw.entityType }
402
+ : {}),
403
+ ...(typeof raw.entityUid === "string"
404
+ ? { entityUid: raw.entityUid }
405
+ : {}),
181
406
  };
182
407
  }
183
408
  return null;
@@ -226,7 +451,7 @@ function backoffDelayMs(attempt, opts) {
226
451
  * full jitter (transient, per-user rate limit); every other non-OK response
227
452
  * (bad creds, disabled flow) is a hard error surfaced immediately.
228
453
  */
229
- export async function mintMachineTokens(config, creds, retry) {
454
+ async function mintMachineTokensUnlocked(config, creds, retry) {
230
455
  const machineCreds = creds ?? loadMachineCreds();
231
456
  if (!machineCreds) {
232
457
  throw new CognitoAuthError(`No machine credentials found at ${machineCredsFilePath()}`);
@@ -275,11 +500,14 @@ export async function mintMachineTokens(config, creds, retry) {
275
500
  accessToken: result.AccessToken,
276
501
  idToken: result.IdToken,
277
502
  // Machine creds never expire — expiry is handled by re-minting, so the
278
- // refresh token (when Cognito returns one at all) is never exercised.
279
- refreshToken: result.RefreshToken ?? "",
503
+ // refresh token (when Cognito returns one at all) is never retained.
504
+ refreshToken: "",
280
505
  expiresAt: Date.now() + (result.ExpiresIn ?? 3600) * 1000,
281
506
  tokenType: "Bearer",
282
507
  };
508
+ if (!machineTokenPairMatchesIdentity(tokens, machineCreds, clientId)) {
509
+ throw new CognitoAuthError("Machine token mint returned tokens that do not match the requested machine identity");
510
+ }
283
511
  saveCachedTokens(tokens);
284
512
  return tokens;
285
513
  }
@@ -287,63 +515,112 @@ export async function mintMachineTokens(config, creds, retry) {
287
515
  // loop — but keeps the function total for the type checker.
288
516
  throw lastError ?? new CognitoAuthError("Machine token mint failed: throttled");
289
517
  }
290
- // In-process machine-token memo (sync-runner mint-storm fix).
291
- //
292
- // `getValidAccessToken` runs on EVERY vault request, and a busy sync fans out
293
- // many requests concurrently (presign primes, STS vend, the per-file tombstone
294
- // HEAD verifies). The on-disk cache alone does NOT bound mints: at run start
295
- // the disk token is often within its expiry buffer, so N concurrent callers
296
- // each independently observe "expiring" and each fire USER_PASSWORD_AUTH —
297
- // bursting Cognito's per-user limit (TooManyRequestsException) on large vaults
298
- // (observed: 56 throttled mints over 314 files on the Infraredi box, 2026-06-26).
299
- //
300
- // This module-level memo makes mints O(1) per process:
301
- // - hot path: a live in-memory token serves every later request with no disk
302
- // read and no network — a 300-file sync does ONE mint, not one-per-request;
303
- // - single-flight: a concurrent burst of cache-misses (the run-start fan-out)
304
- // collapses onto a single in-flight mint instead of one mint per caller.
305
- // One machine identity per process is assumed (true for agent boxes), so a
306
- // single slot suffices.
518
+ /**
519
+ * Mint and persist a fresh machine session while holding the process-shared
520
+ * token lock. The unlocked implementation is private so public callers cannot
521
+ * race another process's cache write.
522
+ */
523
+ export async function mintMachineTokens(config, creds, retry) {
524
+ return withRefreshLock(() => mintMachineTokensUnlocked(config, creds, retry));
525
+ }
307
526
  let inProcessMachineTokens = null;
308
- let inFlightMachineMint = null;
527
+ const inFlightMachineMints = new Map();
528
+ function machineIdentityKey(config, creds) {
529
+ // Include every credential value that can change the effective mint. Hashing
530
+ // keeps the long-lived secret out of diagnostic heap strings while retaining
531
+ // exact-key behavior when a creds file is reprovisioned in place.
532
+ return crypto
533
+ .createHash("sha256")
534
+ .update(JSON.stringify({
535
+ username: creds.username,
536
+ secret: creds.secret,
537
+ clientId: creds.clientId ?? config.clientId,
538
+ region: creds.region ?? config.region,
539
+ entityType: creds.entityType ?? "agent",
540
+ entityUid: creds.entityUid ?? null,
541
+ }))
542
+ .digest("hex");
543
+ }
309
544
  /**
310
545
  * Return a valid (non-expiring) machine session, re-minting on demand.
311
546
  * Cache-hit path never touches the network — and after the first mint, the
312
547
  * in-process memo serves every subsequent request for the life of the token.
313
548
  */
314
549
  export async function getValidMachineTokens(config) {
550
+ const machineCreds = loadMachineCreds();
551
+ if (!machineCreds) {
552
+ throw new CognitoAuthError(`No machine credentials found at ${machineCredsFilePath()}`);
553
+ }
554
+ const expectedClientId = machineCreds.clientId ?? config.clientId;
555
+ const identityKey = machineIdentityKey(config, machineCreds);
315
556
  // Hot path: a live in-memory token serves every request after the first.
316
- if (inProcessMachineTokens && !isExpiring(inProcessMachineTokens, 120)) {
317
- return inProcessMachineTokens;
557
+ if (inProcessMachineTokens?.identityKey === identityKey &&
558
+ !isExpiring(inProcessMachineTokens.tokens, 120) &&
559
+ machineTokenPairMatchesIdentity(inProcessMachineTokens.tokens, machineCreds, expectedClientId)) {
560
+ return inProcessMachineTokens.tokens;
318
561
  }
319
562
  // Single-flight: collapse a concurrent burst of misses onto one resolution.
320
563
  // The assignment below completes before the first `await` yields, so sibling
321
564
  // callers in the same tick see the in-flight promise and reuse it.
322
- if (inFlightMachineMint)
323
- return inFlightMachineMint;
324
- inFlightMachineMint = (async () => {
325
- const machineCreds = loadMachineCreds();
326
- const cached = loadCachedTokens();
327
- if (machineCreds && cached && !isExpiring(cached, 120)) {
328
- // Compare against the client we'd actually mint with (creds-file clientId
329
- // wins over config), and require the cached ID token to prove this exact
330
- // agent identity. Opaque/missing/human-shaped claims are treated as stale.
331
- const expectedClientId = machineCreds.clientId ?? config.clientId;
332
- if (cachedTokensMatchMachineIdentity(cached, machineCreds, expectedClientId)) {
333
- return cached;
565
+ const existingResolution = inFlightMachineMints.get(identityKey);
566
+ if (existingResolution)
567
+ return existingResolution;
568
+ const resolution = (async () => {
569
+ const resolveUnderLock = async () => {
570
+ // Re-read only after acquiring the process-shared lock. A different hq
571
+ // process may have minted and saved this machine's session while this
572
+ // caller was waiting.
573
+ const cached = loadCachedTokens();
574
+ if (cached && !isExpiring(cached, 120)) {
575
+ // Compare against the client we'd actually mint with (creds-file
576
+ // clientId wins over config), and require the cached ID token to prove
577
+ // the exact machine identity. Opaque/missing/human-shaped claims are
578
+ // treated as stale.
579
+ if (machineTokenPairMatchesIdentity(cached, machineCreds, expectedClientId)) {
580
+ if (cached.refreshToken === "")
581
+ return cached;
582
+ const normalized = { ...cached, refreshToken: "" };
583
+ saveCachedTokens(normalized);
584
+ return normalized;
585
+ }
334
586
  }
587
+ // mintMachineTokens persists before returning, so the complete shared
588
+ // cache read -> USER_PASSWORD_AUTH mint -> save sequence is serialized.
589
+ return mintMachineTokensUnlocked(config, machineCreds);
590
+ };
591
+ try {
592
+ return await withRefreshLock(resolveUnderLock);
593
+ }
594
+ catch (err) {
595
+ if (!(err instanceof RefreshLockTimeoutError))
596
+ throw err;
597
+ // Never mint unlocked. If the holder managed to persist a matching fresh
598
+ // session, reuse it; otherwise surface a retryable auth failure.
599
+ const cached = loadCachedTokens();
600
+ if (cached && !isExpiring(cached, 120)) {
601
+ if (machineTokenPairMatchesIdentity(cached, machineCreds, expectedClientId)) {
602
+ // The successful path always leaves a machine cache with no refresh
603
+ // token. If this is legacy state, only normalize it while holding the
604
+ // lock; after a timeout we must not race the live holder's write.
605
+ if (cached.refreshToken === "")
606
+ return cached;
607
+ }
608
+ }
609
+ throw new CognitoAuthError("Machine token mint is held by another process and no fresh matching session is available; retry shortly.");
335
610
  }
336
- return mintMachineTokens(config, machineCreds ?? undefined);
337
611
  })();
612
+ inFlightMachineMints.set(identityKey, resolution);
338
613
  try {
339
- const tokens = await inFlightMachineMint;
340
- inProcessMachineTokens = tokens;
614
+ const tokens = await resolution;
615
+ inProcessMachineTokens = { identityKey, tokens };
341
616
  return tokens;
342
617
  }
343
618
  finally {
344
619
  // Clear on both success and failure: a rejected mint must not wedge later
345
620
  // calls onto a permanently-failed promise.
346
- inFlightMachineMint = null;
621
+ if (inFlightMachineMints.get(identityKey) === resolution) {
622
+ inFlightMachineMints.delete(identityKey);
623
+ }
347
624
  }
348
625
  }
349
626
  // ---------------------------------------------------------------------------
@@ -503,36 +780,118 @@ async function exchangeCodeForTokens(config, code, verifier, port) {
503
780
  tokenType: "Bearer",
504
781
  };
505
782
  }
783
+ /** Whether the cached session was minted by `config`'s app client. */
784
+ function cachedIsForClient(cached, config) {
785
+ const clientId = decodeAccessTokenClientId(cached.accessToken);
786
+ return clientId !== null && clientId === config.clientId;
787
+ }
506
788
  /**
507
789
  * Use the refresh token to obtain a fresh access token without user interaction.
508
- * The refresh token itself is NOT rotated by Cognito on the refresh grant, so
509
- * we preserve the existing one in the result.
790
+ *
791
+ * Concurrency-safe. Several hq processes on a box refresh the same cached
792
+ * session at once, so the whole read→refresh→save cycle runs under a
793
+ * cross-process lock, and inside the lock we (a) reuse a token another refresher
794
+ * just produced if it's still fresh, and (b) otherwise refresh with the FRESHEST
795
+ * token on disk rather than the caller's snapshot. This keeps the shared cache
796
+ * consistent, and stays correct if the app client uses single-use (rotated)
797
+ * tokens — a stale snapshot would otherwise be an already-consumed token
798
+ * (`invalid_grant`). NOTE: rotation is NOT how boxes stay signed in — a rotated
799
+ * token inherits the original's remaining lifetime, so it does not extend the
800
+ * session (AWS). Durable never-log-out is per-box machine identity (ADR-0009).
510
801
  */
511
- export async function refreshTokens(config, currentRefreshToken) {
512
- const body = new URLSearchParams({
513
- grant_type: "refresh_token",
514
- client_id: config.clientId,
515
- refresh_token: currentRefreshToken,
516
- });
517
- const res = await fetch(`${authBaseUrl(config)}/oauth2/token`, {
518
- method: "POST",
519
- headers: { "Content-Type": "application/x-www-form-urlencoded" },
520
- body: body.toString(),
521
- });
522
- if (!res.ok) {
523
- const text = await res.text();
524
- throw new CognitoAuthError(`Refresh failed (${res.status}): ${text}`);
525
- }
526
- const data = (await res.json());
527
- const tokens = {
528
- accessToken: data.access_token,
529
- idToken: data.id_token,
530
- refreshToken: data.refresh_token ?? currentRefreshToken,
531
- expiresAt: Date.now() + data.expires_in * 1000,
532
- tokenType: "Bearer",
802
+ export async function refreshTokens(config, currentRefreshToken, currentAccessToken) {
803
+ const doRefresh = async () => {
804
+ const cached = loadCachedTokens();
805
+ // Reuse a token another refresher just produced — but ONLY when it belongs
806
+ // to THIS app client (a direct caller may target a different pool than the
807
+ // shared cache holds) and is still fresh (a fresh access token has ~1h, so
808
+ // >300s of life means "someone already refreshed", not the caller's own
809
+ // expiring token).
810
+ if (cached && cachedIsForClient(cached, config) && !isExpiring(cached, 300)) {
811
+ return cached;
812
+ }
813
+ // Prefer the freshest refresh token on disk over the caller's snapshot when
814
+ // it's for this client — a concurrent rotation may have consumed the
815
+ // snapshot. Otherwise (different client / no cache) use the supplied token.
816
+ const refreshToken = cached && cachedIsForClient(cached, config)
817
+ ? cached.refreshToken
818
+ : currentRefreshToken;
819
+ const sessionFingerprint = cached && cachedIsForClient(cached, config)
820
+ ? accessTokenFingerprint(cached.accessToken)
821
+ : currentAccessToken
822
+ ? accessTokenFingerprint(currentAccessToken)
823
+ : undefined;
824
+ const body = new URLSearchParams({
825
+ grant_type: "refresh_token",
826
+ client_id: config.clientId,
827
+ refresh_token: refreshToken,
828
+ });
829
+ let lastFailure;
830
+ for (let attempt = 0; attempt < 2; attempt++) {
831
+ let res;
832
+ try {
833
+ res = await fetch(`${authBaseUrl(config)}/oauth2/token`, {
834
+ method: "POST",
835
+ headers: { "Content-Type": "application/x-www-form-urlencoded" },
836
+ body: body.toString(),
837
+ });
838
+ }
839
+ catch (err) {
840
+ lastFailure = new CognitoRefreshError(`Cognito refresh request failed: ${err instanceof Error ? err.message : String(err)}`, false, undefined, sessionFingerprint);
841
+ if (attempt === 0)
842
+ continue;
843
+ throw lastFailure;
844
+ }
845
+ if (!res.ok) {
846
+ const text = await res.text();
847
+ const retryable = res.status === 401 ||
848
+ res.status === 408 ||
849
+ res.status === 429 ||
850
+ res.status >= 500;
851
+ const requiresReauth = res.status === 401 ||
852
+ (res.status >= 400 &&
853
+ res.status < 500 &&
854
+ ![408, 429].includes(res.status));
855
+ lastFailure = new CognitoRefreshError(requiresReauth
856
+ ? "Your HQ session needs a quick refresh. Sign in again to keep sync moving."
857
+ : `Cognito refresh failed (${res.status}): ${text}`, requiresReauth, res.status, sessionFingerprint);
858
+ if (retryable && attempt === 0)
859
+ continue;
860
+ throw lastFailure;
861
+ }
862
+ const data = (await res.json());
863
+ const tokens = {
864
+ accessToken: data.access_token,
865
+ idToken: data.id_token,
866
+ // Persist a rotated refresh token if the endpoint returns one (single-use
867
+ // clients); otherwise keep the freshest token we sent. This is for cache
868
+ // correctness, not longevity; rotation does not extend session lifetime.
869
+ refreshToken: data.refresh_token ?? refreshToken,
870
+ expiresAt: Date.now() + data.expires_in * 1000,
871
+ tokenType: "Bearer",
872
+ };
873
+ saveCachedTokens(tokens);
874
+ return tokens;
875
+ }
876
+ throw (lastFailure ??
877
+ new CognitoRefreshError("Cognito refresh failed after retry", false));
533
878
  };
534
- saveCachedTokens(tokens);
535
- return tokens;
879
+ try {
880
+ return await withRefreshLock(doRefresh);
881
+ }
882
+ catch (err) {
883
+ if (!(err instanceof RefreshLockTimeoutError))
884
+ throw err;
885
+ // A live holder never released within the window (pathological — e.g. a
886
+ // hung network). Prefer whatever it may have persisted for this client; do
887
+ // NOT refresh unlocked (that reintroduces the rotation race). If nothing
888
+ // usable is cached, surface the failure so the caller retries next cycle.
889
+ const cached = loadCachedTokens();
890
+ if (cached && cachedIsForClient(cached, config) && !isExpiring(cached)) {
891
+ return cached;
892
+ }
893
+ throw new CognitoRefreshError("Token refresh is held by another process and no fresh cached session is available; retry shortly.", false);
894
+ }
536
895
  }
537
896
  /**
538
897
  * High-level helper: return a non-expired access token, refreshing or
@@ -568,7 +927,7 @@ export async function getValidAccessToken(config, options = {}) {
568
927
  if (cached) {
569
928
  const cachedClientId = decodeAccessTokenClientId(cached.accessToken);
570
929
  if (cachedClientId !== null && cachedClientId !== config.clientId) {
571
- clearCachedTokens();
930
+ invalidateCachedTokensByFingerprint(accessTokenFingerprint(cached.accessToken));
572
931
  cached = null;
573
932
  }
574
933
  }
@@ -576,10 +935,15 @@ export async function getValidAccessToken(config, options = {}) {
576
935
  return cached.accessToken;
577
936
  if (cached) {
578
937
  try {
579
- const refreshed = await refreshTokens(config, cached.refreshToken);
938
+ const refreshed = await refreshTokens(config, cached.refreshToken, cached.accessToken);
580
939
  return refreshed.accessToken;
581
940
  }
582
- catch {
941
+ catch (err) {
942
+ if (err instanceof CognitoRefreshError && err.requiresReauth) {
943
+ invalidateCachedTokensByFingerprint(err.sessionFingerprint ?? accessTokenFingerprint(cached.accessToken));
944
+ }
945
+ if (!interactive)
946
+ throw err;
583
947
  // fall through to interactive login
584
948
  }
585
949
  }