@opengeni/worker-bundle 0.7.0 → 0.7.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. package/dist/index.d.ts +182 -67
  2. package/dist/index.js +7127 -1654
  3. package/dist/index.js.map +1 -1
  4. package/package.json +19 -19
  5. package/src/activities/agent-turn.ts +4405 -1089
  6. package/src/activities/capabilities.ts +43 -8
  7. package/src/activities/codex-capacity.ts +223 -0
  8. package/src/activities/codex-rotation.ts +550 -54
  9. package/src/activities/common.ts +7 -1
  10. package/src/activities/context-compaction.ts +152 -118
  11. package/src/activities/documents.ts +32 -27
  12. package/src/activities/environment.ts +318 -75
  13. package/src/activities/file-upload-reaper.ts +79 -0
  14. package/src/activities/git-credential-renewal.ts +183 -0
  15. package/src/activities/goals.ts +142 -123
  16. package/src/activities/packs.ts +55 -5
  17. package/src/activities/parent-wake.ts +228 -41
  18. package/src/activities/recording.ts +154 -122
  19. package/src/activities/redaction.ts +3 -1
  20. package/src/activities/rig-verification.ts +441 -0
  21. package/src/activities/run-input.ts +110 -122
  22. package/src/activities/sandbox-lease.ts +186 -51
  23. package/src/activities/scheduled-tasks.ts +287 -171
  24. package/src/activities/session-state.ts +168 -157
  25. package/src/activities/streaming.ts +143 -19
  26. package/src/activities/turn-attempt-fenced.ts +14 -0
  27. package/src/activities/types.ts +102 -32
  28. package/src/activities/workflow-wake.ts +44 -0
  29. package/src/activities/workspace-capture.ts +964 -0
  30. package/src/activities.ts +75 -35
  31. package/src/concurrency.ts +11 -0
  32. package/src/http.ts +16 -6
  33. package/src/index.ts +322 -49
  34. package/src/observability-metrics.ts +441 -8
  35. package/src/op-journal.ts +46 -0
  36. package/src/sandbox-resume.ts +431 -201
  37. package/src/sandbox-routing.ts +209 -22
  38. package/src/workflow-wake-contract.ts +5 -0
  39. package/src/workflows/activities.ts +58 -7
  40. package/src/workflows/file-upload-reaper.ts +12 -0
  41. package/src/workflows/rig-verification.ts +28 -0
  42. package/src/workflows/scheduled-tasks.ts +12 -26
  43. package/src/workflows/session.ts +330 -173
  44. package/src/workflows/workflow-wake.ts +12 -0
  45. package/src/workflows.ts +9 -7
@@ -6,9 +6,65 @@
6
6
  // `selectCodexCredentialForTurn` precedence gate (pin > active). Keeping the
7
7
  // decision pure makes the whole rotation correctness story unit-testable in
8
8
  // isolation (see codex-rotation.test.ts).
9
- import type { CodexAccountStatus } from "@opengeni/db";
9
+ import type {
10
+ CodexAccountStatus,
11
+ CodexCredentialLeaseSelectionContext,
12
+ CodexLeaseAccountStatus,
13
+ CodexPinSource,
14
+ } from "@opengeni/db";
10
15
 
11
- export type CodexRotationStrategy = "most_remaining" | "round_robin" | "drain_then_next";
16
+ export type CodexRotationAccount = CodexAccountStatus | CodexLeaseAccountStatus;
17
+
18
+ export type CodexRotationStrategy =
19
+ | "most_remaining"
20
+ | "round_robin"
21
+ | "drain_then_next"
22
+ | "sharded";
23
+
24
+ /**
25
+ * How a session's codex pin governs THIS turn's account selection (pure). Encodes the
26
+ * policy-pin LIFECYCLE rule: a 'policy' pin is meaningful ONLY while the sharded policy
27
+ * is active.
28
+ * • "manual" — a manual pin: honored under EVERY strategy; never moved or cleared.
29
+ * • "sharded" — the sharded policy is active and the pin is non-manual: assign / keep
30
+ * / re-shard the deterministic home (covers an unpinned first turn AND
31
+ * an existing policy pin).
32
+ * • "clearStale" — a 'policy' pin while the sharded policy is NOT active: IGNORE it
33
+ * (never honor it as a sticky pin — that is the no-escape trap) and
34
+ * clear it lazily so the session converges to the active strategy.
35
+ * • "unpinned" — no pin (or none that applies): follow the active strategy / workspace
36
+ * active pointer, unchanged.
37
+ */
38
+ export type CodexPinDisposition = "manual" | "sharded" | "clearStale" | "unpinned";
39
+
40
+ /** Classify a session's codex pin against the active rotation regime (see {@link CodexPinDisposition}). */
41
+ export function classifyCodexPin(args: {
42
+ pinnedCredentialId: string | null;
43
+ pinSource: CodexPinSource | null;
44
+ strategy: CodexRotationStrategy;
45
+ rotationEnabled: boolean;
46
+ }): CodexPinDisposition {
47
+ const { pinnedCredentialId, pinSource, strategy, rotationEnabled } = args;
48
+ const pinned = pinnedCredentialId != null;
49
+ // A manual pin is sacrosanct under every strategy — checked FIRST so sharded never
50
+ // touches it. DEFENSE-IN-DEPTH (fail-safe toward sacredness): a pin whose source is
51
+ // anything OTHER than the explicit 'policy' — including a NULL source (a pre-backfill
52
+ // row, or any pin an unforeseen path wrote without labeling it) — is treated as
53
+ // MANUAL. An unlabeled pin must NEVER be policy-moved; only an explicitly-'policy' pin
54
+ // is re-shardable.
55
+ if (pinned && pinSource !== "policy") {
56
+ return "manual";
57
+ }
58
+ const shardedActive = rotationEnabled && strategy === "sharded";
59
+ if (shardedActive) {
60
+ return "sharded";
61
+ }
62
+ // A policy pin outside the sharded regime is stale → clear it.
63
+ if (pinned && pinSource === "policy") {
64
+ return "clearStale";
65
+ }
66
+ return "unpinned";
67
+ }
12
68
 
13
69
  export type RotationDecision =
14
70
  // The chosen account. `moved` ⇒ it differs from the current active pointer, so
@@ -24,6 +80,13 @@ export type RotationDecision =
24
80
  // No connected accounts at all (preserves today's relogin-fail path).
25
81
  | { kind: "none" };
26
82
 
83
+ export type CodexTurnLeaseSelection = {
84
+ credentialId: string | null;
85
+ decision: RotationDecision;
86
+ /** Policy/manual homes never move the workspace-global legacy pointer. */
87
+ advanceActivePointer?: boolean;
88
+ };
89
+
27
90
  // Invariant 4 (NO THRASH / BOUNDED) safety floors. An all-capped idle MUST be a
28
91
  // positive, bounded wait — a null / elapsed / unknown reset can NEVER collapse it
29
92
  // into a 0 or a past instant (which the caller would turn into a 0 continueDelayMs
@@ -39,9 +102,20 @@ export const MIN_IDLE_MS = 60_000; // 60s
39
102
  */
40
103
  export const DEFAULT_RESET_COOLDOWN_MS = 60_000; // 60s
41
104
 
42
- /** The worse-window used percent: weekly binds as hard as 5h, so take the max. null ⇒ 0. */
43
- function bindingUsedPct(acct: CodexAccountStatus): number {
44
- return Math.max(acct.primaryUsedPercent ?? 0, acct.secondaryUsedPercent ?? 0);
105
+ function effectiveWindowUsed(usedPercent: number | null, resetAt: Date | null, now: Date): number {
106
+ // A completed provider window is eligible immediately even if its last cache
107
+ // sample still says 100%. This is the five-hour reset boundary—not a TTL
108
+ // heuristic—and avoids stranding a reset subscription while another remains
109
+ // healthy. A future/unknown reset keeps the cached percentage authoritative.
110
+ return resetAt && resetAt.getTime() <= now.getTime() ? 0 : (usedPercent ?? 0);
111
+ }
112
+
113
+ /** The worse live window: weekly binds as hard as 5h. */
114
+ function bindingUsedPct(acct: CodexRotationAccount, now: Date): number {
115
+ return Math.max(
116
+ effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now),
117
+ effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now),
118
+ );
45
119
  }
46
120
 
47
121
  /**
@@ -49,13 +123,15 @@ function bindingUsedPct(acct: CodexAccountStatus): number {
49
123
  * buildCodexUsageWindowFromCache's `remaining = 100 - percent`, taking the MIN
50
124
  * across both windows (the scarcer of 5h/weekly). null percent ⇒ 100 remaining.
51
125
  */
52
- function bindingRemaining(acct: CodexAccountStatus): number {
53
- const primaryRemaining = 100 - (acct.primaryUsedPercent ?? 0);
54
- const secondaryRemaining = 100 - (acct.secondaryUsedPercent ?? 0);
126
+ function bindingRemaining(acct: CodexRotationAccount, now: Date): number {
127
+ const primaryRemaining =
128
+ 100 - effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now);
129
+ const secondaryRemaining =
130
+ 100 - effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now);
55
131
  return Math.min(primaryRemaining, secondaryRemaining);
56
132
  }
57
133
 
58
- function cooling(acct: CodexAccountStatus, now: Date): boolean {
134
+ function cooling(acct: CodexRotationAccount, now: Date): boolean {
59
135
  return acct.exhaustedUntil != null && acct.exhaustedUntil.getTime() > now.getTime();
60
136
  }
61
137
 
@@ -66,7 +142,7 @@ function cooling(acct: CodexAccountStatus, now: Date): boolean {
66
142
  * A null (never-probed) set is UNKNOWN: never credited as covering (Tier 2 only),
67
143
  * but — critically — never excluded, so failover is fully preserved.
68
144
  */
69
- function covers(acct: CodexAccountStatus, usedConnectors: string[]): boolean {
145
+ function covers(acct: CodexRotationAccount, usedConnectors: string[]): boolean {
70
146
  if (usedConnectors.length === 0) {
71
147
  return true;
72
148
  }
@@ -83,7 +159,7 @@ function covers(acct: CodexAccountStatus, usedConnectors: string[]): boolean {
83
159
  * connector" note). Empty when the account covers (superset) or nothing was used.
84
160
  * A null (unknown) set surfaces ALL used connectors — we can't prove it covers them.
85
161
  */
86
- function droppedConnectorsFor(acct: CodexAccountStatus, usedConnectors: string[]): string[] {
162
+ function droppedConnectorsFor(acct: CodexRotationAccount, usedConnectors: string[]): string[] {
87
163
  if (usedConnectors.length === 0) {
88
164
  return [];
89
165
  }
@@ -96,11 +172,139 @@ function droppedConnectorsFor(acct: CodexAccountStatus, usedConnectors: string[]
96
172
  }
97
173
 
98
174
  /**
99
- * Eligible = connected/usable (status "active", excludes needs_relogin/error) AND
100
- * not cooling AND under the near-exhaustion threshold on BOTH windows.
175
+ * Healthy for an already-frozen/in-flight turn = connected/usable (status
176
+ * "active", excludes needs_relogin/error), not cooling, and under the
177
+ * near-exhaustion threshold on BOTH windows. Allocator eligibility is
178
+ * deliberately separate so disabling NEW work cannot break refresh/resume.
179
+ */
180
+ export function isCodexCredentialHealthy(
181
+ acct: CodexRotationAccount,
182
+ nearExhaustionPct: number,
183
+ now: Date,
184
+ ): boolean {
185
+ return (
186
+ acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct, now) < nearExhaustionPct
187
+ );
188
+ }
189
+
190
+ /** New automatic allocation requires both health and explicit allocator eligibility. */
191
+ export function isCodexCredentialEligible(
192
+ acct: CodexRotationAccount,
193
+ nearExhaustionPct: number,
194
+ now: Date,
195
+ ): boolean {
196
+ return acct.allocatorEnabled && isCodexCredentialHealthy(acct, nearExhaustionPct, now);
197
+ }
198
+
199
+ /**
200
+ * Public eligibility predicate (same definition as the private `eligible`): an
201
+ * account is usable this turn iff it is connected/active, not cooling, and under
202
+ * the near-exhaustion threshold on BOTH windows. The worker's sharded home
203
+ * health-check uses this to decide whether a session's existing policy pin is still
204
+ * a valid home or must be re-sharded.
205
+ */
206
+ export function isCodexAccountEligible(
207
+ acct: CodexRotationAccount,
208
+ nearExhaustionPct: number,
209
+ now: Date,
210
+ ): boolean {
211
+ return isCodexCredentialEligible(acct, nearExhaustionPct, now);
212
+ }
213
+
214
+ /** Deterministic 32-bit FNV-1a over a UTF-16 code-unit stream. Pure, allocation-free. */
215
+ function fnv1a32(input: string): number {
216
+ let hash = 0x811c9dc5;
217
+ for (let i = 0; i < input.length; i++) {
218
+ hash ^= input.charCodeAt(i);
219
+ // 32-bit FNV prime multiply via Math.imul; `>>> 0` keeps it unsigned.
220
+ hash = Math.imul(hash, 0x01000193);
221
+ }
222
+ return hash >>> 0;
223
+ }
224
+
225
+ /**
226
+ * Session-sharded HOME account (AM-6): the deterministic account a session runs on
227
+ * under the "sharded" strategy — `stableAccountList[ hash(sessionId) % N ]` over the
228
+ * ELIGIBLE (connected, not cooling, under-threshold) accounts in stable created_at
229
+ * order (`listCodexAccountStatuses`).
230
+ *
231
+ * Why deterministic hash over "least-loaded by session count" (AM-6): it needs ZERO
232
+ * coordination — every worker computes the same home for a given (sessionId,
233
+ * eligible-set), so a burst of concurrent first-turns can't all read the same
234
+ * "least-loaded" account and stampede it (read-then-write skew), and there is no
235
+ * shared round-robin cursor to contend on. It is balanced in expectation.
236
+ *
237
+ * Re-shard (AM-5): pass the just-capped account as already-cooling (or simply let it
238
+ * fall over its threshold) and this reshuffles the SURVIVORS deterministically — the
239
+ * sessions that shared a capped account spread across the remaining pool by their own
240
+ * hashes instead of all re-concentrating on one first-eligible failover.
241
+ *
242
+ * Returns null when NO account is eligible (the caller idles until reset).
101
243
  */
102
- function eligible(acct: CodexAccountStatus, nearExhaustionPct: number, now: Date): boolean {
103
- return acct.status === "active" && !cooling(acct, now) && bindingUsedPct(acct) < nearExhaustionPct;
244
+ export function shardCredentialForSession(args: {
245
+ sessionId: string;
246
+ accounts: CodexRotationAccount[];
247
+ nearExhaustionPct: number;
248
+ now: Date;
249
+ }): string | null {
250
+ const { sessionId, accounts, nearExhaustionPct, now } = args;
251
+ const eligibles = accounts.filter((acct) =>
252
+ isCodexCredentialEligible(acct, nearExhaustionPct, now),
253
+ );
254
+ if (eligibles.length === 0) {
255
+ return null;
256
+ }
257
+ const index = fnv1a32(sessionId) % eligibles.length;
258
+ return eligibles[index]!.id;
259
+ }
260
+
261
+ /** earliestResetAt across ALL connected accounts — exported for the sharded all-capped idle. */
262
+ export function earliestCodexReset(
263
+ accounts: CodexRotationAccount[],
264
+ nearExhaustionPct: number,
265
+ now: Date,
266
+ ): Date {
267
+ return earliestReset(accounts, nearExhaustionPct, now);
268
+ }
269
+
270
+ /**
271
+ * The proactive SHARDED home decision (pure — zero I/O, like {@link chooseRotationActive}).
272
+ * Decides the account a session should run on THIS turn under the "sharded" strategy,
273
+ * given its current POLICY pin (null on the first turn) and the accounts snapshot:
274
+ *
275
+ * - keepPin: the current policy pin is still ELIGIBLE → run there, no rewrite. This
276
+ * is the steady-state cache-warm path (a session stays on its one home account).
277
+ * - reshard: no policy pin yet (first-turn lazy assignment, AM-7) OR the policy pin
278
+ * is capped/ineligible (proactive re-shard, AM-4) → deterministically shard over the
279
+ * ELIGIBLE set (AM-6) and durably (re)write the pin (`rewritePin:true`, AM-3/AM-5).
280
+ * - allCapped: no account is eligible → the caller idles until the earliest reset.
281
+ *
282
+ * A MANUAL pin is handled by the CALLER (it never reaches here); this function only
283
+ * governs policy homes. Pure so the assignment/re-shard/keep logic is unit-testable
284
+ * without a worker/db env; the caller wraps it with a self-heal usage refresh between
285
+ * two evaluations exactly like chooseRotationActive's allCapped path.
286
+ */
287
+ export function chooseShardedHome(args: {
288
+ sessionId: string;
289
+ currentPolicyPin: string | null;
290
+ accounts: CodexRotationAccount[];
291
+ nearExhaustionPct: number;
292
+ now: Date;
293
+ }):
294
+ | { kind: "home"; credentialId: string; rewritePin: boolean }
295
+ | { kind: "allCapped"; earliestResetAt: Date } {
296
+ const { sessionId, currentPolicyPin, accounts, nearExhaustionPct, now } = args;
297
+ const pinRow = currentPolicyPin
298
+ ? (accounts.find((acct) => acct.id === currentPolicyPin) ?? null)
299
+ : null;
300
+ if (pinRow && isCodexCredentialEligible(pinRow, nearExhaustionPct, now)) {
301
+ return { kind: "home", credentialId: currentPolicyPin!, rewritePin: false };
302
+ }
303
+ const home = shardCredentialForSession({ sessionId, accounts, nearExhaustionPct, now });
304
+ if (home == null) {
305
+ return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
306
+ }
307
+ return { kind: "home", credentialId: home, rewritePin: true };
104
308
  }
105
309
 
106
310
  /**
@@ -115,7 +319,11 @@ function eligible(acct: CodexAccountStatus, nearExhaustionPct: number, now: Date
115
319
  * default cooldown (now + DEFAULT_RESET_COOLDOWN_MS), so the caller always idles a
116
320
  * bounded positive time and re-checks (which refreshes usage and self-heals).
117
321
  */
118
- export function availableAt(acct: CodexAccountStatus, nearExhaustionPct: number, now: Date): Date {
322
+ export function availableAt(
323
+ acct: CodexRotationAccount,
324
+ nearExhaustionPct: number,
325
+ now: Date,
326
+ ): Date {
119
327
  const nowMs = now.getTime();
120
328
  // An unknown/elapsed block clears after a default cooldown, never in the past.
121
329
  const defaultClear = new Date(nowMs + DEFAULT_RESET_COOLDOWN_MS);
@@ -131,8 +339,14 @@ export function availableAt(acct: CodexAccountStatus, nearExhaustionPct: number,
131
339
  // reset is unknown → default cooldown (never a past instant).
132
340
  candidates.push(resetAt != null && resetAt.getTime() > nowMs ? resetAt : defaultClear);
133
341
  };
134
- windowClear((acct.primaryUsedPercent ?? 0) >= nearExhaustionPct, acct.primaryResetAt);
135
- windowClear((acct.secondaryUsedPercent ?? 0) >= nearExhaustionPct, acct.secondaryResetAt);
342
+ windowClear(
343
+ effectiveWindowUsed(acct.primaryUsedPercent, acct.primaryResetAt, now) >= nearExhaustionPct,
344
+ acct.primaryResetAt,
345
+ );
346
+ windowClear(
347
+ effectiveWindowUsed(acct.secondaryUsedPercent, acct.secondaryResetAt, now) >= nearExhaustionPct,
348
+ acct.secondaryResetAt,
349
+ );
136
350
  // Ineligible for a non-quota reason (needs_relogin / error) with no known block, or
137
351
  // a cleared cooldown: still idle a bounded cooldown before re-check — never the past.
138
352
  if (candidates.length === 0) {
@@ -143,12 +357,54 @@ export function availableAt(acct: CodexAccountStatus, nearExhaustionPct: number,
143
357
  }
144
358
 
145
359
  /** earliestResetAt across ALL connected accounts (min of each account's availableAt). */
146
- function earliestReset(accounts: CodexAccountStatus[], nearExhaustionPct: number, now: Date): Date {
360
+ function earliestReset(
361
+ accounts: CodexRotationAccount[],
362
+ nearExhaustionPct: number,
363
+ now: Date,
364
+ ): Date {
147
365
  return accounts
148
366
  .map((acct) => availableAt(acct, nearExhaustionPct, now))
149
367
  .reduce((a, b) => (b.getTime() < a.getTime() ? b : a));
150
368
  }
151
369
 
370
+ /**
371
+ * Earliest provider/cooldown timestamp that is actually authoritative. The
372
+ * ranker's `availableAt` deliberately synthesizes a positive default for
373
+ * unknown/stale state; durable capacity waits must distinguish that bounded
374
+ * refresh backoff from a real provider reset timer.
375
+ */
376
+ export function authoritativeCodexCapacityResetAt(
377
+ accounts: CodexRotationAccount[],
378
+ nearExhaustionPct: number,
379
+ now: Date,
380
+ ): Date | null {
381
+ const future: Date[] = [];
382
+ for (const account of accounts) {
383
+ if (!account.allocatorEnabled || account.status !== "active") continue;
384
+ if (account.exhaustedUntil && account.exhaustedUntil.getTime() > now.getTime()) {
385
+ future.push(account.exhaustedUntil);
386
+ }
387
+ if (
388
+ (account.primaryUsedPercent ?? 0) >= nearExhaustionPct &&
389
+ account.primaryResetAt &&
390
+ account.primaryResetAt.getTime() > now.getTime()
391
+ ) {
392
+ future.push(account.primaryResetAt);
393
+ }
394
+ if (
395
+ (account.secondaryUsedPercent ?? 0) >= nearExhaustionPct &&
396
+ account.secondaryResetAt &&
397
+ account.secondaryResetAt.getTime() > now.getTime()
398
+ ) {
399
+ future.push(account.secondaryResetAt);
400
+ }
401
+ }
402
+ if (future.length === 0) return null;
403
+ return future.reduce((earliest, candidate) =>
404
+ candidate.getTime() < earliest.getTime() ? candidate : earliest,
405
+ );
406
+ }
407
+
152
408
  /**
153
409
  * The bounded all-capped idle delay: clamp(earliestResetAt − now) into
154
410
  * [MIN_IDLE_MS, maxMs]. NEVER 0 and NEVER negative, so a null / elapsed / unknown
@@ -237,7 +493,7 @@ export function chooseRotationActive(args: {
237
493
  rotationStrategy: CodexRotationStrategy;
238
494
  activeCredentialId: string | null;
239
495
  priorCredentialId: string | null;
240
- accounts: CodexAccountStatus[];
496
+ accounts: CodexRotationAccount[];
241
497
  nearExhaustionPct: number;
242
498
  now: Date;
243
499
  // P4 (Part B): the connectors the leaving session has access to (the leaving
@@ -246,25 +502,36 @@ export function chooseRotationActive(args: {
246
502
  // P3 (every account trivially covers → Tier 1 == all eligibles). Self-gating.
247
503
  usedConnectors?: string[];
248
504
  }): RotationDecision {
249
- const { rotationStrategy, activeCredentialId, priorCredentialId, accounts, nearExhaustionPct, now } = args;
505
+ const {
506
+ rotationStrategy,
507
+ activeCredentialId,
508
+ priorCredentialId,
509
+ accounts,
510
+ nearExhaustionPct,
511
+ now,
512
+ } = args;
513
+ const allocatableAccounts = accounts.filter((account) => account.allocatorEnabled);
250
514
  const usedConnectors = args.usedConnectors ?? [];
251
515
 
252
- if (accounts.length === 0) {
516
+ if (allocatableAccounts.length === 0) {
253
517
  return { kind: "none" };
254
518
  }
255
519
 
256
- const eligibles = accounts.filter((acct) => eligible(acct, nearExhaustionPct, now));
257
-
258
- // Healthy-active fast path (minimal churn, all strategies): a rotation-enabled
259
- // session whose active account is still eligible does NOT rotate — no pointer
260
- // move, no switch event. Steady-state stays as cheap as a non-rotation turn
261
- // save the in-memory ranking. (Skipped for round_robin/drain which anchor on
262
- // the prior account, but most_remaining is the default + correctness path.)
263
- const activeRow = activeCredentialId ? accounts.find((acct) => acct.id === activeCredentialId) ?? null : null;
520
+ const eligibles = allocatableAccounts.filter((acct) =>
521
+ isCodexCredentialEligible(acct, nearExhaustionPct, now),
522
+ );
264
523
 
265
- const decide = (chosen: CodexAccountStatus | undefined): RotationDecision => {
524
+ // The active pointer is a cursor/manual preference, NOT a sticky lease. In
525
+ // particular, most_remaining must rank the whole eligible pool every turn;
526
+ // keeping a healthy active account until 90% was the production monopolization
527
+ // defect fixed by OPE-21. drain_then_next remains the one explicitly sticky
528
+ // strategy, and a manual pin is handled by the caller as explicit policy.
529
+ const decide = (chosen: CodexRotationAccount | undefined): RotationDecision => {
266
530
  if (!chosen) {
267
- return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
531
+ return {
532
+ kind: "allCapped",
533
+ earliestResetAt: earliestReset(allocatableAccounts, nearExhaustionPct, now),
534
+ };
268
535
  }
269
536
  // P4: surface the dropped-connector note when the chosen account doesn't cover
270
537
  // the session's used connectors (a Tier-2/unknown failover pick). Empty ⇒ omit.
@@ -281,46 +548,275 @@ export function chooseRotationActive(args: {
281
548
  // Next eligible AFTER the prior account in list order (wrap around). When the
282
549
  // prior account isn't found, start from the head.
283
550
  if (eligibles.length === 0) {
284
- return { kind: "allCapped", earliestResetAt: earliestReset(accounts, nearExhaustionPct, now) };
551
+ return {
552
+ kind: "allCapped",
553
+ earliestResetAt: earliestReset(allocatableAccounts, nearExhaustionPct, now),
554
+ };
285
555
  }
286
- const priorIdx = priorCredentialId ? accounts.findIndex((acct) => acct.id === priorCredentialId) : -1;
287
- const ordered = priorIdx >= 0
288
- ? [...accounts.slice(priorIdx + 1), ...accounts.slice(0, priorIdx + 1)]
289
- : accounts;
290
- const chosen = ordered.find((acct) => eligible(acct, nearExhaustionPct, now));
556
+ const priorIdx = priorCredentialId
557
+ ? allocatableAccounts.findIndex((acct) => acct.id === priorCredentialId)
558
+ : -1;
559
+ const ordered =
560
+ priorIdx >= 0
561
+ ? [
562
+ ...allocatableAccounts.slice(priorIdx + 1),
563
+ ...allocatableAccounts.slice(0, priorIdx + 1),
564
+ ]
565
+ : allocatableAccounts;
566
+ const chosen = ordered.find((acct) => isCodexCredentialEligible(acct, nearExhaustionPct, now));
291
567
  return decide(chosen);
292
568
  }
293
569
 
294
570
  if (rotationStrategy === "drain_then_next") {
295
571
  // Stay on the prior account while it is eligible (drain it), else first eligible.
296
- const priorRow = priorCredentialId ? accounts.find((acct) => acct.id === priorCredentialId) : undefined;
297
- if (priorRow && eligible(priorRow, nearExhaustionPct, now)) {
572
+ const priorRow = priorCredentialId
573
+ ? accounts.find((acct) => acct.id === priorCredentialId)
574
+ : undefined;
575
+ if (priorRow && isCodexCredentialEligible(priorRow, nearExhaustionPct, now)) {
298
576
  return decide(priorRow);
299
577
  }
300
578
  return decide(eligibles[0]);
301
579
  }
302
580
 
303
- // most_remaining (default + the correctness path).
304
- // Healthy-active fast path UNCHANGED (no-thrash): a session whose active account
305
- // is still eligible never switches — not even for connector coverage. Coverage
306
- // matters ONLY on the must-move branch below.
307
- if (activeRow && eligible(activeRow, nearExhaustionPct, now)) {
308
- return { kind: "active", credentialId: activeRow.id, moved: false };
309
- }
310
- // Active is capped/near-cap/cooling/missing → must move. Two-tier coverage pick
311
- // over the SAME eligible set (prefer-not-require):
581
+ // most_remaining (default + the correctness path). Two-tier coverage pick over
582
+ // the SAME eligible set (prefer-not-require):
312
583
  // Tier 1 — eligibles whose connector set COVERS the session's used set.
313
584
  // Tier 2 — ALL eligibles, only if Tier 1 is empty (failover fully preserved).
314
- // Within the chosen tier, max bindingRemaining, ties broken by list (created_at)
315
- // order via a stable reduce — byte-identical to P3 when usedConnectors is empty
316
- // (Tier 1 == all eligibles).
585
+ // Within the chosen tier:
586
+ // 1. least active leases (concurrent turns spread before quota metadata moves),
587
+ // 2. most remaining binding quota,
588
+ // 3. fewest historical selections,
589
+ // 4. least-recently selected, then stable created_at/id input order.
590
+ // Base CodexAccountStatus values (pure legacy tests/reactive rank) default the
591
+ // lease/cursor metadata to zero/null, preserving deterministic stable ties.
317
592
  const covering = eligibles.filter((acct) => covers(acct, usedConnectors));
318
593
  const pool = covering.length > 0 ? covering : eligibles;
319
- const chosen = pool.reduce<CodexAccountStatus | undefined>((best, acct) => {
594
+ const activeLeases = (acct: CodexRotationAccount): number =>
595
+ "activeLeaseCount" in acct ? acct.activeLeaseCount : 0;
596
+ const selections = (acct: CodexRotationAccount): number =>
597
+ "selectionCount" in acct ? acct.selectionCount : 0;
598
+ const lastSelected = (acct: CodexRotationAccount): number =>
599
+ "lastSelectedAt" in acct && acct.lastSelectedAt ? acct.lastSelectedAt.getTime() : 0;
600
+ const chosen = pool.reduce<CodexRotationAccount | undefined>((best, acct) => {
320
601
  if (!best) {
321
602
  return acct;
322
603
  }
323
- return bindingRemaining(acct) > bindingRemaining(best) ? acct : best;
604
+ if (activeLeases(acct) !== activeLeases(best)) {
605
+ return activeLeases(acct) < activeLeases(best) ? acct : best;
606
+ }
607
+ if (bindingRemaining(acct, now) !== bindingRemaining(best, now)) {
608
+ return bindingRemaining(acct, now) > bindingRemaining(best, now) ? acct : best;
609
+ }
610
+ if (selections(acct) !== selections(best)) {
611
+ return selections(acct) < selections(best) ? acct : best;
612
+ }
613
+ return lastSelected(acct) < lastSelected(best) ? acct : best;
324
614
  }, undefined);
325
615
  return decide(chosen);
326
616
  }
617
+
618
+ /**
619
+ * Exact pre-0053 selector used while the lease cutover is off.
620
+ *
621
+ * Old workers keep a healthy `most_remaining` active pointer sticky and know
622
+ * nothing about lease counts or fairness cursors. A compatible worker must do
623
+ * the same until BOTH the deployment flag and workspace cutover are enabled;
624
+ * otherwise a rolling fleet would run two allocators at once. When the active
625
+ * pointer is no longer eligible, zeroing the additive metadata delegates to the
626
+ * same capacity/connector ranking that the old selector used.
627
+ */
628
+ export function chooseLegacyRotationActive(
629
+ args: Parameters<typeof chooseRotationActive>[0],
630
+ ): RotationDecision {
631
+ if (args.rotationStrategy === "most_remaining") {
632
+ const active = args.activeCredentialId
633
+ ? args.accounts.find((account) => account.id === args.activeCredentialId)
634
+ : undefined;
635
+ if (active && isCodexCredentialEligible(active, args.nearExhaustionPct, args.now)) {
636
+ return { kind: "active", credentialId: active.id, moved: false };
637
+ }
638
+ }
639
+ return chooseRotationActive({
640
+ ...args,
641
+ accounts: args.accounts.map((account) => ({
642
+ ...account,
643
+ activeLeaseCount: 0,
644
+ selectionCount: 0,
645
+ lastSelectedAt: null,
646
+ })),
647
+ });
648
+ }
649
+
650
+ /**
651
+ * Full pure policy at the transaction boundary. A live same-turn lease is
652
+ * idempotent; the rollout-off/manual path preserves pin>pointer behavior; a
653
+ * healthy pin wins only while eligible, so quota/auth quarantine cannot trap a
654
+ * session on one subscription.
655
+ */
656
+ export function selectCodexCredentialLeaseForTurn(args: {
657
+ context: CodexCredentialLeaseSelectionContext;
658
+ leasingEnabled: boolean;
659
+ sessionId: string;
660
+ sessionPinnedCredentialId: string | null;
661
+ sessionPinSource: CodexPinSource | null;
662
+ sessionLastCredentialId: string | null;
663
+ /** Same-turn checkpoint/frozen account; temporary disable cannot move it. */
664
+ continuationCredentialId?: string | null;
665
+ nearExhaustionPct: number;
666
+ now: Date;
667
+ }): CodexTurnLeaseSelection {
668
+ const {
669
+ accounts,
670
+ activeCredentialId,
671
+ rotationEnabled,
672
+ leaseRotationEnabled,
673
+ rotationStrategy,
674
+ existingCredentialId,
675
+ } = args.context;
676
+ const leasingEnabled = args.leasingEnabled && leaseRotationEnabled;
677
+ const strategy = rotationStrategy as CodexRotationStrategy;
678
+ const existing = existingCredentialId
679
+ ? accounts.find((account) => account.id === existingCredentialId)
680
+ : undefined;
681
+ if (existing && isCodexCredentialHealthy(existing, args.nearExhaustionPct, args.now)) {
682
+ return {
683
+ credentialId: existing.id,
684
+ advanceActivePointer: false,
685
+ decision: {
686
+ kind: "active",
687
+ credentialId: existing.id,
688
+ moved: existing.id !== activeCredentialId,
689
+ },
690
+ };
691
+ }
692
+
693
+ const continuation = args.continuationCredentialId
694
+ ? accounts.find((account) => account.id === args.continuationCredentialId)
695
+ : undefined;
696
+ if (continuation && isCodexCredentialHealthy(continuation, args.nearExhaustionPct, args.now)) {
697
+ return {
698
+ credentialId: continuation.id,
699
+ advanceActivePointer: false,
700
+ decision: {
701
+ kind: "active",
702
+ credentialId: continuation.id,
703
+ moved: continuation.id !== activeCredentialId,
704
+ },
705
+ };
706
+ }
707
+
708
+ const connectedIds = new Set(
709
+ accounts.filter((account) => account.allocatorEnabled).map((account) => account.id),
710
+ );
711
+
712
+ const pinDisposition = classifyCodexPin({
713
+ pinnedCredentialId: args.sessionPinnedCredentialId,
714
+ pinSource: args.sessionPinSource,
715
+ strategy,
716
+ rotationEnabled,
717
+ });
718
+
719
+ // A manual pin is user intent, not a policy hint. Never silently fail it over.
720
+ // We still reject allocator-disabled rows for a NEW turn; healthy live/frozen
721
+ // same-turn continuity already returned above before this admission filter.
722
+ if (pinDisposition === "manual" && args.sessionPinnedCredentialId) {
723
+ const pinned = accounts.find((account) => account.id === args.sessionPinnedCredentialId);
724
+ if (!pinned?.allocatorEnabled) {
725
+ return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
726
+ }
727
+ if (isCodexCredentialHealthy(pinned, args.nearExhaustionPct, args.now)) {
728
+ return {
729
+ credentialId: pinned.id,
730
+ decision: { kind: "active", credentialId: pinned.id, moved: false },
731
+ advanceActivePointer: false,
732
+ };
733
+ }
734
+ return {
735
+ credentialId: null,
736
+ decision: {
737
+ kind: "allCapped",
738
+ earliestResetAt: availableAt(pinned, args.nearExhaustionPct, args.now),
739
+ },
740
+ advanceActivePointer: false,
741
+ };
742
+ }
743
+
744
+ // OPE-31 policy homes compose with OPE-21 by sharding only the candidate list
745
+ // handed to this selector. A future OPE-32 filter may therefore choose one
746
+ // primary/fallback pool before this ranker runs; accounts from different pools
747
+ // are never union-ranked here.
748
+ if (pinDisposition === "sharded") {
749
+ if (accounts.length === 0) {
750
+ return { credentialId: null, decision: { kind: "none" }, advanceActivePointer: false };
751
+ }
752
+ const shard = chooseShardedHome({
753
+ sessionId: args.sessionId,
754
+ currentPolicyPin: args.sessionPinSource === "policy" ? args.sessionPinnedCredentialId : null,
755
+ accounts,
756
+ nearExhaustionPct: args.nearExhaustionPct,
757
+ now: args.now,
758
+ });
759
+ if (shard.kind === "allCapped") {
760
+ return {
761
+ credentialId: null,
762
+ decision: shard,
763
+ advanceActivePointer: false,
764
+ };
765
+ }
766
+ return {
767
+ credentialId: shard.credentialId,
768
+ decision: {
769
+ kind: "active",
770
+ credentialId: shard.credentialId,
771
+ moved: shard.credentialId !== activeCredentialId,
772
+ },
773
+ advanceActivePointer: false,
774
+ };
775
+ }
776
+
777
+ // A stale policy pin is intentionally ignored here and cleared by the caller
778
+ // after the atomic selection. Rotation-off/unpinned behavior otherwise keeps
779
+ // the pre-lease pin > active-pointer contract.
780
+ if (!rotationEnabled) {
781
+ const credentialId =
782
+ activeCredentialId && connectedIds.has(activeCredentialId) ? activeCredentialId : null;
783
+ return {
784
+ credentialId,
785
+ decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" },
786
+ };
787
+ }
788
+
789
+ // Before the workspace cutover bit is enabled (or after the deployment kill
790
+ // switch is turned off), preserve the exact legacy policy used by old workers:
791
+ // a pin stays sticky; otherwise an enabled workspace runs the pure legacy
792
+ // rotation ranker without touching lease/cursor state.
793
+ if (!leasingEnabled && pinDisposition !== "clearStale" && args.sessionPinnedCredentialId) {
794
+ const credentialId = connectedIds.has(args.sessionPinnedCredentialId)
795
+ ? args.sessionPinnedCredentialId
796
+ : activeCredentialId && connectedIds.has(activeCredentialId)
797
+ ? activeCredentialId
798
+ : null;
799
+ return {
800
+ credentialId,
801
+ decision: credentialId ? { kind: "active", credentialId, moved: false } : { kind: "none" },
802
+ };
803
+ }
804
+
805
+ const priorId = args.sessionLastCredentialId ?? activeCredentialId;
806
+ const choose = leasingEnabled ? chooseRotationActive : chooseLegacyRotationActive;
807
+ const decision = choose({
808
+ rotationStrategy: strategy,
809
+ activeCredentialId,
810
+ // Per-session continuity is the round-robin/drain cursor. Falling back to
811
+ // the workspace pointer is only correct when this session has never run.
812
+ priorCredentialId: priorId,
813
+ accounts,
814
+ nearExhaustionPct: args.nearExhaustionPct,
815
+ now: args.now,
816
+ usedConnectors: accounts.find((account) => account.id === priorId)?.connectorNamespaces ?? [],
817
+ });
818
+ return {
819
+ credentialId: decision.kind === "active" ? decision.credentialId : null,
820
+ decision,
821
+ };
822
+ }